Loading...

Cardinality Aggregations in Elasticsearch

Used:

  • elasticsearch v6.2.4

This post demonstrates the usage of the Cardinality feature.

A single-value metrics aggregation that calculates an approximate count of distinct values.

For example I needed to check how many users are using my Elasticsearch via Kibana cluster. Kibana queries by users are of type MultiSearchRequest. The underlying data is audit data from X-Pack security.

POST six-audit-2018.06.01/_search?size=0
{
  "query": {
    "match": {
      "request": "MultiSearchRequest"
    }
  }, 
  "aggs": {
    "users_count": {
      "cardinality": {
        "field": "principal"
      }
    }
  }
}

Example output:

{ 
  "aggregations": {
    "users_count": {
      "value": 40
    }
  }
}

You can combine it with other aggregations like the bucket date_histogram aggregation. If you want to know the number of users distributed over the day in 24 hours.

POST six-audit-2018.06.01/_search?size=0
{
  "query": {
    "match": {
      "request": "MultiSearchRequest"
    }
  },
  "aggs": {
    "users_over_time": {
      "date_histogram": {
        "field": "@timestamp",
        "interval": "1h"
      },
      "aggs": {
        "users_count": {
          "cardinality": {
            "field": "principal"
          }
        }
      }
    }
  }
}

Example output:

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 26001,
    "max_score": 0,
    "hits": []
  },
  "aggregations": {
    "users_over_time": {
      "buckets": [
        {
          "key_as_string": "2018-06-01T00:00:00.000Z",
          "key": 1527811200000,
          "doc_count": 1172,
          "users_count": {
            "value": 2
          }
        },
        {
          "key_as_string": "2018-06-01T01:00:00.000Z",
          "key": 1527814800000,
          "doc_count": 744,
          "users_count": {
            "value": 3
          }
        },
        {
          "key_as_string": "2018-06-01T02:00:00.000Z",
          "key": 1527818400000,
          "doc_count": 1287,
          "users_count": {
            "value": 2
          }
        },
        {
          "key_as_string": "2018-06-01T03:00:00.000Z",
          "key": 1527822000000,
          "doc_count": 710,
          "users_count": {
            "value": 2
          }
        },
        {
          "key_as_string": "2018-06-01T04:00:00.000Z",
          "key": 1527825600000,
          "doc_count": 385,
          "users_count": {
            "value": 2
          }
        },
        {
          "key_as_string": "2018-06-01T05:00:00.000Z",
          "key": 1527829200000,
          "doc_count": 162,
          "users_count": {
            "value": 6
          }
        },
        {
          "key_as_string": "2018-06-01T06:00:00.000Z",
          "key": 1527832800000,
          "doc_count": 160,
          "users_count": {
            "value": 9
          }
        },
        {
          "key_as_string": "2018-06-01T07:00:00.000Z",
          "key": 1527836400000,
          "doc_count": 869,
          "users_count": {
            "value": 9
          }
        },
        {
          "key_as_string": "2018-06-01T08:00:00.000Z",
          "key": 1527840000000,
          "doc_count": 1190,
          "users_count": {
            "value": 13
          }
        },
        {
          "key_as_string": "2018-06-01T09:00:00.000Z",
          "key": 1527843600000,
          "doc_count": 862,
          "users_count": {
            "value": 12
          }
        },
        {
          "key_as_string": "2018-06-01T10:00:00.000Z",
          "key": 1527847200000,
          "doc_count": 772,
          "users_count": {
            "value": 10
          }
        },
        {
          "key_as_string": "2018-06-01T11:00:00.000Z",
          "key": 1527850800000,
          "doc_count": 1000,
          "users_count": {
            "value": 11
          }
        },
        {
          "key_as_string": "2018-06-01T12:00:00.000Z",
          "key": 1527854400000,
          "doc_count": 946,
          "users_count": {
            "value": 11
          }
        },
        {
          "key_as_string": "2018-06-01T13:00:00.000Z",
          "key": 1527858000000,
          "doc_count": 1334,
          "users_count": {
            "value": 17
          }
        },
        {
          "key_as_string": "2018-06-01T14:00:00.000Z",
          "key": 1527861600000,
          "doc_count": 2475,
          "users_count": {
            "value": 20
          }
        },
        {
          "key_as_string": "2018-06-01T15:00:00.000Z",
          "key": 1527865200000,
          "doc_count": 2158,
          "users_count": {
            "value": 11
          }
        },
        {
          "key_as_string": "2018-06-01T16:00:00.000Z",
          "key": 1527868800000,
          "doc_count": 1708,
          "users_count": {
            "value": 9
          }
        },
        {
          "key_as_string": "2018-06-01T17:00:00.000Z",
          "key": 1527872400000,
          "doc_count": 1801,
          "users_count": {
            "value": 6
          }
        },
        {
          "key_as_string": "2018-06-01T18:00:00.000Z",
          "key": 1527876000000,
          "doc_count": 1692,
          "users_count": {
            "value": 5
          }
        },
        {
          "key_as_string": "2018-06-01T19:00:00.000Z",
          "key": 1527879600000,
          "doc_count": 890,
          "users_count": {
            "value": 5
          }
        },
        {
          "key_as_string": "2018-06-01T20:00:00.000Z",
          "key": 1527883200000,
          "doc_count": 731,
          "users_count": {
            "value": 7
          }
        },
        {
          "key_as_string": "2018-06-01T21:00:00.000Z",
          "key": 1527886800000,
          "doc_count": 488,
          "users_count": {
            "value": 3
          }
        },
        {
          "key_as_string": "2018-06-01T22:00:00.000Z",
          "key": 1527890400000,
          "doc_count": 1167,
          "users_count": {
            "value": 3
          }
        },
        {
          "key_as_string": "2018-06-01T23:00:00.000Z",
          "key": 1527894000000,
          "doc_count": 1298,
          "users_count": {
            "value": 4
          }
        }
      ]
    }
  }
}

The peak with 20 concurrent users is from 14:00 to 15:00.