Loading...

Test Pipeline

Example Pipeline

PUT _ingest/pipeline/logs
{
  "description": "Logs Processor",
  "processors": [
    {
      "convert": {
        "field": "duration",
        "type": "integer",
        "ignore_missing": true
      }
    },
    {
      "convert": {
        "field": "message_count",
        "type": "integer",
        "ignore_missing": true
      }
    },
    {
      "convert": {
        "field": "message_duration",
        "type": "integer",
        "ignore_missing": true
      }
    },
    {
      "convert": {
        "field": "response_status",
        "type": "integer",
        "ignore_missing": true
      }
    },
    {
      "script": {
        "lang": "painless",
        "inline": "ctx.host = ctx.beat.hostname"
      }
    },
    {
      "date": {
        "field": "datetime",
        "target_field": "@timestamp",
        "formats": [
          "yyyy-MM-dd'T'HH:mm:ss.SSSZ",
          "ISO8601",
          "yyyy-MM-dd HH:mm:ss,SSS",
          "yyyy-MM-dd, HH:mm:ss.SSS",
          "yyy-MM-dd HH:mm:ss",
          "yyyy-MM-dd",
          "'['dd/MMM/yyyy:HH:mm:ss Z']'"
        ],
        "timezone": "Europe/Zurich"
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error",
        "value": " on operation: "
      }
    }
  ]
}

Test Pipeline with example data

POST _ingest/pipeline/logs/_simulate
{
  "docs": [
    {
      "_index": "index",
      "_type": "type",
      "_id": "id",
      "_source": {
        "datetime": "2017-05-11T23:59:45.248+0200",
        "logger": "mapper.metrics",
        "metric": {
          "transactionUuid": "4053715f-43df-4fe6-afe6-b99c2c7ec06b",
          "threadName": "EJB default - 3",
          "type": "metric",
          "method": "financialAdvice",
          "name": "Monster Inc",
          "totalDuration": 501,
          "result": "OK"
        }
      }
    }
  ]
}

Ingested Output with error since no filebeat hostname was given

{
  "docs": [
    {
      "doc": {
        "_id": "id",
        "_index": "index",
        "_type": "type",
        "_source": {
          "datetime": "2017-05-11T23:59:45.248+0200",
          "metric": {
            "transactionUuid": "4053715f-43df-4fe6-afe6-b99c2c7ec06b",
            "name": "Monster Inc",
            "method": "financialAdvice",
            "type": "metric",
            "result": "OK",
            "threadName": "EJB default - 3",
            "totalDuration": 501
          },
          "logger": "mapper.metrics",
          "error": " on operation: script"
        },
        "_ingest": {
          "timestamp": "2017-05-12T21:24:31.598Z"
        }
      }
    }
  ]
}