[Logstash] logstash filter date 조금 알아보기

문의가 들어 왔습니다.

여러 필드에 대해서 date format 이 다른데 어떻게 적용을 해야 하나요?

 

그래서 소스코드를 열어 보고 아래와 같이 해보라고 했습니다.

date {
...
}

date {
...
}

결국 date {...} 를 필드 별로 선언을 해주면 되는 내용입니다.

 

공식 문서)

https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html

 

Common Options)

https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-common-options

 

Date Filter Configuration Options)

Setting	Input type	Required
locale	string	No
match	array	No
tag_on_failure	array	No
target	string	No
timezone	string	No

전체 옵션이 필수가 아니긴 합니다.

그래도 꼭 아셔야 하는 설정은 match, target 입니다.

 

- match 의 첫 번째 값은 field 명이고, 그 이후는 format 들이 되겠습니다.

(공식 문서에 잘 나와 있습니다.)

 

An array with field name first, and format patterns following, [ field, formats... ]

If your time field has multiple possible formats, you can do this:

 

match => [ "logdate",      "MMM dd yyyy HH:mm:ss",      "MMM d yyyy HH:mm:ss",      "ISO8601" ]

 

- target 은 지정을 하지 않게 되면 기본 @timestamp 필드로 설정이 됩니다. 변경 하고자 하면 target 에 원하시는 field name 을 넣으시면 됩니다.

 

예제)

date {
    match => ["time" , "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"]
    target => "@timestamp"
}

date {
    match => ["localtime" , "yyyy-MM-dd HH:mm:ssZ"]
    target => "time"
}

DateFilter.java)

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.logstash.filters;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.Instant;
import org.logstash.Event;
import org.logstash.ext.JrubyEventExtLibrary.RubyEvent;
import org.logstash.filters.parser.CasualISO8601Parser;
import org.logstash.filters.parser.JodaParser;
import org.logstash.filters.parser.TimestampParser;
import org.logstash.filters.parser.TimestampParserFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class DateFilter {
  private static Logger logger = LogManager.getLogger();
  private final String sourceField;
  private final String[] tagOnFailure;
  private RubyResultHandler successHandler;
  private RubyResultHandler failureHandler;
  private final List<ParserExecutor> executors = new ArrayList<>();
  private final ResultSetter setter;

  public interface RubyResultHandler {
    void handle(RubyEvent event);
  }

  public DateFilter(String sourceField, String targetField, List<String> tagOnFailure, RubyResultHandler successHandler, RubyResultHandler failureHandler) {
    this(sourceField, targetField, tagOnFailure);
    this.successHandler = successHandler;
    this.failureHandler = failureHandler;
  }

  public DateFilter(String sourceField, String targetField, List<String> tagOnFailure) {
    this.sourceField = sourceField;
    this.tagOnFailure = tagOnFailure.toArray(new String[0]);
    if (targetField.equals("@timestamp")) {
      this.setter = new TimestampSetter();
    } else {
      this.setter = new FieldSetter(targetField);
    }
  }

  public void acceptFilterConfig(String format, String locale, String timezone) {
    TimestampParser parser = TimestampParserFactory.makeParser(format, locale, timezone);
    logger.debug("Date filter with format={}, locale={}, timezone={} built as {}", format, locale, timezone, parser.getClass().getName());
    if (parser instanceof JodaParser || parser instanceof CasualISO8601Parser) {
      executors.add(new TextParserExecutor(parser, timezone));
    } else {
      executors.add(new NumericParserExecutor(parser));
    }
  }

 public List<RubyEvent> receive(List<RubyEvent> rubyEvents) {
    for (RubyEvent rubyEvent : rubyEvents) {
      Event event = rubyEvent.getEvent();

      switch (executeParsers(event)) {
        case FIELD_VALUE_IS_NULL_OR_FIELD_NOT_PRESENT:
        case IGNORED:
          continue;
        case SUCCESS:
          if (successHandler != null) {
            successHandler.handle(rubyEvent);
          }
          break;
        case FAIL: // fall through
        default:
          for (String t : tagOnFailure) {
            event.tag(t);
          }
          if (failureHandler != null) {
            failureHandler.handle(rubyEvent);
          }
      }
    }
    return rubyEvents;
  }

  public ParseExecutionResult executeParsers(Event event) {
    Object input = event.getField(sourceField);
    if (event.isCancelled()) { return ParseExecutionResult.IGNORED; }
    if (input == null) { return ParseExecutionResult.FIELD_VALUE_IS_NULL_OR_FIELD_NOT_PRESENT; }

    for (ParserExecutor executor : executors) {
      try {
        Instant instant = executor.execute(input, event);
        setter.set(event, instant);
        return ParseExecutionResult.SUCCESS;
      } catch (IllegalArgumentException | IOException e) {
        // do nothing, try next ParserExecutor
      }
    }
    return ParseExecutionResult.FAIL;
  }
}