문의가 들어 왔습니다.
여러 필드에 대해서 date format 이 다른데 어떻게 적용을 해야 하나요?
그래서 소스코드를 열어 보고 아래와 같이 해보라고 했습니다.
date {
...
}
date {
...
}
결국 date {...} 를 필드 별로 선언을 해주면 되는 내용입니다.
공식 문서)
https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html
Common Options)
https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-common-options
Date Filter Configuration Options)
|
Setting
|
Input type
|
Required
|
|
locale
|
string
|
No
|
|
match
|
array
|
No
|
|
tag_on_failure
|
array
|
No
|
|
target
|
string
|
No
|
|
timezone
|
string
|
No
|
전체 옵션이 필수가 아니긴 합니다.
그래도 꼭 아셔야 하는 설정은 match, target 입니다.
- match 의 첫 번째 값은 field 명이고, 그 이후는 format 들이 되겠습니다.
(공식 문서에 잘 나와 있습니다.)
An array with field name first, and format patterns following, [ field, formats... ]
If your time field has multiple possible formats, you can do this:
|
match => [ "logdate",
"MMM dd yyyy HH:mm:ss",
"MMM d yyyy HH:mm:ss",
"ISO8601" ]
|
- target 은 지정을 하지 않게 되면 기본 @timestamp 필드로 설정이 됩니다. 변경 하고자 하면 target 에 원하시는 field name 을 넣으시면 됩니다.
예제)
date {
match => ["time" , "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ"]
target => "@timestamp"
}
date {
match => ["localtime" , "yyyy-MM-dd HH:mm:ssZ"]
target => "time"
}
DateFilter.java)
더보기
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.logstash.filters;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.Instant;
import org.logstash.Event;
import org.logstash.ext.JrubyEventExtLibrary.RubyEvent;
import org.logstash.filters.parser.CasualISO8601Parser;
import org.logstash.filters.parser.JodaParser;
import org.logstash.filters.parser.TimestampParser;
import org.logstash.filters.parser.TimestampParserFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class DateFilter {
private static Logger logger = LogManager.getLogger();
private final String sourceField;
private final String[] tagOnFailure;
private RubyResultHandler successHandler;
private RubyResultHandler failureHandler;
private final List<ParserExecutor> executors = new ArrayList<>();
private final ResultSetter setter;
public interface RubyResultHandler {
void handle(RubyEvent event);
}
public DateFilter(String sourceField, String targetField, List<String> tagOnFailure, RubyResultHandler successHandler, RubyResultHandler failureHandler) {
this(sourceField, targetField, tagOnFailure);
this.successHandler = successHandler;
this.failureHandler = failureHandler;
}
public DateFilter(String sourceField, String targetField, List<String> tagOnFailure) {
this.sourceField = sourceField;
this.tagOnFailure = tagOnFailure.toArray(new String[0]);
if (targetField.equals("@timestamp")) {
this.setter = new TimestampSetter();
} else {
this.setter = new FieldSetter(targetField);
}
}
public void acceptFilterConfig(String format, String locale, String timezone) {
TimestampParser parser = TimestampParserFactory.makeParser(format, locale, timezone);
logger.debug("Date filter with format={}, locale={}, timezone={} built as {}", format, locale, timezone, parser.getClass().getName());
if (parser instanceof JodaParser || parser instanceof CasualISO8601Parser) {
executors.add(new TextParserExecutor(parser, timezone));
} else {
executors.add(new NumericParserExecutor(parser));
}
}
public List<RubyEvent> receive(List<RubyEvent> rubyEvents) {
for (RubyEvent rubyEvent : rubyEvents) {
Event event = rubyEvent.getEvent();
switch (executeParsers(event)) {
case FIELD_VALUE_IS_NULL_OR_FIELD_NOT_PRESENT:
case IGNORED:
continue;
case SUCCESS:
if (successHandler != null) {
successHandler.handle(rubyEvent);
}
break;
case FAIL: // fall through
default:
for (String t : tagOnFailure) {
event.tag(t);
}
if (failureHandler != null) {
failureHandler.handle(rubyEvent);
}
}
}
return rubyEvents;
}
public ParseExecutionResult executeParsers(Event event) {
Object input = event.getField(sourceField);
if (event.isCancelled()) { return ParseExecutionResult.IGNORED; }
if (input == null) { return ParseExecutionResult.FIELD_VALUE_IS_NULL_OR_FIELD_NOT_PRESENT; }
for (ParserExecutor executor : executors) {
try {
Instant instant = executor.execute(input, event);
setter.set(event, instant);
return ParseExecutionResult.SUCCESS;
} catch (IllegalArgumentException | IOException e) {
// do nothing, try next ParserExecutor
}
}
return ParseExecutionResult.FAIL;
}
}
