1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.layout;
18
19 import java.nio.charset.Charset;
20 import java.util.ArrayList;
21 import java.util.Calendar;
22 import java.util.GregorianCalendar;
23 import java.util.HashMap;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.SortedMap;
27 import java.util.TreeMap;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30
31 import org.apache.logging.log4j.Level;
32 import org.apache.logging.log4j.LoggingException;
33 import org.apache.logging.log4j.core.LogEvent;
34 import org.apache.logging.log4j.core.appender.TlsSyslogFrame;
35 import org.apache.logging.log4j.core.config.Configuration;
36 import org.apache.logging.log4j.core.config.plugins.Plugin;
37 import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
38 import org.apache.logging.log4j.core.config.plugins.PluginConfiguration;
39 import org.apache.logging.log4j.core.config.plugins.PluginElement;
40 import org.apache.logging.log4j.core.config.plugins.PluginFactory;
41 import org.apache.logging.log4j.core.net.Facility;
42 import org.apache.logging.log4j.core.net.Priority;
43 import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
44 import org.apache.logging.log4j.core.pattern.PatternConverter;
45 import org.apache.logging.log4j.core.pattern.PatternFormatter;
46 import org.apache.logging.log4j.core.pattern.PatternParser;
47 import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter;
48 import org.apache.logging.log4j.core.util.Charsets;
49 import org.apache.logging.log4j.core.util.NetUtils;
50 import org.apache.logging.log4j.core.util.Patterns;
51 import org.apache.logging.log4j.message.Message;
52 import org.apache.logging.log4j.message.StructuredDataId;
53 import org.apache.logging.log4j.message.StructuredDataMessage;
54 import org.apache.logging.log4j.util.Strings;
55
56
57
58
59
60
61
62 @Plugin(name = "Rfc5424Layout", category = "Core", elementType = "layout", printObject = true)
63 public final class Rfc5424Layout extends AbstractStringLayout {
64
65 private static final String LF = "\n";
66
67
68
69
70 public static final int DEFAULT_ENTERPRISE_NUMBER = 18060;
71
72
73
74 public static final String DEFAULT_ID = "Audit";
75
76
77
78 public static final Pattern NEWLINE_PATTERN = Pattern.compile("\\r?\\n");
79
80
81
82 public static final Pattern PARAM_VALUE_ESCAPE_PATTERN = Pattern.compile("[\\\"\\]\\\\]");
83
84 public static final String DEFAULT_MDCID = "mdc";
85 private static final int TWO_DIGITS = 10;
86 private static final int THREE_DIGITS = 100;
87 private static final int MILLIS_PER_MINUTE = 60000;
88 private static final int MINUTES_PER_HOUR = 60;
89
90 private static final String COMPONENT_KEY = "RFC5424-Converter";
91
92 private final Facility facility;
93 private final String defaultId;
94 private final int enterpriseNumber;
95 private final boolean includeMdc;
96 private final String mdcId;
97 private final StructuredDataId mdcSdId;
98 private final String localHostName;
99 private final String appName;
100 private final String messageId;
101 private final String configName;
102 private final String mdcPrefix;
103 private final String eventPrefix;
104 private final List<String> mdcExcludes;
105 private final List<String> mdcIncludes;
106 private final List<String> mdcRequired;
107 private final ListChecker checker;
108 private final ListChecker noopChecker = new NoopChecker();
109 private final boolean includeNewLine;
110 private final String escapeNewLine;
111 private final boolean useTlsMessageFormat;
112
113 private long lastTimestamp = -1;
114 private String timestamppStr;
115
116 private final List<PatternFormatter> exceptionFormatters;
117 private final Map<String, FieldFormatter> fieldFormatters;
118
119 private Rfc5424Layout(final Configuration config, final Facility facility, final String id, final int ein,
120 final boolean includeMDC, final boolean includeNL, final String escapeNL, final String mdcId,
121 final String mdcPrefix, final String eventPrefix,
122 final String appName, final String messageId, final String excludes, final String includes,
123 final String required, final Charset charset, final String exceptionPattern,
124 final boolean useTLSMessageFormat, final LoggerFields[] loggerFields) {
125 super(charset);
126 final PatternParser exceptionParser = createPatternParser(config, ThrowablePatternConverter.class);
127 exceptionFormatters = exceptionPattern == null ? null : exceptionParser.parse(exceptionPattern, false, false);
128 this.facility = facility;
129 this.defaultId = id == null ? DEFAULT_ID : id;
130 this.enterpriseNumber = ein;
131 this.includeMdc = includeMDC;
132 this.includeNewLine = includeNL;
133 this.escapeNewLine = escapeNL == null ? null : Matcher.quoteReplacement(escapeNL);
134 this.mdcId = mdcId;
135 this.mdcSdId = new StructuredDataId(mdcId, enterpriseNumber, null, null);
136 this.mdcPrefix = mdcPrefix;
137 this.eventPrefix = eventPrefix;
138 this.appName = appName;
139 this.messageId = messageId;
140 this.useTlsMessageFormat = useTLSMessageFormat;
141 this.localHostName = NetUtils.getLocalHostname();
142 ListChecker c = null;
143 if (excludes != null) {
144 final String[] array = excludes.split(Patterns.COMMA_SEPARATOR);
145 if (array.length > 0) {
146 c = new ExcludeChecker();
147 mdcExcludes = new ArrayList<String>(array.length);
148 for (final String str : array) {
149 mdcExcludes.add(str.trim());
150 }
151 } else {
152 mdcExcludes = null;
153 }
154 } else {
155 mdcExcludes = null;
156 }
157 if (includes != null) {
158 final String[] array = includes.split(Patterns.COMMA_SEPARATOR);
159 if (array.length > 0) {
160 c = new IncludeChecker();
161 mdcIncludes = new ArrayList<String>(array.length);
162 for (final String str : array) {
163 mdcIncludes.add(str.trim());
164 }
165 } else {
166 mdcIncludes = null;
167 }
168 } else {
169 mdcIncludes = null;
170 }
171 if (required != null) {
172 final String[] array = required.split(Patterns.COMMA_SEPARATOR);
173 if (array.length > 0) {
174 mdcRequired = new ArrayList<String>(array.length);
175 for (final String str : array) {
176 mdcRequired.add(str.trim());
177 }
178 } else {
179 mdcRequired = null;
180 }
181
182 } else {
183 mdcRequired = null;
184 }
185 this.checker = c != null ? c : noopChecker;
186 final String name = config == null ? null : config.getName();
187 configName = name != null && name.length() > 0 ? name : null;
188 this.fieldFormatters = createFieldFormatters(loggerFields, config);
189 }
190
191 private Map<String, FieldFormatter> createFieldFormatters(final LoggerFields[] loggerFields,
192 final Configuration config) {
193 final Map<String, FieldFormatter> sdIdMap = new HashMap<String, FieldFormatter>();
194
195 if (loggerFields != null) {
196 for (final LoggerFields lField : loggerFields) {
197 final StructuredDataId key = lField.getSdId() == null ? mdcSdId : lField.getSdId();
198 final Map<String, List<PatternFormatter>> sdParams = new HashMap<String, List<PatternFormatter>>();
199 final Map<String, String> fields = lField.getMap();
200 if (!fields.isEmpty()) {
201 final PatternParser fieldParser = createPatternParser(config, null);
202
203 for (final Map.Entry<String, String> entry : fields.entrySet()) {
204 final List<PatternFormatter> formatters = fieldParser.parse(entry.getValue(), false, false);
205 sdParams.put(entry.getKey(), formatters);
206 }
207 final FieldFormatter fieldFormatter = new FieldFormatter(sdParams,
208 lField.getDiscardIfAllFieldsAreEmpty());
209 sdIdMap.put(key.toString(), fieldFormatter);
210 }
211 }
212 }
213 return sdIdMap.size() > 0 ? sdIdMap : null;
214 }
215
216
217
218
219
220
221
222
223 private static PatternParser createPatternParser(final Configuration config,
224 final Class<? extends PatternConverter> filterClass) {
225 if (config == null) {
226 return new PatternParser(config, PatternLayout.KEY, LogEventPatternConverter.class, filterClass);
227 }
228 PatternParser parser = config.getComponent(COMPONENT_KEY);
229 if (parser == null) {
230 parser = new PatternParser(config, PatternLayout.KEY, ThrowablePatternConverter.class);
231 config.addComponent(COMPONENT_KEY, parser);
232 parser = (PatternParser) config.getComponent(COMPONENT_KEY);
233 }
234 return parser;
235 }
236
237
238
239
240
241
242
243
244 @Override
245 public Map<String, String> getContentFormat() {
246 final Map<String, String> result = new HashMap<String, String>();
247 result.put("structured", "true");
248 result.put("formatType", "RFC5424");
249 return result;
250 }
251
252
253
254
255
256
257
258 @Override
259 public String toSerializable(final LogEvent event) {
260 final StringBuilder buf = new StringBuilder();
261 appendPriority(buf, event.getLevel());
262 appendTimestamp(buf, event.getTimeMillis());
263 appendSpace(buf);
264 appendHostName(buf);
265 appendSpace(buf);
266 appendAppName(buf);
267 appendSpace(buf);
268 appendProcessId(buf);
269 appendSpace(buf);
270 appendMessageId(buf, event.getMessage());
271 appendSpace(buf);
272 appendStructuredElements(buf, event);
273 appendMessage(buf, event);
274 if (useTlsMessageFormat) {
275 return new TlsSyslogFrame(buf.toString()).toString();
276 }
277 return buf.toString();
278 }
279
280 private void appendPriority(final StringBuilder buffer, final Level logLevel) {
281 buffer.append('<');
282 buffer.append(Priority.getPriority(facility, logLevel));
283 buffer.append(">1 ");
284 }
285
286 private void appendTimestamp(final StringBuilder buffer, final long milliseconds) {
287 buffer.append(computeTimeStampString(milliseconds));
288 }
289
290 private void appendSpace(final StringBuilder buffer) {
291 buffer.append(' ');
292 }
293
294 private void appendHostName(final StringBuilder buffer) {
295 buffer.append(localHostName);
296 }
297
298 private void appendAppName(final StringBuilder buffer) {
299 if (appName != null) {
300 buffer.append(appName);
301 } else if (configName != null) {
302 buffer.append(configName);
303 } else {
304 buffer.append('-');
305 }
306 }
307
308 private void appendProcessId(final StringBuilder buffer) {
309 buffer.append(getProcId());
310 }
311
312 private void appendMessageId(final StringBuilder buffer, final Message message) {
313 final boolean isStructured = message instanceof StructuredDataMessage;
314 final String type = isStructured ? ((StructuredDataMessage) message).getType() : null;
315 if (type != null) {
316 buffer.append(type);
317 } else if (messageId != null) {
318 buffer.append(messageId);
319 } else {
320 buffer.append('-');
321 }
322 }
323
324 private void appendMessage(final StringBuilder buffer, final LogEvent event) {
325 final Message message = event.getMessage();
326
327 final String text = (message instanceof StructuredDataMessage) ? message.getFormat() : message.getFormattedMessage();
328
329 if (text != null && text.length() > 0) {
330 buffer.append(' ').append(escapeNewlines(text, escapeNewLine));
331 }
332
333 if (exceptionFormatters != null && event.getThrown() != null) {
334 final StringBuilder exception = new StringBuilder(LF);
335 for (final PatternFormatter formatter : exceptionFormatters) {
336 formatter.format(event, exception);
337 }
338 buffer.append(escapeNewlines(exception.toString(), escapeNewLine));
339 }
340 if (includeNewLine) {
341 buffer.append(LF);
342 }
343 }
344
345 private void appendStructuredElements(final StringBuilder buffer, final LogEvent event) {
346 final Message message = event.getMessage();
347 final boolean isStructured = message instanceof StructuredDataMessage;
348
349 if (!isStructured && (fieldFormatters!= null && fieldFormatters.isEmpty()) && !includeMdc) {
350 buffer.append('-');
351 return;
352 }
353
354 final Map<String, StructuredDataElement> sdElements = new HashMap<String, StructuredDataElement>();
355 final Map<String, String> contextMap = event.getContextMap();
356
357 if (mdcRequired != null) {
358 checkRequired(contextMap);
359 }
360
361 if (fieldFormatters != null) {
362 for (final Map.Entry<String, FieldFormatter> sdElement: fieldFormatters.entrySet()) {
363 final String sdId = sdElement.getKey();
364 final StructuredDataElement elem = sdElement.getValue().format(event);
365 sdElements.put(sdId, elem);
366 }
367 }
368
369 if (includeMdc && contextMap.size() > 0) {
370 if (sdElements.containsKey(mdcSdId.toString())) {
371 final StructuredDataElement union = sdElements.get(mdcSdId.toString());
372 union.union(contextMap);
373 sdElements.put(mdcSdId.toString(), union);
374 } else {
375 final StructuredDataElement formattedContextMap = new StructuredDataElement(contextMap, false);
376 sdElements.put(mdcSdId.toString(), formattedContextMap);
377 }
378 }
379
380 if (isStructured) {
381 final StructuredDataMessage data = (StructuredDataMessage) message;
382 final Map<String, String> map = data.getData();
383 final StructuredDataId id = data.getId();
384 final String sdId = getId(id);
385
386 if (sdElements.containsKey(sdId)) {
387 final StructuredDataElement union = sdElements.get(id.toString());
388 union.union(map);
389 sdElements.put(sdId, union);
390 } else {
391 final StructuredDataElement formattedData = new StructuredDataElement(map, false);
392 sdElements.put(sdId, formattedData);
393 }
394 }
395
396 if (sdElements.isEmpty()) {
397 buffer.append('-');
398 return;
399 }
400
401 for (final Map.Entry<String, StructuredDataElement> entry: sdElements.entrySet()) {
402 formatStructuredElement(entry.getKey(), mdcPrefix, entry.getValue(), buffer, checker);
403 }
404 }
405
406 private String escapeNewlines(final String text, final String escapeNewLine) {
407 if (null == escapeNewLine) {
408 return text;
409 }
410 return NEWLINE_PATTERN.matcher(text).replaceAll(escapeNewLine);
411 }
412
413 protected String getProcId() {
414 return "-";
415 }
416
417 protected List<String> getMdcExcludes() {
418 return mdcExcludes;
419 }
420
421 protected List<String> getMdcIncludes() {
422 return mdcIncludes;
423 }
424
425 private String computeTimeStampString(final long now) {
426 long last;
427 synchronized (this) {
428 last = lastTimestamp;
429 if (now == lastTimestamp) {
430 return timestamppStr;
431 }
432 }
433
434 final StringBuilder buffer = new StringBuilder();
435 final Calendar cal = new GregorianCalendar();
436 cal.setTimeInMillis(now);
437 buffer.append(Integer.toString(cal.get(Calendar.YEAR)));
438 buffer.append('-');
439 pad(cal.get(Calendar.MONTH) + 1, TWO_DIGITS, buffer);
440 buffer.append('-');
441 pad(cal.get(Calendar.DAY_OF_MONTH), TWO_DIGITS, buffer);
442 buffer.append('T');
443 pad(cal.get(Calendar.HOUR_OF_DAY), TWO_DIGITS, buffer);
444 buffer.append(':');
445 pad(cal.get(Calendar.MINUTE), TWO_DIGITS, buffer);
446 buffer.append(':');
447 pad(cal.get(Calendar.SECOND), TWO_DIGITS, buffer);
448
449 final int millis = cal.get(Calendar.MILLISECOND);
450 if (millis != 0) {
451 buffer.append('.');
452 pad(millis, THREE_DIGITS, buffer);
453 }
454
455 int tzmin = (cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET)) / MILLIS_PER_MINUTE;
456 if (tzmin == 0) {
457 buffer.append('Z');
458 } else {
459 if (tzmin < 0) {
460 tzmin = -tzmin;
461 buffer.append('-');
462 } else {
463 buffer.append('+');
464 }
465 final int tzhour = tzmin / MINUTES_PER_HOUR;
466 tzmin -= tzhour * MINUTES_PER_HOUR;
467 pad(tzhour, TWO_DIGITS, buffer);
468 buffer.append(':');
469 pad(tzmin, TWO_DIGITS, buffer);
470 }
471 synchronized (this) {
472 if (last == lastTimestamp) {
473 lastTimestamp = now;
474 timestamppStr = buffer.toString();
475 }
476 }
477 return buffer.toString();
478 }
479
480 private void pad(final int val, int max, final StringBuilder buf) {
481 while (max > 1) {
482 if (val < max) {
483 buf.append('0');
484 }
485 max = max / TWO_DIGITS;
486 }
487 buf.append(Integer.toString(val));
488 }
489
490 private void formatStructuredElement(final String id, final String prefix, final StructuredDataElement data,
491 final StringBuilder sb, final ListChecker checker) {
492 if ((id == null && defaultId == null) || data.discard()) {
493 return;
494 }
495
496 sb.append('[');
497 sb.append(id);
498 if (!mdcSdId.toString().equals(id)) {
499 appendMap(prefix, data.getFields(), sb, noopChecker);
500 } else {
501 appendMap(prefix, data.getFields(), sb, checker);
502 }
503 sb.append(']');
504 }
505
506 private String getId(final StructuredDataId id) {
507 final StringBuilder sb = new StringBuilder();
508 if (id == null || id.getName() == null) {
509 sb.append(defaultId);
510 } else {
511 sb.append(id.getName());
512 }
513 int ein = id != null ? id.getEnterpriseNumber() : enterpriseNumber;
514 if (ein < 0) {
515 ein = enterpriseNumber;
516 }
517 if (ein >= 0) {
518 sb.append('@').append(ein);
519 }
520 return sb.toString();
521 }
522
523 private void checkRequired(final Map<String, String> map) {
524 for (final String key : mdcRequired) {
525 final String value = map.get(key);
526 if (value == null) {
527 throw new LoggingException("Required key " + key + " is missing from the " + mdcId);
528 }
529 }
530 }
531
532 private void appendMap(final String prefix, final Map<String, String> map, final StringBuilder sb,
533 final ListChecker checker) {
534 final SortedMap<String, String> sorted = new TreeMap<String, String>(map);
535 for (final Map.Entry<String, String> entry : sorted.entrySet()) {
536 if (checker.check(entry.getKey()) && entry.getValue() != null) {
537 sb.append(' ');
538 if (prefix != null) {
539 sb.append(prefix);
540 }
541 sb.append(escapeNewlines(escapeSDParams(entry.getKey()), escapeNewLine)).append("=\"")
542 .append(escapeNewlines(escapeSDParams(entry.getValue()), escapeNewLine)).append('"');
543 }
544 }
545 }
546
547 private String escapeSDParams(final String value) {
548 return PARAM_VALUE_ESCAPE_PATTERN.matcher(value).replaceAll("\\\\$0");
549 }
550
551
552
553
554 private interface ListChecker {
555 boolean check(String key);
556 }
557
558
559
560
561 private class IncludeChecker implements ListChecker {
562 @Override
563 public boolean check(final String key) {
564 return mdcIncludes.contains(key);
565 }
566 }
567
568
569
570
571 private class ExcludeChecker implements ListChecker {
572 @Override
573 public boolean check(final String key) {
574 return !mdcExcludes.contains(key);
575 }
576 }
577
578
579
580
581 private class NoopChecker implements ListChecker {
582 @Override
583 public boolean check(final String key) {
584 return true;
585 }
586 }
587
588 @Override
589 public String toString() {
590 final StringBuilder sb = new StringBuilder();
591 sb.append("facility=").append(facility.name());
592 sb.append(" appName=").append(appName);
593 sb.append(" defaultId=").append(defaultId);
594 sb.append(" enterpriseNumber=").append(enterpriseNumber);
595 sb.append(" newLine=").append(includeNewLine);
596 sb.append(" includeMDC=").append(includeMdc);
597 sb.append(" messageId=").append(messageId);
598 return sb.toString();
599 }
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625 @PluginFactory
626 public static Rfc5424Layout createLayout(
627 @PluginAttribute(value = "facility", defaultString = "LOCAL0") final Facility facility,
628 @PluginAttribute("id") final String id,
629 @PluginAttribute(value = "enterpriseNumber", defaultInt = DEFAULT_ENTERPRISE_NUMBER) final int enterpriseNumber,
630 @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC,
631 @PluginAttribute(value = "mdcId", defaultString = DEFAULT_MDCID) final String mdcId,
632 @PluginAttribute("mdcPrefix") final String mdcPrefix,
633 @PluginAttribute("eventPrefix") final String eventPrefix,
634 @PluginAttribute(value = "newLine", defaultBoolean = false) final boolean newLine,
635 @PluginAttribute("newLineEscape") final String escapeNL,
636 @PluginAttribute("appName") final String appName,
637 @PluginAttribute("messageId") final String msgId,
638 @PluginAttribute("mdcExcludes") final String excludes,
639 @PluginAttribute("mdcIncludes") String includes,
640 @PluginAttribute("mdcRequired") final String required,
641 @PluginAttribute("exceptionPattern") final String exceptionPattern,
642 @PluginAttribute(value = "useTlsMessageFormat", defaultBoolean = false) final boolean useTlsMessageFormat,
643 @PluginElement("LoggerFields") final LoggerFields[] loggerFields,
644 @PluginConfiguration final Configuration config) {
645 final Charset charset = Charsets.UTF_8;
646 if (includes != null && excludes != null) {
647 LOGGER.error("mdcIncludes and mdcExcludes are mutually exclusive. Includes wil be ignored");
648 includes = null;
649 }
650
651 return new Rfc5424Layout(config, facility, id, enterpriseNumber, includeMDC, newLine, escapeNL, mdcId, mdcPrefix,
652 eventPrefix, appName, msgId, excludes, includes, required, charset, exceptionPattern,
653 useTlsMessageFormat, loggerFields);
654 }
655
656 private class FieldFormatter {
657
658 private final Map<String, List<PatternFormatter>> delegateMap;
659 private final boolean discardIfEmpty;
660
661 public FieldFormatter(final Map<String, List<PatternFormatter>> fieldMap, final boolean discardIfEmpty) {
662 this.discardIfEmpty = discardIfEmpty;
663 this.delegateMap = fieldMap;
664 }
665
666 public StructuredDataElement format(final LogEvent event) {
667 final Map<String, String> map = new HashMap<String, String>();
668
669 for (final Map.Entry<String, List<PatternFormatter>> entry : delegateMap.entrySet()) {
670 final StringBuilder buffer = new StringBuilder();
671 for (final PatternFormatter formatter : entry.getValue()) {
672 formatter.format(event, buffer);
673 }
674 map.put(entry.getKey(), buffer.toString());
675 }
676 return new StructuredDataElement(map, discardIfEmpty);
677 }
678 }
679
680 private class StructuredDataElement {
681
682 private final Map<String, String> fields;
683 private final boolean discardIfEmpty;
684
685 public StructuredDataElement(final Map<String, String> fields, final boolean discardIfEmpty) {
686 this.discardIfEmpty = discardIfEmpty;
687 this.fields = fields;
688 }
689
690 boolean discard() {
691 if (discardIfEmpty == false) {
692 return false;
693 }
694 boolean foundNotEmptyValue = false;
695 for (final Map.Entry<String, String> entry : fields.entrySet()) {
696 if (Strings.isNotEmpty(entry.getValue())) {
697 foundNotEmptyValue = true;
698 break;
699 }
700 }
701 return !foundNotEmptyValue;
702 }
703
704 void union(final Map<String, String> fields) {
705 this.fields.putAll(fields);
706 }
707
708 Map<String, String> getFields() {
709 return this.fields;
710 }
711 }
712 }