1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.layout;
18
19 import java.nio.charset.Charset;
20 import java.nio.charset.StandardCharsets;
21 import java.util.ArrayList;
22 import java.util.Calendar;
23 import java.util.GregorianCalendar;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.SortedMap;
28 import java.util.TreeMap;
29 import java.util.regex.Matcher;
30 import java.util.regex.Pattern;
31
32 import org.apache.logging.log4j.Level;
33 import org.apache.logging.log4j.LoggingException;
34 import org.apache.logging.log4j.core.Layout;
35 import org.apache.logging.log4j.core.LogEvent;
36 import org.apache.logging.log4j.core.appender.TlsSyslogFrame;
37 import org.apache.logging.log4j.core.config.Configuration;
38 import org.apache.logging.log4j.core.config.Node;
39 import org.apache.logging.log4j.core.config.plugins.Plugin;
40 import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
41 import org.apache.logging.log4j.core.config.plugins.PluginConfiguration;
42 import org.apache.logging.log4j.core.config.plugins.PluginElement;
43 import org.apache.logging.log4j.core.config.plugins.PluginFactory;
44 import org.apache.logging.log4j.core.net.Facility;
45 import org.apache.logging.log4j.core.net.Priority;
46 import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
47 import org.apache.logging.log4j.core.pattern.PatternConverter;
48 import org.apache.logging.log4j.core.pattern.PatternFormatter;
49 import org.apache.logging.log4j.core.pattern.PatternParser;
50 import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter;
51 import org.apache.logging.log4j.core.util.NetUtils;
52 import org.apache.logging.log4j.core.util.Patterns;
53 import org.apache.logging.log4j.message.Message;
54 import org.apache.logging.log4j.message.StructuredDataId;
55 import org.apache.logging.log4j.message.StructuredDataMessage;
56 import org.apache.logging.log4j.util.StringBuilders;
57 import org.apache.logging.log4j.util.Strings;
58
59
60
61
62
63
64 @Plugin(name = "Rfc5424Layout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
65 public final class Rfc5424Layout extends AbstractStringLayout {
66
67
68
69
70 public static final int DEFAULT_ENTERPRISE_NUMBER = 18060;
71
72
73
74 public static final String DEFAULT_ID = "Audit";
75
76
77
78 public static final Pattern NEWLINE_PATTERN = Pattern.compile("\\r?\\n");
79
80
81
82 public static final Pattern PARAM_VALUE_ESCAPE_PATTERN = Pattern.compile("[\\\"\\]\\\\]");
83
84
85
86
87 public static final String DEFAULT_MDCID = "mdc";
88
89 private static final String LF = "\n";
90 private static final int TWO_DIGITS = 10;
91 private static final int THREE_DIGITS = 100;
92 private static final int MILLIS_PER_MINUTE = 60000;
93 private static final int MINUTES_PER_HOUR = 60;
94 private static final String COMPONENT_KEY = "RFC5424-Converter";
95
96 private final Facility facility;
97 private final String defaultId;
98 private final int enterpriseNumber;
99 private final boolean includeMdc;
100 private final String mdcId;
101 private final StructuredDataId mdcSdId;
102 private final String localHostName;
103 private final String appName;
104 private final String messageId;
105 private final String configName;
106 private final String mdcPrefix;
107 private final String eventPrefix;
108 private final List<String> mdcExcludes;
109 private final List<String> mdcIncludes;
110 private final List<String> mdcRequired;
111 private final ListChecker listChecker;
112 private final ListChecker noopChecker = new NoopChecker();
113 private final boolean includeNewLine;
114 private final String escapeNewLine;
115 private final boolean useTlsMessageFormat;
116
117 private long lastTimestamp = -1;
118 private String timestamppStr;
119
120 private final List<PatternFormatter> exceptionFormatters;
121 private final Map<String, FieldFormatter> fieldFormatters;
122 private final String procId;
123
124 private Rfc5424Layout(final Configuration config, final Facility facility, final String id, final int ein,
125 final boolean includeMDC, final boolean includeNL, final String escapeNL, final String mdcId,
126 final String mdcPrefix, final String eventPrefix, final String appName, final String messageId,
127 final String excludes, final String includes, final String required, final Charset charset,
128 final String exceptionPattern, final boolean useTLSMessageFormat, final LoggerFields[] loggerFields) {
129 super(charset);
130 final PatternParser exceptionParser = createPatternParser(config, ThrowablePatternConverter.class);
131 exceptionFormatters = exceptionPattern == null ? null : exceptionParser.parse(exceptionPattern, false, false);
132 this.facility = facility;
133 this.defaultId = id == null ? DEFAULT_ID : id;
134 this.enterpriseNumber = ein;
135 this.includeMdc = includeMDC;
136 this.includeNewLine = includeNL;
137 this.escapeNewLine = escapeNL == null ? null : Matcher.quoteReplacement(escapeNL);
138 this.mdcId = mdcId;
139 this.mdcSdId = new StructuredDataId(mdcId, enterpriseNumber, null, null);
140 this.mdcPrefix = mdcPrefix;
141 this.eventPrefix = eventPrefix;
142 this.appName = appName;
143 this.messageId = messageId;
144 this.useTlsMessageFormat = useTLSMessageFormat;
145 this.localHostName = NetUtils.getLocalHostname();
146 ListChecker c = null;
147 if (excludes != null) {
148 final String[] array = excludes.split(Patterns.COMMA_SEPARATOR);
149 if (array.length > 0) {
150 c = new ExcludeChecker();
151 mdcExcludes = new ArrayList<>(array.length);
152 for (final String str : array) {
153 mdcExcludes.add(str.trim());
154 }
155 } else {
156 mdcExcludes = null;
157 }
158 } else {
159 mdcExcludes = null;
160 }
161 if (includes != null) {
162 final String[] array = includes.split(Patterns.COMMA_SEPARATOR);
163 if (array.length > 0) {
164 c = new IncludeChecker();
165 mdcIncludes = new ArrayList<>(array.length);
166 for (final String str : array) {
167 mdcIncludes.add(str.trim());
168 }
169 } else {
170 mdcIncludes = null;
171 }
172 } else {
173 mdcIncludes = null;
174 }
175 if (required != null) {
176 final String[] array = required.split(Patterns.COMMA_SEPARATOR);
177 if (array.length > 0) {
178 mdcRequired = new ArrayList<>(array.length);
179 for (final String str : array) {
180 mdcRequired.add(str.trim());
181 }
182 } else {
183 mdcRequired = null;
184 }
185
186 } else {
187 mdcRequired = null;
188 }
189 this.listChecker = c != null ? c : noopChecker;
190 final String name = config == null ? null : config.getName();
191 configName = Strings.isNotEmpty(name) ? name : null;
192 this.fieldFormatters = createFieldFormatters(loggerFields, config);
193
194 this.procId = "-";
195 }
196
197 private Map<String, FieldFormatter> createFieldFormatters(final LoggerFields[] loggerFields,
198 final Configuration config) {
199 final Map<String, FieldFormatter> sdIdMap = new HashMap<>();
200
201 if (loggerFields != null) {
202 for (final LoggerFields lField : loggerFields) {
203 final StructuredDataId key = lField.getSdId() == null ? mdcSdId : lField.getSdId();
204 final Map<String, List<PatternFormatter>> sdParams = new HashMap<>();
205 final Map<String, String> fields = lField.getMap();
206 if (!fields.isEmpty()) {
207 final PatternParser fieldParser = createPatternParser(config, null);
208
209 for (final Map.Entry<String, String> entry : fields.entrySet()) {
210 final List<PatternFormatter> formatters = fieldParser.parse(entry.getValue(), false, false);
211 sdParams.put(entry.getKey(), formatters);
212 }
213 final FieldFormatter fieldFormatter = new FieldFormatter(sdParams,
214 lField.getDiscardIfAllFieldsAreEmpty());
215 sdIdMap.put(key.toString(), fieldFormatter);
216 }
217 }
218 }
219 return sdIdMap.size() > 0 ? sdIdMap : null;
220 }
221
222
223
224
225
226
227
228
229 private static PatternParser createPatternParser(final Configuration config,
230 final Class<? extends PatternConverter> filterClass) {
231 if (config == null) {
232 return new PatternParser(config, PatternLayout.KEY, LogEventPatternConverter.class, filterClass);
233 }
234 PatternParser parser = config.getComponent(COMPONENT_KEY);
235 if (parser == null) {
236 parser = new PatternParser(config, PatternLayout.KEY, ThrowablePatternConverter.class);
237 config.addComponent(COMPONENT_KEY, parser);
238 parser = config.getComponent(COMPONENT_KEY);
239 }
240 return parser;
241 }
242
243
244
245
246
247
248
249
250
251
252 @Override
253 public Map<String, String> getContentFormat() {
254 final Map<String, String> result = new HashMap<>();
255 result.put("structured", "true");
256 result.put("formatType", "RFC5424");
257 return result;
258 }
259
260
261
262
263
264
265
266 @Override
267 public String toSerializable(final LogEvent event) {
268 final StringBuilder buf = getStringBuilder();
269 appendPriority(buf, event.getLevel());
270 appendTimestamp(buf, event.getTimeMillis());
271 appendSpace(buf);
272 appendHostName(buf);
273 appendSpace(buf);
274 appendAppName(buf);
275 appendSpace(buf);
276 appendProcessId(buf);
277 appendSpace(buf);
278 appendMessageId(buf, event.getMessage());
279 appendSpace(buf);
280 appendStructuredElements(buf, event);
281 appendMessage(buf, event);
282 if (useTlsMessageFormat) {
283 return new TlsSyslogFrame(buf.toString()).toString();
284 }
285 return buf.toString();
286 }
287
288 private void appendPriority(final StringBuilder buffer, final Level logLevel) {
289 buffer.append('<');
290 buffer.append(Priority.getPriority(facility, logLevel));
291 buffer.append(">1 ");
292 }
293
294 private void appendTimestamp(final StringBuilder buffer, final long milliseconds) {
295 buffer.append(computeTimeStampString(milliseconds));
296 }
297
298 private void appendSpace(final StringBuilder buffer) {
299 buffer.append(' ');
300 }
301
302 private void appendHostName(final StringBuilder buffer) {
303 buffer.append(localHostName);
304 }
305
306 private void appendAppName(final StringBuilder buffer) {
307 if (appName != null) {
308 buffer.append(appName);
309 } else if (configName != null) {
310 buffer.append(configName);
311 } else {
312 buffer.append('-');
313 }
314 }
315
316 private void appendProcessId(final StringBuilder buffer) {
317 buffer.append(getProcId());
318 }
319
320 private void appendMessageId(final StringBuilder buffer, final Message message) {
321 final boolean isStructured = message instanceof StructuredDataMessage;
322 final String type = isStructured ? ((StructuredDataMessage) message).getType() : null;
323 if (type != null) {
324 buffer.append(type);
325 } else if (messageId != null) {
326 buffer.append(messageId);
327 } else {
328 buffer.append('-');
329 }
330 }
331
332 private void appendMessage(final StringBuilder buffer, final LogEvent event) {
333 final Message message = event.getMessage();
334
335 final String text = (message instanceof StructuredDataMessage) ? message.getFormat() : message
336 .getFormattedMessage();
337
338 if (text != null && text.length() > 0) {
339 buffer.append(' ').append(escapeNewlines(text, escapeNewLine));
340 }
341
342 if (exceptionFormatters != null && event.getThrown() != null) {
343 final StringBuilder exception = new StringBuilder(LF);
344 for (final PatternFormatter formatter : exceptionFormatters) {
345 formatter.format(event, exception);
346 }
347 buffer.append(escapeNewlines(exception.toString(), escapeNewLine));
348 }
349 if (includeNewLine) {
350 buffer.append(LF);
351 }
352 }
353
354 private void appendStructuredElements(final StringBuilder buffer, final LogEvent event) {
355 final Message message = event.getMessage();
356 final boolean isStructured = message instanceof StructuredDataMessage;
357
358 if (!isStructured && (fieldFormatters != null && fieldFormatters.isEmpty()) && !includeMdc) {
359 buffer.append('-');
360 return;
361 }
362
363 final Map<String, StructuredDataElement> sdElements = new HashMap<>();
364 final Map<String, String> contextMap = event.getContextMap();
365
366 if (mdcRequired != null) {
367 checkRequired(contextMap);
368 }
369
370 if (fieldFormatters != null) {
371 for (final Map.Entry<String, FieldFormatter> sdElement : fieldFormatters.entrySet()) {
372 final String sdId = sdElement.getKey();
373 final StructuredDataElement elem = sdElement.getValue().format(event);
374 sdElements.put(sdId, elem);
375 }
376 }
377
378 if (includeMdc && contextMap.size() > 0) {
379 final String mdcSdIdStr = mdcSdId.toString();
380 final StructuredDataElement union = sdElements.get(mdcSdIdStr);
381 if (union != null) {
382 union.union(contextMap);
383 sdElements.put(mdcSdIdStr, union);
384 } else {
385 final StructuredDataElement formattedContextMap = new StructuredDataElement(contextMap, false);
386 sdElements.put(mdcSdIdStr, formattedContextMap);
387 }
388 }
389
390 if (isStructured) {
391 final StructuredDataMessage data = (StructuredDataMessage) message;
392 final Map<String, String> map = data.getData();
393 final StructuredDataId id = data.getId();
394 final String sdId = getId(id);
395
396 if (sdElements.containsKey(sdId)) {
397 final StructuredDataElement union = sdElements.get(id.toString());
398 union.union(map);
399 sdElements.put(sdId, union);
400 } else {
401 final StructuredDataElement formattedData = new StructuredDataElement(map, false);
402 sdElements.put(sdId, formattedData);
403 }
404 }
405
406 if (sdElements.isEmpty()) {
407 buffer.append('-');
408 return;
409 }
410
411 for (final Map.Entry<String, StructuredDataElement> entry : sdElements.entrySet()) {
412 formatStructuredElement(entry.getKey(), mdcPrefix, entry.getValue(), buffer, listChecker);
413 }
414 }
415
416 private String escapeNewlines(final String text, final String replacement) {
417 if (null == replacement) {
418 return text;
419 }
420 return NEWLINE_PATTERN.matcher(text).replaceAll(replacement);
421 }
422
423 protected String getProcId() {
424 return procId;
425 }
426
427 protected List<String> getMdcExcludes() {
428 return mdcExcludes;
429 }
430
431 protected List<String> getMdcIncludes() {
432 return mdcIncludes;
433 }
434
435 private String computeTimeStampString(final long now) {
436 long last;
437 synchronized (this) {
438 last = lastTimestamp;
439 if (now == lastTimestamp) {
440 return timestamppStr;
441 }
442 }
443
444 final StringBuilder buffer = new StringBuilder();
445 final Calendar cal = new GregorianCalendar();
446 cal.setTimeInMillis(now);
447 buffer.append(Integer.toString(cal.get(Calendar.YEAR)));
448 buffer.append('-');
449 pad(cal.get(Calendar.MONTH) + 1, TWO_DIGITS, buffer);
450 buffer.append('-');
451 pad(cal.get(Calendar.DAY_OF_MONTH), TWO_DIGITS, buffer);
452 buffer.append('T');
453 pad(cal.get(Calendar.HOUR_OF_DAY), TWO_DIGITS, buffer);
454 buffer.append(':');
455 pad(cal.get(Calendar.MINUTE), TWO_DIGITS, buffer);
456 buffer.append(':');
457 pad(cal.get(Calendar.SECOND), TWO_DIGITS, buffer);
458 buffer.append('.');
459 pad(cal.get(Calendar.MILLISECOND), THREE_DIGITS, buffer);
460
461 int tzmin = (cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET)) / MILLIS_PER_MINUTE;
462 if (tzmin == 0) {
463 buffer.append('Z');
464 } else {
465 if (tzmin < 0) {
466 tzmin = -tzmin;
467 buffer.append('-');
468 } else {
469 buffer.append('+');
470 }
471 final int tzhour = tzmin / MINUTES_PER_HOUR;
472 tzmin -= tzhour * MINUTES_PER_HOUR;
473 pad(tzhour, TWO_DIGITS, buffer);
474 buffer.append(':');
475 pad(tzmin, TWO_DIGITS, buffer);
476 }
477 synchronized (this) {
478 if (last == lastTimestamp) {
479 lastTimestamp = now;
480 timestamppStr = buffer.toString();
481 }
482 }
483 return buffer.toString();
484 }
485
486 private void pad(final int val, int max, final StringBuilder buf) {
487 while (max > 1) {
488 if (val < max) {
489 buf.append('0');
490 }
491 max = max / TWO_DIGITS;
492 }
493 buf.append(Integer.toString(val));
494 }
495
496 private void formatStructuredElement(final String id, final String prefix, final StructuredDataElement data,
497 final StringBuilder sb, final ListChecker checker) {
498 if ((id == null && defaultId == null) || data.discard()) {
499 return;
500 }
501
502 sb.append('[');
503 sb.append(id);
504 if (!mdcSdId.toString().equals(id)) {
505 appendMap(prefix, data.getFields(), sb, noopChecker);
506 } else {
507 appendMap(prefix, data.getFields(), sb, checker);
508 }
509 sb.append(']');
510 }
511
512 private String getId(final StructuredDataId id) {
513 final StringBuilder sb = new StringBuilder();
514 if (id == null || id.getName() == null) {
515 sb.append(defaultId);
516 } else {
517 sb.append(id.getName());
518 }
519 int ein = id != null ? id.getEnterpriseNumber() : enterpriseNumber;
520 if (ein < 0) {
521 ein = enterpriseNumber;
522 }
523 if (ein >= 0) {
524 sb.append('@').append(ein);
525 }
526 return sb.toString();
527 }
528
529 private void checkRequired(final Map<String, String> map) {
530 for (final String key : mdcRequired) {
531 final String value = map.get(key);
532 if (value == null) {
533 throw new LoggingException("Required key " + key + " is missing from the " + mdcId);
534 }
535 }
536 }
537
538 private void appendMap(final String prefix, final Map<String, String> map, final StringBuilder sb,
539 final ListChecker checker) {
540 final SortedMap<String, String> sorted = new TreeMap<>(map);
541 for (final Map.Entry<String, String> entry : sorted.entrySet()) {
542 if (checker.check(entry.getKey()) && entry.getValue() != null) {
543 sb.append(' ');
544 if (prefix != null) {
545 sb.append(prefix);
546 }
547 final String safeKey = escapeNewlines(escapeSDParams(entry.getKey()), escapeNewLine);
548 final String safeValue = escapeNewlines(escapeSDParams(entry.getValue()), escapeNewLine);
549 StringBuilders.appendKeyDqValue(sb, safeKey, safeValue);
550 }
551 }
552 }
553
554 private String escapeSDParams(final String value) {
555 return PARAM_VALUE_ESCAPE_PATTERN.matcher(value).replaceAll("\\\\$0");
556 }
557
558
559
560
561 private interface ListChecker {
562 boolean check(String key);
563 }
564
565
566
567
568 private class IncludeChecker implements ListChecker {
569 @Override
570 public boolean check(final String key) {
571 return mdcIncludes.contains(key);
572 }
573 }
574
575
576
577
578 private class ExcludeChecker implements ListChecker {
579 @Override
580 public boolean check(final String key) {
581 return !mdcExcludes.contains(key);
582 }
583 }
584
585
586
587
588 private class NoopChecker implements ListChecker {
589 @Override
590 public boolean check(final String key) {
591 return true;
592 }
593 }
594
595 @Override
596 public String toString() {
597 final StringBuilder sb = new StringBuilder();
598 sb.append("facility=").append(facility.name());
599 sb.append(" appName=").append(appName);
600 sb.append(" defaultId=").append(defaultId);
601 sb.append(" enterpriseNumber=").append(enterpriseNumber);
602 sb.append(" newLine=").append(includeNewLine);
603 sb.append(" includeMDC=").append(includeMdc);
604 sb.append(" messageId=").append(messageId);
605 return sb.toString();
606 }
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632 @PluginFactory
633 public static Rfc5424Layout createLayout(
634
635 @PluginAttribute(value = "facility", defaultString = "LOCAL0") final Facility facility,
636 @PluginAttribute("id") final String id,
637 @PluginAttribute(value = "enterpriseNumber", defaultInt = DEFAULT_ENTERPRISE_NUMBER)
638 final int enterpriseNumber,
639 @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC,
640 @PluginAttribute(value = "mdcId", defaultString = DEFAULT_MDCID) final String mdcId,
641 @PluginAttribute("mdcPrefix") final String mdcPrefix,
642 @PluginAttribute("eventPrefix") final String eventPrefix,
643 @PluginAttribute(value = "newLine", defaultBoolean = false) final boolean newLine,
644 @PluginAttribute("newLineEscape") final String escapeNL,
645 @PluginAttribute("appName") final String appName,
646 @PluginAttribute("messageId") final String msgId,
647 @PluginAttribute("mdcExcludes") final String excludes,
648 @PluginAttribute("mdcIncludes") String includes,
649 @PluginAttribute("mdcRequired") final String required,
650 @PluginAttribute("exceptionPattern") final String exceptionPattern,
651
652 @PluginAttribute(value = "useTlsMessageFormat", defaultBoolean = false) final boolean useTlsMessageFormat,
653 @PluginElement("LoggerFields") final LoggerFields[] loggerFields,
654 @PluginConfiguration final Configuration config) {
655
656 if (includes != null && excludes != null) {
657 LOGGER.error("mdcIncludes and mdcExcludes are mutually exclusive. Includes wil be ignored");
658 includes = null;
659 }
660
661 return new Rfc5424Layout(config, facility, id, enterpriseNumber, includeMDC, newLine, escapeNL, mdcId,
662 mdcPrefix, eventPrefix, appName, msgId, excludes, includes, required, StandardCharsets.UTF_8,
663 exceptionPattern, useTlsMessageFormat, loggerFields);
664 }
665
666 private class FieldFormatter {
667
668 private final Map<String, List<PatternFormatter>> delegateMap;
669 private final boolean discardIfEmpty;
670
671 public FieldFormatter(final Map<String, List<PatternFormatter>> fieldMap, final boolean discardIfEmpty) {
672 this.discardIfEmpty = discardIfEmpty;
673 this.delegateMap = fieldMap;
674 }
675
676 public StructuredDataElement format(final LogEvent event) {
677 final Map<String, String> map = new HashMap<>();
678
679 for (final Map.Entry<String, List<PatternFormatter>> entry : delegateMap.entrySet()) {
680 final StringBuilder buffer = new StringBuilder();
681 for (final PatternFormatter formatter : entry.getValue()) {
682 formatter.format(event, buffer);
683 }
684 map.put(entry.getKey(), buffer.toString());
685 }
686 return new StructuredDataElement(map, discardIfEmpty);
687 }
688 }
689
690 private class StructuredDataElement {
691
692 private final Map<String, String> fields;
693 private final boolean discardIfEmpty;
694
695 public StructuredDataElement(final Map<String, String> fields, final boolean discardIfEmpty) {
696 this.discardIfEmpty = discardIfEmpty;
697 this.fields = fields;
698 }
699
700 boolean discard() {
701 if (discardIfEmpty == false) {
702 return false;
703 }
704 boolean foundNotEmptyValue = false;
705 for (final Map.Entry<String, String> entry : fields.entrySet()) {
706 if (Strings.isNotEmpty(entry.getValue())) {
707 foundNotEmptyValue = true;
708 break;
709 }
710 }
711 return !foundNotEmptyValue;
712 }
713
714 void union(final Map<String, String> addFields) {
715 this.fields.putAll(addFields);
716 }
717
718 Map<String, String> getFields() {
719 return this.fields;
720 }
721 }
722 }