1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.layout;
18
19 import java.nio.charset.Charset;
20 import java.nio.charset.StandardCharsets;
21 import java.util.ArrayList;
22 import java.util.Calendar;
23 import java.util.GregorianCalendar;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.SortedMap;
28 import java.util.TreeMap;
29 import java.util.regex.Matcher;
30 import java.util.regex.Pattern;
31
32 import org.apache.logging.log4j.Level;
33 import org.apache.logging.log4j.LoggingException;
34 import org.apache.logging.log4j.core.Layout;
35 import org.apache.logging.log4j.core.LogEvent;
36 import org.apache.logging.log4j.core.appender.TlsSyslogFrame;
37 import org.apache.logging.log4j.core.config.Configuration;
38 import org.apache.logging.log4j.core.config.Node;
39 import org.apache.logging.log4j.core.config.plugins.Plugin;
40 import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
41 import org.apache.logging.log4j.core.config.plugins.PluginConfiguration;
42 import org.apache.logging.log4j.core.config.plugins.PluginElement;
43 import org.apache.logging.log4j.core.config.plugins.PluginFactory;
44 import org.apache.logging.log4j.core.net.Facility;
45 import org.apache.logging.log4j.core.net.Priority;
46 import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
47 import org.apache.logging.log4j.core.pattern.PatternConverter;
48 import org.apache.logging.log4j.core.pattern.PatternFormatter;
49 import org.apache.logging.log4j.core.pattern.PatternParser;
50 import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter;
51 import org.apache.logging.log4j.core.util.NetUtils;
52 import org.apache.logging.log4j.core.util.Patterns;
53 import org.apache.logging.log4j.message.Message;
54 import org.apache.logging.log4j.message.StructuredDataId;
55 import org.apache.logging.log4j.message.StructuredDataMessage;
56 import org.apache.logging.log4j.util.StringBuilders;
57 import org.apache.logging.log4j.util.Strings;
58
59
60
61
62
63
64 @Plugin(name = "Rfc5424Layout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
65 public final class Rfc5424Layout extends AbstractStringLayout {
66
67
68
69
70 public static final int DEFAULT_ENTERPRISE_NUMBER = 18060;
71
72
73
74 public static final String DEFAULT_ID = "Audit";
75
76
77
78 public static final Pattern NEWLINE_PATTERN = Pattern.compile("\\r?\\n");
79
80
81
82 public static final Pattern PARAM_VALUE_ESCAPE_PATTERN = Pattern.compile("[\\\"\\]\\\\]");
83
84
85
86
87 public static final String DEFAULT_MDCID = "mdc";
88
89 private static final long serialVersionUID = 1L;
90
91 private static final String LF = "\n";
92 private static final int TWO_DIGITS = 10;
93 private static final int THREE_DIGITS = 100;
94 private static final int MILLIS_PER_MINUTE = 60000;
95 private static final int MINUTES_PER_HOUR = 60;
96 private static final String COMPONENT_KEY = "RFC5424-Converter";
97
98 private final Facility facility;
99 private final String defaultId;
100 private final int enterpriseNumber;
101 private final boolean includeMdc;
102 private final String mdcId;
103 private final StructuredDataId mdcSdId;
104 private final String localHostName;
105 private final String appName;
106 private final String messageId;
107 private final String configName;
108 private final String mdcPrefix;
109 private final String eventPrefix;
110 private final List<String> mdcExcludes;
111 private final List<String> mdcIncludes;
112 private final List<String> mdcRequired;
113 private final ListChecker checker;
114 private final ListChecker noopChecker = new NoopChecker();
115 private final boolean includeNewLine;
116 private final String escapeNewLine;
117 private final boolean useTlsMessageFormat;
118
119 private long lastTimestamp = -1;
120 private String timestamppStr;
121
122 private final List<PatternFormatter> exceptionFormatters;
123 private final Map<String, FieldFormatter> fieldFormatters;
124
125 private Rfc5424Layout(final Configuration config, final Facility facility, final String id, final int ein,
126 final boolean includeMDC, final boolean includeNL, final String escapeNL, final String mdcId,
127 final String mdcPrefix, final String eventPrefix, final String appName, final String messageId,
128 final String excludes, final String includes, final String required, final Charset charset,
129 final String exceptionPattern, final boolean useTLSMessageFormat, final LoggerFields[] loggerFields) {
130 super(charset);
131 final PatternParser exceptionParser = createPatternParser(config, ThrowablePatternConverter.class);
132 exceptionFormatters = exceptionPattern == null ? null : exceptionParser.parse(exceptionPattern, false, false);
133 this.facility = facility;
134 this.defaultId = id == null ? DEFAULT_ID : id;
135 this.enterpriseNumber = ein;
136 this.includeMdc = includeMDC;
137 this.includeNewLine = includeNL;
138 this.escapeNewLine = escapeNL == null ? null : Matcher.quoteReplacement(escapeNL);
139 this.mdcId = mdcId;
140 this.mdcSdId = new StructuredDataId(mdcId, enterpriseNumber, null, null);
141 this.mdcPrefix = mdcPrefix;
142 this.eventPrefix = eventPrefix;
143 this.appName = appName;
144 this.messageId = messageId;
145 this.useTlsMessageFormat = useTLSMessageFormat;
146 this.localHostName = NetUtils.getLocalHostname();
147 ListChecker c = null;
148 if (excludes != null) {
149 final String[] array = excludes.split(Patterns.COMMA_SEPARATOR);
150 if (array.length > 0) {
151 c = new ExcludeChecker();
152 mdcExcludes = new ArrayList<>(array.length);
153 for (final String str : array) {
154 mdcExcludes.add(str.trim());
155 }
156 } else {
157 mdcExcludes = null;
158 }
159 } else {
160 mdcExcludes = null;
161 }
162 if (includes != null) {
163 final String[] array = includes.split(Patterns.COMMA_SEPARATOR);
164 if (array.length > 0) {
165 c = new IncludeChecker();
166 mdcIncludes = new ArrayList<>(array.length);
167 for (final String str : array) {
168 mdcIncludes.add(str.trim());
169 }
170 } else {
171 mdcIncludes = null;
172 }
173 } else {
174 mdcIncludes = null;
175 }
176 if (required != null) {
177 final String[] array = required.split(Patterns.COMMA_SEPARATOR);
178 if (array.length > 0) {
179 mdcRequired = new ArrayList<>(array.length);
180 for (final String str : array) {
181 mdcRequired.add(str.trim());
182 }
183 } else {
184 mdcRequired = null;
185 }
186
187 } else {
188 mdcRequired = null;
189 }
190 this.checker = c != null ? c : noopChecker;
191 final String name = config == null ? null : config.getName();
192 configName = Strings.isNotEmpty(name) ? name : null;
193 this.fieldFormatters = createFieldFormatters(loggerFields, config);
194 }
195
196 private Map<String, FieldFormatter> createFieldFormatters(final LoggerFields[] loggerFields,
197 final Configuration config) {
198 final Map<String, FieldFormatter> sdIdMap = new HashMap<>();
199
200 if (loggerFields != null) {
201 for (final LoggerFields lField : loggerFields) {
202 final StructuredDataId key = lField.getSdId() == null ? mdcSdId : lField.getSdId();
203 final Map<String, List<PatternFormatter>> sdParams = new HashMap<>();
204 final Map<String, String> fields = lField.getMap();
205 if (!fields.isEmpty()) {
206 final PatternParser fieldParser = createPatternParser(config, null);
207
208 for (final Map.Entry<String, String> entry : fields.entrySet()) {
209 final List<PatternFormatter> formatters = fieldParser.parse(entry.getValue(), false, false);
210 sdParams.put(entry.getKey(), formatters);
211 }
212 final FieldFormatter fieldFormatter = new FieldFormatter(sdParams,
213 lField.getDiscardIfAllFieldsAreEmpty());
214 sdIdMap.put(key.toString(), fieldFormatter);
215 }
216 }
217 }
218 return sdIdMap.size() > 0 ? sdIdMap : null;
219 }
220
221
222
223
224
225
226
227
228 private static PatternParser createPatternParser(final Configuration config,
229 final Class<? extends PatternConverter> filterClass) {
230 if (config == null) {
231 return new PatternParser(config, PatternLayout.KEY, LogEventPatternConverter.class, filterClass);
232 }
233 PatternParser parser = config.getComponent(COMPONENT_KEY);
234 if (parser == null) {
235 parser = new PatternParser(config, PatternLayout.KEY, ThrowablePatternConverter.class);
236 config.addComponent(COMPONENT_KEY, parser);
237 parser = config.getComponent(COMPONENT_KEY);
238 }
239 return parser;
240 }
241
242
243
244
245
246
247
248
249
250
251 @Override
252 public Map<String, String> getContentFormat() {
253 final Map<String, String> result = new HashMap<>();
254 result.put("structured", "true");
255 result.put("formatType", "RFC5424");
256 return result;
257 }
258
259
260
261
262
263
264
265 @Override
266 public String toSerializable(final LogEvent event) {
267 final StringBuilder buf = getStringBuilder();
268 appendPriority(buf, event.getLevel());
269 appendTimestamp(buf, event.getTimeMillis());
270 appendSpace(buf);
271 appendHostName(buf);
272 appendSpace(buf);
273 appendAppName(buf);
274 appendSpace(buf);
275 appendProcessId(buf);
276 appendSpace(buf);
277 appendMessageId(buf, event.getMessage());
278 appendSpace(buf);
279 appendStructuredElements(buf, event);
280 appendMessage(buf, event);
281 if (useTlsMessageFormat) {
282 return new TlsSyslogFrame(buf.toString()).toString();
283 }
284 return buf.toString();
285 }
286
287 private void appendPriority(final StringBuilder buffer, final Level logLevel) {
288 buffer.append('<');
289 buffer.append(Priority.getPriority(facility, logLevel));
290 buffer.append(">1 ");
291 }
292
293 private void appendTimestamp(final StringBuilder buffer, final long milliseconds) {
294 buffer.append(computeTimeStampString(milliseconds));
295 }
296
297 private void appendSpace(final StringBuilder buffer) {
298 buffer.append(' ');
299 }
300
301 private void appendHostName(final StringBuilder buffer) {
302 buffer.append(localHostName);
303 }
304
305 private void appendAppName(final StringBuilder buffer) {
306 if (appName != null) {
307 buffer.append(appName);
308 } else if (configName != null) {
309 buffer.append(configName);
310 } else {
311 buffer.append('-');
312 }
313 }
314
315 private void appendProcessId(final StringBuilder buffer) {
316 buffer.append(getProcId());
317 }
318
319 private void appendMessageId(final StringBuilder buffer, final Message message) {
320 final boolean isStructured = message instanceof StructuredDataMessage;
321 final String type = isStructured ? ((StructuredDataMessage) message).getType() : null;
322 if (type != null) {
323 buffer.append(type);
324 } else if (messageId != null) {
325 buffer.append(messageId);
326 } else {
327 buffer.append('-');
328 }
329 }
330
331 private void appendMessage(final StringBuilder buffer, final LogEvent event) {
332 final Message message = event.getMessage();
333
334 final String text = (message instanceof StructuredDataMessage) ? message.getFormat() : message
335 .getFormattedMessage();
336
337 if (text != null && text.length() > 0) {
338 buffer.append(' ').append(escapeNewlines(text, escapeNewLine));
339 }
340
341 if (exceptionFormatters != null && event.getThrown() != null) {
342 final StringBuilder exception = new StringBuilder(LF);
343 for (final PatternFormatter formatter : exceptionFormatters) {
344 formatter.format(event, exception);
345 }
346 buffer.append(escapeNewlines(exception.toString(), escapeNewLine));
347 }
348 if (includeNewLine) {
349 buffer.append(LF);
350 }
351 }
352
353 private void appendStructuredElements(final StringBuilder buffer, final LogEvent event) {
354 final Message message = event.getMessage();
355 final boolean isStructured = message instanceof StructuredDataMessage;
356
357 if (!isStructured && (fieldFormatters != null && fieldFormatters.isEmpty()) && !includeMdc) {
358 buffer.append('-');
359 return;
360 }
361
362 final Map<String, StructuredDataElement> sdElements = new HashMap<>();
363 final Map<String, String> contextMap = event.getContextMap();
364
365 if (mdcRequired != null) {
366 checkRequired(contextMap);
367 }
368
369 if (fieldFormatters != null) {
370 for (final Map.Entry<String, FieldFormatter> sdElement : fieldFormatters.entrySet()) {
371 final String sdId = sdElement.getKey();
372 final StructuredDataElement elem = sdElement.getValue().format(event);
373 sdElements.put(sdId, elem);
374 }
375 }
376
377 if (includeMdc && contextMap.size() > 0) {
378 final String mdcSdIdStr = mdcSdId.toString();
379 final StructuredDataElement union = sdElements.get(mdcSdIdStr);
380 if (union != null) {
381 union.union(contextMap);
382 sdElements.put(mdcSdIdStr, union);
383 } else {
384 final StructuredDataElement formattedContextMap = new StructuredDataElement(contextMap, false);
385 sdElements.put(mdcSdIdStr, formattedContextMap);
386 }
387 }
388
389 if (isStructured) {
390 final StructuredDataMessage data = (StructuredDataMessage) message;
391 final Map<String, String> map = data.getData();
392 final StructuredDataId id = data.getId();
393 final String sdId = getId(id);
394
395 if (sdElements.containsKey(sdId)) {
396 final StructuredDataElement union = sdElements.get(id.toString());
397 union.union(map);
398 sdElements.put(sdId, union);
399 } else {
400 final StructuredDataElement formattedData = new StructuredDataElement(map, false);
401 sdElements.put(sdId, formattedData);
402 }
403 }
404
405 if (sdElements.isEmpty()) {
406 buffer.append('-');
407 return;
408 }
409
410 for (final Map.Entry<String, StructuredDataElement> entry : sdElements.entrySet()) {
411 formatStructuredElement(entry.getKey(), mdcPrefix, entry.getValue(), buffer, checker);
412 }
413 }
414
415 private String escapeNewlines(final String text, final String escapeNewLine) {
416 if (null == escapeNewLine) {
417 return text;
418 }
419 return NEWLINE_PATTERN.matcher(text).replaceAll(escapeNewLine);
420 }
421
422 protected String getProcId() {
423 return "-";
424 }
425
426 protected List<String> getMdcExcludes() {
427 return mdcExcludes;
428 }
429
430 protected List<String> getMdcIncludes() {
431 return mdcIncludes;
432 }
433
434 private String computeTimeStampString(final long now) {
435 long last;
436 synchronized (this) {
437 last = lastTimestamp;
438 if (now == lastTimestamp) {
439 return timestamppStr;
440 }
441 }
442
443 final StringBuilder buffer = new StringBuilder();
444 final Calendar cal = new GregorianCalendar();
445 cal.setTimeInMillis(now);
446 buffer.append(Integer.toString(cal.get(Calendar.YEAR)));
447 buffer.append('-');
448 pad(cal.get(Calendar.MONTH) + 1, TWO_DIGITS, buffer);
449 buffer.append('-');
450 pad(cal.get(Calendar.DAY_OF_MONTH), TWO_DIGITS, buffer);
451 buffer.append('T');
452 pad(cal.get(Calendar.HOUR_OF_DAY), TWO_DIGITS, buffer);
453 buffer.append(':');
454 pad(cal.get(Calendar.MINUTE), TWO_DIGITS, buffer);
455 buffer.append(':');
456 pad(cal.get(Calendar.SECOND), TWO_DIGITS, buffer);
457 buffer.append('.');
458 pad(cal.get(Calendar.MILLISECOND), THREE_DIGITS, buffer);
459
460 int tzmin = (cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET)) / MILLIS_PER_MINUTE;
461 if (tzmin == 0) {
462 buffer.append('Z');
463 } else {
464 if (tzmin < 0) {
465 tzmin = -tzmin;
466 buffer.append('-');
467 } else {
468 buffer.append('+');
469 }
470 final int tzhour = tzmin / MINUTES_PER_HOUR;
471 tzmin -= tzhour * MINUTES_PER_HOUR;
472 pad(tzhour, TWO_DIGITS, buffer);
473 buffer.append(':');
474 pad(tzmin, TWO_DIGITS, buffer);
475 }
476 synchronized (this) {
477 if (last == lastTimestamp) {
478 lastTimestamp = now;
479 timestamppStr = buffer.toString();
480 }
481 }
482 return buffer.toString();
483 }
484
485 private void pad(final int val, int max, final StringBuilder buf) {
486 while (max > 1) {
487 if (val < max) {
488 buf.append('0');
489 }
490 max = max / TWO_DIGITS;
491 }
492 buf.append(Integer.toString(val));
493 }
494
495 private void formatStructuredElement(final String id, final String prefix, final StructuredDataElement data,
496 final StringBuilder sb, final ListChecker checker) {
497 if ((id == null && defaultId == null) || data.discard()) {
498 return;
499 }
500
501 sb.append('[');
502 sb.append(id);
503 if (!mdcSdId.toString().equals(id)) {
504 appendMap(prefix, data.getFields(), sb, noopChecker);
505 } else {
506 appendMap(prefix, data.getFields(), sb, checker);
507 }
508 sb.append(']');
509 }
510
511 private String getId(final StructuredDataId id) {
512 final StringBuilder sb = new StringBuilder();
513 if (id == null || id.getName() == null) {
514 sb.append(defaultId);
515 } else {
516 sb.append(id.getName());
517 }
518 int ein = id != null ? id.getEnterpriseNumber() : enterpriseNumber;
519 if (ein < 0) {
520 ein = enterpriseNumber;
521 }
522 if (ein >= 0) {
523 sb.append('@').append(ein);
524 }
525 return sb.toString();
526 }
527
528 private void checkRequired(final Map<String, String> map) {
529 for (final String key : mdcRequired) {
530 final String value = map.get(key);
531 if (value == null) {
532 throw new LoggingException("Required key " + key + " is missing from the " + mdcId);
533 }
534 }
535 }
536
537 private void appendMap(final String prefix, final Map<String, String> map, final StringBuilder sb,
538 final ListChecker checker) {
539 final SortedMap<String, String> sorted = new TreeMap<>(map);
540 for (final Map.Entry<String, String> entry : sorted.entrySet()) {
541 if (checker.check(entry.getKey()) && entry.getValue() != null) {
542 sb.append(' ');
543 if (prefix != null) {
544 sb.append(prefix);
545 }
546 final String safeKey = escapeNewlines(escapeSDParams(entry.getKey()), escapeNewLine);
547 final String safeValue = escapeNewlines(escapeSDParams(entry.getValue()), escapeNewLine);
548 StringBuilders.appendKeyDqValue(sb, safeKey, safeValue);
549 }
550 }
551 }
552
553 private String escapeSDParams(final String value) {
554 return PARAM_VALUE_ESCAPE_PATTERN.matcher(value).replaceAll("\\\\$0");
555 }
556
557
558
559
560 private interface ListChecker {
561 boolean check(String key);
562 }
563
564
565
566
567 private class IncludeChecker implements ListChecker {
568 @Override
569 public boolean check(final String key) {
570 return mdcIncludes.contains(key);
571 }
572 }
573
574
575
576
577 private class ExcludeChecker implements ListChecker {
578 @Override
579 public boolean check(final String key) {
580 return !mdcExcludes.contains(key);
581 }
582 }
583
584
585
586
587 private class NoopChecker implements ListChecker {
588 @Override
589 public boolean check(final String key) {
590 return true;
591 }
592 }
593
594 @Override
595 public String toString() {
596 final StringBuilder sb = new StringBuilder();
597 sb.append("facility=").append(facility.name());
598 sb.append(" appName=").append(appName);
599 sb.append(" defaultId=").append(defaultId);
600 sb.append(" enterpriseNumber=").append(enterpriseNumber);
601 sb.append(" newLine=").append(includeNewLine);
602 sb.append(" includeMDC=").append(includeMdc);
603 sb.append(" messageId=").append(messageId);
604 return sb.toString();
605 }
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631 @PluginFactory
632 public static Rfc5424Layout createLayout(
633
634 @PluginAttribute(value = "facility", defaultString = "LOCAL0") final Facility facility,
635 @PluginAttribute("id") final String id,
636 @PluginAttribute(value = "enterpriseNumber", defaultInt = DEFAULT_ENTERPRISE_NUMBER)
637 final int enterpriseNumber,
638 @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC,
639 @PluginAttribute(value = "mdcId", defaultString = DEFAULT_MDCID) final String mdcId,
640 @PluginAttribute("mdcPrefix") final String mdcPrefix,
641 @PluginAttribute("eventPrefix") final String eventPrefix,
642 @PluginAttribute(value = "newLine", defaultBoolean = false) final boolean newLine,
643 @PluginAttribute("newLineEscape") final String escapeNL,
644 @PluginAttribute("appName") final String appName,
645 @PluginAttribute("messageId") final String msgId,
646 @PluginAttribute("mdcExcludes") final String excludes,
647 @PluginAttribute("mdcIncludes") String includes,
648 @PluginAttribute("mdcRequired") final String required,
649 @PluginAttribute("exceptionPattern") final String exceptionPattern,
650
651 @PluginAttribute(value = "useTlsMessageFormat", defaultBoolean = false) final boolean useTlsMessageFormat,
652 @PluginElement("LoggerFields") final LoggerFields[] loggerFields,
653 @PluginConfiguration final Configuration config) {
654
655 if (includes != null && excludes != null) {
656 LOGGER.error("mdcIncludes and mdcExcludes are mutually exclusive. Includes wil be ignored");
657 includes = null;
658 }
659
660 return new Rfc5424Layout(config, facility, id, enterpriseNumber, includeMDC, newLine, escapeNL, mdcId,
661 mdcPrefix, eventPrefix, appName, msgId, excludes, includes, required, StandardCharsets.UTF_8,
662 exceptionPattern, useTlsMessageFormat, loggerFields);
663 }
664
665 private class FieldFormatter {
666
667 private final Map<String, List<PatternFormatter>> delegateMap;
668 private final boolean discardIfEmpty;
669
670 public FieldFormatter(final Map<String, List<PatternFormatter>> fieldMap, final boolean discardIfEmpty) {
671 this.discardIfEmpty = discardIfEmpty;
672 this.delegateMap = fieldMap;
673 }
674
675 public StructuredDataElement format(final LogEvent event) {
676 final Map<String, String> map = new HashMap<>();
677
678 for (final Map.Entry<String, List<PatternFormatter>> entry : delegateMap.entrySet()) {
679 final StringBuilder buffer = new StringBuilder();
680 for (final PatternFormatter formatter : entry.getValue()) {
681 formatter.format(event, buffer);
682 }
683 map.put(entry.getKey(), buffer.toString());
684 }
685 return new StructuredDataElement(map, discardIfEmpty);
686 }
687 }
688
689 private class StructuredDataElement {
690
691 private final Map<String, String> fields;
692 private final boolean discardIfEmpty;
693
694 public StructuredDataElement(final Map<String, String> fields, final boolean discardIfEmpty) {
695 this.discardIfEmpty = discardIfEmpty;
696 this.fields = fields;
697 }
698
699 boolean discard() {
700 if (discardIfEmpty == false) {
701 return false;
702 }
703 boolean foundNotEmptyValue = false;
704 for (final Map.Entry<String, String> entry : fields.entrySet()) {
705 if (Strings.isNotEmpty(entry.getValue())) {
706 foundNotEmptyValue = true;
707 break;
708 }
709 }
710 return !foundNotEmptyValue;
711 }
712
713 void union(final Map<String, String> fields) {
714 this.fields.putAll(fields);
715 }
716
717 Map<String, String> getFields() {
718 return this.fields;
719 }
720 }
721 }