1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.layout;
18
19 import java.nio.charset.Charset;
20 import java.nio.charset.StandardCharsets;
21 import java.util.ArrayList;
22 import java.util.Calendar;
23 import java.util.GregorianCalendar;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.SortedMap;
28 import java.util.TreeMap;
29 import java.util.regex.Matcher;
30 import java.util.regex.Pattern;
31
32 import org.apache.logging.log4j.Level;
33 import org.apache.logging.log4j.LoggingException;
34 import org.apache.logging.log4j.core.Layout;
35 import org.apache.logging.log4j.core.LogEvent;
36 import org.apache.logging.log4j.core.appender.TlsSyslogFrame;
37 import org.apache.logging.log4j.core.config.Configuration;
38 import org.apache.logging.log4j.core.config.Node;
39 import org.apache.logging.log4j.core.config.plugins.Plugin;
40 import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
41 import org.apache.logging.log4j.core.config.plugins.PluginConfiguration;
42 import org.apache.logging.log4j.core.config.plugins.PluginElement;
43 import org.apache.logging.log4j.core.config.plugins.PluginFactory;
44 import org.apache.logging.log4j.core.net.Facility;
45 import org.apache.logging.log4j.core.net.Priority;
46 import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
47 import org.apache.logging.log4j.core.pattern.PatternConverter;
48 import org.apache.logging.log4j.core.pattern.PatternFormatter;
49 import org.apache.logging.log4j.core.pattern.PatternParser;
50 import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter;
51 import org.apache.logging.log4j.core.util.NetUtils;
52 import org.apache.logging.log4j.core.util.Patterns;
53 import org.apache.logging.log4j.message.Message;
54 import org.apache.logging.log4j.message.StructuredDataId;
55 import org.apache.logging.log4j.message.StructuredDataMessage;
56 import org.apache.logging.log4j.util.StringBuilders;
57 import org.apache.logging.log4j.util.Strings;
58
59
60
61
62
63
64
65 @Plugin(name = "Rfc5424Layout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
66 public final class Rfc5424Layout extends AbstractStringLayout {
67
68 private static final long serialVersionUID = 1L;
69
70 private static final String LF = "\n";
71
72
73
74
75 public static final int DEFAULT_ENTERPRISE_NUMBER = 18060;
76
77
78
79 public static final String DEFAULT_ID = "Audit";
80
81
82
83 public static final Pattern NEWLINE_PATTERN = Pattern.compile("\\r?\\n");
84
85
86
87 public static final Pattern PARAM_VALUE_ESCAPE_PATTERN = Pattern.compile("[\\\"\\]\\\\]");
88
89 public static final String DEFAULT_MDCID = "mdc";
90 private static final int TWO_DIGITS = 10;
91 private static final int THREE_DIGITS = 100;
92 private static final int MILLIS_PER_MINUTE = 60000;
93 private static final int MINUTES_PER_HOUR = 60;
94
95 private static final String COMPONENT_KEY = "RFC5424-Converter";
96
97 private final Facility facility;
98 private final String defaultId;
99 private final int enterpriseNumber;
100 private final boolean includeMdc;
101 private final String mdcId;
102 private final StructuredDataId mdcSdId;
103 private final String localHostName;
104 private final String appName;
105 private final String messageId;
106 private final String configName;
107 private final String mdcPrefix;
108 private final String eventPrefix;
109 private final List<String> mdcExcludes;
110 private final List<String> mdcIncludes;
111 private final List<String> mdcRequired;
112 private final ListChecker checker;
113 private final ListChecker noopChecker = new NoopChecker();
114 private final boolean includeNewLine;
115 private final String escapeNewLine;
116 private final boolean useTlsMessageFormat;
117
118 private long lastTimestamp = -1;
119 private String timestamppStr;
120
121 private final List<PatternFormatter> exceptionFormatters;
122 private final Map<String, FieldFormatter> fieldFormatters;
123
124 private Rfc5424Layout(final Configuration config, final Facility facility, final String id, final int ein,
125 final boolean includeMDC, final boolean includeNL, final String escapeNL, final String mdcId,
126 final String mdcPrefix, final String eventPrefix,
127 final String appName, final String messageId, final String excludes, final String includes,
128 final String required, final Charset charset, final String exceptionPattern,
129 final boolean useTLSMessageFormat, final LoggerFields[] loggerFields) {
130 super(charset);
131 final PatternParser exceptionParser = createPatternParser(config, ThrowablePatternConverter.class);
132 exceptionFormatters = exceptionPattern == null ? null : exceptionParser.parse(exceptionPattern, false, false);
133 this.facility = facility;
134 this.defaultId = id == null ? DEFAULT_ID : id;
135 this.enterpriseNumber = ein;
136 this.includeMdc = includeMDC;
137 this.includeNewLine = includeNL;
138 this.escapeNewLine = escapeNL == null ? null : Matcher.quoteReplacement(escapeNL);
139 this.mdcId = mdcId;
140 this.mdcSdId = new StructuredDataId(mdcId, enterpriseNumber, null, null);
141 this.mdcPrefix = mdcPrefix;
142 this.eventPrefix = eventPrefix;
143 this.appName = appName;
144 this.messageId = messageId;
145 this.useTlsMessageFormat = useTLSMessageFormat;
146 this.localHostName = NetUtils.getLocalHostname();
147 ListChecker c = null;
148 if (excludes != null) {
149 final String[] array = excludes.split(Patterns.COMMA_SEPARATOR);
150 if (array.length > 0) {
151 c = new ExcludeChecker();
152 mdcExcludes = new ArrayList<>(array.length);
153 for (final String str : array) {
154 mdcExcludes.add(str.trim());
155 }
156 } else {
157 mdcExcludes = null;
158 }
159 } else {
160 mdcExcludes = null;
161 }
162 if (includes != null) {
163 final String[] array = includes.split(Patterns.COMMA_SEPARATOR);
164 if (array.length > 0) {
165 c = new IncludeChecker();
166 mdcIncludes = new ArrayList<>(array.length);
167 for (final String str : array) {
168 mdcIncludes.add(str.trim());
169 }
170 } else {
171 mdcIncludes = null;
172 }
173 } else {
174 mdcIncludes = null;
175 }
176 if (required != null) {
177 final String[] array = required.split(Patterns.COMMA_SEPARATOR);
178 if (array.length > 0) {
179 mdcRequired = new ArrayList<>(array.length);
180 for (final String str : array) {
181 mdcRequired.add(str.trim());
182 }
183 } else {
184 mdcRequired = null;
185 }
186
187 } else {
188 mdcRequired = null;
189 }
190 this.checker = c != null ? c : noopChecker;
191 final String name = config == null ? null : config.getName();
192 configName = Strings.isNotEmpty(name) ? name : null;
193 this.fieldFormatters = createFieldFormatters(loggerFields, config);
194 }
195
196 private Map<String, FieldFormatter> createFieldFormatters(final LoggerFields[] loggerFields,
197 final Configuration config) {
198 final Map<String, FieldFormatter> sdIdMap = new HashMap<>();
199
200 if (loggerFields != null) {
201 for (final LoggerFields lField : loggerFields) {
202 final StructuredDataId key = lField.getSdId() == null ? mdcSdId : lField.getSdId();
203 final Map<String, List<PatternFormatter>> sdParams = new HashMap<>();
204 final Map<String, String> fields = lField.getMap();
205 if (!fields.isEmpty()) {
206 final PatternParser fieldParser = createPatternParser(config, null);
207
208 for (final Map.Entry<String, String> entry : fields.entrySet()) {
209 final List<PatternFormatter> formatters = fieldParser.parse(entry.getValue(), false, false);
210 sdParams.put(entry.getKey(), formatters);
211 }
212 final FieldFormatter fieldFormatter = new FieldFormatter(sdParams,
213 lField.getDiscardIfAllFieldsAreEmpty());
214 sdIdMap.put(key.toString(), fieldFormatter);
215 }
216 }
217 }
218 return sdIdMap.size() > 0 ? sdIdMap : null;
219 }
220
221
222
223
224
225
226
227
228 private static PatternParser createPatternParser(final Configuration config,
229 final Class<? extends PatternConverter> filterClass) {
230 if (config == null) {
231 return new PatternParser(config, PatternLayout.KEY, LogEventPatternConverter.class, filterClass);
232 }
233 PatternParser parser = config.getComponent(COMPONENT_KEY);
234 if (parser == null) {
235 parser = new PatternParser(config, PatternLayout.KEY, ThrowablePatternConverter.class);
236 config.addComponent(COMPONENT_KEY, parser);
237 parser = (PatternParser) config.getComponent(COMPONENT_KEY);
238 }
239 return parser;
240 }
241
242
243
244
245
246
247
248
249
250
251 @Override
252 public Map<String, String> getContentFormat() {
253 final Map<String, String> result = new HashMap<>();
254 result.put("structured", "true");
255 result.put("formatType", "RFC5424");
256 return result;
257 }
258
259
260
261
262
263
264
265 @Override
266 public String toSerializable(final LogEvent event) {
267 final StringBuilder buf = new StringBuilder();
268 appendPriority(buf, event.getLevel());
269 appendTimestamp(buf, event.getTimeMillis());
270 appendSpace(buf);
271 appendHostName(buf);
272 appendSpace(buf);
273 appendAppName(buf);
274 appendSpace(buf);
275 appendProcessId(buf);
276 appendSpace(buf);
277 appendMessageId(buf, event.getMessage());
278 appendSpace(buf);
279 appendStructuredElements(buf, event);
280 appendMessage(buf, event);
281 if (useTlsMessageFormat) {
282 return new TlsSyslogFrame(buf.toString()).toString();
283 }
284 return buf.toString();
285 }
286
287 private void appendPriority(final StringBuilder buffer, final Level logLevel) {
288 buffer.append('<');
289 buffer.append(Priority.getPriority(facility, logLevel));
290 buffer.append(">1 ");
291 }
292
293 private void appendTimestamp(final StringBuilder buffer, final long milliseconds) {
294 buffer.append(computeTimeStampString(milliseconds));
295 }
296
297 private void appendSpace(final StringBuilder buffer) {
298 buffer.append(' ');
299 }
300
301 private void appendHostName(final StringBuilder buffer) {
302 buffer.append(localHostName);
303 }
304
305 private void appendAppName(final StringBuilder buffer) {
306 if (appName != null) {
307 buffer.append(appName);
308 } else if (configName != null) {
309 buffer.append(configName);
310 } else {
311 buffer.append('-');
312 }
313 }
314
315 private void appendProcessId(final StringBuilder buffer) {
316 buffer.append(getProcId());
317 }
318
319 private void appendMessageId(final StringBuilder buffer, final Message message) {
320 final boolean isStructured = message instanceof StructuredDataMessage;
321 final String type = isStructured ? ((StructuredDataMessage) message).getType() : null;
322 if (type != null) {
323 buffer.append(type);
324 } else if (messageId != null) {
325 buffer.append(messageId);
326 } else {
327 buffer.append('-');
328 }
329 }
330
331 private void appendMessage(final StringBuilder buffer, final LogEvent event) {
332 final Message message = event.getMessage();
333
334 final String text = (message instanceof StructuredDataMessage) ? message.getFormat() : message.getFormattedMessage();
335
336 if (text != null && text.length() > 0) {
337 buffer.append(' ').append(escapeNewlines(text, escapeNewLine));
338 }
339
340 if (exceptionFormatters != null && event.getThrown() != null) {
341 final StringBuilder exception = new StringBuilder(LF);
342 for (final PatternFormatter formatter : exceptionFormatters) {
343 formatter.format(event, exception);
344 }
345 buffer.append(escapeNewlines(exception.toString(), escapeNewLine));
346 }
347 if (includeNewLine) {
348 buffer.append(LF);
349 }
350 }
351
352 private void appendStructuredElements(final StringBuilder buffer, final LogEvent event) {
353 final Message message = event.getMessage();
354 final boolean isStructured = message instanceof StructuredDataMessage;
355
356 if (!isStructured && (fieldFormatters!= null && fieldFormatters.isEmpty()) && !includeMdc) {
357 buffer.append('-');
358 return;
359 }
360
361 final Map<String, StructuredDataElement> sdElements = new HashMap<>();
362 final Map<String, String> contextMap = event.getContextMap();
363
364 if (mdcRequired != null) {
365 checkRequired(contextMap);
366 }
367
368 if (fieldFormatters != null) {
369 for (final Map.Entry<String, FieldFormatter> sdElement: fieldFormatters.entrySet()) {
370 final String sdId = sdElement.getKey();
371 final StructuredDataElement elem = sdElement.getValue().format(event);
372 sdElements.put(sdId, elem);
373 }
374 }
375
376 if (includeMdc && contextMap.size() > 0) {
377 final String mdcSdIdStr = mdcSdId.toString();
378 final StructuredDataElement union = sdElements.get(mdcSdIdStr);
379 if (union != null) {
380 union.union(contextMap);
381 sdElements.put(mdcSdIdStr, union);
382 } else {
383 final StructuredDataElement formattedContextMap = new StructuredDataElement(contextMap, false);
384 sdElements.put(mdcSdIdStr, formattedContextMap);
385 }
386 }
387
388 if (isStructured) {
389 final StructuredDataMessage data = (StructuredDataMessage) message;
390 final Map<String, String> map = data.getData();
391 final StructuredDataId id = data.getId();
392 final String sdId = getId(id);
393
394 if (sdElements.containsKey(sdId)) {
395 final StructuredDataElement union = sdElements.get(id.toString());
396 union.union(map);
397 sdElements.put(sdId, union);
398 } else {
399 final StructuredDataElement formattedData = new StructuredDataElement(map, false);
400 sdElements.put(sdId, formattedData);
401 }
402 }
403
404 if (sdElements.isEmpty()) {
405 buffer.append('-');
406 return;
407 }
408
409 for (final Map.Entry<String, StructuredDataElement> entry: sdElements.entrySet()) {
410 formatStructuredElement(entry.getKey(), mdcPrefix, entry.getValue(), buffer, checker);
411 }
412 }
413
414 private String escapeNewlines(final String text, final String escapeNewLine) {
415 if (null == escapeNewLine) {
416 return text;
417 }
418 return NEWLINE_PATTERN.matcher(text).replaceAll(escapeNewLine);
419 }
420
421 protected String getProcId() {
422 return "-";
423 }
424
425 protected List<String> getMdcExcludes() {
426 return mdcExcludes;
427 }
428
429 protected List<String> getMdcIncludes() {
430 return mdcIncludes;
431 }
432
433 private String computeTimeStampString(final long now) {
434 long last;
435 synchronized (this) {
436 last = lastTimestamp;
437 if (now == lastTimestamp) {
438 return timestamppStr;
439 }
440 }
441
442 final StringBuilder buffer = new StringBuilder();
443 final Calendar cal = new GregorianCalendar();
444 cal.setTimeInMillis(now);
445 buffer.append(Integer.toString(cal.get(Calendar.YEAR)));
446 buffer.append('-');
447 pad(cal.get(Calendar.MONTH) + 1, TWO_DIGITS, buffer);
448 buffer.append('-');
449 pad(cal.get(Calendar.DAY_OF_MONTH), TWO_DIGITS, buffer);
450 buffer.append('T');
451 pad(cal.get(Calendar.HOUR_OF_DAY), TWO_DIGITS, buffer);
452 buffer.append(':');
453 pad(cal.get(Calendar.MINUTE), TWO_DIGITS, buffer);
454 buffer.append(':');
455 pad(cal.get(Calendar.SECOND), TWO_DIGITS, buffer);
456 buffer.append('.');
457 pad(cal.get(Calendar.MILLISECOND), THREE_DIGITS, buffer);
458
459 int tzmin = (cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET)) / MILLIS_PER_MINUTE;
460 if (tzmin == 0) {
461 buffer.append('Z');
462 } else {
463 if (tzmin < 0) {
464 tzmin = -tzmin;
465 buffer.append('-');
466 } else {
467 buffer.append('+');
468 }
469 final int tzhour = tzmin / MINUTES_PER_HOUR;
470 tzmin -= tzhour * MINUTES_PER_HOUR;
471 pad(tzhour, TWO_DIGITS, buffer);
472 buffer.append(':');
473 pad(tzmin, TWO_DIGITS, buffer);
474 }
475 synchronized (this) {
476 if (last == lastTimestamp) {
477 lastTimestamp = now;
478 timestamppStr = buffer.toString();
479 }
480 }
481 return buffer.toString();
482 }
483
484 private void pad(final int val, int max, final StringBuilder buf) {
485 while (max > 1) {
486 if (val < max) {
487 buf.append('0');
488 }
489 max = max / TWO_DIGITS;
490 }
491 buf.append(Integer.toString(val));
492 }
493
494 private void formatStructuredElement(final String id, final String prefix, final StructuredDataElement data,
495 final StringBuilder sb, final ListChecker checker) {
496 if ((id == null && defaultId == null) || data.discard()) {
497 return;
498 }
499
500 sb.append('[');
501 sb.append(id);
502 if (!mdcSdId.toString().equals(id)) {
503 appendMap(prefix, data.getFields(), sb, noopChecker);
504 } else {
505 appendMap(prefix, data.getFields(), sb, checker);
506 }
507 sb.append(']');
508 }
509
510 private String getId(final StructuredDataId id) {
511 final StringBuilder sb = new StringBuilder();
512 if (id == null || id.getName() == null) {
513 sb.append(defaultId);
514 } else {
515 sb.append(id.getName());
516 }
517 int ein = id != null ? id.getEnterpriseNumber() : enterpriseNumber;
518 if (ein < 0) {
519 ein = enterpriseNumber;
520 }
521 if (ein >= 0) {
522 sb.append('@').append(ein);
523 }
524 return sb.toString();
525 }
526
527 private void checkRequired(final Map<String, String> map) {
528 for (final String key : mdcRequired) {
529 final String value = map.get(key);
530 if (value == null) {
531 throw new LoggingException("Required key " + key + " is missing from the " + mdcId);
532 }
533 }
534 }
535
536 private void appendMap(final String prefix, final Map<String, String> map, final StringBuilder sb,
537 final ListChecker checker) {
538 final SortedMap<String, String> sorted = new TreeMap<>(map);
539 for (final Map.Entry<String, String> entry : sorted.entrySet()) {
540 if (checker.check(entry.getKey()) && entry.getValue() != null) {
541 sb.append(' ');
542 if (prefix != null) {
543 sb.append(prefix);
544 }
545 final String safeKey = escapeNewlines(escapeSDParams(entry.getKey()), escapeNewLine);
546 final String safeValue = escapeNewlines(escapeSDParams(entry.getValue()), escapeNewLine);
547 StringBuilders.appendKeyDqValue(sb, safeKey, safeValue);
548 }
549 }
550 }
551
552 private String escapeSDParams(final String value) {
553 return PARAM_VALUE_ESCAPE_PATTERN.matcher(value).replaceAll("\\\\$0");
554 }
555
556
557
558
559 private interface ListChecker {
560 boolean check(String key);
561 }
562
563
564
565
566 private class IncludeChecker implements ListChecker {
567 @Override
568 public boolean check(final String key) {
569 return mdcIncludes.contains(key);
570 }
571 }
572
573
574
575
576 private class ExcludeChecker implements ListChecker {
577 @Override
578 public boolean check(final String key) {
579 return !mdcExcludes.contains(key);
580 }
581 }
582
583
584
585
586 private class NoopChecker implements ListChecker {
587 @Override
588 public boolean check(final String key) {
589 return true;
590 }
591 }
592
593 @Override
594 public String toString() {
595 final StringBuilder sb = new StringBuilder();
596 sb.append("facility=").append(facility.name());
597 sb.append(" appName=").append(appName);
598 sb.append(" defaultId=").append(defaultId);
599 sb.append(" enterpriseNumber=").append(enterpriseNumber);
600 sb.append(" newLine=").append(includeNewLine);
601 sb.append(" includeMDC=").append(includeMdc);
602 sb.append(" messageId=").append(messageId);
603 return sb.toString();
604 }
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630 @PluginFactory
631 public static Rfc5424Layout createLayout(
632 @PluginAttribute(value = "facility", defaultString = "LOCAL0") final Facility facility,
633 @PluginAttribute("id") final String id,
634 @PluginAttribute(value = "enterpriseNumber", defaultInt = DEFAULT_ENTERPRISE_NUMBER) final int enterpriseNumber,
635 @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC,
636 @PluginAttribute(value = "mdcId", defaultString = DEFAULT_MDCID) final String mdcId,
637 @PluginAttribute("mdcPrefix") final String mdcPrefix,
638 @PluginAttribute("eventPrefix") final String eventPrefix,
639 @PluginAttribute(value = "newLine", defaultBoolean = false) final boolean newLine,
640 @PluginAttribute("newLineEscape") final String escapeNL,
641 @PluginAttribute("appName") final String appName,
642 @PluginAttribute("messageId") final String msgId,
643 @PluginAttribute("mdcExcludes") final String excludes,
644 @PluginAttribute("mdcIncludes") String includes,
645 @PluginAttribute("mdcRequired") final String required,
646 @PluginAttribute("exceptionPattern") final String exceptionPattern,
647 @PluginAttribute(value = "useTlsMessageFormat", defaultBoolean = false) final boolean useTlsMessageFormat,
648 @PluginElement("LoggerFields") final LoggerFields[] loggerFields,
649 @PluginConfiguration final Configuration config) {
650 if (includes != null && excludes != null) {
651 LOGGER.error("mdcIncludes and mdcExcludes are mutually exclusive. Includes wil be ignored");
652 includes = null;
653 }
654
655 return new Rfc5424Layout(config, facility, id, enterpriseNumber, includeMDC, newLine, escapeNL, mdcId, mdcPrefix,
656 eventPrefix, appName, msgId, excludes, includes, required, StandardCharsets.UTF_8, exceptionPattern,
657 useTlsMessageFormat, loggerFields);
658 }
659
660 private class FieldFormatter {
661
662 private final Map<String, List<PatternFormatter>> delegateMap;
663 private final boolean discardIfEmpty;
664
665 public FieldFormatter(final Map<String, List<PatternFormatter>> fieldMap, final boolean discardIfEmpty) {
666 this.discardIfEmpty = discardIfEmpty;
667 this.delegateMap = fieldMap;
668 }
669
670 public StructuredDataElement format(final LogEvent event) {
671 final Map<String, String> map = new HashMap<>();
672
673 for (final Map.Entry<String, List<PatternFormatter>> entry : delegateMap.entrySet()) {
674 final StringBuilder buffer = new StringBuilder();
675 for (final PatternFormatter formatter : entry.getValue()) {
676 formatter.format(event, buffer);
677 }
678 map.put(entry.getKey(), buffer.toString());
679 }
680 return new StructuredDataElement(map, discardIfEmpty);
681 }
682 }
683
684 private class StructuredDataElement {
685
686 private final Map<String, String> fields;
687 private final boolean discardIfEmpty;
688
689 public StructuredDataElement(final Map<String, String> fields, final boolean discardIfEmpty) {
690 this.discardIfEmpty = discardIfEmpty;
691 this.fields = fields;
692 }
693
694 boolean discard() {
695 if (discardIfEmpty == false) {
696 return false;
697 }
698 boolean foundNotEmptyValue = false;
699 for (final Map.Entry<String, String> entry : fields.entrySet()) {
700 if (Strings.isNotEmpty(entry.getValue())) {
701 foundNotEmptyValue = true;
702 break;
703 }
704 }
705 return !foundNotEmptyValue;
706 }
707
708 void union(final Map<String, String> fields) {
709 this.fields.putAll(fields);
710 }
711
712 Map<String, String> getFields() {
713 return this.fields;
714 }
715 }
716 }