View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache license, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License. You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the license for the specific language governing permissions and
15   * limitations under the license.
16   */
17  package org.apache.logging.log4j.core.async;
18  
19  import java.util.List;
20  
21  import org.apache.logging.log4j.Level;
22  import org.apache.logging.log4j.Marker;
23  import org.apache.logging.log4j.ThreadContext;
24  import org.apache.logging.log4j.ThreadContext.ContextStack;
25  import org.apache.logging.log4j.core.ContextDataInjector;
26  import org.apache.logging.log4j.core.Logger;
27  import org.apache.logging.log4j.core.LoggerContext;
28  import org.apache.logging.log4j.core.config.Configuration;
29  import org.apache.logging.log4j.core.config.LoggerConfig;
30  import org.apache.logging.log4j.core.config.Property;
31  import org.apache.logging.log4j.core.config.ReliabilityStrategy;
32  import org.apache.logging.log4j.core.impl.ContextDataFactory;
33  import org.apache.logging.log4j.core.impl.ContextDataInjectorFactory;
34  import org.apache.logging.log4j.core.util.Clock;
35  import org.apache.logging.log4j.core.util.ClockFactory;
36  import org.apache.logging.log4j.core.util.NanoClock;
37  import org.apache.logging.log4j.message.Message;
38  import org.apache.logging.log4j.message.MessageFactory;
39  import org.apache.logging.log4j.message.ReusableMessage;
40  import org.apache.logging.log4j.spi.AbstractLogger;
41  import org.apache.logging.log4j.status.StatusLogger;
42  import org.apache.logging.log4j.util.StackLocatorUtil;
43  import org.apache.logging.log4j.util.StringMap;
44  
45  import com.lmax.disruptor.EventTranslatorVararg;
46  import com.lmax.disruptor.dsl.Disruptor;
47  
48  /**
49   * AsyncLogger is a logger designed for high throughput and low latency logging. It does not perform any I/O in the
50   * calling (application) thread, but instead hands off the work to another thread as soon as possible. The actual
51   * logging is performed in the background thread. It uses the LMAX Disruptor library for inter-thread communication. (<a
52   * href="http://lmax-exchange.github.com/disruptor/" >http://lmax-exchange.github.com/disruptor/</a>)
53   * <p>
54   * To use AsyncLogger, specify the System property
55   * {@code -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector} before you obtain a
56   * Logger, and all Loggers returned by LogManager.getLogger will be AsyncLoggers.
57   * <p>
58   * Note that for performance reasons, this logger does not include source location by default. You need to specify
59   * {@code includeLocation="true"} in the configuration or any %class, %location or %line conversion patterns in your
60   * log4j.xml configuration will produce either a "?" character or no output at all.
61   * <p>
62   * For best performance, use AsyncLogger with the RandomAccessFileAppender or RollingRandomAccessFileAppender, with
63   * immediateFlush=false. These appenders have built-in support for the batching mechanism used by the Disruptor library,
64   * and they will flush to disk at the end of each batch. This means that even with immediateFlush=false, there will
65   * never be any items left in the buffer; all log events will all be written to disk in a very efficient manner.
66   */
67  public class AsyncLogger extends Logger implements EventTranslatorVararg<RingBufferLogEvent> {
68      // Implementation note: many methods in this class are tuned for performance. MODIFY WITH CARE!
69      // Specifically, try to keep the hot methods to 35 bytecodes or less:
70      // this is within the MaxInlineSize threshold and makes these methods candidates for
71      // immediate inlining instead of waiting until they are designated "hot enough".
72  
73      private static final StatusLogger LOGGER = StatusLogger.getLogger();
74      private static final Clock CLOCK = ClockFactory.getClock(); // not reconfigurable
75      private static final ContextDataInjector CONTEXT_DATA_INJECTOR = ContextDataInjectorFactory.createInjector();
76  
77      private static final ThreadNameCachingStrategy THREAD_NAME_CACHING_STRATEGY = ThreadNameCachingStrategy.create();
78  
79      private final ThreadLocal<RingBufferLogEventTranslator> threadLocalTranslator = new ThreadLocal<>();
80      private final AsyncLoggerDisruptor loggerDisruptor;
81  
82      private volatile boolean includeLocation; // reconfigurable
83      private volatile NanoClock nanoClock; // reconfigurable
84  
85      /**
86       * Constructs an {@code AsyncLogger} with the specified context, name and message factory.
87       *
88       * @param context context of this logger
89       * @param name name of this logger
90       * @param messageFactory message factory of this logger
91       * @param loggerDisruptor helper class that logging can be delegated to. This object owns the Disruptor.
92       */
93      public AsyncLogger(final LoggerContext context, final String name, final MessageFactory messageFactory,
94              final AsyncLoggerDisruptor loggerDisruptor) {
95          super(context, name, messageFactory);
96          this.loggerDisruptor = loggerDisruptor;
97          includeLocation = privateConfig.loggerConfig.isIncludeLocation();
98          nanoClock = context.getConfiguration().getNanoClock();
99      }
100 
101     /*
102      * (non-Javadoc)
103      *
104      * @see org.apache.logging.log4j.core.Logger#updateConfiguration(org.apache.logging.log4j.core.config.Configuration)
105      */
106     @Override
107     protected void updateConfiguration(final Configuration newConfig) {
108         nanoClock = newConfig.getNanoClock();
109         includeLocation = newConfig.getLoggerConfig(name).isIncludeLocation();
110         super.updateConfiguration(newConfig);
111     }
112 
113     // package protected for unit tests
114     NanoClock getNanoClock() {
115         return nanoClock;
116     }
117 
118     private RingBufferLogEventTranslator getCachedTranslator() {
119         RingBufferLogEventTranslator result = threadLocalTranslator.get();
120         if (result == null) {
121             result = new RingBufferLogEventTranslator();
122             threadLocalTranslator.set(result);
123         }
124         return result;
125     }
126 
127     @Override
128     public void logMessage(final String fqcn, final Level level, final Marker marker, final Message message,
129             final Throwable thrown) {
130         getTranslatorType().log(fqcn, level, marker, message, thrown);
131     }
132 
133     abstract class TranslatorType {
134 
135         abstract void log(final String fqcn, final Level level, final Marker marker,
136                           final Message message, final Throwable thrown);
137     }
138 
139     private final TranslatorType threadLocalTranslatorType = new TranslatorType() {
140         @Override
141         void log(String fqcn, Level level, Marker marker, Message message, Throwable thrown) {
142             logWithThreadLocalTranslator(fqcn, level, marker, message, thrown);
143         }
144     };
145 
146     private final TranslatorType varargTranslatorType = new TranslatorType() {
147         @Override
148         void log(String fqcn, Level level, Marker marker, Message message, Throwable thrown) {
149             // LOG4J2-1172: avoid storing non-JDK classes in ThreadLocals to avoid memory leaks in web apps
150             logWithVarargTranslator(fqcn, level, marker, message, thrown);
151         }
152     };
153 
154     private TranslatorType getTranslatorType() {
155         return loggerDisruptor.isUseThreadLocals() ? threadLocalTranslatorType : varargTranslatorType;
156     }
157 
158     private boolean isReused(final Message message) {
159         return message instanceof ReusableMessage;
160     }
161 
162     /**
163      * Enqueues the specified log event data for logging in a background thread.
164      * <p>
165      * This re-uses a {@code RingBufferLogEventTranslator} instance cached in a {@code ThreadLocal} to avoid creating
166      * unnecessary objects with each event.
167      *
168      * @param fqcn fully qualified name of the caller
169      * @param level level at which the caller wants to log the message
170      * @param marker message marker
171      * @param message the log message
172      * @param thrown a {@code Throwable} or {@code null}
173      */
174     private void logWithThreadLocalTranslator(final String fqcn, final Level level, final Marker marker,
175             final Message message, final Throwable thrown) {
176         // Implementation note: this method is tuned for performance. MODIFY WITH CARE!
177 
178         final RingBufferLogEventTranslator translator = getCachedTranslator();
179         initTranslator(translator, fqcn, level, marker, message, thrown);
180         initTranslatorThreadValues(translator);
181         publish(translator);
182     }
183 
184     private void publish(final RingBufferLogEventTranslator translator) {
185         if (!loggerDisruptor.tryPublish(translator)) {
186             handleRingBufferFull(translator);
187         }
188     }
189 
190     private void handleRingBufferFull(final RingBufferLogEventTranslator translator) {
191         if (AbstractLogger.getRecursionDepth() > 1) { // LOG4J2-1518, LOG4J2-2031
192             // If queue is full AND we are in a recursive call, call appender directly to prevent deadlock
193             AsyncQueueFullMessageUtil.logWarningToStatusLogger();
194             logMessageInCurrentThread(translator.fqcn, translator.level, translator.marker, translator.message,
195                     translator.thrown);
196             return;
197         }
198         final EventRoute eventRoute = loggerDisruptor.getEventRoute(translator.level);
199         switch (eventRoute) {
200             case ENQUEUE:
201                 loggerDisruptor.enqueueLogMessageWhenQueueFull(translator);
202                 break;
203             case SYNCHRONOUS:
204                 logMessageInCurrentThread(translator.fqcn, translator.level, translator.marker, translator.message,
205                         translator.thrown);
206                 break;
207             case DISCARD:
208                 break;
209             default:
210                 throw new IllegalStateException("Unknown EventRoute " + eventRoute);
211         }
212     }
213 
214     private void initTranslator(final RingBufferLogEventTranslator translator, final String fqcn,
215             final Level level, final Marker marker, final Message message, final Throwable thrown) {
216 
217         translator.setBasicValues(this, name, marker, fqcn, level, message, //
218                 // don't construct ThrowableProxy until required
219                 thrown,
220 
221                 // needs shallow copy to be fast (LOG4J2-154)
222                 ThreadContext.getImmutableStack(), //
223 
224                 // location (expensive to calculate)
225                 calcLocationIfRequested(fqcn), //
226                 CLOCK, //
227                 nanoClock //
228         );
229     }
230 
231     private void initTranslatorThreadValues(final RingBufferLogEventTranslator translator) {
232         // constant check should be optimized out when using default (CACHED)
233         if (THREAD_NAME_CACHING_STRATEGY == ThreadNameCachingStrategy.UNCACHED) {
234             translator.updateThreadValues();
235         }
236     }
237 
238     /**
239      * Returns the caller location if requested, {@code null} otherwise.
240      *
241      * @param fqcn fully qualified caller name.
242      * @return the caller location if requested, {@code null} otherwise.
243      */
244     private StackTraceElement calcLocationIfRequested(final String fqcn) {
245         // location: very expensive operation. LOG4J2-153:
246         // Only include if "includeLocation=true" is specified,
247         // exclude if not specified or if "false" was specified.
248         return includeLocation ? StackLocatorUtil.calcLocation(fqcn) : null;
249     }
250 
251     /**
252      * Enqueues the specified log event data for logging in a background thread.
253      * <p>
254      * This creates a new varargs Object array for each invocation, but does not store any non-JDK classes in a
255      * {@code ThreadLocal} to avoid memory leaks in web applications (see LOG4J2-1172).
256      *
257      * @param fqcn fully qualified name of the caller
258      * @param level level at which the caller wants to log the message
259      * @param marker message marker
260      * @param message the log message
261      * @param thrown a {@code Throwable} or {@code null}
262      */
263     private void logWithVarargTranslator(final String fqcn, final Level level, final Marker marker,
264             final Message message, final Throwable thrown) {
265         // Implementation note: candidate for optimization: exceeds 35 bytecodes.
266 
267         final Disruptor<RingBufferLogEvent> disruptor = loggerDisruptor.getDisruptor();
268         if (disruptor == null) {
269             LOGGER.error("Ignoring log event after Log4j has been shut down.");
270             return;
271         }
272         // if the Message instance is reused, there is no point in freezing its message here
273         if (!isReused(message)) {
274             InternalAsyncUtil.makeMessageImmutable(message);
275         }
276         StackTraceElement location = null;
277         // calls the translateTo method on this AsyncLogger
278         if (!disruptor.getRingBuffer().tryPublishEvent(this,
279                 this, // asyncLogger: 0
280                 (location = calcLocationIfRequested(fqcn)), // location: 1
281                 fqcn, // 2
282                 level, // 3
283                 marker, // 4
284                 message, // 5
285                 thrown)) { // 6
286             handleRingBufferFull(location, fqcn, level, marker, message, thrown);
287         }
288     }
289 
290     /*
291      * (non-Javadoc)
292      *
293      * @see com.lmax.disruptor.EventTranslatorVararg#translateTo(java.lang.Object, long, java.lang.Object[])
294      */
295     @Override
296     public void translateTo(final RingBufferLogEvent event, final long sequence, final Object... args) {
297         // Implementation note: candidate for optimization: exceeds 35 bytecodes.
298         final AsyncLogger asyncLogger = (AsyncLogger) args[0];
299         final StackTraceElement location = (StackTraceElement) args[1];
300         final String fqcn = (String) args[2];
301         final Level level = (Level) args[3];
302         final Marker marker = (Marker) args[4];
303         final Message message = (Message) args[5];
304         final Throwable thrown = (Throwable) args[6];
305 
306         // needs shallow copy to be fast (LOG4J2-154)
307         final ContextStack contextStack = ThreadContext.getImmutableStack();
308 
309         final Thread currentThread = Thread.currentThread();
310         final String threadName = THREAD_NAME_CACHING_STRATEGY.getThreadName();
311         event.setValues(asyncLogger, asyncLogger.getName(), marker, fqcn, level, message, thrown,
312                 // config properties are taken care of in the EventHandler thread
313                 // in the AsyncLogger#actualAsyncLog method
314                 CONTEXT_DATA_INJECTOR.injectContextData(null, (StringMap) event.getContextData()),
315                 contextStack, currentThread.getId(), threadName, currentThread.getPriority(), location,
316                 CLOCK, nanoClock);
317     }
318 
319     /**
320      * LOG4J2-471: prevent deadlock when RingBuffer is full and object being logged calls Logger.log() from its
321      * toString() method
322      *
323      * @param fqcn fully qualified caller name
324      * @param level log level
325      * @param marker optional marker
326      * @param message log message
327      * @param thrown optional exception
328      */
329     void logMessageInCurrentThread(final String fqcn, final Level level, final Marker marker,
330             final Message message, final Throwable thrown) {
331         // bypass RingBuffer and invoke Appender directly
332         final ReliabilityStrategy strategy = privateConfig.loggerConfig.getReliabilityStrategy();
333         strategy.log(this, getName(), fqcn, marker, level, message, thrown);
334     }
335 
336     private void handleRingBufferFull(final StackTraceElement location,
337                                       final String fqcn,
338                                       final Level level,
339                                       final Marker marker,
340                                       final Message msg,
341                                       final Throwable thrown) {
342         if (AbstractLogger.getRecursionDepth() > 1) { // LOG4J2-1518, LOG4J2-2031
343             // If queue is full AND we are in a recursive call, call appender directly to prevent deadlock
344             AsyncQueueFullMessageUtil.logWarningToStatusLogger();
345             logMessageInCurrentThread(fqcn, level, marker, msg, thrown);
346             return;
347         }
348         final EventRoute eventRoute = loggerDisruptor.getEventRoute(level);
349         switch (eventRoute) {
350             case ENQUEUE:
351                 loggerDisruptor.enqueueLogMessageWhenQueueFull(this,
352                         this, // asyncLogger: 0
353                         location, // location: 1
354                         fqcn, // 2
355                         level, // 3
356                         marker, // 4
357                         msg, // 5
358                         thrown); // 6
359                 break;
360             case SYNCHRONOUS:
361                 logMessageInCurrentThread(fqcn, level, marker, msg, thrown);
362                 break;
363             case DISCARD:
364                 break;
365             default:
366                 throw new IllegalStateException("Unknown EventRoute " + eventRoute);
367         }
368     }
369 
370     /**
371      * This method is called by the EventHandler that processes the RingBufferLogEvent in a separate thread.
372      * Merges the contents of the configuration map into the contextData, after replacing any variables in the property
373      * values with the StrSubstitutor-supplied actual values.
374      *
375      * @param event the event to log
376      */
377     public void actualAsyncLog(final RingBufferLogEvent event) {
378         final LoggerConfig privateConfigLoggerConfig = privateConfig.loggerConfig;
379         final List<Property> properties = privateConfigLoggerConfig.getPropertyList();
380 
381         if (properties != null) {
382             onPropertiesPresent(event, properties);
383         }
384 
385         privateConfigLoggerConfig.getReliabilityStrategy().log(this, event);
386     }
387 
388     @SuppressWarnings("ForLoopReplaceableByForEach") // Avoid iterator allocation
389     private void onPropertiesPresent(final RingBufferLogEvent event, final List<Property> properties) {
390         StringMap contextData = getContextData(event);
391         for (int i = 0, size = properties.size(); i < size; i++) {
392             final Property prop = properties.get(i);
393             if (contextData.getValue(prop.getName()) != null) {
394                 continue; // contextMap overrides config properties
395             }
396             final String value = prop.isValueNeedsLookup() //
397                     ? privateConfig.config.getStrSubstitutor().replace(event, prop.getValue()) //
398                     : prop.getValue();
399             contextData.putValue(prop.getName(), value);
400         }
401         event.setContextData(contextData);
402     }
403 
404     private static StringMap getContextData(final RingBufferLogEvent event) {
405         StringMap contextData = (StringMap) event.getContextData();
406         if (contextData.isFrozen()) {
407             final StringMap temp = ContextDataFactory.createContextData();
408             temp.putAll(contextData);
409             return temp;
410         }
411         return contextData;
412     }
413 }