View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache license, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License. You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the license for the specific language governing permissions and
15   * limitations under the license.
16   */
17  package org.apache.logging.log4j.core.appender;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.io.RandomAccessFile;
23  import java.io.Serializable;
24  import java.lang.reflect.Method;
25  import java.nio.ByteBuffer;
26  import java.nio.ByteOrder;
27  import java.nio.MappedByteBuffer;
28  import java.nio.channels.FileChannel;
29  import java.security.AccessController;
30  import java.security.PrivilegedActionException;
31  import java.security.PrivilegedExceptionAction;
32  import java.util.HashMap;
33  import java.util.Map;
34  import java.util.Objects;
35  
36  import org.apache.logging.log4j.core.Layout;
37  import org.apache.logging.log4j.core.util.Closer;
38  import org.apache.logging.log4j.core.util.NullOutputStream;
39  
40  //Lines too long...
41  //CHECKSTYLE:OFF
42  /**
43   * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
44   * memory and writes to this memory region.
45   * <p>
46   *
47   * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
48   *      http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>;
49   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
50   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
51   * @see <a
52   *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
53   *      http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>;
54   *
55   * @since 2.1
56   */
57  //CHECKSTYLE:ON
58  public class MemoryMappedFileManager extends OutputStreamManager {
59      /**
60       * Default length of region to map.
61       */
62      static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
63      private static final int MAX_REMAP_COUNT = 10;
64      private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
65      private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
66  
67      private final boolean isForce;
68      private final int regionLength;
69      private final String advertiseURI;
70      private final RandomAccessFile randomAccessFile;
71      private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
72      private MappedByteBuffer mappedBuffer;
73      private long mappingOffset;
74  
75      protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
76              final boolean force, final long position, final int regionLength, final String advertiseURI,
77              final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
78          super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0]));
79          this.isForce = force;
80          this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
81          this.regionLength = regionLength;
82          this.advertiseURI = advertiseURI;
83          this.isEndOfBatch.set(Boolean.FALSE);
84          this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
85          this.byteBuffer = mappedBuffer;
86          this.mappingOffset = position;
87      }
88  
89      /**
90       * Returns the MemoryMappedFileManager.
91       *
92       * @param fileName The name of the file to manage.
93       * @param append true if the file should be appended to, false if it should be overwritten.
94       * @param isForce true if the contents should be flushed to disk on every write
95       * @param regionLength The mapped region length.
96       * @param advertiseURI the URI to use when advertising the file
97       * @param layout The layout.
98       * @return A MemoryMappedFileManager for the File.
99       */
100     public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
101             final boolean isForce, final int regionLength, final String advertiseURI,
102             final Layout<? extends Serializable> layout) {
103         return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
104                 advertiseURI, layout), FACTORY);
105     }
106 
107     public Boolean isEndOfBatch() {
108         return isEndOfBatch.get();
109     }
110 
111     public void setEndOfBatch(final boolean endOfBatch) {
112         this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
113     }
114 
115     @Override
116     protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
117         while (length > mappedBuffer.remaining()) {
118             final int chunk = mappedBuffer.remaining();
119             mappedBuffer.put(bytes, offset, chunk);
120             offset += chunk;
121             length -= chunk;
122             remap();
123         }
124         mappedBuffer.put(bytes, offset, length);
125 
126         // no need to call flush() if force is true,
127         // already done in AbstractOutputStreamAppender.append
128     }
129 
130     private synchronized void remap() {
131         final long offset = this.mappingOffset + mappedBuffer.position();
132         final int length = mappedBuffer.remaining() + regionLength;
133         try {
134             unsafeUnmap(mappedBuffer);
135             final long fileLength = randomAccessFile.length() + regionLength;
136             LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
137                     regionLength, fileLength);
138 
139             final long startNanos = System.nanoTime();
140             randomAccessFile.setLength(fileLength);
141             final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
142             LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
143                     millis);
144 
145             mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
146             this.byteBuffer = mappedBuffer;
147             mappingOffset = offset;
148         } catch (final Exception ex) {
149             logError("Unable to remap", ex);
150         }
151     }
152 
153     @Override
154     public synchronized void flush() {
155         mappedBuffer.force();
156     }
157 
158     @Override
159     public synchronized boolean closeOutputStream() {
160         final long position = mappedBuffer.position();
161         final long length = mappingOffset + position;
162         try {
163             unsafeUnmap(mappedBuffer);
164         } catch (final Exception ex) {
165             logError("Unable to unmap MappedBuffer", ex);
166         }
167         try {
168             LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
169                     length, mappingOffset, position);
170             randomAccessFile.setLength(length);
171             randomAccessFile.close();
172             return true;
173         } catch (final IOException ex) {
174             logError("Unable to close MemoryMappedFile", ex);
175             return false;
176         }
177     }
178 
179     public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
180             final int size) throws IOException {
181         for (int i = 1;; i++) {
182             try {
183                 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
184 
185                 final long startNanos = System.nanoTime();
186                 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
187                 map.order(ByteOrder.nativeOrder());
188 
189                 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
190                 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
191 
192                 return map;
193             } catch (final IOException e) {
194                 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
195                     throw e;
196                 }
197                 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
198                 if (i < MAX_REMAP_COUNT) {
199                     Thread.yield();
200                 } else {
201                     try {
202                         Thread.sleep(1);
203                     } catch (final InterruptedException ignored) {
204                         Thread.currentThread().interrupt();
205                         throw e;
206                     }
207                 }
208             }
209         }
210     }
211 
212     private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
213         LOGGER.debug("MMapAppender unmapping old buffer...");
214         final long startNanos = System.nanoTime();
215         AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
216             @Override
217             public Object run() throws Exception {
218                 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
219                 getCleanerMethod.setAccessible(true);
220                 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
221                 final Method cleanMethod = cleaner.getClass().getMethod("clean");
222                 cleanMethod.invoke(cleaner);
223                 return null;
224             }
225         });
226         final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
227         LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
228     }
229 
230     /**
231      * Returns the name of the File being managed.
232      *
233      * @return The name of the File being managed.
234      */
235     public String getFileName() {
236         return getName();
237     }
238 
239     /**
240      * Returns the length of the memory mapped region.
241      *
242      * @return the length of the mapped region
243      */
244     public int getRegionLength() {
245         return regionLength;
246     }
247 
248     /**
249      * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
250      * {@code false} otherwise.
251      *
252      * @return whether each write should be force-sync'ed
253      */
254     public boolean isImmediateFlush() {
255         return isForce;
256     }
257 
258     /**
259      * Gets this FileManager's content format specified by:
260      * <p>
261      * Key: "fileURI" Value: provided "advertiseURI" param.
262      * </p>
263      *
264      * @return Map of content format keys supporting FileManager
265      */
266     @Override
267     public Map<String, String> getContentFormat() {
268         final Map<String, String> result = new HashMap<>(super.getContentFormat());
269         result.put("fileURI", advertiseURI);
270         return result;
271     }
272 
273     @Override
274     protected void flushBuffer(final ByteBuffer buffer) {
275         // do nothing (do not call drain() to avoid spurious remapping)
276     }
277 
278     @Override
279     public ByteBuffer getByteBuffer() {
280         return mappedBuffer;
281     }
282 
283     @Override
284     public ByteBuffer drain(final ByteBuffer buf) {
285         remap();
286         return mappedBuffer;
287     }
288 
289     /**
290      * Factory Data.
291      */
292     private static class FactoryData {
293         private final boolean append;
294         private final boolean force;
295         private final int regionLength;
296         private final String advertiseURI;
297         private final Layout<? extends Serializable> layout;
298 
299         /**
300          * Constructor.
301          *
302          * @param append Append to existing file or truncate.
303          * @param force forces the memory content to be written to the storage device on every event
304          * @param regionLength length of the mapped region
305          * @param advertiseURI the URI to use when advertising the file
306          * @param layout The layout.
307          */
308         public FactoryData(final boolean append, final boolean force, final int regionLength,
309                 final String advertiseURI, final Layout<? extends Serializable> layout) {
310             this.append = append;
311             this.force = force;
312             this.regionLength = regionLength;
313             this.advertiseURI = advertiseURI;
314             this.layout = layout;
315         }
316     }
317 
318     /**
319      * Factory to create a MemoryMappedFileManager.
320      */
321     private static class MemoryMappedFileManagerFactory
322             implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
323 
324         /**
325          * Create a MemoryMappedFileManager.
326          *
327          * @param name The name of the File.
328          * @param data The FactoryData
329          * @return The MemoryMappedFileManager for the File.
330          */
331         @SuppressWarnings("resource")
332         @Override
333         public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
334             final File file = new File(name);
335             final File parent = file.getParentFile();
336             if (null != parent && !parent.exists()) {
337                 parent.mkdirs();
338             }
339             if (!data.append) {
340                 file.delete();
341             }
342 
343             final boolean writeHeader = !data.append || !file.exists();
344             final OutputStream os = NullOutputStream.getInstance();
345             RandomAccessFile raf = null;
346             try {
347                 raf = new RandomAccessFile(name, "rw");
348                 final long position = (data.append) ? raf.length() : 0;
349                 raf.setLength(position + data.regionLength);
350                 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
351                         data.advertiseURI, data.layout, writeHeader);
352             } catch (final Exception ex) {
353                 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
354                 Closer.closeSilently(raf);
355             }
356             return null;
357         }
358     }
359 }