1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.appender;
18
19 import java.io.File;
20 import java.io.IOException;
21 import java.io.OutputStream;
22 import java.io.RandomAccessFile;
23 import java.io.Serializable;
24 import java.lang.reflect.Method;
25 import java.nio.ByteOrder;
26 import java.nio.MappedByteBuffer;
27 import java.nio.channels.FileChannel;
28 import java.security.AccessController;
29 import java.security.PrivilegedActionException;
30 import java.security.PrivilegedExceptionAction;
31 import java.util.HashMap;
32 import java.util.Map;
33
34 import org.apache.logging.log4j.core.Layout;
35 import org.apache.logging.log4j.core.util.Assert;
36 import org.apache.logging.log4j.core.util.Closer;
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 public class MemoryMappedFileManager extends OutputStreamManager {
53 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
54 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
55
56 private final boolean isForce;
57 private final int regionLength;
58 private final String advertiseURI;
59 private final RandomAccessFile randomAccessFile;
60 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
61 private MappedByteBuffer mappedBuffer;
62 private long mappingOffset;
63
64 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
65 final boolean force, final long position, final int regionLength, final String advertiseURI,
66 final Layout<? extends Serializable> layout) throws IOException {
67 super(os, fileName, layout);
68 this.isForce = force;
69 this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
70 this.regionLength = regionLength;
71 this.advertiseURI = advertiseURI;
72 this.isEndOfBatch.set(Boolean.FALSE);
73 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
74 this.mappingOffset = position;
75 }
76
77
78
79
80
81
82
83
84
85
86
87
88 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
89 final boolean isForce, final int regionLength, final String advertiseURI,
90 final Layout<? extends Serializable> layout) {
91 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
92 advertiseURI, layout), FACTORY);
93 }
94
95 public Boolean isEndOfBatch() {
96 return isEndOfBatch.get();
97 }
98
99 public void setEndOfBatch(final boolean isEndOfBatch) {
100 this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
101 }
102
103 @Override
104 protected synchronized void write(final byte[] bytes, int offset, int length) {
105 super.write(bytes, offset, length);
106
107 while (length > mappedBuffer.remaining()) {
108 final int chunk = mappedBuffer.remaining();
109 mappedBuffer.put(bytes, offset, chunk);
110 offset += chunk;
111 length -= chunk;
112 remap();
113 }
114 mappedBuffer.put(bytes, offset, length);
115
116
117
118 }
119
120 private synchronized void remap() {
121 final long offset = this.mappingOffset + mappedBuffer.position();
122 final int length = mappedBuffer.remaining() + regionLength;
123 try {
124 unsafeUnmap(mappedBuffer);
125 final long fileLength = randomAccessFile.length() + regionLength;
126 LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
127
128 long startNanos = System.nanoTime();
129 randomAccessFile.setLength(fileLength);
130 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
131 LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
132
133 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
134 mappingOffset = offset;
135 } catch (final Exception ex) {
136 LOGGER.error("Unable to remap " + getName() + ". " + ex);
137 }
138 }
139
140 @Override
141 public synchronized void flush() {
142 mappedBuffer.force();
143 }
144
145 @Override
146 public synchronized void close() {
147 final long position = mappedBuffer.position();
148 final long length = mappingOffset + position;
149 try {
150 unsafeUnmap(mappedBuffer);
151 } catch (final Exception ex) {
152 LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
153 }
154 try {
155 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
156 length, mappingOffset, position);
157 randomAccessFile.setLength(length);
158 randomAccessFile.close();
159 } catch (final IOException ex) {
160 LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
161 }
162 }
163
164 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
165 final int size) throws IOException {
166 for (int i = 1;; i++) {
167 try {
168 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
169
170 final long startNanos = System.nanoTime();
171 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
172 map.order(ByteOrder.nativeOrder());
173
174 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
175 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
176
177 return map;
178 } catch (final IOException e) {
179 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
180 throw e;
181 }
182 LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
183 if (i < 10) {
184 Thread.yield();
185 } else {
186 try {
187 Thread.sleep(1);
188 } catch (final InterruptedException ignored) {
189 Thread.currentThread().interrupt();
190 throw e;
191 }
192 }
193 }
194 }
195 }
196
197 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
198 LOGGER.debug("MMapAppender unmapping old buffer...");
199 final long startNanos = System.nanoTime();
200 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
201 @Override
202 public Object run() throws Exception {
203 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
204 getCleanerMethod.setAccessible(true);
205 final Object cleaner = getCleanerMethod.invoke(mbb);
206 final Method cleanMethod = cleaner.getClass().getMethod("clean");
207 cleanMethod.invoke(cleaner);
208 return null;
209 }
210 });
211 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
212 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
213 }
214
215
216
217
218
219
220 public String getFileName() {
221 return getName();
222 }
223
224
225
226
227
228
229 public int getRegionLength() {
230 return regionLength;
231 }
232
233
234
235
236
237
238
239 public boolean isImmediateFlush() {
240 return isForce;
241 }
242
243
244 static class DummyOutputStream extends OutputStream {
245 @Override
246 public void write(final int b) throws IOException {
247 }
248
249 @Override
250 public void write(final byte[] b, final int off, final int len) throws IOException {
251 }
252 }
253
254
255
256
257
258
259
260
261
262 @Override
263 public Map<String, String> getContentFormat() {
264 final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
265 result.put("fileURI", advertiseURI);
266 return result;
267 }
268
269
270
271
272 private static class FactoryData {
273 private final boolean append;
274 private final boolean force;
275 private final int regionLength;
276 private final String advertiseURI;
277 private final Layout<? extends Serializable> layout;
278
279
280
281
282
283
284
285
286 public FactoryData(final boolean append, final boolean force, final int regionLength,
287 final String advertiseURI, final Layout<? extends Serializable> layout) {
288 this.append = append;
289 this.force = force;
290 this.regionLength = regionLength;
291 this.advertiseURI = advertiseURI;
292 this.layout = layout;
293 }
294 }
295
296
297
298
299 private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
300
301
302
303
304
305
306
307
308 @SuppressWarnings("resource")
309 @Override
310 public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
311 final File file = new File(name);
312 final File parent = file.getParentFile();
313 if (null != parent && !parent.exists()) {
314 parent.mkdirs();
315 }
316 if (!data.append) {
317 file.delete();
318 }
319
320 final OutputStream os = new DummyOutputStream();
321 RandomAccessFile raf = null;
322 try {
323 raf = new RandomAccessFile(name, "rw");
324 final long position = (data.append) ? raf.length() : 0;
325 raf.setLength(position + data.regionLength);
326 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
327 data.advertiseURI, data.layout);
328 } catch (final Exception ex) {
329 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
330 Closer.closeSilently(raf);
331 }
332 return null;
333 }
334 }
335 }