/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using Constants = Lucene.Net.Util.Constants; namespace Lucene.Net.Store { /// File-based implementation that uses /// mmap for reading, and /// for writing. /// ///

NOTE: memory mapping uses up a portion of the /// virtual memory address space in your process equal to the /// size of the file being mapped. Before using this class, /// be sure your have plenty of virtual address space, e.g. by /// using a 64 bit JRE, or a 32 bit JRE with indexes that are /// guaranteed to fit within the address space. /// On 32 bit platforms also consult /// if you have problems with mmap failing because of fragmented /// address space. If you get an OutOfMemoryException, it is recommened /// to reduce the chunk size, until it works. /// ///

Due to /// this bug in Sun's JRE, MMapDirectory's /// is unable to close the underlying OS file handle. Only when GC /// finally collects the underlying objects, which could be quite /// some time later, will the file handle be closed. /// ///

This will consume additional transient disk usage: on Windows, /// attempts to delete or overwrite the files will result in an /// exception; on other platforms, which typically have a "delete on /// last close" semantics, while such operations will succeed, the bytes /// are still consuming space on disk. For many applications this /// limitation is not a problem (e.g. if you have plenty of disk space, /// and you don't rely on overwriting files on Windows) but it's still /// an important limitation to be aware of. /// ///

This class supplies the workaround mentioned in the bug report /// (disabled by default, see ), which may fail on /// non-Sun JVMs. It forcefully unmaps the buffer on close by using /// an undocumented internal cleanup functionality. /// is true, if the workaround /// can be enabled (with no guarantees). ///

public class MMapDirectory:FSDirectory { private class AnonymousClassPrivilegedExceptionAction // : SupportClass.IPriviligedAction // {{Aroush-2.9}} { public AnonymousClassPrivilegedExceptionAction(byte[] buffer, MMapDirectory enclosingInstance) { InitBlock(buffer, enclosingInstance); } private void InitBlock(byte[] buffer, MMapDirectory enclosingInstance) { this.buffer = buffer; this.enclosingInstance = enclosingInstance; } private byte[] buffer; private MMapDirectory enclosingInstance; public MMapDirectory Enclosing_Instance { get { return enclosingInstance; } } public virtual System.Object Run() { // {{Aroush-2.9 /* System.Reflection.MethodInfo getCleanerMethod = buffer.GetType().GetMethod("cleaner", (Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES); getCleanerMethod.SetAccessible(true); System.Object cleaner = getCleanerMethod.Invoke(buffer, (System.Object[]) Lucene.Net.Store.MMapDirectory.NO_PARAMS); if (cleaner != null) { cleaner.GetType().GetMethod("clean", (Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES).Invoke(cleaner, (System.Object[]) Lucene.Net.Store.MMapDirectory.NO_PARAMS); } */ //System.Diagnostics.Debug.Fail("Port issue:", "sun.misc.Cleaner()"); // {{Aroush-2.9}} throw new NotImplementedException("Port issue: sun.misc.Cleaner()"); // Aroush-2.9}} //return null; } } private void InitBlock() { maxBBuf = Constants.JRE_IS_64BIT?System.Int32.MaxValue:(256 * 1024 * 1024); } /// Create a new MMapDirectory for the named location. /// /// /// the path of the directory /// /// the lock factory to use, or null for the default. /// /// IOException public MMapDirectory(System.IO.DirectoryInfo path, LockFactory lockFactory) : base(path, lockFactory) { InitBlock(); } /// Create a new MMapDirectory for the named location and the default lock factory. /// /// /// the path of the directory /// /// IOException public MMapDirectory(System.IO.DirectoryInfo path) : base(path, null) { InitBlock(); } private bool useUnmapHack = false; private int maxBBuf; /// true, if this platform supports unmapping mmaped files. public static bool UNMAP_SUPPORTED; /// Enables or disables the workaround for unmapping the buffers /// from address space after closing , that is /// mentioned in the bug report. This hack may fail on non-Sun JVMs. /// It forcefully unmaps the buffer on close by using /// an undocumented internal cleanup functionality. ///

NOTE: Enabling this is completely unsupported /// by Java and may lead to JVM crashs if IndexInput /// is closed while another thread is still accessing it (SIGSEGV). ///

/// IllegalArgumentException if /// is false and the workaround cannot be enabled. /// public virtual bool UseUnmap { get { return useUnmapHack; } set { if (value && !UNMAP_SUPPORTED) throw new System.ArgumentException("Unmap hack not supported on this platform!"); this.useUnmapHack = value; } } /// Try to unmap the buffer, this method silently fails if no support /// for that in the JVM. On Windows, this leads to the fact, /// that mmapped files cannot be modified or deleted. /// internal void CleanMapping(System.IO.MemoryStream buffer) { if (useUnmapHack) { try { // {{Aroush-2.9}} Not converted: java.security.AccessController.doPrivileged() //System.Diagnostics.Debug.Fail("Port issue:", "java.security.AccessController.doPrivileged()"); // {{Aroush-2.9}} throw new NotImplementedException("Port issue: java.security.AccessController.doPrivileged()"); // AccessController.DoPrivileged(new AnonymousClassPrivilegedExceptionAction(buffer, this)); } catch (System.Exception e) { System.IO.IOException ioe = new System.IO.IOException("unable to unmap the mapped buffer", e.InnerException); throw ioe; } } } /// Gets or sets the maximum chunk size (default is for /// 64 bit JVMs and 256 MiBytes for 32 bit JVMs) used for memory mapping. /// Especially on 32 bit platform, the address space can be very fragmented, /// so large index files cannot be mapped. /// Using a lower chunk size makes the directory implementation a little /// bit slower (as the correct chunk must be resolved on each seek) /// but the chance is higher that mmap does not fail. On 64 bit /// Java platforms, this parameter should always be , /// as the adress space is big enough. /// public virtual int MaxChunkSize { get { return maxBBuf; } set { if (value <= 0) throw new System.ArgumentException("Maximum chunk size for mmap must be >0"); this.maxBBuf = value; } } private class MMapIndexInput : IndexInput { private void InitBlock(MMapDirectory enclosingInstance) { this.enclosingInstance = enclosingInstance; } private MMapDirectory enclosingInstance; public MMapDirectory Enclosing_Instance { get { return enclosingInstance; } } private System.IO.MemoryStream buffer; private long length; private bool isClone; private bool isDisposed; internal MMapIndexInput(MMapDirectory enclosingInstance, System.IO.FileStream raf) { byte[] data = new byte[raf.Length]; raf.Read(data, 0, (int) raf.Length); InitBlock(enclosingInstance); this.length = raf.Length; this.buffer = new System.IO.MemoryStream(data); } public override byte ReadByte() { try { return (byte) buffer.ReadByte(); } catch (ObjectDisposedException) { throw new System.IO.IOException("read past EOF"); } } public override void ReadBytes(byte[] b, int offset, int len) { try { buffer.Read(b, offset, len); } catch (ObjectDisposedException) { throw new System.IO.IOException("read past EOF"); } } public override long FilePointer { get { return buffer.Position; } } public override void Seek(long pos) { buffer.Seek(pos, System.IO.SeekOrigin.Begin); } public override long Length() { return length; } public override System.Object Clone() { if (buffer == null) throw new AlreadyClosedException("MMapIndexInput already closed"); MMapIndexInput clone = (MMapIndexInput) base.Clone(); clone.isClone = true; // clone.buffer = buffer.duplicate(); // {{Aroush-1.9}} return clone; } protected override void Dispose(bool isDisposing) { if (isDisposed) return; if (isDisposing) { if (isClone || buffer == null) return; // unmap the buffer (if enabled) and at least unset it for GC try { Enclosing_Instance.CleanMapping(buffer); } finally { buffer = null; } } isDisposed = true; } } // Because Java's ByteBuffer uses an int to address the // values, it's necessary to access a file > // Integer.MAX_VALUE in size using multiple byte buffers. protected internal class MultiMMapIndexInput:IndexInput, System.ICloneable { private void InitBlock(MMapDirectory enclosingInstance) { this.enclosingInstance = enclosingInstance; } private MMapDirectory enclosingInstance; public MMapDirectory Enclosing_Instance { get { return enclosingInstance; } } private System.IO.MemoryStream[] buffers; private int[] bufSizes; // keep here, ByteBuffer.size() method is optional private long length; private bool isDisposed; private int curBufIndex; private int maxBufSize; private System.IO.MemoryStream curBuf; // redundant for speed: buffers[curBufIndex] private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position()) private bool isClone = false; public MultiMMapIndexInput(MMapDirectory enclosingInstance, System.IO.FileStream raf, int maxBufSize) { InitBlock(enclosingInstance); this.length = raf.Length; this.maxBufSize = maxBufSize; if (maxBufSize <= 0) throw new System.ArgumentException("Non positive maxBufSize: " + maxBufSize); if ((length / maxBufSize) > System.Int32.MaxValue) { throw new System.ArgumentException("RandomAccessFile too big for maximum buffer size: " + raf.ToString()); } int nrBuffers = (int) (length / maxBufSize); if (((long) nrBuffers * maxBufSize) < length) nrBuffers++; this.buffers = new System.IO.MemoryStream[nrBuffers]; this.bufSizes = new int[nrBuffers]; long bufferStart = 0; System.IO.FileStream rafc = raf; for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { byte[] data = new byte[rafc.Length]; raf.Read(data, 0, (int) rafc.Length); int bufSize = (length > (bufferStart + maxBufSize))?maxBufSize:(int) (length - bufferStart); this.buffers[bufNr] = new System.IO.MemoryStream(data); this.bufSizes[bufNr] = bufSize; bufferStart += bufSize; } Seek(0L); } public override byte ReadByte() { // Performance might be improved by reading ahead into an array of // e.g. 128 bytes and readByte() from there. if (curAvail == 0) { curBufIndex++; if (curBufIndex >= buffers.Length) throw new System.IO.IOException("read past EOF"); curBuf = buffers[curBufIndex]; curBuf.Seek(0, System.IO.SeekOrigin.Begin); curAvail = bufSizes[curBufIndex]; } curAvail--; return (byte) curBuf.ReadByte(); } public override void ReadBytes(byte[] b, int offset, int len) { while (len > curAvail) { curBuf.Read(b, offset, curAvail); len -= curAvail; offset += curAvail; curBufIndex++; if (curBufIndex >= buffers.Length) throw new System.IO.IOException("read past EOF"); curBuf = buffers[curBufIndex]; curBuf.Seek(0, System.IO.SeekOrigin.Begin); curAvail = bufSizes[curBufIndex]; } curBuf.Read(b, offset, len); curAvail -= len; } public override long FilePointer { get { return ((long) curBufIndex*maxBufSize) + curBuf.Position; } } public override void Seek(long pos) { curBufIndex = (int) (pos / maxBufSize); curBuf = buffers[curBufIndex]; int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize)); curBuf.Seek(bufOffset, System.IO.SeekOrigin.Begin); curAvail = bufSizes[curBufIndex] - bufOffset; } public override long Length() { return length; } public override System.Object Clone() { MultiMMapIndexInput clone = (MultiMMapIndexInput) base.Clone(); clone.isClone = true; clone.buffers = new System.IO.MemoryStream[buffers.Length]; // No need to clone bufSizes. // Since most clones will use only one buffer, duplicate() could also be // done lazy in clones, e.g. when adapting curBuf. for (int bufNr = 0; bufNr < buffers.Length; bufNr++) { clone.buffers[bufNr] = buffers[bufNr]; // clone.buffers[bufNr] = buffers[bufNr].duplicate(); // {{Aroush-1.9}} how do we clone?! } try { clone.Seek(FilePointer); } catch (System.IO.IOException ioe) { System.SystemException newException = new System.SystemException(ioe.Message, ioe); throw newException; } return clone; } protected override void Dispose(bool disposing) { if (isDisposed) return; if (isClone || buffers == null) return; try { for (int bufNr = 0; bufNr < buffers.Length; bufNr++) { // unmap the buffer (if enabled) and at least unset it for GC try { Enclosing_Instance.CleanMapping(buffers[bufNr]); } finally { buffers[bufNr] = null; } } } finally { buffers = null; } isDisposed = true; } } /// Creates an IndexInput for the file with the given name. public override IndexInput OpenInput(System.String name, int bufferSize) { EnsureOpen(); System.String path = System.IO.Path.Combine(Directory.FullName, name); System.IO.FileStream raf = new System.IO.FileStream(path, System.IO.FileMode.Open, System.IO.FileAccess.Read); try { return (raf.Length <= (long) maxBBuf)?(IndexInput) new MMapIndexInput(this, raf):(IndexInput) new MultiMMapIndexInput(this, raf, maxBBuf); } finally { raf.Close(); } } /// Creates an IndexOutput for the file with the given name. public override IndexOutput CreateOutput(System.String name) { InitOutput(name); return new SimpleFSDirectory.SimpleFSIndexOutput(new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name))); } static MMapDirectory() { { bool v; try { // {{Aroush-2.9 /* System.Type.GetType("sun.misc.Cleaner"); // {{Aroush-2.9}} port issue? System.Type.GetType("java.nio.DirectByteBuffer").GetMethod("cleaner", (NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) NO_PARAM_TYPES); */ //System.Diagnostics.Debug.Fail("Port issue:", "sun.misc.Cleaner.clean()"); // {{Aroush-2.9}} throw new NotImplementedException("Port issue: sun.misc.Cleaner.clean()"); // Aroush-2.9}} //v = true; } catch (System.Exception) { v = false; } UNMAP_SUPPORTED = v; } } } }