/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using Directory = Lucene.Net.Store.Directory;
using IndexInput = Lucene.Net.Store.IndexInput;
using IndexOutput = Lucene.Net.Store.IndexOutput;
namespace Lucene.Net.Index
{
[Serializable]
sealed public class SegmentInfos : System.Collections.ArrayList
{
private class AnonymousClassFindSegmentsFile : FindSegmentsFile
{
private void InitBlock(SegmentInfos enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
private SegmentInfos enclosingInstance;
public SegmentInfos Enclosing_Instance
{
get
{
return enclosingInstance;
}
}
internal AnonymousClassFindSegmentsFile(SegmentInfos enclosingInstance, Lucene.Net.Store.Directory Param1) : base(Param1)
{
InitBlock(enclosingInstance);
}
protected internal override System.Object DoBody(System.String segmentFileName)
{
Enclosing_Instance.Read(directory, segmentFileName);
return null;
}
}
private class AnonymousClassFindSegmentsFile1 : FindSegmentsFile
{
internal AnonymousClassFindSegmentsFile1(Lucene.Net.Store.Directory Param1) : base(Param1)
{
}
protected internal override System.Object DoBody(System.String segmentFileName)
{
IndexInput input = directory.OpenInput(segmentFileName);
int format = 0;
long version = 0;
try
{
format = input.ReadInt();
if (format < 0)
{
if (format < Lucene.Net.Index.SegmentInfos.CURRENT_FORMAT)
throw new CorruptIndexException("Unknown format version: " + format);
version = input.ReadLong(); // read version
}
}
finally
{
input.Close();
}
if (format < 0)
return (long) version;
// We cannot be sure about the format of the file.
// Therefore we have to read the whole file and cannot simply seek to the version entry.
SegmentInfos sis = new SegmentInfos();
sis.Read(directory, segmentFileName);
return (long) sis.GetVersion();
}
}
/// The file format version, a negative number.
/* Works since counter, the old 1st entry, is always >= 0 */
public const int FORMAT = - 1;
/// This format adds details used for lockless commits. It differs
/// slightly from the previous format in that file names
/// are never re-used (write once). Instead, each file is
/// written to the next generation. For example,
/// segments_1, segments_2, etc. This allows us to not use
/// a commit lock. See file
/// formats for details.
///
public const int FORMAT_LOCKLESS = - 2;
/// This format adds a "hasSingleNormFile" flag into each segment info.
/// See LUCENE-756
/// for details.
///
public const int FORMAT_SINGLE_NORM_FILE = - 3;
/// This format allows multiple segments to share a single
/// vectors and stored fields file.
///
public const int FORMAT_SHARED_DOC_STORE = - 4;
/* This must always point to the most recent file format. */
private static readonly int CURRENT_FORMAT = FORMAT_SHARED_DOC_STORE;
public int counter = 0; // used to name new segments
/// counts how often the index has been changed by adding or deleting docs.
/// starting with the current time in milliseconds forces to create unique version numbers.
///
private long version = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
private long generation = 0; // generation of the "segments_N" for the next commit
private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
// or wrote; this is normally the same as generation except if
// there was an IOException that had interrupted a commit
/// If non-null, information about loading segments_N files
///
///
private static System.IO.StreamWriter infoStream;
public SegmentInfo Info(int i)
{
return (SegmentInfo) this[i];
}
/// Get the generation (N) of the current segments_N file
/// from a list of files.
///
///
/// -- array of file names to check
///
public static long GetCurrentSegmentGeneration(System.String[] files)
{
if (files == null)
{
return - 1;
}
long max = - 1;
for (int i = 0; i < files.Length; i++)
{
System.String file = System.IO.Path.GetFileName(files[i]);
if (file.StartsWith(IndexFileNames.SEGMENTS) && !file.Equals(IndexFileNames.SEGMENTS_GEN))
{
long gen = GenerationFromSegmentsFileName(file);
if (gen > max)
{
max = gen;
}
}
}
return max;
}
/// Get the generation (N) of the current segments_N file
/// in the directory.
///
///
/// -- directory to search for the latest segments_N file
///
public static long GetCurrentSegmentGeneration(Directory directory)
{
System.String[] files = directory.List();
if (files == null)
{
throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
}
return GetCurrentSegmentGeneration(files);
}
/// Get the filename of the current segments_N file
/// from a list of files.
///
///
/// -- array of file names to check
///
public static System.String GetCurrentSegmentFileName(System.String[] files)
{
return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(files));
}
/// Get the filename of the current segments_N file
/// in the directory.
///
///
/// -- directory to search for the latest segments_N file
///
public static System.String GetCurrentSegmentFileName(Directory directory)
{
return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(directory));
}
/// Get the segments_N filename in use by this segment infos.
public System.String GetCurrentSegmentFileName()
{
return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", lastGeneration);
}
/// Parse the generation off the segments file name and
/// return it.
///
public static long GenerationFromSegmentsFileName(System.String fileName)
{
if (fileName.Equals(IndexFileNames.SEGMENTS))
{
return 0;
}
else if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
return SupportClass.Number.ToInt64(fileName.Substring(1 + IndexFileNames.SEGMENTS.Length));
}
else
{
throw new System.ArgumentException("fileName \"" + fileName + "\" is not a segments file");
}
}
/// Get the next segments_N filename that will be written.
public System.String GetNextSegmentFileName()
{
long nextGeneration;
if (generation == - 1)
{
nextGeneration = 1;
}
else
{
nextGeneration = generation + 1;
}
return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", nextGeneration);
}
/// Read a particular segmentFileName. Note that this may
/// throw an IOException if a commit is in process.
///
///
/// -- directory containing the segments file
///
/// -- segment file to load
///
/// CorruptIndexException if the index is corrupt
/// IOException if there is a low-level IO error
public void Read(Directory directory, System.String segmentFileName)
{
bool success = false;
// Clear any previous segments:
Clear();
IndexInput input = directory.OpenInput(segmentFileName);
generation = GenerationFromSegmentsFileName(segmentFileName);
lastGeneration = generation;
try
{
int format = input.ReadInt();
if (format < 0)
{
// file contains explicit format info
// check that it is a format we can understand
if (format < CURRENT_FORMAT)
throw new CorruptIndexException("Unknown format version: " + format);
version = input.ReadLong(); // read version
counter = input.ReadInt(); // read counter
}
else
{
// file is in old format without explicit format info
counter = format;
}
for (int i = input.ReadInt(); i > 0; i--)
{
// read segmentInfos
Add(new SegmentInfo(directory, format, input));
}
if (format >= 0)
{
// in old format the version number may be at the end of the file
if (input.GetFilePointer() >= input.Length())
version = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
// old file format without version number
else
version = input.ReadLong(); // read version
}
success = true;
}
finally
{
input.Close();
if (!success)
{
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
Clear();
}
}
}
/// This version of read uses the retry logic (for lock-less
/// commits) to find the right segments file to load.
///
/// CorruptIndexException if the index is corrupt
/// IOException if there is a low-level IO error
public void Read(Directory directory)
{
generation = lastGeneration = - 1;
new AnonymousClassFindSegmentsFile(this, directory).Run();
}
public void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == - 1)
{
generation = 1;
}
else
{
generation++;
}
IndexOutput output = directory.CreateOutput(segmentFileName);
bool success = false;
try
{
output.WriteInt(CURRENT_FORMAT); // write FORMAT
output.WriteLong(++version); // every write changes
// the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(output);
}
}
finally
{
try
{
output.Close();
success = true;
}
finally
{
if (!success)
{
// Try not to leave a truncated segments_N file in
// the index:
directory.DeleteFile(segmentFileName);
}
}
}
try
{
output = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
output.WriteInt(FORMAT_LOCKLESS);
output.WriteLong(generation);
output.WriteLong(generation);
}
finally
{
output.Close();
}
}
catch (System.IO.IOException e)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
lastGeneration = generation;
}
/// Returns a copy of this instance, also copying each
/// SegmentInfo.
///
public override System.Object Clone()
{
SegmentInfos si = new SegmentInfos();
for (int i = 0; i < base.Count; i++)
{
si.Add(((SegmentInfo) base[i]).Clone());
}
si.generation = this.generation;
si.lastGeneration = this.lastGeneration;
return si;
}
private SegmentInfos(SegmentInfos si) : base(si)
{
}
public SegmentInfos()
{
}
/// version number when this SegmentInfos was generated.
public long GetVersion()
{
return version;
}
public long GetGeneration()
{
return generation;
}
public long GetLastGeneration()
{
return lastGeneration;
}
/// Current version number from segments file.
/// CorruptIndexException if the index is corrupt
/// IOException if there is a low-level IO error
public static long ReadCurrentVersion(Directory directory)
{
return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run());
}
/// If non-null, information about retries when loading
/// the segments file will be printed to this.
///
public static void SetInfoStream(System.IO.StreamWriter infoStream)
{
SegmentInfos.infoStream = infoStream;
}
/* Advanced configuration of retry logic in loading
segments_N file */
private static int defaultGenFileRetryCount = 10;
private static int defaultGenFileRetryPauseMsec = 50;
private static int defaultGenLookaheadCount = 10;
/// Advanced: set how many times to try loading the
/// segments.gen file contents to determine current segment
/// generation. This file is only referenced when the
/// primary method (listing the directory) fails.
///
public static void SetDefaultGenFileRetryCount(int count)
{
defaultGenFileRetryCount = count;
}
///
///
public static int GetDefaultGenFileRetryCount()
{
return defaultGenFileRetryCount;
}
/// Advanced: set how many milliseconds to pause in between
/// attempts to load the segments.gen file.
///
public static void SetDefaultGenFileRetryPauseMsec(int msec)
{
defaultGenFileRetryPauseMsec = msec;
}
///
///
public static int GetDefaultGenFileRetryPauseMsec()
{
return defaultGenFileRetryPauseMsec;
}
/// Advanced: set how many times to try incrementing the
/// gen when loading the segments file. This only runs if
/// the primary (listing directory) and secondary (opening
/// segments.gen file) methods fail to find the segments
/// file.
///
public static void SetDefaultGenLookaheadCount(int count)
{
defaultGenLookaheadCount = count;
}
///
///
public static int GetDefaultGenLookahedCount()
{
return defaultGenLookaheadCount;
}
///
///
public static System.IO.StreamWriter GetInfoStream()
{
return infoStream;
}
private static void Message(System.String message)
{
if (infoStream != null)
{
infoStream.WriteLine("SIS [" + SupportClass.ThreadClass.Current().Name + "]: " + message);
}
}
/// Utility class for executing code that needs to do
/// something with the current segments file. This is
/// necessary with lock-less commits because from the time
/// you locate the current segments file name, until you
/// actually open it, read its contents, or check modified
/// time, etc., it could have been deleted due to a writer
/// commit finishing.
///
public abstract class FindSegmentsFile
{
internal System.IO.FileInfo fileDirectory;
internal Directory directory;
public FindSegmentsFile(System.IO.FileInfo directory)
{
this.fileDirectory = directory;
}
public FindSegmentsFile(Directory directory)
{
this.directory = directory;
}
public System.Object Run()
{
System.String segmentFileName = null;
long lastGen = - 1;
long gen = 0;
int genLookaheadCount = 0;
System.IO.IOException exc = null;
bool retry = false;
int method = 0;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try the first two in parallel, and
// fall back to the third when necessary.
while (true)
{
if (0 == method)
{
// Method 1: list the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents (NOTE: NFS clients often have such stale
// caching):
System.String[] files = null;
long genA = - 1;
if (directory != null)
files = directory.List();
else
{
files = System.IO.Directory.GetFileSystemEntries(fileDirectory.FullName);
}
if (files != null)
genA = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
Lucene.Net.Index.SegmentInfos.Message("directory listing genA=" + genA);
// Method 2: open segments.gen and read its
// contents. Then we take the larger of the two
// gen's. This way, if either approach is hitting
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = - 1;
if (directory != null)
{
for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
{
IndexInput genInput = null;
try
{
genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
}
catch (System.IO.FileNotFoundException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: FileNotFoundException " + e);
break;
}
catch (System.IO.IOException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
}
catch (UnauthorizedAccessException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: UnauthorizedAccessException " + e);
System.Threading.Thread.Sleep(10);
}
if (genInput != null)
{
try
{
int version = genInput.ReadInt();
if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
{
long gen0 = genInput.ReadLong();
long gen1 = genInput.ReadLong();
Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
if (gen0 == gen1)
{
// The file is consistent.
genB = gen0;
break;
}
}
}
catch (System.IO.IOException err2)
{
// will retry
}
finally
{
genInput.Close();
}
}
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
}
catch (System.Threading.ThreadInterruptedException e)
{
// will retry
}
}
}
Lucene.Net.Index.SegmentInfos.Message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
// Pick the larger of the two gen's:
if (genA > genB)
gen = genA;
else
gen = genB;
if (gen == - 1)
{
// Neither approach found a generation
System.String s;
if (files != null)
{
s = "";
for (int i = 0; i < files.Length; i++)
s += (" " + files[i]);
}
else
s = " null";
throw new System.IO.FileNotFoundException("no segments* file found in " + directory + ": files:" + s);
}
}
// Third method (fallback if first & second methods
// are not reliable): since both directory cache and
// file contents cache seem to be stale, just
// advance the generation.
if (1 == method || (0 == method && lastGen == gen && retry))
{
method = 1;
if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
{
gen++;
genLookaheadCount++;
Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
}
}
if (lastGen == gen)
{
// This means we're about to try the same
// segments_N last tried. This is allowed,
// exactly once, because writer could have been in
// the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file
// twice in a row, so this must be a real
// error. We throw the original exception we
// got.
throw exc;
}
else
{
retry = true;
}
}
else
{
// Segment file has advanced since our last loop, so
// reset retry:
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try
{
System.Object v = DoBody(segmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
}
return v;
}
catch (System.IO.IOException err)
{
// Save the original root cause:
if (exc == null)
{
exc = err;
}
Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
if (!retry && gen > 1)
{
// This is our first time trying this segments
// file (because retry is false), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
bool prevExists;
if (directory != null)
prevExists = directory.FileExists(prevSegmentFileName);
else
{
bool tmpBool;
if (System.IO.File.Exists(new System.IO.FileInfo(fileDirectory.FullName + System.IO.Path.DirectorySeparatorChar + prevSegmentFileName).FullName))
tmpBool = true;
else
tmpBool = System.IO.Directory.Exists(new System.IO.FileInfo(fileDirectory.FullName + System.IO.Path.DirectorySeparatorChar + prevSegmentFileName).FullName);
prevExists = tmpBool;
}
if (prevExists)
{
Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
try
{
System.Object v = DoBody(prevSegmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
}
return v;
}
catch (System.IO.IOException err2)
{
Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
}
}
}
}
}
}
/// Subclass must implement this. The assumption is an
/// IOException will be thrown if something goes wrong
/// during the processing that could have been caused by
/// a writer committing.
///
protected internal abstract System.Object DoBody(System.String segmentFileName);
}
/// Returns a new SegmentInfos containg the SegmentInfo
/// instances in the specified range first (inclusive) to
/// last (exclusive), so total number of segments returned
/// is last-first.
///
public SegmentInfos Range(int first, int last)
{
SegmentInfos infos = new SegmentInfos();
infos.AddRange((System.Collections.IList) ((System.Collections.ArrayList) this).GetRange(first, last - first));
return infos;
}
}
}