/* * Copyright 2004 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using Directory = Lucene.Net.Store.Directory; using IndexInput = Lucene.Net.Store.IndexInput; namespace Lucene.Net.Index { /// $Id: TermVectorsReader.cs,v 1.2 2005/10/06 19:29:56 dsd Exp $ /// class TermVectorsReader : System.ICloneable { private FieldInfos fieldInfos; private IndexInput tvx; private IndexInput tvd; private IndexInput tvf; private int size; private int tvdFormat; private int tvfFormat; public /*internal*/ TermVectorsReader(Directory d, System.String segment, FieldInfos fieldInfos) { if (d.FileExists(segment + TermVectorsWriter.TVX_EXTENSION)) { tvx = d.OpenInput(segment + TermVectorsWriter.TVX_EXTENSION); CheckValidFormat(tvx); tvd = d.OpenInput(segment + TermVectorsWriter.TVD_EXTENSION); tvdFormat = CheckValidFormat(tvd); tvf = d.OpenInput(segment + TermVectorsWriter.TVF_EXTENSION); tvfFormat = CheckValidFormat(tvf); size = (int) tvx.Length() / 8; } this.fieldInfos = fieldInfos; } private int CheckValidFormat(IndexInput in_Renamed) { int format = in_Renamed.ReadInt(); if (format > TermVectorsWriter.FORMAT_VERSION) { throw new System.IO.IOException("Incompatible format version: " + format + " expected " + TermVectorsWriter.FORMAT_VERSION + " or less"); } return format; } internal virtual void Close() { // make all effort to close up. Keep the first exception // and throw it as a new one. System.IO.IOException keep = null; if (tvx != null) try { tvx.Close(); } catch (System.IO.IOException e) { if (keep == null) keep = e; } if (tvd != null) try { tvd.Close(); } catch (System.IO.IOException e) { if (keep == null) keep = e; } if (tvf != null) try { tvf.Close(); } catch (System.IO.IOException e) { if (keep == null) keep = e; } if (keep != null) { throw new System.IO.IOException(keep.StackTrace); } } /// /// The number of documents in the reader /// internal virtual int Size() { return size; } /// Retrieve the term vector for the given document and field /// The document number to retrieve the vector for /// /// The field within the document to retrieve /// /// The TermFreqVector for the document and field or null if there is no termVector for this field. /// /// IOException if there is an error reading the term vector files public /*internal*/ virtual TermFreqVector Get(int docNum, System.String field) { // Check if no term vectors are available for this segment at all int fieldNumber = fieldInfos.FieldNumber(field); TermFreqVector result = null; if (tvx != null) { //We need to account for the FORMAT_SIZE at when seeking in the tvx //We don't need to do this in other seeks because we already have the // file pointer //that was written in another file tvx.Seek((docNum * 8L) + TermVectorsWriter.FORMAT_SIZE); //System.out.println("TVX Pointer: " + tvx.getFilePointer()); long position = tvx.ReadLong(); tvd.Seek(position); int fieldCount = tvd.ReadVInt(); //System.out.println("Num Fields: " + fieldCount); // There are only a few fields per document. We opt for a full scan // rather then requiring that they be ordered. We need to read through // all of the fields anyway to get to the tvf pointers. int number = 0; int found = - 1; for (int i = 0; i < fieldCount; i++) { if (tvdFormat == TermVectorsWriter.FORMAT_VERSION) number = tvd.ReadVInt(); else number += tvd.ReadVInt(); if (number == fieldNumber) found = i; } // This field, although valid in the segment, was not found in this // document if (found != - 1) { // Compute position in the tvf file position = 0; for (int i = 0; i <= found; i++) position += tvd.ReadVLong(); result = ReadTermVector(field, position); } else { //System.out.println("Field not found"); } } else { //System.out.println("No tvx file"); } return result; } /// Return all term vectors stored for this document or null if the could not be read in. /// /// /// The document number to retrieve the vector for /// /// All term frequency vectors /// /// IOException if there is an error reading the term vector files internal virtual TermFreqVector[] Get(int docNum) { TermFreqVector[] result = null; // Check if no term vectors are available for this segment at all if (tvx != null) { //We need to offset by tvx.Seek((docNum * 8L) + TermVectorsWriter.FORMAT_SIZE); long position = tvx.ReadLong(); tvd.Seek(position); int fieldCount = tvd.ReadVInt(); // No fields are vectorized for this document if (fieldCount != 0) { int number = 0; System.String[] fields = new System.String[fieldCount]; for (int i = 0; i < fieldCount; i++) { if (tvdFormat == TermVectorsWriter.FORMAT_VERSION) number = tvd.ReadVInt(); else number += tvd.ReadVInt(); fields[i] = fieldInfos.FieldName(number); } // Compute position in the tvf file position = 0; long[] tvfPointers = new long[fieldCount]; for (int i = 0; i < fieldCount; i++) { position += tvd.ReadVLong(); tvfPointers[i] = position; } result = ReadTermVectors(fields, tvfPointers); } } else { //System.out.println("No tvx file"); } return result; } private SegmentTermVector[] ReadTermVectors(System.String[] fields, long[] tvfPointers) { SegmentTermVector[] res = new SegmentTermVector[fields.Length]; for (int i = 0; i < fields.Length; i++) { res[i] = ReadTermVector(fields[i], tvfPointers[i]); } return res; } /// /// The field to read in /// /// The pointer within the tvf file where we should start reading /// /// The TermVector located at that position /// /// IOException private SegmentTermVector ReadTermVector(System.String field, long tvfPointer) { // Now read the data from specified position //We don't need to offset by the FORMAT here since the pointer already includes the offset tvf.Seek(tvfPointer); int numTerms = tvf.ReadVInt(); //System.out.println("Num Terms: " + numTerms); // If no terms - return a constant empty termvector. However, this should never occur! if (numTerms == 0) return new SegmentTermVector(field, null, null); bool storePositions; bool storeOffsets; if (tvfFormat == TermVectorsWriter.FORMAT_VERSION) { byte bits = tvf.ReadByte(); storePositions = (bits & TermVectorsWriter.STORE_POSITIONS_WITH_TERMVECTOR) != 0; storeOffsets = (bits & TermVectorsWriter.STORE_OFFSET_WITH_TERMVECTOR) != 0; } else { tvf.ReadVInt(); storePositions = false; storeOffsets = false; } System.String[] terms = new System.String[numTerms]; int[] termFreqs = new int[numTerms]; // we may not need these, but declare them int[][] positions = null; TermVectorOffsetInfo[][] offsets = null; if (storePositions) positions = new int[numTerms][]; if (storeOffsets) offsets = new TermVectorOffsetInfo[numTerms][]; int start = 0; int deltaLength = 0; int totalLength = 0; char[] buffer = new char[]{}; System.String previousString = ""; for (int i = 0; i < numTerms; i++) { start = tvf.ReadVInt(); deltaLength = tvf.ReadVInt(); totalLength = start + deltaLength; if (buffer.Length < totalLength) { buffer = new char[totalLength]; for (int j = 0; j < previousString.Length; j++) // copy contents buffer[j] = previousString[j]; } tvf.ReadChars(buffer, start, deltaLength); terms[i] = new System.String(buffer, 0, totalLength); previousString = terms[i]; int freq = tvf.ReadVInt(); termFreqs[i] = freq; if (storePositions) { //read in the positions int[] pos = new int[freq]; positions[i] = pos; int prevPosition = 0; for (int j = 0; j < freq; j++) { pos[j] = prevPosition + tvf.ReadVInt(); prevPosition = pos[j]; } } if (storeOffsets) { TermVectorOffsetInfo[] offs = new TermVectorOffsetInfo[freq]; offsets[i] = offs; int prevOffset = 0; for (int j = 0; j < freq; j++) { int startOffset = prevOffset + tvf.ReadVInt(); int endOffset = startOffset + tvf.ReadVInt(); offs[j] = new TermVectorOffsetInfo(startOffset, endOffset); prevOffset = endOffset; } } } SegmentTermVector tv; if (storePositions || storeOffsets) { tv = new SegmentTermPositionVector(field, terms, termFreqs, positions, offsets); } else { tv = new SegmentTermVector(field, terms, termFreqs); } return tv; } public virtual System.Object Clone() { if (tvx == null || tvd == null || tvf == null) return null; TermVectorsReader clone = null; try { clone = (TermVectorsReader) base.MemberwiseClone(); } catch (System.Exception e) { } clone.tvx = (IndexInput) tvx.Clone(); clone.tvd = (IndexInput) tvd.Clone(); clone.tvf = (IndexInput) tvf.Clone(); return clone; } } }