如何在HIGH_COMPRESSION 4.8中使用HIGH_COMPRESSION

Question

我正在尽量压缩索引大小，有什么帮助吗？压缩

public class LuceneIndexer
    {
        private Analyzer _analyzer = new ArabicAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
        private string _indexPath;
        private Directory _indexDirectory;
        public IndexWriter _indexWriter;

        public LuceneIndexer(string indexPath)
        {
            this._indexPath = indexPath;
            _indexDirectory = new SimpleFSDirectory(new System.IO.DirectoryInfo(_indexPath));
        }

        public void BuildCompleteIndex(IEnumerable<Document> documents)
        {
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.Net.Util.LuceneVersion.LUCENE_48, _analyzer) { OpenMode = OpenMode.CREATE_OR_APPEND };
            indexWriterConfig.MaxBufferedDocs = 2;
            indexWriterConfig.RAMBufferSizeMB = 128;
            indexWriterConfig.MaxThreadStates = 2;

            _indexWriter = new IndexWriter(_indexDirectory, indexWriterConfig);

            _indexWriter.AddDocuments(documents);

            _indexWriter.Flush(true, true);
            _indexWriter.Commit();

            _indexWriter.Dispose();
        }

        
        public IEnumerable<Document> Search(string searchTerm, string searchField, int limit)
        {
            IndexReader indexReader = DirectoryReader.Open(_indexDirectory);
            var searcher = new IndexSearcher(indexReader);
            var termQuery = new TermQuery(new Term(searchField, searchTerm)); // Lucene.Net.Util.LuceneVersion.LUCENE_48, searchField, _analyzer
            var hits = searcher.Search(termQuery, limit).ScoreDocs;

            var documents = new List<Document>();
            foreach (var hit in hits)
            {
                documents.Add(searcher.Doc(hit.Doc));
            }

            _analyzer.Dispose();
            return documents;
        }

    }

Answer 1

首先要知道的是，"Lucene指数“有很多方面。当不使用复合文件时，这将在创建的各种文件中得到体现。只要看看其中的两个，我们就可以讨论倒排的索引，也就是所谓的Just，也可以讨论存储的文档。据我所知，在这两种方法中，没有任何关于倒排索引压缩的可随时可调的设置。

HIGH_COMPRESSION模式与存储字段相关。如果您不是在存储字段，并且只使用Lucene.Net创建一个反向索引，那么对存储字段进行高压缩的工作不会减少“Lucene.Net”的大小。

也就是说，如果您正在存储字段并希望对存储的字段数据使用高压缩，那么您将需要创建您自己的编解码器，该编解码器为存储字段打开了高压缩。要做到这一点，您首先需要一个打开高压缩的存储字段类。下面是这两个类，后面是一个单元测试，它使用了我为您编写的这个新的编解码器。我还没有在大量的数据上尝试使用这段代码来查看这种效果，我把它留给您作为练习，但是这将为使用高压缩存储字段指明方向。

/*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

public sealed class Lucene41StoredFieldsHighCompressionFormat : CompressingStoredFieldsFormat {
        /// <summary>
        /// Sole constructor. </summary>
        public Lucene41StoredFieldsHighCompressionFormat()
            : base("Lucene41StoredFieldsHighCompression", CompressionMode.HIGH_COMPRESSION, 1 << 14) {
        }
    }

以下是使用这种高压缩格式的自定义编解码器：

/*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    using Lucene40LiveDocsFormat = Lucene.Net.Codecs.Lucene40.Lucene40LiveDocsFormat;
    using Lucene41StoredFieldsFormat = Lucene.Net.Codecs.Lucene41.Lucene41StoredFieldsFormat;
    using Lucene42NormsFormat = Lucene.Net.Codecs.Lucene42.Lucene42NormsFormat;
    using Lucene42TermVectorsFormat = Lucene.Net.Codecs.Lucene42.Lucene42TermVectorsFormat;
    using PerFieldDocValuesFormat = Lucene.Net.Codecs.PerField.PerFieldDocValuesFormat;
    using PerFieldPostingsFormat = Lucene.Net.Codecs.PerField.PerFieldPostingsFormat;

    /// <summary>
    /// Implements the Lucene 4.6 index format, with configurable per-field postings
    /// and docvalues formats.
    /// <para/>
    /// If you want to reuse functionality of this codec in another codec, extend
    /// <see cref="FilterCodec"/>.
    /// <para/>
    /// See <see cref="Lucene.Net.Codecs.Lucene46"/> package documentation for file format details.
    /// <para/>
    /// @lucene.experimental 
    /// </summary>
    // NOTE: if we make largish changes in a minor release, easier to just make Lucene46Codec or whatever
    // if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
    // (it writes a minor version, etc).
    [CodecName("Lucene46HighCompression")]
    public class Lucene46HighCompressionCodec : Codec {
        private readonly StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsHighCompressionFormat();    //<--This is the only line different then the stock Lucene46Codec
        private readonly TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
        private readonly FieldInfosFormat fieldInfosFormat = new Lucene46FieldInfosFormat();
        private readonly SegmentInfoFormat segmentInfosFormat = new Lucene46SegmentInfoFormat();
        private readonly LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();

        private readonly PostingsFormat postingsFormat;

        private class PerFieldPostingsFormatAnonymousInnerClassHelper : PerFieldPostingsFormat {
            private readonly Lucene46HighCompressionCodec outerInstance;

            public PerFieldPostingsFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
                this.outerInstance = outerInstance;
            }

            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            public override PostingsFormat GetPostingsFormatForField(string field) {
                return outerInstance.GetPostingsFormatForField(field);
            }
        }

        private readonly DocValuesFormat docValuesFormat;

        private class PerFieldDocValuesFormatAnonymousInnerClassHelper : PerFieldDocValuesFormat {
            private readonly Lucene46HighCompressionCodec outerInstance;

            public PerFieldDocValuesFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
                this.outerInstance = outerInstance;
            }

            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            public override DocValuesFormat GetDocValuesFormatForField(string field) {
                return outerInstance.GetDocValuesFormatForField(field);
            }
        }

        /// <summary>
        /// Sole constructor. </summary>
        public Lucene46HighCompressionCodec()
            : base() {
            postingsFormat = new PerFieldPostingsFormatAnonymousInnerClassHelper(this);
            docValuesFormat = new PerFieldDocValuesFormatAnonymousInnerClassHelper(this);
        }

        public override sealed StoredFieldsFormat StoredFieldsFormat => fieldsFormat;

        public override sealed TermVectorsFormat TermVectorsFormat => vectorsFormat;

        public override sealed PostingsFormat PostingsFormat => postingsFormat;

        public override sealed FieldInfosFormat FieldInfosFormat => fieldInfosFormat;

        public override sealed SegmentInfoFormat SegmentInfoFormat => segmentInfosFormat;

        public override sealed LiveDocsFormat LiveDocsFormat => liveDocsFormat;

        /// <summary>
        /// Returns the postings format that should be used for writing
        /// new segments of <paramref name="field"/>.
        /// <para/>
        /// The default implementation always returns "Lucene41"
        /// </summary>
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public virtual PostingsFormat GetPostingsFormatForField(string field) {
            // LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
            if (defaultFormat == null) {
                defaultFormat = Lucene.Net.Codecs.PostingsFormat.ForName("Lucene41");
            }
            return defaultFormat;
        }

        /// <summary>
        /// Returns the docvalues format that should be used for writing
        /// new segments of <paramref name="field"/>.
        /// <para/>
        /// The default implementation always returns "Lucene45"
        /// </summary>
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public virtual DocValuesFormat GetDocValuesFormatForField(string field) {
            // LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
            if (defaultDVFormat == null) {
                defaultDVFormat = Lucene.Net.Codecs.DocValuesFormat.ForName("Lucene45");
            }
            return defaultDVFormat;
        }

        public override sealed DocValuesFormat DocValuesFormat => docValuesFormat;

        // LUCENENET specific - lazy initialize the codecs to ensure we get the correct type if overridden.
        private PostingsFormat defaultFormat;
        private DocValuesFormat defaultDVFormat;

        private readonly NormsFormat normsFormat = new Lucene42NormsFormat();

        public override sealed NormsFormat NormsFormat => normsFormat;
    }

感谢@NightOwl888 888，我现在知道您还需要在启动时注册新的Codec，如下所示：

Codec.SetCodecFactory(new DefaultCodecFactory {
    CustomCodecTypes = new Type[] { typeof(Lucene46HighCompressionCodec) }
});

下面是一个单元测试，演示如何使用高压缩编码：

public class TestCompression {


        [Fact]
        public void HighCompression() {
            FxTest.Setup();

            Directory indexDir = new RAMDirectory();

            Analyzer standardAnalyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48);

            IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, standardAnalyzer);
            indexConfig.Codec = new Lucene46HighCompressionCodec();     //<--------Install the High Compression codec.

            indexConfig.UseCompoundFile = true;

            IndexWriter writer = new IndexWriter(indexDir, indexConfig);

            //souce: https://github.com/apache/lucenenet/blob/Lucene.Net_4_8_0_beta00006/src/Lucene.Net/Search/SearcherFactory.cs
            SearcherManager searcherManager = new SearcherManager(writer, applyAllDeletes: true, new SearchWarmer());

            Document doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "001", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Unique gifts are great gifts.", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "002", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Everyone is gifted.", Field.Store.YES));
            writer.AddDocument(doc);

            doc = new Document();
            doc.Add(new StringField("examplePrimaryKey", "003", Field.Store.YES));
            doc.Add(new TextField("exampleField", "Gifts are meant to be shared.", Field.Store.YES));
            writer.AddDocument(doc);

            writer.Commit();

            searcherManager.MaybeRefreshBlocking();
            IndexSearcher indexSearcher = searcherManager.Acquire();
            try {
                QueryParser parser = new QueryParser(LuceneVersion.LUCENE_48, "exampleField", standardAnalyzer);
                Query query = parser.Parse("everyone");

                TopDocs topDocs = indexSearcher.Search(query, int.MaxValue);

                int numMatchingDocs = topDocs.ScoreDocs.Length;
                Assert.Equal(1, numMatchingDocs);


                Document docRead = indexSearcher.Doc(topDocs.ScoreDocs[0].Doc);
                string primaryKey = docRead.Get("examplePrimaryKey");
                Assert.Equal("002", primaryKey);

            } finally {
                searcherManager.Release(indexSearcher);
            }

        }

    }

虽然我最初的反应是通过一个Lucene.Net github问题，但我在这里提供的答案是，它将对Lucene.Net社区有更好的可视性，希望它也能帮助其他人。对于那些感兴趣的人，有更多的背景信息，有关使用自定义编解码器接近该问题的线程结束。