Full-text search for database entities with Lucene.NET

public class LuceneWriter<T> : ILuceneWriter<T> where T: class
{
    private readonly ILuceneIndex<T> _index;
 
    public LuceneWriter(ILuceneIndex<T> index)
    {
        _index = index;
    }
 
    public void AddOrUpdateRange(IEnumerable<T> entities)
    {
        if (entities is null)
        {
            throw new ArgumentNullException();
        }
 
        var writer = _index.IndexWriter;
 
        foreach (var entity in entities)
        {
            var indexTerm = new Term(
                _index.IdFieldKey, 
                _index.GetEntityId(entity).ToString());
            var newDocument = _index.GetDocumentFromEntity(entity);
            writer.UpdateDocument(indexTerm, newDocument);
        }
        writer.Commit();
    }
 
    public void Remove(T entity)
    {
        if (entity is null)
        {
            throw new ArgumentNullException();
        }
 
        var writer = _index.IndexWriter;
        var indexTerm = new Term(
            _index.IdFieldKey, 
            _index.GetEntityId(entity).ToString());
        writer.DeleteDocuments(indexTerm);
        writer.Commit();
    }
 
    public IDictionary<int, string> GetIndexedFieldNames()
    {
        return _index.FieldValues
            .ToDictionary(
                x => x, 
                x => _index.GetFieldKeyByFieldId(x));
    }
}

public class EmployeeIndex : ILuceneIndex<Employee>
{
    private static IndexWriter _indexWriter;
 
    public EmployeeIndex(
        IConfigurationManager configurationManager)
    {
        var dictionaryPath = configurationManager
            .GetAppSetting("LuceneDirectoryPath") + 
                           "/EmployeeIndex";
 
        if (System.IO.Directory.Exists(dictionaryPath))
        {
            System.IO.Directory.Delete(dictionaryPath, true);
        }
 
        Directory = FSDirectory.Open(dictionaryPath);
        Analyzer = new CaseInsensitiveWhitespaceAnalyzer();
 
        //Singleton pattern for IndexWriter to ensure only one
        //IndexWriter exists per index.
        _indexWriter = _indexWriter ?? new IndexWriter(
            Directory, 
            Analyzer, 
            IndexWriter.MaxFieldLength.UNLIMITED);
    }
 
    public IndexWriter IndexWriter => _indexWriter;
 
    public Directory Directory { get; }
 
    public Analyzer Analyzer { get; }
 
    public string IdFieldKey => IndexField.Id.ToString();
 
    public IEnumerable<string> FieldKeys => 
        Enum.GetNames(typeof(IndexField));
 
    public IEnumerable<int> FieldValues =>
        Enum.GetValues(typeof(IndexField))
            .Cast<IndexField>().Cast<int>().ToList();
 
    public Document GetDocumentFromEntity(Employee entity)
    {
        if (entity is null)
        {
            throw new ArgumentNullException();
        }
 
        var employee = entity;
 
        if (employee.Skills is null ||
            employee.UserCourses is null ||
            employee.UserCertificates is null ||
            employee.SchoolEducations is null)
        {
            throw new ArgumentException(
                "Related entity was not eagerly loaded and " +
                "could not be loaded lazily either.");
        }
 
        var doc = new Document();
 
        doc.Add(new Field(
            IndexField.Id.ToString(), 
            employee.Id.ToString(), 
            Field.Store.YES, Field.Index.NOT_ANALYZED));
 
        doc.Add(new Field(
            IndexField.FirstName.ToString(), 
            employee.FirstName ?? "", 
            Field.Store.YES, Field.Index.ANALYZED));
 
        doc.Add(new Field(
            IndexField.LastName.ToString(), 
            employee.LastName ?? "", 
            Field.Store.YES, Field.Index.ANALYZED));
 
        employee.Skills.ToList().ForEach(s =>
        {
            var field = new Field(
                IndexField.Skill.ToString(),
                s.Knowledge.Name + " {" + s.Id + "}",
                Field.Store.YES,
                Field.Index.ANALYZED);
 
            //Boost any matches that will happen on this field
            //by the Skill's SkillGrade (x1-x5)
            field.Boost = s.SkillGrade.Value > 0 ? 
                s.SkillGrade.Value : 1;
 
            doc.Add(field);
        });
 
        employee.UserCourses.ToList().ForEach(c =>
            doc.Add(new Field(
                IndexField.Course.ToString(),
                c.Course.Name + " {" + c.Id + "}",
                Field.Store.YES, Field.Index.ANALYZED)));
 
        employee.UserCertificates.ToList().ForEach(c =>
            doc.Add(new Field(
                IndexField.Certificate.ToString(),
                c.Certificate.Name + " {" + c.Id + "}",
                Field.Store.YES, Field.Index.ANALYZED)));
 
        employee.SchoolEducations.ToList().ForEach(e =>
            doc.Add(new Field(
                IndexField.Course.ToString(),
                $"{e.Institute} {e.Specialism} {e.Degree}" + 
                " {" + e.Id + "}",
                Field.Store.YES, Field.Index.ANALYZED)));
 
        return doc;
    }
 
    public string GetFieldKeyByFieldId(int fieldId)
    {
        if (!Enum.IsDefined(typeof(IndexField), fieldId))
        {
            throw new ArgumentException(
                $"There's no Enum value corresponding to " +
                $"value {fieldId}.");
        }
 
        return ((IndexField) fieldId).ToString();
    }
 
    public int GetEntityId(Employee entity)
    {
        if (entity is null)
        {
            throw new ArgumentNullException();
        }
 
        return entity.Id;
    }
 
    public enum IndexField
    {
        Id,
        Education,
        Skill,
        Course,
        Certificate,
        FirstName,
        LastName
    }
}

public class CaseInsensitiveWhitespaceAnalyzer : Analyzer
{
    public override TokenStream TokenStream(
        string fieldName, TextReader reader)
    {
        var result = new WhitespaceTokenizer(reader);
        return new LowerCaseFilter(result);
    }
 
    public override TokenStream ReusableTokenStream(
        string fieldName, TextReader reader)
    {
        var tokenizer = (Tokenizer)PreviousTokenStream;
        if (tokenizer == null)
        {
            var result = new WhitespaceTokenizer(reader);
            return new LowerCaseFilter(result);
        }
        else
        {
            tokenizer.Reset(reader);
        }
 
        return tokenizer;
    }
 
    public override int GetPositionIncrementGap(string fieldName)
    {
        return 1;
    }
}

public class LuceneQueryBuilder<T> : 
        ILuceneQueryBuilder<T> where T : class
{
    private readonly ILuceneIndex<T> _index;
    private readonly ILuceneConfiguration _configuration;
 
    public LuceneQueryBuilder(
        ILuceneIndex<T> index,
        ILuceneConfiguration configuration
    )
    {
        _index = index;
        _configuration = configuration;
    }
 
    public Query GetQuery(
        string searchString, 
        int? fieldIdToSearch, 
        IEnumerable<int> documentIdsToSearch)
    {
        var query = new BooleanQuery();
 
        IEnumerable<string> fieldsToSearch;
        if (fieldIdToSearch is null)
        {
            fieldsToSearch = _index.FieldKeys.ToList();
        }
        else
        {
            fieldsToSearch = new List<string>
            {
                _index.GetFieldKeyByFieldId(
                    (int)fieldIdToSearch)
            };
        }
 
        var searchStringParts = 
            GetAnalyzedSearchStringParts(searchString);
 
        if (!searchStringParts.Any())
        {
            return null;
        }
 
        foreach (var searchStringPart in searchStringParts)
        {
            var partSubQueries = new List<Query>();
 
            if (searchStringPart.SearchModifier == 
                SearchModifier.Phrase)
            {
                partSubQueries.AddRange(GetPhraseQueries(
                    searchStringPart.Text, fieldsToSearch));
            }
 
            if (searchStringPart.SearchModifier == 
                SearchModifier.Wildcard)
            {
                partSubQueries.AddRange(GetAllTerms(
                        searchStringPart.Text, fieldsToSearch)
                    .Select(term => new WildcardQuery(term)));
            }
 
            if (searchStringPart.SearchModifier == 
                SearchModifier.None)
            {
                partSubQueries.AddRange(GetAllTerms(
                        searchStringPart.Text, fieldsToSearch)
                    .Select(term =>
                    {
                        var termQuery = new TermQuery(term);
                        termQuery.Boost = 10; //Boost exact matches
                        return termQuery;
                    }));
            }
 
            //For search string without modifier, add additional
            //WildcardQuery if string length is long enough
            //to avoid too broad searches like *C* when user
            //quickly searches for users with Skill "C"
            if (searchStringPart.SearchModifier == 
                SearchModifier.None &&
                searchStringPart.Text.Length >= 
                _configuration.MinLengthForWildCardQuery)
            {
                //Add asterisks for Wildcard-Query.
                partSubQueries.AddRange(GetAllTerms("*" + 
                    searchStringPart.Text + "*", fieldsToSearch)
                    .Select(term => new WildcardQuery(term)));
            }
 
            //For search string without modifier, add additional
            //FuzzyQuery if string length is long enough
            //to avoid too broad searches like Java --> Jira
            if (searchStringPart.SearchModifier == 
                SearchModifier.None &&
                searchStringPart.Text.Length >= 
                _configuration.MinLengthForFuzzyQuery)
            {
                partSubQueries.AddRange(GetAllTerms(
                        searchStringPart.Text, fieldsToSearch)
                    .Select(term => new FuzzyQuery(
                        term,
                        _configuration.MinFuzzyQuerySimilarity,
                        _configuration.FuzzyQueryPrefixLength)));
            }
 
            //Combine all queries into one query so that at
            //least one of these queries must match.
            var partSearchQuery = new BooleanQuery();
            var partQuery = new BooleanQuery();
            partSubQueries.ForEach(q => partSearchQuery.Add(
                q, Occur.SHOULD));
            partQuery.Add(partSearchQuery, Occur.MUST);
 
            //If only specific documents should be searched,
            //add Term queries on the ID field
            if (documentIdsToSearch != null)
            {
                var idQueries = documentIdsToSearch
                    .Select(id => new TermQuery(new Term(
                        _index.IdFieldKey, id.ToString())));
 
                var idQuery = new BooleanQuery();
 
                idQueries.ToList().ForEach(q => idQuery.Add(
                    q, Occur.SHOULD));
                partQuery.Add(idQuery, Occur.MUST);
            }
 
            //Overall query is a combination of all search
            //string parts' combined queries.
            query.Add(partSearchQuery, Occur.MUST);
        }
 
        return query;
    }
 
    private List<AnalyzedSearchStringPart> 
        GetAnalyzedSearchStringParts(string searchString)
    {
        var stringParts = new List<AnalyzedSearchStringPart>();
 
        searchString = searchString.ToLowerInvariant();
 
        //Extract any exact phrases
        while (searchString.Contains("\""))
        {
            var start = searchString.IndexOf("\"", 
                StringComparison.Ordinal) + 1;
            var end = searchString.IndexOf("\"", start, 
                StringComparison.Ordinal);
 
            var part = searchString.Substring(
                start, end - start);
 
            if (!string.IsNullOrEmpty(part))
            {
                stringParts.Add(new AnalyzedSearchStringPart
                {
                    Text = part,
                    SearchModifier = SearchModifier.Phrase
                });
            }
 
            //cut out part including double quotes.
            searchString = searchString.Replace(
                $"\"{part}\"", "");
 
            //If number of double quotes is uneven and one
            //remains, ignore it.
            if (searchString.Count(f => f == '"') == 1)
            {
                searchString = searchString.Replace("\"", "");
            }
        }
 
        //Split by spaces and analyze each bit
        foreach (var part in searchString.Split(' '))
        {
            var trimmed = part.Trim();
            if (trimmed.Length == 0)
            {
                continue;
            }
 
            //If part contains asterisks and anything else than
            //asterisks, it's a Wildcard part
            //Only-asterisks ("*", "**"...) are ignored.
            if (trimmed.Contains("*") && trimmed
                    .Replace("*", "").Length > 0)
            {
                stringParts.Add(new AnalyzedSearchStringPart
                {
                    Text = trimmed,
                    SearchModifier = SearchModifier.Wildcard
                });
            }
 
            //Everything else is a regular search term
            else
            {
                stringParts.Add(new AnalyzedSearchStringPart
                {
                    Text = trimmed,
                    SearchModifier = SearchModifier.None
                });
            }
        }
 
        return stringParts;
    }
 
    private IEnumerable<Query> GetPhraseQueries(
        string searchString, IEnumerable<string> fieldNames)
    {
        var queries = new List<PhraseQuery>();
 
        foreach (var fieldName in fieldNames)
        {
            var phraseWords = searchString.Split(' ');
            var phraseQuery = new PhraseQuery();
 
            foreach (var phraseWord in phraseWords)
            {
                phraseQuery.Add(new Term(fieldName, phraseWord));
            }
 
            queries.Add(phraseQuery);
        }
 
        return queries;
    }
 
    //Shorthand to get Terms for each field.
    private IEnumerable<Term> GetAllTerms(
        string searchString, IEnumerable<string> fieldNames)
    {
        return fieldNames.Select(fieldName => 
            new Term(fieldName, searchString));
    }
}

public class LuceneReader<T> : ILuceneReader<T> where T : class
{
    private readonly ILuceneIndex<T> _index;
    private readonly ILuceneQueryBuilder<T> _queryBuilder;
    private readonly ILuceneConfiguration _configuration;
 
    public LuceneReader(
        ILuceneIndex<T> index,
        ILuceneQueryBuilder<T> queryBuilder,
        ILuceneConfiguration configuration)
    {
        _index = index;
        _queryBuilder = queryBuilder;
        _configuration = configuration;
    }
 
    public LuceneSearchResult<T> Search(
        string searchString,
        bool doLoadAllResults,
        int? fieldIdToSearch = null,
        IEnumerable<int> documentIdsToSearch = null)
    {
        var searchResultItems = 
            new List<LuceneSearchResultItem<T>>();
        int totalHits;
        var query = _queryBuilder.GetQuery(
            searchString, 
            fieldIdToSearch, 
            documentIdsToSearch);
 
        if (query is null)
        {
            throw new ArgumentException(
                $"No meaningful query could be derived from " +
                $"the given searchString: {searchString}");
        }
 
        using (var indexReader = IndexReader.Open(
            _index.Directory, true))
        using (var indexSearcher = new IndexSearcher(
            _index.Directory))
        {
            //Force calculating of scores.
            indexSearcher.SetDefaultFieldSortScoring(true, true); 
            
            TopDocs searchResult;
 
            if (doLoadAllResults)
            {
                //Hack to load all results: set result count
                //to number of Documents in Index.
                searchResult = indexSearcher.Search(
                    query, 
                    indexReader.NumDocs());
            }
            else
            {
                searchResult = indexSearcher.Search(
                    query, 
                    _configuration.DefaultResultCountPerEntity);
            }
 
            totalHits = searchResult.TotalHits;
 
            searchResult.ScoreDocs.ToList().ForEach(x =>
            {
                var document = indexSearcher.Doc(x.Doc);
                var searchResultItem = GetResultItemFromDocument(
                    document, query, x.Score);
 
                searchResultItems.Add(searchResultItem);
            });
        }
 
        return new LuceneSearchResult<T>
        {
            SearchResultItems = searchResultItems,
            TotalResultCount = totalHits
        };
    }
 
    //Extract the relevant information about the search.
    private LuceneSearchResultItem<T> GetResultItemFromDocument(
        Document document, Query query, float score)
    {
        var scorer = new QueryScorer(query);
        var formatter = new SimpleHTMLFormatter(
            "<span class=\"highlighted-match\">", "</span>");
 
        //Makes sure always the entire content of a
        //field is fetched.
        var fragmenter = new NullFragmenter();
        var highlighter = new Highlighter(formatter, scorer)
        {
            TextFragmenter = fragmenter
        };
 
        var searchResultItem = new LuceneSearchResultItem<T>
        {
            Id = int.Parse(document.Get(_index.IdFieldKey)),
            Document = document,
            Score = score
        };
 
        //For each field check if there were any search matches
        //on that field.
        document.GetFields().ToList().ForEach(field =>
        {
            var fieldKey = field.Name;
            var fieldValue = field.StringValue;
            var tokenStream = _index.Analyzer.TokenStream(
                fieldKey, new StringReader(fieldValue));
 
            //For each field get value with HTML markup around
            //matches. Ignore fields with a score of 0 (no match)
            //As NullFragmenter anyway only creates one Fragment
            //per Field, maxNumberOfFragments can be set to 1.
            var matches = highlighter.GetBestTextFragments(
                    tokenStream, fieldValue, false, 1)
                .Where(m => m.Score > 0)
                .Select(m => GetMatchFromFragment(m, fieldKey));
 
            searchResultItem.Matches.AddRange(matches);
        });
 
        return searchResultItem;
    }
 
    //Create a LuceneMatch instance for every field which had
    //one or more matches.
    private LuceneMatch GetMatchFromFragment(
        TextFragment fragment, string fieldKey)
    {
        var matchText = fragment.ToString();
 
        var match = new LuceneMatch
        {
            FieldKey = fieldKey,
            MatchText = matchText
        };
 
        //If field value contains an entity ID, extract it,
        //parse it and store it in separate property.
        var endsWithCurlyBracesWithNumbersRegex = 
            new Regex(@".*\s[{]\d*[}]");
        if (endsWithCurlyBracesWithNumbersRegex.IsMatch(matchText))
        {
            var position = matchText
                .LastIndexOf("{", StringComparison.Ordinal);
            match.MatchText = matchText
                .Substring(0, position).Trim();
 
            var idAsString = matchText
                .Substring(position).Trim('{', '}');
            var couldParse = int.TryParse(
                idAsString, out var parsedId);
 
            match.FieldEntityId = couldParse ? parsedId : 
                null as int?;
        }
 
        return match;
    }
}