namespace SpyFuResources

    {

        public class IndexBuilder

        {

            static string indexDirectoryPath = "C:\\code\\ProofOfConcept\\";

            // static List<string> wordsFilePaths = new List<string>();

            static int searchDateId = -1;

            static List<IFieldable> fields = SpyFuResourcesIndexConfig.Fields;

​

            // example shell cmd: C:\bin\lucene\domainname\indexbuilder\indexbuilder.exe domainname -sd 20140801 -p E:\lucene\domain_name -wf C:\bin\lucene\domainname\wordlist1.txt -wf C:\bin\lucene\domainname\wordlist2.txt

​

            public static void Main(string[] args)

            {

                #region boilerplate

​

                DateTime start = DateTime.Now;

                ProcessArgs(args);

                if (searchDateId == -1) throw new Exception("Please specify a search date id with -sd ########");

                SpyFu.Standard.AppSettingsParsed.CentralConfig.Load();

                ReportStatus("Initialization", "Started");

​

                if (!Directory.Exists(indexDirectoryPath)) Directory.CreateDirectory(indexDirectoryPath);

                new IndexInfo(searchDateId, IndexType.DomainName, Region.US).SaveToFile(indexDirectoryPath.TrimEnd(new[] { '\\' }) + "\\index.info");

                HelperMethods.AddEveryoneFilePermissions(indexDirectoryPath.TrimEnd(new[] { '\\' }) + "\\index.info");

​

                #endregion

​

                //SpyFuResourcesIndexConfig.PopulateWords(wordsFilePaths);

​

                #region Index Writing

                IndexWriter indexWriter = new IndexWriter(Lucene.Net.Store.FSDirectory.Open(new DirectoryInfo(indexDirectoryPath)), SpyFuResourcesIndexConfig.Analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);

                Document doc = new Document();

                foreach (IFieldable field in IndexBuilder.fields) doc.Add(field);

                int docsAdded = 0;

​

                try

                {

                    ///

                    /// Get data from remote source

                    ///

​

                    // Will this wait for a response?

                    var glossaryTermList = GlossaryTermModels.GetList();

                    int totalDocs = glossaryTermList.Count();

​

                    ///

                    /// Take data, entry by entry, and map to Lucene field value

                    ///

​

                        // how do I iterate over the GlossaryTerm 

                        foreach (var glossaryTerm in glossaryTermList)

                        {

                            foreach (IFieldable field in SpyFuResourcesIndexConfig.Fields)

                            {

                                string val = SpyFuResourcesIndexConfig.luceneToCustomMapping[field.Name](glossaryTerm);

                                doc.GetField(field.Name).SetValue(val);

                            }

                        }

                    ///

                    /// Add new Document to master Document

                    /// 

​

                    indexWriter.AddDocument(doc);

                    if (++docsAdded % 100000 == 0)

                    {

                        Console.WriteLine(docsAdded);

                        ReportStatus("Adding Documents", "In Progress", (int)Math.Round(100 * (docsAdded / (double)totalDocs)));

                    }

                    ReportStatus("Adding Documents", "Completed", 100);

                }

​

​

                catch (Exception ex)

                {

                    Console.WriteLine(ex.Message);

                    ReportStatus("", "Error");

                    Console.WriteLine("Docs Added: {0}", docsAdded);

                }

                finally

                {

                    Console.WriteLine("All documents have been added to the index (took {0})\nOptimizing...", DateTime.Now - start);

                    ReportStatus("Optimizing", "Started");

                    indexWriter.Optimize();

                    ReportStatus("Optimizing", "Completed");

                    indexWriter.Dispose();

                }

​

                #endregion Index Writing

​

                IndexInfo indexInfo = IndexInfo.LoadFromFile(indexDirectoryPath.TrimEnd(new[] { '\\' }) + "\\index.info");

                indexInfo.Size = new System.IO.DirectoryInfo(indexDirectoryPath).GetIndexSize();

                indexInfo.SaveToFile(indexDirectoryPath.TrimEnd(new[] { '\\' }) + "\\index.info");

            }

​

            // Added "SpyFuResourcesIndexConfig" before analyzer

            public void CreateIndex(int searchDateId, SpyFuResourcesIndexConfig.Analyzer analyzer)

            {

                var startTime = DateTime.Now;

                var lastTime = DateTime.Now;

​

                #region Index Writing

​

                var _idx = new IndexWriter(FSDirectory.Open(new DirectoryInfo(IndexDirectoryPath)), analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);

                new IndexInfo(searchDateId, IndexType.KeywordToDomain, KeywordToDomainIndexConfig.Db.DbName.ToLower().Equals("honeybadger") ? Region.US : Region.UK).SaveToFile(IndexDirectoryPath.TrimEnd('\\') + "\\index.info");

                HelperMethods.AddEveryoneFilePermissions(IndexDirectoryPath.TrimEnd('\\') + "\\index.info");

​

                // Why define after use?

​

                //These are supposed to increase performance.  First, let's see what they default to.

                Console.WriteLine("MaxBufferedDocs:" + _idx.GetMaxBufferedDocs());

                Console.WriteLine("MaxMergeDocs:" + _idx.MaxMergeDocs);

                Console.WriteLine("MergeFactor:" + _idx.MergeFactor);

                _idx.MergeFactor = 1000;

                _idx.MaxMergeDocs = int.MaxValue;

                _idx.SetMaxBufferedDocs(1000);

                //End of performance enhancers.

​

                // Pre-populate the document with placeholder field values

                Document doc = new Document();

                foreach (IFieldable field in DomainToKeywordIndexConfig.Fields) doc.Add(field);

                int docsAdded = 0;

​

                try

                {

                    #region Setting Field Values

​

                    foreach (IFieldable field in KeywordToDomainIndexConfig.Fields)

                    {

                        string fieldValue;

​

                        if (KeywordToDomainIndexConfig.luceneToCustomMapping.ContainsKey(field.Name))

                        {

                            fieldValue = KeywordToDomainIndexConfig.luceneToCustomMapping[field.Name](prevDomain);

                        }

                        else

                        {

                            string sqlFieldName = KeywordToDomainIndexConfig.LuceneToSqlMapping.ContainsKey(field.Name) ? KeywordToDomainIndexConfig.LuceneToSqlMapping[field.Name] : field.Name;

                            try

                            {

                                fieldValue = dr[sqlFieldName].ToString();

                            }

                            catch (IndexOutOfRangeException)

                            {

                                fieldValue = string.Empty;

                            }

                        }

​

                        if (field is Field)

                        {

                            doc.GetField(field.Name).SetValue(fieldValue);

​

                            if (KeywordToDomainIndexConfig.customFieldWeighters.ContainsKey(field.Name))

                            {

                                doc.GetField(field.Name).Boost = KeywordToDomainIndexConfig.customFieldWeighters[field.Name](dr);

                            }

                        }

                        else // NumericField

                        {

                            double num;

                            if (!double.TryParse(fieldValue, out num))

                            {

                                num = 0.0;

                            }

​

                            ((NumericField)doc.GetFieldable(field.Name)).SetDoubleValue(num);

​

                            if (KeywordToDomainIndexConfig.customFieldWeighters.ContainsKey(field.Name))

                            {

                                ((NumericField)doc.GetFieldable(field.Name)).Boost = KeywordToDomainIndexConfig.customFieldWeighters[field.Name](dr);

                            }

                        }

                    }

​

                    #endregion

​

​

                }

                catch (Exception ex)

                {

                    Console.WriteLine(ex.Message);

                    Console.WriteLine("Docs Added: {0}", docsAdded);

                }

                finally

                {

                }

​

                #endregion

            }

​

            private static void DisplayHelp()

            {

                Console.WriteLine("\nAccepted arguments:\n");

                Console.WriteLine("-sd [########] : search date id (REQUIRED)");

                Console.WriteLine("-cs [db connection string] (OPTIONAL - default is \"SERVER=honeybadger;User=ds;PASSWORD=honv4Rm1nt;DATABASE=SearchFu\")");

                Console.WriteLine("-p [target index directory] (REQUIRED)");

                Console.WriteLine("-wf [file path to a newline-separated text file containing words] (REQUIRED)");

                Console.WriteLine("\nLast value entered will be used, except for word file paths, which will keep track of,");

                Console.WriteLine("and utilize, multiple entries.");

                Environment.Exit(1);

            }

​

            private static void ProcessArgs(string[] args)

            {

                for (int i = 0; i < args.Length; i++)

                {

                    switch (args[i].ToLower())

                    {

                        case "-sd":

                            IndexBuilder.searchDateId = Int32.Parse(args[++i]);

                            break;

                        case "-cs":

                            IndexBuilder.connectionString = args[++i];

                            break;

                        case "-p":

                            IndexBuilder.indexDirectoryPath = args[++i];

                            break;

                        case "-wf":

                            IndexBuilder.wordsFilePaths.Add(args[++i]);

                            break;

                        case "/?":

                            DisplayHelp();

                            break;

                    }

                }

            }

​

            private static void ReportStatus(string task, string status, int? percent = null)

            {

                var parameters = new Dictionary<string, object>()

                {

                    { "@IndexType", "DomainName" },

                    { "@Status", status },

                    { "@TaskName", task },

                    { "@SearchDateId", searchDateId }

                };

                if (percent != null) parameters.Add("@StatusPct", percent.ToString());

​

                SpyFu.Standard.AppSettingsParsed.DatabaseSettings.DbUser.ExecNonQueryProc("Lucene_Insert_Status", parameters);

            }

        }

    }