namespace Gspann.Web.Helpers

{

    public class HTMLCrawler : IComputedIndexField

    {

        public string FieldName { get; set; }

        public string ReturnType { get; set; }

​

        public object ComputeFieldValue(IIndexable indexable)

        {

            Assert.ArgumentNotNull(indexable, "indexable");

            string url = null;

            try

            {

​

                Item item = indexable as SitecoreIndexableItem;

                using (new DatabaseSwitcher(item.Database))

                {

                    if (item != null && item.IsPageForCrawling)

                    {

                        itemUrl = LinkManager.GetItemUrl(item, new UrlOptions()

                        {

                            AlwaysIncludeServerUrl = true,

                            LanguageEmbedding = LanguageEmbedding.Never

                        }).Replace("/sitecore/shell/MyWebsite/Home", "");

​

                        // Http request the page

                        using (var client = new WebClient())

                        {

                            string webContent = client.DownloadString(itemUrl);

                            HtmlDocument htmlDocument = new HtmlDocument();

                            htmlDocument.LoadHtml(webContent); //using HtmlAgilityPack  - Install this nuget package. 

                            string crawlableContent = GetOnlyContent(htmlDocument);

​

                            //** GetOnlyContent is your custom method. only you have full HTML it is upto you on how you want to store it. 

                            //There are lot of methods available to strip html tags from string. ***

                            return crawlableContent;

                        }

                    }

                }

            }

​

            catch (Exception ex)

            {

​

            }

​

            return null;

        }

    }

}

​

​