using System;

using System.Collections.Generic;

using System.Linq;

using System.Threading;

using System.Threading.Tasks;

using System.Net.Http;

using System.IO;

//using HtmlAgilityPack;

using AngleSharp.Html.Parser;

using AngleSharp.Html.Dom;

using AngleSharp.Dom;

using AngleSharp.Text;

​

//https://stackoverflow.com/questions/2038104/parsing-html-to-get-content-using-c-sharp

public class Program

{

    //try too:

    //https://dev.to/anjankant/visual-studio-learn-free-webscriping-in-easy-10-steps-data-mining-htmlagilitypack-linq-618

    static async Task Main(string[] args)

    {

        Console.WriteLine("Hello World");

        var p = new ScrapieWeb();

        await Task.Run(() => p.ScrapeWebsite());

    }

​

}

public class ScrapieWeb

{

        //https://dev.to/rachelsoderberg/create-a-simple-web-scraper-in-c-1l1m

    private string Title { get; set; }

    private string Url { get; set; }

​

    private string siteUrl = "https://www.oceannetworks.ca/news-and-stories/stories/";

    public string[] QueryTerms { get; } =

    {

        "Ocean",

        "Nature",

        "Pollution"

    };

    private void CleanUpResults(IElement result)

    {

        //https://twicardiology.libsyn.com/rss

        //https://stackoverflow.com/questions/576267/how-can-i-get-started-making-a-c-sharp-rss-reader

        //AngleSharp.Text => at StringExtensions.cs ReplaceFirst

        //Before date:

        //"</div></div></div></div><div><div class="sc-360731aa-1 csMORw">"

        //Before title:

        //"</div></div><div class="sc-360731aa-2 bVbofe">"

        //Before Href

        //"</div><a class="sc-95424b5b-0 hyEmzW read-more-btn" target="_self" rel="" font-size="0.875rem" href=""

        string htmlResult = result.InnerHtml.ReplaceFirst("        <span class=\"field-content\"><div><a href=\"", "https://www.oceannetworks.ca");

        htmlResult = htmlResult.ReplaceFirst("\">", "*");

        htmlResult = htmlResult.ReplaceFirst("</a></div>\n<div class=\"article-title-top\">", "-");

        htmlResult = htmlResult.ReplaceFirst("</div>\n<hr></span>  ", "");

    // Split Results: See Next Step

    }

    //

​

    private void SplitResults(string htmlResult)

    {

        string[] splitResults = htmlResult.Split('*');

        Url = splitResults[0];

        Title = splitResults[1];

    }

​

    internal async Task ScrapeWebsite()

    {

        CancellationTokenSource cancellationToken = new CancellationTokenSource();

        HttpClient httpClient = new HttpClient();

        HttpResponseMessage request = await httpClient.GetAsync(siteUrl);

        cancellationToken.Token.ThrowIfCancellationRequested();

        Stream response = await request.Content.ReadAsStreamAsync();

        cancellationToken.Token.ThrowIfCancellationRequested();

        HtmlParser parser = new HtmlParser();

        IHtmlDocument document = parser.ParseDocument(response);

        //Add connection between initial scrape, and parsing of results

        Console.WriteLine("Hello from ScrapeWebsite");

        GetScrapeResults(document);

    }

​

    private void GetScrapeResults(IHtmlDocument document)

    {

        IEnumerable<AngleSharp.Dom.IElement> articleLink = null;

        // AngleSharp.Dom ^^^^^^^^^^^^ 

        foreach (var term in QueryTerms)

        {

            //articleLink = document.All.Where(x => x.ClassName == "views-field views-field-nothing" && (x.ParentElement.InnerHtml.Contains(term) || x.ParentElement.InnerHtml.Contains(term.ToLower())));

            //https://gist.github.com/CodeCommissions/43a5aac117873dabcb1c9bdbf98cbd39

            articleLink = document.All.Where(x => x.ClassName == "views-field views-field-nothing" && (x.ParentElement.InnerHtml.Contains(term) || x.ParentElement.InnerHtml.Contains(term.ToLower()))).Skip(1);

        }

        Console.WriteLine("Hello from GetScrapeResults");

        if (articleLink.Any())

        {

            PrintResults(articleLink);

        }

    }

​

    public void PrintResults(IEnumerable<IElement> articleLink)

    {

        //Every element needs to be cleaned and displayed

        foreach (var element in articleLink)

        {

            CleanUpResults(element);

            //rtb_debugDisplay.AppendText($"{Title} - {Url}{Environment.NewLine}");

            Console.WriteLine($"{Title} - {Url}{Environment.NewLine}");

        }

    }

}