using System;

using System.Collections.Generic;

using System.Linq;

//using System.Threading;

//using System.Threading.Tasks;

//using System.Net.Http;

//using System.Globalization;

//using System.IO;

using System.Text;

using EpisodesTWC_Crawler;

using HtmlAgilityPack;

​

//using EpisodesTWC_Crawler;

//https://stackoverflow.com/questions/2038104/parsing-html-to-get-content-using-c-sharp

public class Program

{

    public static void Main()

    {

        var categoriesLinks = PageCrawler.GetEpisodesOfTWC("https://www.medscape.com/index/list_10295_0");

        //Console.WriteLine("Found {0} categories", categoriesLinks.Count); //var books = GetBookDetails(categoriesLinks);

        //exportToCSV(categoriesLinks);

        //foreach (var category in categoriesLinks)

        //{

            //category.ExportToCSV();

        //}

    }

}

​

namespace EpisodesTWC_Crawler

{

    public static class PageCrawler

    {

        static HtmlDocument GetDocument(string url)

        {

            HtmlWeb web = new HtmlWeb();

            web.OverrideEncoding = Encoding.UTF8;

            HtmlDocument doc = web.Load(url);

            return doc;

        }

​

        public static List<EpisodesTWC> GetEpisodesOfTWC(string url)

        {

            var episodesList = new List<EpisodesTWC>();

            // var bookLinks = new List<string>();

            HtmlDocument doc = GetDocument(url);

            var linkToNextPage = doc.DocumentNode.SelectSingleNode("//div/a[text() = 'Next']");

            linkToNextPage.Dump();

            HtmlNodeCollection linkNodes = doc.DocumentNode.SelectNodes("//div[@id='archives']/ul/li");

            linkNodes.Dump();

            var baseUri = new Uri(url);

            //GetEpisodesOfThisPage(episodesList, linkNodes, baseUri);

            while (linkToNextPage != null)

            {

                baseUri = new Uri(linkToNextPage.Attributes["href"].Value);

                //GetEpisodesOfThisPage(episodesList, linkNodes, baseUri);

                linkToNextPage = doc.DocumentNode.SelectSingleNode("//div/a[text() = 'Next']");

            }

​

            return episodesList;

        }

​

        static void GetEpisodesOfThisPage(List<EpisodesTWC> episodesList, HtmlNodeCollection linkNodes, Uri baseUri)

        {

            foreach (var episodes in linkNodes)

            {

                string href = episodes.Attributes["href"].Value;

                var fullUrl = new Uri(baseUri, href).AbsoluteUri;

                Console.WriteLine("Cataloging {0}: ", episodes.InnerText.Trim());

                //var episode = EpisodesTWC.Create(episodes.InnerText.Trim(), fullUrl);

                //var bookLinks = GetBookLinks(fullUrl);

                //episode.Books = GetEpisodeDetails(bookLinks);

                //Console.WriteLine("\tFound {0} books in this category\n", episode.Books.Count);

                //episodesList.Add(episode);

            }

        }

    }

​

    public class EpisodesTWC

    {

        public static EpisodesTWC Create(string title, string link, string teaser)

        {

            return new EpisodesTWC(title, link, teaser);

        }

​

        private EpisodesTWC(string title, string link, string teaser)

        {

            Title = title;

            Link = link;

            Teaser = teaser;

            Episode = new Episode();

        }

​

        public string Title { get; set; }

        public string Link { get; set; }

        public string Teaser { get; }

        public Episode Episode { get; set; }

​

        public void ExportToCSV()

        {

            //using var writer = new StreamWriter($"./BooksOfCategory_{Title.Trim()}.csv");

            //using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);

            //csv.WriteRecords(Episode);

            Episode.Dump();

        }

    }

​

    public class Episode()

    {

        private Episode(string title, string price) : this()

        {

            Title = title;

            Price = price;

        }

​

        public static Episode NullBook = new("", "");

        public static Episode Create(string title, string price) => new Episode(title, price);

        public string Title { get; set; }

        public string Price { get; set; }

    }

}