// Read about this and other fiddles on .NET Fiddle Newsletter https://dotnetfiddle.substack.com/

//

// This fiddle demonstrates how to use HtmlAgilityPack to parse HTML pages

​

using System;

using System.Net;

using System.Collections.Generic;

​

using System.Text.RegularExpressions;

using System.Web;

​

using HtmlAgilityPack;

​

public class Program

{

    public static void Main()

    {

        string url = "https://www.imdb.com/list/ls016522954/?ref_=nv_tvv_dvd&sort=release_date,desc&st_dt=&mode=detail&page=1";

​

        using (var client = new WebClient())

        {

            string html = client.DownloadString(url);

​

            var doc = new HtmlDocument();

            doc.LoadHtml(html);

​

            var xpath = "//div[@class='lister-item mode-detail']";

            var movieNodes = doc.DocumentNode.SelectNodes(xpath);

            var movies = new List<Movie>();

            //Do latest 10 movies

            for (int i = 0; i < 10; i++)

            {

                var movie = ParseMovieUsingXpath(movieNodes[i]);

                //movie can also be parsed by traversing node tree like node.ChildNodes[0]....

                movies.Add(movie);

            }

            Console.WriteLine("Parsed movies are:\n");

            FiddleHelper.Dump(movies);

        }       

    }

    public static Movie ParseMovieUsingXpath(HtmlNode htmlNode)

    {   

        var title = htmlNode.SelectSingleNode(".//h3[@class='lister-item-header']/a").InnerText.CleanParsedText();

        var runtime = htmlNode.SelectSingleNode(".//span[@class='runtime']").InnerText.CleanParsedText();

        var genre = htmlNode.SelectSingleNode(".//span[@class='genre']").InnerText.CleanParsedText();

        //for dvdReleaseDate IMDB uses two different formtats so need to check if first one exists first

        var releaseDateNode =  htmlNode.SelectSingleNode(".//div[@class='list-description']/p/b");

        string releaseDate;

        if (releaseDateNode != null)

        {

            releaseDate = releaseDateNode.InnerText.CleanParsedText();

        }

        else

        {

            releaseDate =  htmlNode.SelectSingleNode(".//div[@class='list-description']/p").InnerText.CleanParsedText();

        }

​

        //releaseDate looks like this at this time 'VOD Release Date: Tuesday, December 29', so need to get only the Decemeber 29 part.

        releaseDate = releaseDate.Substring(releaseDate.LastIndexOf(", ") + 2);

        return new Movie

                    {

                        Title = title,

                        Runtime = runtime,

                        Genre = genre,

                        ReleaseDate = releaseDate

                    };

    }

    public class Movie

    {

        public string Title {get ; set;}

        public string Genre {get; set;}

        public string Runtime {get; set;}

        public string ReleaseDate {get; set;}

    }

}

​

public static class StringExtensions

{

    static Regex TextCleanRegex = new Regex("\\s{2,}");

    //This method removes all the junk which may be in html values

    public static string CleanParsedText(this string text)

    {

        if (String.IsNullOrWhiteSpace(text))

            return null;

​

        var cleanText = HttpUtility.HtmlDecode(text);

        cleanText = TextCleanRegex.Replace(cleanText, " ");

        cleanText = cleanText.Trim();

        return cleanText;

    }

}