Scraper Demo Solution | C# Online Compiler

Scraper Demo Solution by gabriel.uribe

// @nuget: HtmlAgilityPack
// @nuget: Fizzler.Systems.HtmlAgilityPack

#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using Fizzler.Systems.HtmlAgilityPack;
using HtmlAgilityPack;

namespace ScraperDemo
{
    public class Movie {
	    public string Title { get; init; }
	    public string? ImageUrl { get; init; }
		public int Year { get; init; }
        public string Description { get; init; }
        public bool IsPremium { get; init; }
    }

    class Program
    {
        static void Main(string[] args)
        {
			// Goal: Fix the bugs below around parsing movies from the html, and then filter out movies that are premium and weren't released in 2021.
			
	        string html =
                @"<!DOCTYPE html>
					<html>
					<body>
						<div id='content'>
							<div class='movie'>
								<h1 class='title'>Titanic</h1>
								<img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
								<p class='release-year'>1997</p>
								<p class='description'>Titanic: Directed by James Cameron. With Leonardo DiCaprio, Kate Winslet, Billy Zane, Kathy Bates. A seventeen-year-old aristocrat falls in love with a kind but poor artist aboard the luxurious, ill-fated R.M.S. Titanic.</p>
							</div>

							<div class='movie'>
								<h1 class='title'>A Boy Called Christmas</h1>
								<p class='release-year'>2021</p>
								<p class='description'>An ordinary young boy called Nikolas sets out on an adventure into the snowy north in search of his father who is on a quest to discover the fabled village of the elves, Elfhelm. Taking with him a headstrong reindeer called Blitzen and a loyal pet mouse, Nikolas soon meets his destiny.</p>
							</div>

							<div class='movie'>
								<h1 class='title'>Good on Paper</h1>
								<img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
								<p class='release-year'>2021</p>
								<p class='description'>After years of putting her career ahead of love, stand-up comic Andrea Singer stumbles upon the perfect guy. On paper, Dennis checks all the boxes, but Andrea's friend, Margot, is convinced he's not all he appears to be.</p>
							</div>

							<div class='show'>
								<h1 class='name'>Jupiter's Legacy</h1>
								<img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
								<p class='release-year'>2021</p>
								<p class='description'>The super-powered children of superheroes struggle to live up to the legendary feats of their parents.</p>
								<p class='premium'>Subscription required</p>
							</div>
						</div>
						
					</body>
					</html> ";

            HtmlDocument htmlDoc = new HtmlDocument();
            htmlDoc.LoadHtml(html);

            // Step 1. Fix the runtime error.
            // Step 2. Fix the bug causing no movies to get returned.
            // Step 3. Fix the bug with incorrect descriptions.
            // Step 4. Parse release year for each movie.
            // Step 5. Only include movies released in 2021.
			
			IEnumerable<HtmlNode> movieNodes = htmlDoc.DocumentNode
				.QuerySelector("div#content")
	            .QuerySelectorAll("div.movie"); 

            IReadOnlyList<Movie> movies = movieNodes
	            .Select((HtmlNode div, int index) =>
	            {
					Console.WriteLine($"Parsing movie ({index + 1}/{movieNodes.Count()})");
					
					string title = div.QuerySelector("h1.title").InnerText;
					string? imageUrl = div.QuerySelector("img")?.Attributes["src"].Value;
					string description = div.QuerySelector("p.description").InnerText;
					bool isPremium = div.QuerySelector("p.premium") != null;
					int releaseYear = Int32.Parse(div.QuerySelector("p.release-year").InnerText);
					
					Console.WriteLine($"Parsed data for movie: {title}");
		            return new Movie()
		            {
			            Title = title,
			            ImageUrl = imageUrl,
			            Description = description,
			            IsPremium = isPremium,
						Year = releaseYear
		            };
	            })
	            .Where(movie => !movie.IsPremium && movie.Year == 2021)
	            .ToList();

            foreach (var movie in movies)
            {
	            // Step 6. Print all values from the movie object.
	            Console.WriteLine(@"
					Title: " + movie.Title + @"
					Image URL: " + movie.ImageUrl + @"
					Description: " + movie.Description + @"
					IsPremium: " + movie.IsPremium + @"
					Year: " + movie.Year + @"
				");
            }

            // Final output should look like:

			// Title: A Boy Called Christmas
			// Image URL:
			// Description: An ordinary young boy called Nikolas sets out on an adventure into the snowy north in search of his father who is on a quest to discover the fabled village of the elves, Elfhelm. Taking with him a headstrong reindeer called Blitzen and a loyal pet mouse, Nikolas soon meets his destiny.
			// IsPremium: False
			// Year: 2021

			// Title: Good on Paper
            // Image URL: https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg
            // Description: After years of putting her career ahead of love, stand-up comic Andrea Singer stumbles upon the perfect guy. On paper, Dennis checks all the boxes, but Andrea's friend, Margot, is convinced he's not all he appears to be.
            // IsPremium: False
			// Year: 2021
        }
    }
}

​x
 
// @nuget: HtmlAgilityPack
// @nuget: Fizzler.Systems.HtmlAgilityPack
​
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using Fizzler.Systems.HtmlAgilityPack;
using HtmlAgilityPack;
​
namespace ScraperDemo
{
    public class Movie {
        public string Title { get; init; }
        public string? ImageUrl { get; init; }
        public int Year { get; init; }
        public string Description { get; init; }
        public bool IsPremium { get; init; }
    }
​
    class Program
    {
        static void Main(string[] args)
        {
            // Goal: Fix the bugs below around parsing movies from the html, and then filter out movies that are premium and weren't released in 2021.
            
            string html =
                @"<!DOCTYPE html>
                    <html>
                    <body>
                        <div id='content'>
                            <div class='movie'>
                                <h1 class='title'>Titanic</h1>
                                <img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
                                <p class='release-year'>1997</p>
                                <p class='description'>Titanic: Directed by James Cameron. With Leonardo DiCaprio, Kate Winslet, Billy Zane, Kathy Bates. A seventeen-year-old aristocrat falls in love with a kind but poor artist aboard the luxurious, ill-fated R.M.S. Titanic.</p>
                            </div>
​
                            <div class='movie'>
                                <h1 class='title'>A Boy Called Christmas</h1>
                                <p class='release-year'>2021</p>
                                <p class='description'>An ordinary young boy called Nikolas sets out on an adventure into the snowy north in search of his father who is on a quest to discover the fabled village of the elves, Elfhelm. Taking with him a headstrong reindeer called Blitzen and a loyal pet mouse, Nikolas soon meets his destiny.</p>
                            </div>
​
                            <div class='movie'>
                                <h1 class='title'>Good on Paper</h1>
                                <img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
                                <p class='release-year'>2021</p>
                                <p class='description'>After years of putting her career ahead of love, stand-up comic Andrea Singer stumbles upon the perfect guy. On paper, Dennis checks all the boxes, but Andrea's friend, Margot, is convinced he's not all he appears to be.</p>
                            </div>
​
                            <div class='show'>
                                <h1 class='name'>Jupiter's Legacy</h1>
                                <img src='https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg' />
                                <p class='release-year'>2021</p>
                                <p class='description'>The super-powered children of superheroes struggle to live up to the legendary feats of their parents.</p>
                                <p class='premium'>Subscription required</p>
                            </div>
                        </div>
                        
                    </body>
                    </html> ";
​
            HtmlDocument htmlDoc = new HtmlDocument();
            htmlDoc.LoadHtml(html);
​
            // Step 1. Fix the runtime error.
            // Step 2. Fix the bug causing no movies to get returned.
            // Step 3. Fix the bug with incorrect descriptions.
            // Step 4. Parse release year for each movie.
            // Step 5. Only include movies released in 2021.
            
            IEnumerable<HtmlNode> movieNodes = htmlDoc.DocumentNode
                .QuerySelector("div#content")
                .QuerySelectorAll("div.movie"); 
​
            IReadOnlyList<Movie> movies = movieNodes
                .Select((HtmlNode div, int index) =>
                {
                    Console.WriteLine($"Parsing movie ({index + 1}/{movieNodes.Count()})");
                    
                    string title = div.QuerySelector("h1.title").InnerText;
                    string? imageUrl = div.QuerySelector("img")?.Attributes["src"].Value;
                    string description = div.QuerySelector("p.description").InnerText;
                    bool isPremium = div.QuerySelector("p.premium") != null;
                    int releaseYear = Int32.Parse(div.QuerySelector("p.release-year").InnerText);
                    
                    Console.WriteLine($"Parsed data for movie: {title}");
                    return new Movie()
                    {
                        Title = title,
                        ImageUrl = imageUrl,
                        Description = description,
                        IsPremium = isPremium,
                        Year = releaseYear
                    };
                })
                .Where(movie => !movie.IsPremium && movie.Year == 2021)
                .ToList();
​
            foreach (var movie in movies)
            {
                // Step 6. Print all values from the movie object.
                Console.WriteLine(@"
                    Title: " + movie.Title + @"
                    Image URL: " + movie.ImageUrl + @"
                    Description: " + movie.Description + @"
                    IsPremium: " + movie.IsPremium + @"
                    Year: " + movie.Year + @"
                ");
            }
​
            // Final output should look like:
​
            // Title: A Boy Called Christmas
            // Image URL:
            // Description: An ordinary young boy called Nikolas sets out on an adventure into the snowy north in search of his father who is on a quest to discover the fabled village of the elves, Elfhelm. Taking with him a headstrong reindeer called Blitzen and a loyal pet mouse, Nikolas soon meets his destiny.
            // IsPremium: False
            // Year: 2021
​
            // Title: Good on Paper
            // Image URL: https://m.media-amazon.com/images/M/MV5BMDdmZGU3NDQtY2E5My00ZTliLWIzOTUtMTY4ZGI1YjdiNjk3XkEyXkFqcGdeQXVyNTA4NzY1MzY._V1_FMjpg_UX1000_.jpg
            // Description: After years of putting her career ahead of love, stand-up comic Andrea Singer stumbles upon the perfect guy. On paper, Dennis checks all the boxes, but Andrea's friend, Margot, is convinced he's not all he appears to be.
            // IsPremium: False
            // Year: 2021
        }
    }
}
​
​

View IL Code