HtmlAgilityPack startup fiddle

CSharp · Console

/Program.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;	

using HtmlAgilityPack;

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

	public class CharacterData
	{
		public string Name { get; set; }
		public string ImageUrl { get; set; }
		public string Url { get; set; }
	}

    class TestClass
    {
		public static CharacterData ExtractCharacterNameAndImage(string url)
		{
			//Use the following if you are OK with hardcoding the structure of <div> elements.
			//var tableXpath             = "/html/body/div[1]/div[3]/div[3]/div[2]/table"; 
			//Use the following if you are OK with hardcoding the fact that the relevant table comes first.
			var tableXpath             = "/html/body//table"; 
			var nameXpath              = "tr/td[2]/div[4]";
			var imageXpath             = "tr/td[1]/div[1]/a/img";

			var htmlDoc = new HtmlWeb().Load(url);
			
			var table = htmlDoc.DocumentNode.SelectNodes(tableXpath).First();
			
			var name = table.SelectNodes(nameXpath).Select(n => n.GetDirectInnerText().Trim()).SingleOrDefault();
			var imageUrl = table.SelectNodes(imageXpath).Select(n => n.GetAttributeValue("src", "")).SingleOrDefault();
			
			return new CharacterData { Name = name, ImageUrl = imageUrl, Url = url };
		}
		
        public static void Test()
        {
			var url = "https://myanimelist.net/character/214";

			var data = ExtractCharacterNameAndImage(url);
			var json = JsonConvert.SerializeObject(data, Formatting.Indented);
			
			Console.WriteLine(json);
			
			var url2 = "https://myanimelist.net/character/256";
			var data2 = ExtractCharacterNameAndImage(url2);
			var json2 = JsonConvert.SerializeObject(data2, Formatting.Indented);
			Console.WriteLine(json2);

			var htmlDoc = new HtmlWeb().Load(url);

			var title = string.Concat(htmlDoc.DocumentNode.SelectNodes("/html/head/title").Select(n => n.InnerText.Trim()));
			var index = title.IndexOf("- MyAnimeList.net");
			if (index >= 0)
				title = title.Substring(0, index).Trim();

			Console.WriteLine("\nTitle = {0}", title);

			var nameXpathFromFirefox   = "/html/body/div[1]/div[3]/div[3]/div[2]/table/tbody/tr/td[2]/div[4]";
			var imageXpathFromFirefox  = "/html/body/div[1]/div[3]/div[3]/div[2]/table/tbody/tr/td[1]/div[1]/a/img";
			TestSelect(htmlDoc, nameXpathFromFirefox);
			TestSelect(htmlDoc, imageXpathFromFirefox);
			var nameXpathFromFirefoxFixed   = "/html/body/div[1]/div[3]/div[3]/div[2]/table/tr/td[2]/div[4]";
			var imageXpathFromFirefoxFixed  = "/html/body/div[1]/div[3]/div[3]/div[2]/table/tr/td[1]/div[1]/a/img";
			TestSelect(htmlDoc, nameXpathFromFirefoxFixed);
			TestSelect(htmlDoc, imageXpathFromFirefoxFixed);
		}	
		
		static void TestSelect(HtmlDocument htmlDoc, string xpath)
		{
			Console.WriteLine("\nInput path: " + xpath);
			var splitPath = xpath.Split('/');
			for (int i = 2; i <= splitPath.Length; i++)
			{
				if (splitPath[i-1] == "")
					continue;
				var thisPath = string.Join("/", splitPath, 0, i);
				Console.Write("Testing \"{0}\": ", thisPath);
				var result = htmlDoc.DocumentNode.SelectNodes(thisPath);
				Console.WriteLine("result count = {0}", result == null ? "null" : result.Count.ToString());
			}
		}
	}

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: " + Environment.Version);
			Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}