[Fork] [Fork] [Fork] https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags | C# Online Compiler

[Fork] [Fork] [Fork] https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags by dbc_MinLength

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;	

using HtmlAgilityPack;

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags

    class TestClass
    {
        public static void Test()
        {
			var html = GetHtml();
			
			var doc = new HtmlDocument();
			doc.LoadHtml(html);
			
			var h2nodes =  doc.DocumentNode.SelectNodes("//h2[text() = 'Applicant' or text() = 'Agent']"); // Select both start and end h2 nodes in one query
			for (int i = 0, count = 2*(h2nodes.Count/2); i < count; i+=2)
			{
				var startnode = h2nodes[i];
				var endnode = h2nodes[i+1];

				var query = startnode.SelectNodes("./following::node()") // Select all nodes following startnode
					.TakeWhile(n => n != endnode)                        // Until endnode is reached
					.Where(n => n.Name == "dd");                         // With name "dd".

				var innerTexts = query.Select(n => n.InnerText).ToList();

                // Handle the inner texts however you require
				
				Console.WriteLine("Query:");

				Console.WriteLine(string.Format("{0} nodes found:", innerTexts.Count));
				Console.WriteLine("Inner Texts:");
				Console.WriteLine(JsonConvert.SerializeObject(innerTexts, Formatting.Indented));
				Console.WriteLine("Nodes:");
				foreach (var node in query)
				{
					Console.WriteLine(node.OuterHtml);
					Assert.IsTrue(node.Name == "dd" && !node.InnerText.Contains("Don't Include Me"));
				}
				
				Assert.IsTrue(endnode.Line >= startnode.Line);
				Assert.IsTrue(innerTexts.Count == 10);
			}
		}	
			
		static string GetHtml() =>
			@"<!DOCTYPE html>
<html lang=""en"" >
<head>
    <!-- general !-->
    <meta charset=""utf-8"">
</head>
   <body class=""some example"">
foo
Bar
Alice 
Bob

    <div id=""main-container-z"">

	<h2 class=""DontIncludeMe"">1</h3>
            <h3 class=""DontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>

<!-- Applicants section  -->

    <h2 class=""GridTitle"">Applicant</h2>
    
        
            <h3 class=""DataTitle"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>06/08/2020</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here</dd>
            <dt>To:</dt>
            <dd></dd>
        </dl>
    
        
            <h3 class=""DataTitle"">2</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here1</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>04/08/2010</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here1</dd>
            <dt>To:</dt>
            <dd>06/08/2020</dd>
        </dl>
    



<!-- Agents section  -->

    <h2 class=""GridTitle"">Agent</h2>

	<h2 class=""AlsoDontIncludeMe"">1</h3>
            <h3 class=""AlsoDontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>

    </div>

   </body>
</html>";
	}

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);
			Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}

	public static partial class HtmlAgilityPackExtensions
	{
		public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)
		{
			if (node1 == null || node2 == null)
				throw new ArgumentNullException();
			if (node1.OwnerDocument != node2.OwnerDocument)
				throw new ArgumentException();
			return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;
		}
	}

	public static partial class HtmlAgilityPackExtensions
	{
        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)
        {
            if (start == null)
                yield break;
            if (includeSelf)
                yield return start;
            var current = start.FirstChild;
            while (current != null)
            {
				yield return current;

                // If not a match, get the first child of the current element.
				var next = current.FirstChild;
				
                if (next == null)
                    // If no first child, get the next sibling of the current element.
					next = current.NextSibling;

                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.
                if (next == null)
                {
                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)
                    {
                        next = parent.NextSibling;
                    }
                }

                current = next;
            }
        }
		
        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)
        {
			if (start == null || end == null)
				throw new ArgumentNullException();
			if (start.OwnerDocument != end.OwnerDocument)
				throw new ArgumentException();
			var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);
			if (includeSelf)
				query = query.Concat(new []  { end });
			return query;
		}
	}

​x
 
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;
​
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;    
​
using HtmlAgilityPack;
​
using NUnit.Framework; // Throws NUnit.Framework.AssertionException
​
//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags
​
    class TestClass
    {
        public static void Test()
        {
            var html = GetHtml();
            
            var doc = new HtmlDocument();
            doc.LoadHtml(html);
            
            var h2nodes =  doc.DocumentNode.SelectNodes("//h2[text() = 'Applicant' or text() = 'Agent']"); // Select both start and end h2 nodes in one query
            for (int i = 0, count = 2*(h2nodes.Count/2); i < count; i+=2)
            {
                var startnode = h2nodes[i];
                var endnode = h2nodes[i+1];
​
                var query = startnode.SelectNodes("./following::node()") // Select all nodes following startnode
                    .TakeWhile(n => n != endnode)                        // Until endnode is reached
                    .Where(n => n.Name == "dd");                         // With name "dd".
​
                var innerTexts = query.Select(n => n.InnerText).ToList();
​
                // Handle the inner texts however you require
                
                Console.WriteLine("Query:");
​
                Console.WriteLine(string.Format("{0} nodes found:", innerTexts.Count));
                Console.WriteLine("Inner Texts:");
                Console.WriteLine(JsonConvert.SerializeObject(innerTexts, Formatting.Indented));
                Console.WriteLine("Nodes:");
                foreach (var node in query)
                {
                    Console.WriteLine(node.OuterHtml);
                    Assert.IsTrue(node.Name == "dd" && !node.InnerText.Contains("Don't Include Me"));
                }
                
                Assert.IsTrue(endnode.Line >= startnode.Line);
                Assert.IsTrue(innerTexts.Count == 10);
            }
        }   
            
        static string GetHtml() =>
            @"<!DOCTYPE html>
<html lang=""en"" >
<head>
    <!-- general !-->
    <meta charset=""utf-8"">
</head>
   <body class=""some example"">
foo
Bar
Alice 
Bob
​
    <div id=""main-container-z"">
​
    <h2 class=""DontIncludeMe"">1</h3>
            <h3 class=""DontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>
​
<!-- Applicants section  -->
​
    <h2 class=""GridTitle"">Applicant</h2>
    
        
            <h3 class=""DataTitle"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>06/08/2020</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here</dd>
            <dt>To:</dt>
            <dd></dd>
        </dl>
    
        
            <h3 class=""DataTitle"">2</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here1</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>04/08/2010</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here1</dd>
            <dt>To:</dt>
            <dd>06/08/2020</dd>
        </dl>
    
​
​
​
<!-- Agents section  -->
​
    <h2 class=""GridTitle"">Agent</h2>
​
    <h2 class=""AlsoDontIncludeMe"">1</h3>
            <h3 class=""AlsoDontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>
​
    </div>
​
   </body>
</html>";
    }
​
    public class Program
    {
        public static void Main()
        {
            Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);
            Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
            Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);
            Console.WriteLine();
​
            try
            {
                TestClass.Test();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Failed with unhandled exception: ");
                Console.WriteLine(ex);
                throw;
            }
        }
    }
​
    public static partial class HtmlAgilityPackExtensions
    {
        public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)
        {
            if (node1 == null || node2 == null)
                throw new ArgumentNullException();
            if (node1.OwnerDocument != node2.OwnerDocument)
                throw new ArgumentException();
            return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;
        }
    }
​
    public static partial class HtmlAgilityPackExtensions
    {
        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)
        {
            if (start == null)
                yield break;
            if (includeSelf)
                yield return start;
            var current = start.FirstChild;
            while (current != null)
            {
                yield return current;
​
                // If not a match, get the first child of the current element.
                var next = current.FirstChild;
                
                if (next == null)
                    // If no first child, get the next sibling of the current element.
                    next = current.NextSibling;
​
                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.
                if (next == null)
                {
                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)
                    {
                        next = parent.NextSibling;
                    }
                }
​
                current = next;
            }
        }
        
        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)
        {
            if (start == null || end == null)
                throw new ArgumentNullException();
            if (start.OwnerDocument != end.OwnerDocument)
                throw new ArgumentException();
            var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);
            if (includeSelf)
                query = query.Concat(new []  { end });
            return query;
        }
    }
​

Environment version: .NET 9.0.0 (9.0.0)

Failed with unhandled exception:
NUnit.Framework.AssertionException:   !ReferenceEquals(s1, s2)
  Expected: True
  But was:  False

   at NUnit.Framework.Assert.ReportFailure(String message)
   at NUnit.Framework.Assert.ReportFailure(ConstraintResult result, String message, Object[] args)
   at NUnit.Framework.Assert.That[TActual](TActual actual, IResolveConstraint expression, String message, Object[] args)
   at NUnit.Framework.Assert.IsTrue(Boolean condition, String message, Object[] args)
   at TestClass.TestNewStringCharArray()
   at TestClass.Test()
   at Program.Main()
Unhandled exception. NUnit.Framework.AssertionException:   !ReferenceEquals(s1, s2)
  Expected: True
  But was:  False

   at NUnit.Framework.Assert.ReportFailure(String message)
   at NUnit.Framework.Assert.ReportFailure(ConstraintResult result, String message, Object[] args)
   at NUnit.Framework.Assert.That[TActual](TActual actual, IResolveConstraint expression, String message, Object[] args)
   at NUnit.Framework.Assert.IsTrue(Boolean condition, String message, Object[] args)
   at TestClass.TestNewStringCharArray()
   at TestClass.Test()
   at Program.Main()
Command terminated by signal 6

Last Run:	10:53:00 am
Compile:	0.015s
Execute:	0.2s
Memory:	11.22Mb
CPU:	0.215s

View IL Code