[Fork] [Fork] [Fork] https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags | C# Online Compiler

[Fork] [Fork] [Fork] https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags by dbc_MinLength

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;	

using HtmlAgilityPack;

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags

    class TestClass
    {
        public static void Test()
        {
			var html = GetHtml();
			
			var doc = new HtmlDocument();
			doc.LoadHtml(html);
			
			var startnode = doc.DocumentNode.SelectSingleNode("//h2[text() = 'Applicant']"); // Select the start node.
			// TODO: Handle the case where startnode is null/missing here.

			var endnode = startnode.SelectSingleNode("./following::h2");                     // And select the following end node using whatever criteria you need.
			// TODO: Handle the case where endnode is null/missing here.
			
			var followingXPath = $"./following::dd";                                         // Select nodes following the current node, which will be startnode
			var precedingXPath = $"{endnode.XPath}/preceding::dd";                           // Select nodes preceding the end node explicitly.
			var intersectedXPath = $"{followingXPath}[count(. | {precedingXPath}) = count({precedingXPath})]";

			var query = startnode.SelectNodes(intersectedXPath);

			var innerTexts = query.Select(n => n.InnerText).ToList();

			// Handle the inner texts however you require

			Console.WriteLine("Query:");
			Console.WriteLine(intersectedXPath);

			Console.WriteLine(string.Format("{0} nodes found:", query.Count));
			Console.WriteLine("Inner Texts:");
			Console.WriteLine(JsonConvert.SerializeObject(innerTexts, Formatting.Indented));
			Console.WriteLine("Nodes:");
			foreach (var node in query)
			{
				Console.WriteLine(node.OuterHtml);
				Assert.IsTrue(node.Name == "dd" && !node.InnerText.Contains("Don't Include Me"));
			}

			Assert.IsTrue(endnode.Line >= startnode.Line);
			Assert.IsTrue(innerTexts.Count == 10);
		}	
			
		static string GetHtml() =>
			@"<!DOCTYPE html>
<html lang=""en"" >
<head>
    <!-- general !-->
    <meta charset=""utf-8"">
</head>
   <body class=""some example"">
foo
Bar
Alice 
Bob

    <div id=""main-container-z"">

	<h2 class=""DontIncludeMe"">1</h3>
            <h3 class=""DontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>

<!-- Applicants section  -->

    <h2 class=""GridTitle"">Applicant</h2>
    
        
            <h3 class=""DataTitle"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>06/08/2020</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here</dd>
            <dt>To:</dt>
            <dd></dd>
        </dl>
    
        
            <h3 class=""DataTitle"">2</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here1</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>04/08/2010</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here1</dd>
            <dt>To:</dt>
            <dd>06/08/2020</dd>
        </dl>
    



<!-- Agents section  -->

    <h2 class=""GridTitle"">Agent</h2>

	<h2 class=""AlsoDontIncludeMe"">1</h3>
            <h3 class=""AlsoDontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>

    </div>

   </body>
</html>";
	}

	public class Program
	{
		public static void Main()
		{
			Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);
			Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
			Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);
			Console.WriteLine();

			try
			{
				TestClass.Test();
			}
			catch (Exception ex)
			{
				Console.WriteLine("Failed with unhandled exception: ");
				Console.WriteLine(ex);
				throw;
			}
		}
	}

	public static partial class HtmlAgilityPackExtensions
	{
		public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)
		{
			if (node1 == null || node2 == null)
				throw new ArgumentNullException();
			if (node1.OwnerDocument != node2.OwnerDocument)
				throw new ArgumentException();
			return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;
		}
	}

	public static partial class HtmlAgilityPackExtensions
	{
        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)
        {
            if (start == null)
                yield break;
            if (includeSelf)
                yield return start;
            var current = start.FirstChild;
            while (current != null)
            {
				yield return current;

                // If not a match, get the first child of the current element.
				var next = current.FirstChild;
				
                if (next == null)
                    // If no first child, get the next sibling of the current element.
					next = current.NextSibling;

                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.
                if (next == null)
                {
                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)
                    {
                        next = parent.NextSibling;
                    }
                }

                current = next;
            }
        }
		
        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)
        {
			if (start == null || end == null)
				throw new ArgumentNullException();
			if (start.OwnerDocument != end.OwnerDocument)
				throw new ArgumentException();
			var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);
			if (includeSelf)
				query = query.Concat(new []  { end });
			return query;
		}
	}

​x
 
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.Serialization.Formatters;
using System.IO;
using System.Data;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.Reflection;
​
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;    
​
using HtmlAgilityPack;
​
using NUnit.Framework; // Throws NUnit.Framework.AssertionException
​
//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags
​
    class TestClass
    {
        public static void Test()
        {
            var html = GetHtml();
            
            var doc = new HtmlDocument();
            doc.LoadHtml(html);
            
            var startnode = doc.DocumentNode.SelectSingleNode("//h2[text() = 'Applicant']"); // Select the start node.
            // TODO: Handle the case where startnode is null/missing here.
​
            var endnode = startnode.SelectSingleNode("./following::h2");                     // And select the following end node using whatever criteria you need.
            // TODO: Handle the case where endnode is null/missing here.
            
            var followingXPath = $"./following::dd";                                         // Select nodes following the current node, which will be startnode
            var precedingXPath = $"{endnode.XPath}/preceding::dd";                           // Select nodes preceding the end node explicitly.
            var intersectedXPath = $"{followingXPath}[count(. | {precedingXPath}) = count({precedingXPath})]";
​
            var query = startnode.SelectNodes(intersectedXPath);
​
            var innerTexts = query.Select(n => n.InnerText).ToList();
​
            // Handle the inner texts however you require
​
            Console.WriteLine("Query:");
            Console.WriteLine(intersectedXPath);
​
            Console.WriteLine(string.Format("{0} nodes found:", query.Count));
            Console.WriteLine("Inner Texts:");
            Console.WriteLine(JsonConvert.SerializeObject(innerTexts, Formatting.Indented));
            Console.WriteLine("Nodes:");
            foreach (var node in query)
            {
                Console.WriteLine(node.OuterHtml);
                Assert.IsTrue(node.Name == "dd" && !node.InnerText.Contains("Don't Include Me"));
            }
​
            Assert.IsTrue(endnode.Line >= startnode.Line);
            Assert.IsTrue(innerTexts.Count == 10);
        }   
            
        static string GetHtml() =>
            @"<!DOCTYPE html>
<html lang=""en"" >
<head>
    <!-- general !-->
    <meta charset=""utf-8"">
</head>
   <body class=""some example"">
foo
Bar
Alice 
Bob
​
    <div id=""main-container-z"">
​
    <h2 class=""DontIncludeMe"">1</h3>
            <h3 class=""DontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>
​
<!-- Applicants section  -->
​
    <h2 class=""GridTitle"">Applicant</h2>
    
        
            <h3 class=""DataTitle"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>06/08/2020</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here</dd>
            <dt>To:</dt>
            <dd></dd>
        </dl>
    
        
            <h3 class=""DataTitle"">2</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Name:</dt>
            <dd>Some name here1</dd>
            <dt>Legal Form:</dt>
            <dd></dd>              
            <dt>From:</dt>
            <dd>04/08/2010</dd>
        </dl>
        <dl class=""Grid RightCol"">
            <dt>Address:</dt>
            <dd>Some address here1</dd>
            <dt>To:</dt>
            <dd>06/08/2020</dd>
        </dl>
    
​
​
​
<!-- Agents section  -->
​
    <h2 class=""GridTitle"">Agent</h2>
​
    <h2 class=""AlsoDontIncludeMe"">1</h3>
            <h3 class=""AlsoDontIncludeMe"">1</h3>
        
        <dl class=""Grid LeftCol"">
            <dt>Something:</dt>
            <dd>Don't Include Me</dd>
        </dl>
​
    </div>
​
   </body>
</html>";
    }
​
    public class Program
    {
        public static void Main()
        {
            Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);
            Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);
            Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);
            Console.WriteLine();
​
            try
            {
                TestClass.Test();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Failed with unhandled exception: ");
                Console.WriteLine(ex);
                throw;
            }
        }
    }
​
    public static partial class HtmlAgilityPackExtensions
    {
        public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)
        {
            if (node1 == null || node2 == null)
                throw new ArgumentNullException();
            if (node1.OwnerDocument != node2.OwnerDocument)
                throw new ArgumentException();
            return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;
        }
    }
​
    public static partial class HtmlAgilityPackExtensions
    {
        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)
        {
            if (start == null)
                yield break;
            if (includeSelf)
                yield return start;
            var current = start.FirstChild;
            while (current != null)
            {
                yield return current;
​
                // If not a match, get the first child of the current element.
                var next = current.FirstChild;
                
                if (next == null)
                    // If no first child, get the next sibling of the current element.
                    next = current.NextSibling;
​
                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.
                if (next == null)
                {
                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)
                    {
                        next = parent.NextSibling;
                    }
                }
​
                current = next;
            }
        }
        
        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)
        {
            if (start == null || end == null)
                throw new ArgumentNullException();
            if (start.OwnerDocument != end.OwnerDocument)
                throw new ArgumentException();
            var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);
            if (includeSelf)
                query = query.Concat(new []  { end });
            return query;
        }
    }
​

Jerry Cuban

Cached Result
Last Run:	11:19:32 am
Compile:	0.235s
Execute:	7.089s
Memory:	34.08Mb
CPU:	3.812s

View IL Code