using System;

using System.Collections;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Runtime.Serialization.Formatters;

using System.IO;

using System.Data;

using System.ComponentModel.DataAnnotations;

using System.Globalization;

using System.Reflection;

​

using Newtonsoft.Json;

using Newtonsoft.Json.Linq;

using Newtonsoft.Json.Converters;

using Newtonsoft.Json.Serialization;    

​

using HtmlAgilityPack;

​

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

​

//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags

​

    public static partial class HtmlAgilityPackExtensions

    {

        public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)

        {

            if (node1 == null || node2 == null)

                throw new ArgumentNullException();

            if (node1.OwnerDocument != node2.OwnerDocument)

                throw new ArgumentException();

            return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;

        }

    }

​

    public static partial class HtmlAgilityPackExtensions

    {

        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)

        {

            if (start == null)

                yield break;

            if (includeSelf)

                yield return start;

            var current = start.FirstChild;

            while (current != null)

            {

                yield return current;

​

                // If not a match, get the first child of the current element.

                var next = current.FirstChild;

                if (next == null)

                    // If no first child, get the next sibling of the current element.

                    next = current.NextSibling;

​

                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.

                if (next == null)

                {

                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)

                    {

                        next = parent.NextSibling;

                    }

                }

​

                current = next;

            }

        }

        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)

        {

            if (start == null || end == null)

                throw new ArgumentNullException();

            if (start.OwnerDocument != end.OwnerDocument)

                throw new ArgumentException();

            var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);

            if (includeSelf)

                query = query.Concat(new []  { end });

            return query;

        }

    }

​

    class TestClass

    {

        public static void Test()

        {

            var html = GetHtml();

            var doc = new HtmlDocument();

            doc.LoadHtml(html);

            // Assert that NextNodes() really does return all the nodes in document order

            Assert.IsTrue(doc.DocumentNode.DescendantsAndSelf().SequenceEqual(doc.DocumentNode.NextNodes(true)));

            Assert.IsTrue(doc.DocumentNode.Descendants().SequenceEqual(doc.DocumentNode.NextNodes(false)));

​

            var startnode = doc.DocumentNode.SelectNodes("//h2").Where(n => n.InnerText == "Applicant").FirstOrDefault();

            var endnode = doc.DocumentNode.SelectNodes("//h2").Where(n => n.InnerText == "Agent").FirstOrDefault();

            var query = doc.DocumentNode.Descendants()      // Traverse all nodes

                .SkipWhile(n => n != startnode)             // Skip until we find the start node

                .Skip(1)                                    // Skip it

                .TakeWhile(n => n != endnode);              // And take nodes until we reach the end node 

​

            Console.WriteLine(string.Format("{0} nodes found between {1} and {2} using DocumentNode.Descendants()", query.Count(), startnode.XPath, endnode.XPath));

            var query2 = startnode.NodesUntil(endnode, false);

            Assert.IsTrue(query.SequenceEqual(query2));

            Console.WriteLine("HtmlAgilityPackExtensions.NextNodes() returned identically.");

        }   

        static string GetHtml() =>

            @"<!DOCTYPE html>

<html lang=""en"" >

<head>

    <!-- general !-->

    <meta charset=""utf-8"">

</head>

   <body class=""some example"">

foo

Bar

Alice 

Bob

​

    <div id=""main-container-z"">

​

<!-- Applicants section  -->

​

    <h2 class=""GridTitle"">Applicant</h2>

            <h3 class=""DataTitle"">1</h3>

        <dl class=""Grid LeftCol"">

            <dt>Name:</dt>

            <dd>Some name here</dd>

            <dt>Legal Form:</dt>

            <dd></dd>              

            <dt>From:</dt>

            <dd>06/08/2020</dd>

        </dl>

        <dl class=""Grid RightCol"">

            <dt>Address:</dt>

            <dd>Some address here</dd>

            <dt>To:</dt>

            <dd></dd>

        </dl>

            <h3 class=""DataTitle"">2</h3>

        <dl class=""Grid LeftCol"">

            <dt>Name:</dt>

            <dd>Some name here1</dd>

            <dt>Legal Form:</dt>

            <dd></dd>              

            <dt>From:</dt>

            <dd>04/08/2010</dd>

        </dl>

        <dl class=""Grid RightCol"">

            <dt>Address:</dt>

            <dd>Some address here1</dd>

            <dt>To:</dt>

            <dd>06/08/2020</dd>

        </dl>

​

​

​

<!-- Agents section  -->

​

    <h2 class=""GridTitle"">Agent</h2>

​

    </div>

​

   </body>

</html>";

    }

​

    public class Program

    {

        public static void Main()

        {

            Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);

            Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);

            Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);

            Console.WriteLine();

​

            try

            {

                TestClass.Test();

            }

            catch (Exception ex)

            {

                Console.WriteLine("Failed with unhandled exception: ");

                Console.WriteLine(ex);

                throw;

            }

        }

    }

​