using System;

using System.Collections;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Runtime.Serialization.Formatters;

using System.IO;

using System.Data;

using System.ComponentModel.DataAnnotations;

using System.Globalization;

using System.Reflection;

​

using Newtonsoft.Json;

using Newtonsoft.Json.Linq;

using Newtonsoft.Json.Converters;

using Newtonsoft.Json.Serialization;    

​

using HtmlAgilityPack;

​

using NUnit.Framework; // Throws NUnit.Framework.AssertionException

​

//https://stackoverflow.com/questions/65668692/c-sharp-htmlagilitypack-scrape-data-between-two-tags

​

    class TestClass

    {

        public static void Test()

        {

            var url = @"https://bpp.economie.fgov.be/fo-eregister-view/search/details/721770937_EPV/0/0/1/10/0/0/0/null/null?locale=en";

​

            var doc = new HtmlWeb().Load(url);

            var allh2nodes =  doc.DocumentNode.SelectNodes("//h2"); // Select both start and end h2 nodes in one query

            Console.WriteLine($"allh2nodes.Count = {allh2nodes.Count}.");

            foreach (var node in allh2nodes)

            {

                Console.WriteLine(node.OuterHtml);

            }

            var startnode = doc.DocumentNode.SelectSingleNode("//h2[text() = 'Applicant']"); // Select the start node.

            // TODO: Handle the case where startnode is null/missing here.

​

            var endnode = startnode.SelectSingleNode("./following::h2");                     // And select the following end node using whatever criteria you need.

            // TODO: Handle the case where endnode is null/missing here.

​

            var followingXPath = $"./following::dd";                                         // Select nodes following the current node, which will be startnode

            var precedingXPath = $"{endnode.XPath}/preceding::dd";                           // Select nodes preceding the end node explicitly.

​

            var intersectedXPath = $"{followingXPath}[count(. | {precedingXPath}) = count({precedingXPath})]";

​

            var query = startnode.SelectNodes(intersectedXPath);

​

            var innerTexts = query.Select(n => n.InnerText).ToList();

​

            // Handle the inner texts however you require

​

            Console.WriteLine("Query:");

            Console.WriteLine(intersectedXPath);

​

            Console.WriteLine(string.Format("{0} nodes found:", query.Count));

            Console.WriteLine("Inner Texts:");

            Console.WriteLine(JsonConvert.SerializeObject(innerTexts, Formatting.Indented));

            Console.WriteLine("Nodes:");

            foreach (var node in query)

            {

                Console.WriteLine(node.OuterHtml);

                Assert.IsTrue(node.Name == "dd" && !node.InnerText.Contains("European patent granted for BE") && !node.InnerText.Contains("Payer:"));

            }

​

            Assert.IsTrue(endnode.Line >= startnode.Line);

            Assert.IsTrue(innerTexts.Count > 0);

        }   

    }

​

    public class Program

    {

        public static void Main()

        {

            Console.WriteLine("Environment version: {0}", System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription);

            Console.WriteLine("Json.NET version: " + typeof(JsonSerializer).Assembly.FullName);

            Console.WriteLine("HtmlAgilityPack version: " + typeof(HtmlDocument).Assembly.FullName);

            Console.WriteLine();

​

            try

            {

                TestClass.Test();

            }

            catch (Exception ex)

            {

                Console.WriteLine("Failed with unhandled exception: ");

                Console.WriteLine(ex);

                throw;

            }

        }

    }

​

    public static partial class HtmlAgilityPackExtensions

    {

        public static HtmlNode LowestCommonNode(this HtmlNode node1, HtmlNode node2)

        {

            if (node1 == null || node2 == null)

                throw new ArgumentNullException();

            if (node1.OwnerDocument != node2.OwnerDocument)

                throw new ArgumentException();

            return node1.AncestorsAndSelf().Reverse().Zip(node2.AncestorsAndSelf().Reverse()).Last(p => p.Item1 == p.Item2).Item1;

        }

    }

​

    public static partial class HtmlAgilityPackExtensions

    {

        public static IEnumerable<HtmlNode> NextNodes(this HtmlNode start, bool includeSelf = false)

        {

            if (start == null)

                yield break;

            if (includeSelf)

                yield return start;

            var current = start.FirstChild;

            while (current != null)

            {

                yield return current;

​

                // If not a match, get the first child of the current element.

                var next = current.FirstChild;

                if (next == null)

                    // If no first child, get the next sibling of the current element.

                    next = current.NextSibling;

​

                // If no more siblings, crawl up the list of parents until hitting the root, getting the next sibling of the lowest parent that has more siblings.

                if (next == null)

                {

                    for (var parent = current.ParentNode; parent != null && next == null; parent = parent.ParentNode)

                    {

                        next = parent.NextSibling;

                    }

                }

​

                current = next;

            }

        }

        public static IEnumerable<HtmlNode> NodesUntil(this HtmlNode start, HtmlNode end, bool includeSelf = false)

        {

            if (start == null || end == null)

                throw new ArgumentNullException();

            if (start.OwnerDocument != end.OwnerDocument)

                throw new ArgumentException();

            var query = start.NextNodes(includeSelf).TakeWhile(n => n != end);

            if (includeSelf)

                query = query.Concat(new []  { end });

            return query;

        }

    }

​