using HtmlAgilityPack;

using System;

using System.IO;

using System.Linq;

using System.Net;

​

namespace WebScraping

{

    public class Program

    {

        public static void Main(string[] args)

        {

            HtmlDocument htmlDoc = new HtmlDocument();

            string url = "https://public.oed.com/updates/new-words-list-june-2020/";            

            string urlResponse = URLRequest(url);

​

            //Convert the Raw HTML into an HTML Object

            htmlDoc.LoadHtml(urlResponse);

​

            //Find all A tags in the document

            var anchorNodes = htmlDoc.DocumentNode.SelectNodes("//a");

​

            if (anchorNodes != null)

            {          

​

                foreach (var anchorNode in anchorNodes)

                {

                    string value = anchorNode.InnerText;

                    string separator = ",";

                    int separatorIndex = value.IndexOf(separator);

                    if(separatorIndex >= 0)

                    {

                        string result = value.Substring(0,separatorIndex) + ".com";

                        result = result.Replace("-","");

                        result = result.Replace(" ","");

                        Console.WriteLine(String.Format("{0}", result, anchorNode.GetAttributeValue("href", "")));

                    }

                }

            }

​

        }

​

        //General Function to request data from a Server

        static string URLRequest(string url)

        {

            // Prepare the Request

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

​

            // Set method to GET to retrieve data

            request.Method = "GET";

            request.Timeout = 6000; //60 second timeout

            request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)";

​

            string responseContent = null;

​

            // Get the Response

            using (WebResponse response = request.GetResponse())

            {

                // Retrieve a handle to the Stream

                using (Stream stream = response.GetResponseStream())

                {

                    // Begin reading the Stream

                    using (StreamReader streamreader = new StreamReader(stream))

                    {

                        // Read the Response Stream to the end

                        responseContent = streamreader.ReadToEnd();

                    }

                }

            }

​

            return (responseContent);

        }

    }

}

​