using System;

using System.Diagnostics;

using System.Globalization;

using System.Net;

using System.Reflection;

using System.Text;

using System.Text.RegularExpressions;

using System.Xml.Linq;

using System.Collections.Generic;

using System.Linq;

using System.Threading;

using System.Threading.Tasks;

​

public class Program

{

    public static void Main()

    {

        Action scrape = (async () => await ScrapeManifesto("Carlos Rodarte"));

        scrape();

    }

        public static async Task ScrapeManifesto(string name)

        {

            string url = "https://agilemanifesto.org/display";

            string uriManifesto = $"{url}/index.html";

            string uriManifesto2 = $"{url}/000000001.html";

​

            string html = await (new WebClient().DownloadStringTaskAsync(new Uri(uriManifesto)));

            var uris = html.LRParser(StringExtension.Tokenizer, "href", @".html").Aggregate(new Dictionary<string, string>(), (c, n) =>

            {

                c.Add(n.Key, $"{url}/{(n.Value as string)?.Replace("href", "").Replace("=", "").Replace($"\"", "")}");

                return c;

            });

            List<string> list = new List<string>();

            foreach (var uri in uris)

            {

                try

                {

                    Task<string> t1 = new WebClient().DownloadStringTaskAsync(new Uri(uri.Value));

                    var task = t1.ContinueWith(_ =>

                    {

                        if (t1.Exception != null)

                            t1.Exception.InnerException.Dump();

                        else

                        {

                            string htmlContent = t1.Result;

                            if (htmlContent.Contains(name))

                            {

                                list.Add(uri.Value);

                                Console.WriteLine(uri.Value);

                            }

                        }

                    });

                    task.Wait();

                }

                catch (Exception e)

                {

                    Console.WriteLine($"{e.Message}\r\n{uri.Value}");

                }

            };

        }

​

}

​

public static class StringExtension

{

    public static Dictionary<string, object> Tokenizer(string prefix, string body, string postfix)

    {

        Dictionary<string, object> parsedTokens = new Dictionary<string, object>();

        parsedTokens.Add(Guid.NewGuid().ToString(), String.Format("{0}{1}{2}", prefix, body, postfix)); //+ @";" +      

        return parsedTokens;

    }

​

    /// <summary>

    /// Parses left to right and collects tokens with tokenizer.

    /// </summary>

    /// <param name = "text"></param>

    /// <param name = "tokenizer">Collects tokens as a sublanguage</param>

    /// <param name = "startProductionRule"></param>

    /// <param name = "endProductionRule"></param>

    /// <returns></returns>

    public static TResult LRParser<TResult>(this String text, Func<string, string, string, TResult> tokenizer, string beginToken, string endToken)

        where TResult : IDictionary<string, object>, new()

    {

        TResult parsedTokens = new TResult();

        string matchedBeginToken = String.Empty;

        string matchedEndToken = String.Empty;

        string lookahead = text.LRPostfixify(beginToken, out matchedBeginToken);

        string body = lookahead.LRPrefixify(endToken, out matchedEndToken);

        string nextPass = text.LRPostfixify(endToken);

        if (!string.IsNullOrEmpty(body))

        {

            tokenizer(matchedBeginToken, body, matchedEndToken).Aggregate(parsedTokens, (c, n) =>

            {

                c.Add(n.Key, n.Value);

                return c;

            });

        }

​

        if (!string.IsNullOrEmpty(nextPass))

        {

            nextPass.LRParser(tokenizer, beginToken, endToken).Aggregate(parsedTokens, (c, n) =>

            {

                if (!c.ContainsKey(n.Key))

                    c.Add(n.Key, n.Value);

                return c;

            });

        }

​

        return parsedTokens;

    }

​

    /// <summary>

    /// returns a non-null or nullified postfix value after marker. Marker is a regex .

    /// </summary>

    /// <param name = "word"></param>

    /// <param name = "patternMarker"></param>

    /// <param name = "nullify"></param>

    /// <returns></returns>

    public static string LRPostfixify(this String word, string patternMarker, bool nullify = true)

    {

        string matched = String.Empty;

        if (String.IsNullOrWhiteSpace(word))

            return matched;

        return word.LRPostfixify(patternMarker, out matched, nullify);

    }

​

    public static string LRPostfixify(this String word, string patternMarker, out string matchedMarker, bool nullify = true)

    {

        matchedMarker = String.Empty;

        if (String.IsNullOrWhiteSpace(word) || String.IsNullOrEmpty(patternMarker))

            return String.Empty;

        var match = Regex.Match(word, patternMarker, RegexOptions.IgnoreCase);

        //match.Dump();

        if (match.Length > 0)

        {

            matchedMarker = match.Captures[0].Value;

            string postfixified = word.LRPostfix(matchedMarker);

            return postfixified;

        }

​

        return nullify ? null : word;

    }

​

    public static string LRPrefixify(this String word, string patternMarker, out string matchedMarker, bool nullify = true)

    {

        matchedMarker = String.Empty;

        if (String.IsNullOrWhiteSpace(word) || String.IsNullOrEmpty(patternMarker))

            return matchedMarker;

        var match = Regex.Match(word, patternMarker, RegexOptions.IgnoreCase);

        //match.Dump();

        if (match.Length > 0)

        {

            matchedMarker = match.Captures[0].Value;

            string prefixified = word.LRPrefix(matchedMarker);

            if (String.IsNullOrWhiteSpace(prefixified) || String.IsNullOrEmpty(prefixified))

                return word;

            return prefixified;

        }

​

        return nullify ? null : word;

    }

​

    /// <summary>

    /// Returns a non-null or nullified prefix value marked by marker which is excluded.

    /// </summary>

    /// <param name = "word"></param>

    /// <param name = "marker"></param>

    /// <returns></returns>

    public static string LRPrefix(this String word, string marker, bool nullify = true)

    {

        if (!string.IsNullOrEmpty(marker) && !string.IsNullOrEmpty(word))

        {

            int res = word.IndexOf(marker, 0, StringComparison.OrdinalIgnoreCase);

            if (res > -1)

                return word.Substring(0, word.IndexOf(marker, StringComparison.OrdinalIgnoreCase));

        }

​

        return nullify ? null : word;

    }

​

    /// <summary>

    /// Returns a non-null or nullified postfix value marked by marker which is excluded.

    /// left to right search direction is used.

    /// </summary>

    /// <param name = "word"></param>

    /// <param name = "marker"></param>

    /// <returns></returns>

    public static string LRPostfix(this String word, string marker, bool nullify = true)

    {

        if (!string.IsNullOrEmpty(marker) && !string.IsNullOrEmpty(word))

        {

            int res = word.IndexOf(marker, 0, StringComparison.OrdinalIgnoreCase);

            if (res > -1)

                return word.Substring(word.IndexOf(marker, 0, StringComparison.OrdinalIgnoreCase) + marker.Length);

        }

​

        return nullify ? null : word;

    }

}