using System;

using System.IO;

using System.Net;

using System.Text;

using System.Text.RegularExpressions;

using System.Collections.Generic;

using System.Threading;

​

public class Program

{

    public static void Main()

    {

            string patternForUnits1 = @"class=""units-row-link";

            string patternForUnits2 = @"href=[""\']?([^""\'>]+)[""\']?""";

            string patternForUnits3 = @"\/Small\/(([^""\'>]+)[""\']?.png)""";

            var states = getStates();

            //Cities

            foreach(String stateUri in states) {

                Dictionary<string, List<string>> cities = new Dictionary<string, List<string>>();

                cities.Add(stateUri, getBasicScrapedItems(stateUri));

                Thread.Sleep(3000);

            }

    }

    public static List<string> getStates() 

    {

            var statesUri = new List<string>(); 

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/alberta");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/british-columbia");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/manitoba");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/new-brunswick");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/newfoundland-and-labrador");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/nova-scotia");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/ontario");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/quebec");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/saskatchewan");

            statesUri.Add("http://www.buzzbuzzhome.com/cities/canada/yukon-territory");

            return statesUri;

    }

    public static List<string> getBasicScrapedItems(String strRequest)

    {

        //Cities pattern

        string patternForCities = @"class=""cities-city"" href=[""\']?(\/city\/([^""\'>]+)[""\']?"")";

        //Create a request for the URL. 

        WebRequest request = WebRequest.Create (strRequest);

        // Get the response.

        WebResponse response = request.GetResponse ();

        // Display the status.

        Console.WriteLine (((HttpWebResponse)response).StatusDescription);

        // Get the stream containing content returned by the server.

        Stream dataStream = response.GetResponseStream ();

        // Open the stream using a StreamReader for easy access.

        StreamReader reader = new StreamReader (dataStream);

        // Read the content.

        string responseFromServer = reader.ReadToEnd();

        // Display the content.

        //Console.WriteLine (responseFromServer);

        var scrapedItems = new List<string>();

        foreach (Match match in Regex.Matches(responseFromServer, patternForCities, RegexOptions.IgnoreCase)) {

            scrapedItems.Add(match.Groups[1].Value);

        }

        foreach (String item in scrapedItems) {

            Console.WriteLine("{0}", item);

        }

        // Clean up the streams and the response.

        reader.Close ();

        response.Close ();

        return scrapedItems;

    }

        public static List<string> getMultipleStepScrapedItems(String strRequest, List<String> strPatterns)

    {

        //Cities pattern

        string patternForCities = @"class=""cities-city"" href=[""\']?(\/city\/([^""\'>]+)[""\']?"")";

        var scrapedItems = new List<string>();

​

        //Create a request for the URL. 

        WebRequest request = WebRequest.Create (strRequest);

        // Get the response.

        WebResponse response = request.GetResponse ();

        // Display the status.

        Console.WriteLine (((HttpWebResponse)response).StatusDescription);

        // Get the stream containing content returned by the server.

        Stream dataStream = response.GetResponseStream ();

        // Open the stream using a StreamReader for easy access.

        StreamReader reader = new StreamReader (dataStream);

        // Read the content.

        string responseFromServer = reader.ReadToEnd();

        // Display the content.

        //Console.WriteLine (responseFromServer);

        foreach(String pattern in strPatterns) {

            foreach (Match matchFirstRound in Regex.Matches(responseFromServer, patternForCities, RegexOptions.IgnoreCase)) {

                responseFromServer = responseFromServer.Substring(matchFirstRound.Index);

                foreach (Match matchSecondRound in Regex.Matches(responseFromServer, patternForCities, RegexOptions.IgnoreCase)) {

                    scrapedItems.Add(matchSecondRound.Groups[1].Value);

​

                }

            }

            foreach (String item in scrapedItems) {

                Console.WriteLine("{0}", item);

            }

            // Clean up the streams and the response.

            reader.Close ();

            response.Close ();

        }

        return scrapedItems;

    }

}