C# Online Compiler | .NET Fiddle

<no name> by Genfried

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using NUnit.Framework;
	
	public class Canonicalisation
    {
        private const String IpValidation = "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}$";

        private static Canonicalisation _instance;

        private Canonicalisation(){}

        /// <summary>
        /// Singleton instance
        /// </summary>
        public static Canonicalisation GetInstance()
        {
            return _instance ?? (_instance = new Canonicalisation());
        }

        /// <summary>
        /// Unscapes a string repeatedly to remove all escaped characters. Returns null if the string is invalid.
        /// </summary>
        private static String Unescape(String url)
        {
            if (url == null)
                return null;

            var stringBuilder = new StringBuilder(url);
            url = stringBuilder.ToString();
            for (var x = 0; x < 50; x++)
            {
                // keep iterating to make sure all those encodings are killed.
                url = Uri.UnescapeDataString(url); // Unescape repeatedly until no more percent signs left
            }
            return url;
        }

        /// <summary>
        /// Decode a host, If it is a hostname, return it, if it is an IP decode (encoding, including octal & hex)
        /// </summary>
        private static String DecodeHost(String host)
        {
            try
            {
                var addr = new Uri(host);
                if (host == IpValidation) 
                    return host.ToLower();
                if (host.IndexOf('.') > -1)
                {
                    // most likely a domain 
                    return addr.Host.ToLower();
                }

                if (addr.Host != IpValidation)
                    return addr.Host;

                return addr.Host.ToLower();
            }
            catch
            {
                return host;
            }
        }

        /// <summary>
        /// Removes BackSlashes
        /// </summary>
        public string RemoveBackSlashes(string input)
        {
            var literal = new StringBuilder();
            foreach (var c in input)
            {
                switch (c)
                {
                    case '\'': break;
                    case '\"': break;
                    case '\\': break;
                    case '\0': break;
                    case '\a': break;
                    case '\b': break;
                    case '\f': break;
                    case '\n': break;
                    case '\r': break;
                    case '\t': break;
                    case '\v': break;
                    default:
                        if (Char.GetUnicodeCategory(c) != UnicodeCategory.Control)
                        {
                            literal.Append(c);
                        }
                        else
                        {
                            literal.Append(@"\u");
                            literal.Append(((ushort)c).ToString("x4"));
                        }
                        break;
                }
            }
            return literal.ToString();
        }


        /// <summary>
        /// Returns the canonicalized form of a URL, core logic written by Henrik Sjostrand, heavily modified for v2 by Dave Shanley.
        /// Authors: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com & Dave Shanley <dave@buildabrand.com>
        /// </summary>
        public String CanonicalizeUrl(String queryUrl)
        {
            if (queryUrl == null)
                return null;
            String url = RemoveBackSlashes(queryUrl);
            url = url.Trim(' ');// Remove trailing spaces
            url = Uri.UnescapeDataString(url); // Decode before use

            /* replace consecutive dots with a single dot */
            while ((url.IndexOf("..", StringComparison.Ordinal) != -1))
                url = url.Replace("..", ".");
            /* replace any encoded percentage signs */
            url = url.Replace("(?i)%5C", "%");
            /* unescape url to remove all hex encodings */
            url = Unescape(url);

            /* remove /./ occurences from url */
            url = url.Replace("/./", "/");

            /* resolve /../ occurences in url */
            url = url.Replace("/../", "/");

            /* resolve ./ occurences in url */
            url = url.Replace("./", "/");
            /* use URI class to normalise the URL */

            /* first of all extract the components of the URL to make sure that it has a protocol! */
            if (url.IndexOf("http://", StringComparison.Ordinal) <= -1 && url.IndexOf("https://", StringComparison.Ordinal) <= -1) url = "http://" + url;
            url = url.Replace("[\\t\\n\\r\\f\\e]*", ""); // replace all whitespace and escape characters.
            var theUrl = new Uri(url);
            var host = theUrl.Host;
            var path = theUrl.AbsolutePath;
            var query = theUrl.Query;
            var protocol = GetProtocol(theUrl);
            if (protocol == null || String.IsNullOrEmpty(protocol)) 
                protocol = "http";
            var port = theUrl.Port;
            var user = theUrl.UserInfo;

            /* escape host */
            host = Unescape(host);
            /* remove double slashes from path  */
            path = path.Replace("//", "/");
            /* decode host / IP */
            host = DecodeHost(host);
            var sb = new StringBuilder();
            foreach (char urlChar in host)
            {
                if ((urlChar >= '0' && urlChar <= '9') || (urlChar >= 'a' && urlChar <= 'z') || urlChar == '.' || urlChar == '-')
                    sb.Append(urlChar);
                else
                    sb.Append(Uri.EscapeDataString(urlChar.ToString(CultureInfo.InvariantCulture))); // Escape using UTF-8
            }
            host = sb.ToString();
            /* remove leading and trailing dots */
            while (host.StartsWith("."))
                host = host.Trim('.');
            /* add a trailing slash if the path is empty */
            if (String.IsNullOrEmpty(path))
                host = host + "/";
            /* find and replace any dodgy decoded escape characters */
            var pattern = new Regex("([a-z]{1})([0-9]{2})");
            const string val = "$2";
            while (pattern.IsMatch(host))
            {
                host = pattern.Replace(host,val);
            }

            Uri uri = null; 
            var regex = new Regex("[\\s].*");
            try
            {
                /* only normalise if the host doesn't contain some odd hex */
                if (!host.Contains("%") && !regex.IsMatch(host))
                {
                     return Uri.EscapeUriString(CreateUri(protocol, host, port,path,query).ToString()).Replace("%25","%");
                }
            }
            catch
            {
                try
                {
                    /* only normalise if the host doesn't contain some odd hex */
                    if (!host.Contains("%") && !regex.IsMatch(host))
                    {
                        uri = CreateUri(protocol,host, port,path,query);
                    }

                    /* only use URI normalized URL if it's not a total failure */
                    if (uri != null && !String.IsNullOrEmpty(uri.AbsolutePath.Trim()))
                    {
                        path = uri.AbsolutePath;
                    }
                    /* escape the path */
                    path = Escape(path);

                    /* replace all semi-colons */
                    path = path.Replace("[;]*", "");

                    /* unescape the query */
                    query = Unescape(query);

                    /* re-escape the query */
                    query = Escape(query);


                    /* re-assemble the URL */
                    sb.Clear();
                    sb.Append(protocol + ":");

                    sb.Append("//");
                    sb.Append(user + "@");

                    if (port != -1)
                    {

                        /* remove slash from host */

                        if (host.LastIndexOf("/", host.Length, StringComparison.Ordinal) > 8)
                        {

                            host = host.Substring(0, host.Length - 1);
                        }
                        sb.Append(host);
                        sb.Append(":");
                        sb.Append(port);
                    }
                    else
                    {
                        sb.Append(host);
                    }

                    /* make sure any hashes are re-encoded back to %23*/
                    path = path.Replace("#", "%23");

                    sb.Append(path);
                    const string endtrim = "//";
                    if (sb.ToString().EndsWith(endtrim))
                    {
                        string result = sb.ToString();
                        while (result.EndsWith(endtrim))
                        {
                            result = result.Substring(0, result.Length - endtrim.Length);
                        }
                        sb = new StringBuilder(result);
                    }

                    if (query != null)
                        sb.Append("?" + query);

                    url = sb.ToString();


                }
                catch (Exception e)
                {
                    throw new Exception("Could not canonicalise URL: " + queryUrl, e);
                }
                return url;
            }
            throw new ApplicationException("Application wasn't supposed to be here.");
        }

        /// <summary>
        /// Creates a valid Uri from the indepenent parts
        /// </summary>
        private Uri CreateUri(string protocol, string site, int port,string path,string query)
        {
            var url = "";
            if (site.StartsWith(protocol))
            {
                url += site;
            }
            else
            {
                url += protocol + "://" + site;
            }

            if (port > 0)
            {
                url += ":" + port;
            }

            url += path;

            if (query.StartsWith("?") && !String.IsNullOrWhiteSpace(query))
            {
                url += query;
            }
            else if (!String.IsNullOrWhiteSpace(query))
            {
                url += "?" + query;
            }
            return new Uri(url);
        }

        /// <summary>
        /// Return the protocol from the supplied Url
        /// </summary>
        private static string GetProtocol(Uri theUrl)
        {
            if (theUrl.OriginalString.Contains("://"))
            {
                return theUrl.OriginalString.Split(new[] {"://"}, StringSplitOptions.None)[0];
            }
            return null;
        }

        /// <summary>
        /// Escapes a string by replacing characters having ASCII <=32, >=127, or % with their UTF-8-escaped codes 
        /// Author: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com
        /// </summary>
        private static String Escape(String url)
        {
            if (url == null)
                return null;
            var sb = new StringBuilder();
            foreach (char charUrl in url)
            {
                if (charUrl == ' ')
                    sb.Append("%20");
                else if (charUrl <= 32 || charUrl >= 127 || charUrl == '%')
                {
                    sb.Append(Uri.EscapeDataString(charUrl.ToString(CultureInfo.CurrentCulture)));
                }
                else
                    sb.Append(charUrl);
            }
            return sb.ToString();
        }

        /// <summary>
        /// Parses the query URL into several different url strings, each of which should be looked up 
        /// Authors: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com & Dave Shanley <dave@buildabrand.com>
        /// </summary>
        public List<string> GetLookupUrLs(String queryUrl)
        {
            var urls = new List<string>();
            if (queryUrl != null)
            {
                try
                {
                    var canonicalizedUrl = CanonicalizeUrl(queryUrl);
                    if (canonicalizedUrl != null)
                    {
                        // canonicalizeURL & unescapeUTF8 returns null on invalid strings, and then we don't add any URLs to the lookup list which means we consider this URL safe
                        var url = new Uri(canonicalizedUrl);
                        var host = url.Host;
                        var path = url.AbsolutePath;
                        var query = url.Query;
                        query = "?" + query;

                        // Generate a list of the hosts to test (exact hostname plus up to four truncated hostnames) 
                        var hosts = new List<String> {host};
                        var hostArray = host.Split(new []{"\\."},StringSplitOptions.None);
                        var sb = new StringBuilder();
                        var start = (hostArray.Length < 6 ? 1 : hostArray.Length - 5);
                        var stop = hostArray.Length;
                        for (var i = start; i < stop - 1; i++)
                        {
                            sb = sb.Clear();
                            for (var j = i; j < stop; j++)
                                sb.Append(hostArray[j] + ".");
                            sb.Remove(sb.Length - 1, 1); // Trim trailing dot
                            hosts.Add(sb.ToString());
                        }

                        // Generate a list of paths to test
                        var paths = new List<String> {path + query, path};
                        if (!paths.Contains("/"))
                            paths.Add("/");

                        var pathElement = "/";
                        var st = path.Split('/');
                        foreach (var thisToken in st)
                        {
                            pathElement = pathElement + thisToken + (thisToken.IndexOf(".", StringComparison.Ordinal) == -1 ? "/" : "");
                            if (!paths.Contains(pathElement))
                                paths.Add(pathElement);
                        }

                        foreach (string selectedHost in hosts)
                        {
                            urls.AddRange(paths.Select(t => selectedHost + t));
                        }
                    }
                }
                catch (Exception e)
                {
                    throw new Exception("Could not generate lookup URLs", e);
                }
            }
            return urls;
        }
    }
	
	[TestFixture]
    public class CanonicalisationTest
    {
        
        //Succeeded Tests
        [Test]
        public void TrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%25%32%35"), "http://host/%25");
        }


        [Test]
        public void DoubleTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%25%32%35%25%32%35"), "http://host/%25%25");
        }

        [Test]
        public void NoSlashtrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%2525252525252525"), "http://host/%25");
        }

        [Test]
        public void MiddleUrlTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/asdf%25%32%35asd"), "http://host/asdf%25asd");
        }

        [Test]
        public void PercentageRecognitionTestTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%%%25%32%35asd%%"), "http://host/%25%25%25asd%25%25");
        }

        [Test]
        public void BasicTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/"), "http://www.google.com/");
        }

        [Test]
        public void AdvancedIpTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/"),
                "http://168.188.99.26/.secure/www.ebay.com/");
        }

        [Test]
        public void BasicIpTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/"),
                    "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
        }

        [Test]
        public void IpConversionTest()
        {
            Assert.AreEqual(Canonicalize("http://3279880203/blah"), "http://195.127.0.11/blah");
        }

        [Test]
        public void NoPrototcolTest()
        {
            Assert.AreEqual(Canonicalize("www.google.com/"), "http://www.google.com/");
        }

        [Test]
        public void NoProtocolAndEndingSlashTest()
        {
            Assert.AreEqual(Canonicalize("www.google.com"), "http://www.google.com/");
        }

        [Test]
        public void HashtagTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.evil.com/blah#frag"), "http://www.evil.com/blah");
        }

        [Test]
        public void CapitalizationTest()
        {
            Assert.AreEqual(Canonicalize("http://www.GOOgle.com/"), "http://www.google.com/");
        }

        [Test]
        public void DotTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com.../"), "http://www.google.com/");
        }

        [Test]
        public void BackSlashTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/foo\tbar\rbaz\n2"), "http://www.google.com/foobarbaz2");
        }

        [Test]
        public void UrlParameterBasicTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?"), "http://www.google.com/q?");
        }

        [Test]
        public void UrlparamaterOneParameterTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?r?"), "http://www.google.com/q?r?");
        }

        [Test]
        public void UrlParameterDoubleParameterTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?r?s"), "http://www.google.com/q?r?s");
        }

        [Test]
        public void AdvancedHashTagTestTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo#bar#baz"), "http://evil.com/foo");
        }

        [Test]
        public void SlashTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo);"), "http://evil.com/foo);");
        }

        [Test]
        public void ParameterSlashTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo?bar);"), "http://evil.com/foo?bar);");
        }

        [Test]
        public void NotrailingSlashTest()
        {
            Assert.AreEqual(Canonicalize("http://notrailingslash.com"), "http://notrailingslash.com/");
        }

        [Test]
        public void PortTest()
        {
            Assert.AreEqual(Canonicalize("http://www.gotaport.com:1234/"), "http://www.gotaport.com:1234/");
        }

        [Test]
        public void TrailingSpacesTest()
        {
            Assert.AreEqual(Canonicalize("  http://www.google.com/  "), "http://www.google.com/");
        }

        [Test]
        public void HttpsTest()
        {
            Assert.AreEqual(Canonicalize("https://www.securesite.com/"), "https://www.securesite.com/");
        }
        
        [Test]
        public void TwoSlashesTest()
        {
            Assert.AreEqual(Canonicalize("http://host.com//twoslashes?more//slashes") , "http://host.com/twoslashes?more//slashes");
        }

        private string Canonicalize(string url)
        {
            var canonizer = Canonicalisation.GetInstance();
            return canonizer.CanonicalizeUrl(url);
        }

        //Failed Tests
        [Test]
        public void AdvancedLeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("http://%20leadingspace.com/"), "http://%20leadingspace.com/");
        }
        
        [Test]
        public void BasicLocationTest()
        {
            Assert.AreEqual(Canonicalize("http://host.com/ab%23cd") , "http://host.com/ab%23cd");
        }
        
        [Test]
        public void ConversionTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://host%23.com/%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_55%252B"),
                "http://host%23.com/~a!b@c%23d$e%25f^00&11*22(33)44_55+");
        }
        
        [Test]
        public void NoProtocolLeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("%20leadingspace.com/"), "http://%20leadingspace.com/");
        }
        
        [Test]
        public void UnicodeTest()
        {
            Assert.AreEqual(Canonicalize("http://\x01\x80.com/"), "http://%01%80.com/");
        }
        
        [Test]
        public void DotTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/blah/.."), "http://www.google.com/");
        }
         
        
        [Test]
        public void LeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("http:// leadingspace.com/"), "http://%20leadingspace.com/");
        }
    }

​x
 
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using NUnit.Framework;
    
    public class Canonicalisation
    {
        private const String IpValidation = "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}$";
​
        private static Canonicalisation _instance;
​
        private Canonicalisation(){}
​
        /// <summary>
        /// Singleton instance
        /// </summary>
        public static Canonicalisation GetInstance()
        {
            return _instance ?? (_instance = new Canonicalisation());
        }
​
        /// <summary>
        /// Unscapes a string repeatedly to remove all escaped characters. Returns null if the string is invalid.
        /// </summary>
        private static String Unescape(String url)
        {
            if (url == null)
                return null;
​
            var stringBuilder = new StringBuilder(url);
            url = stringBuilder.ToString();
            for (var x = 0; x < 50; x++)
            {
                // keep iterating to make sure all those encodings are killed.
                url = Uri.UnescapeDataString(url); // Unescape repeatedly until no more percent signs left
            }
            return url;
        }
​
        /// <summary>
        /// Decode a host, If it is a hostname, return it, if it is an IP decode (encoding, including octal & hex)
        /// </summary>
        private static String DecodeHost(String host)
        {
            try
            {
                var addr = new Uri(host);
                if (host == IpValidation) 
                    return host.ToLower();
                if (host.IndexOf('.') > -1)
                {
                    // most likely a domain 
                    return addr.Host.ToLower();
                }
​
                if (addr.Host != IpValidation)
                    return addr.Host;
​
                return addr.Host.ToLower();
            }
            catch
            {
                return host;
            }
        }
​
        /// <summary>
        /// Removes BackSlashes
        /// </summary>
        public string RemoveBackSlashes(string input)
        {
            var literal = new StringBuilder();
            foreach (var c in input)
            {
                switch (c)
                {
                    case '\'': break;
                    case '\"': break;
                    case '\\': break;
                    case '\0': break;
                    case '\a': break;
                    case '\b': break;
                    case '\f': break;
                    case '\n': break;
                    case '\r': break;
                    case '\t': break;
                    case '\v': break;
                    default:
                        if (Char.GetUnicodeCategory(c) != UnicodeCategory.Control)
                        {
                            literal.Append(c);
                        }
                        else
                        {
                            literal.Append(@"\u");
                            literal.Append(((ushort)c).ToString("x4"));
                        }
                        break;
                }
            }
            return literal.ToString();
        }
​
​
        /// <summary>
        /// Returns the canonicalized form of a URL, core logic written by Henrik Sjostrand, heavily modified for v2 by Dave Shanley.
        /// Authors: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com & Dave Shanley <dave@buildabrand.com>
        /// </summary>
        public String CanonicalizeUrl(String queryUrl)
        {
            if (queryUrl == null)
                return null;
            String url = RemoveBackSlashes(queryUrl);
            url = url.Trim(' ');// Remove trailing spaces
            url = Uri.UnescapeDataString(url); // Decode before use
​
            /* replace consecutive dots with a single dot */
            while ((url.IndexOf("..", StringComparison.Ordinal) != -1))
                url = url.Replace("..", ".");
            /* replace any encoded percentage signs */
            url = url.Replace("(?i)%5C", "%");
            /* unescape url to remove all hex encodings */
            url = Unescape(url);
​
            /* remove /./ occurences from url */
            url = url.Replace("/./", "/");
​
            /* resolve /../ occurences in url */
            url = url.Replace("/../", "/");
​
            /* resolve ./ occurences in url */
            url = url.Replace("./", "/");
            /* use URI class to normalise the URL */
​
            /* first of all extract the components of the URL to make sure that it has a protocol! */
            if (url.IndexOf("http://", StringComparison.Ordinal) <= -1 && url.IndexOf("https://", StringComparison.Ordinal) <= -1) url = "http://" + url;
            url = url.Replace("[\\t\\n\\r\\f\\e]*", ""); // replace all whitespace and escape characters.
            var theUrl = new Uri(url);
            var host = theUrl.Host;
            var path = theUrl.AbsolutePath;
            var query = theUrl.Query;
            var protocol = GetProtocol(theUrl);
            if (protocol == null || String.IsNullOrEmpty(protocol)) 
                protocol = "http";
            var port = theUrl.Port;
            var user = theUrl.UserInfo;
​
            /* escape host */
            host = Unescape(host);
            /* remove double slashes from path  */
            path = path.Replace("//", "/");
            /* decode host / IP */
            host = DecodeHost(host);
            var sb = new StringBuilder();
            foreach (char urlChar in host)
            {
                if ((urlChar >= '0' && urlChar <= '9') || (urlChar >= 'a' && urlChar <= 'z') || urlChar == '.' || urlChar == '-')
                    sb.Append(urlChar);
                else
                    sb.Append(Uri.EscapeDataString(urlChar.ToString(CultureInfo.InvariantCulture))); // Escape using UTF-8
            }
            host = sb.ToString();
            /* remove leading and trailing dots */
            while (host.StartsWith("."))
                host = host.Trim('.');
            /* add a trailing slash if the path is empty */
            if (String.IsNullOrEmpty(path))
                host = host + "/";
            /* find and replace any dodgy decoded escape characters */
            var pattern = new Regex("([a-z]{1})([0-9]{2})");
            const string val = "$2";
            while (pattern.IsMatch(host))
            {
                host = pattern.Replace(host,val);
            }
​
            Uri uri = null; 
            var regex = new Regex("[\\s].*");
            try
            {
                /* only normalise if the host doesn't contain some odd hex */
                if (!host.Contains("%") && !regex.IsMatch(host))
                {
                     return Uri.EscapeUriString(CreateUri(protocol, host, port,path,query).ToString()).Replace("%25","%");
                }
            }
            catch
            {
                try
                {
                    /* only normalise if the host doesn't contain some odd hex */
                    if (!host.Contains("%") && !regex.IsMatch(host))
                    {
                        uri = CreateUri(protocol,host, port,path,query);
                    }
​
                    /* only use URI normalized URL if it's not a total failure */
                    if (uri != null && !String.IsNullOrEmpty(uri.AbsolutePath.Trim()))
                    {
                        path = uri.AbsolutePath;
                    }
                    /* escape the path */
                    path = Escape(path);
​
                    /* replace all semi-colons */
                    path = path.Replace("[;]*", "");
​
                    /* unescape the query */
                    query = Unescape(query);
​
                    /* re-escape the query */
                    query = Escape(query);
​
​
                    /* re-assemble the URL */
                    sb.Clear();
                    sb.Append(protocol + ":");
​
                    sb.Append("//");
                    sb.Append(user + "@");
​
                    if (port != -1)
                    {
​
                        /* remove slash from host */
​
                        if (host.LastIndexOf("/", host.Length, StringComparison.Ordinal) > 8)
                        {
​
                            host = host.Substring(0, host.Length - 1);
                        }
                        sb.Append(host);
                        sb.Append(":");
                        sb.Append(port);
                    }
                    else
                    {
                        sb.Append(host);
                    }
​
                    /* make sure any hashes are re-encoded back to %23*/
                    path = path.Replace("#", "%23");
​
                    sb.Append(path);
                    const string endtrim = "//";
                    if (sb.ToString().EndsWith(endtrim))
                    {
                        string result = sb.ToString();
                        while (result.EndsWith(endtrim))
                        {
                            result = result.Substring(0, result.Length - endtrim.Length);
                        }
                        sb = new StringBuilder(result);
                    }
​
                    if (query != null)
                        sb.Append("?" + query);
​
                    url = sb.ToString();
​
​
                }
                catch (Exception e)
                {
                    throw new Exception("Could not canonicalise URL: " + queryUrl, e);
                }
                return url;
            }
            throw new ApplicationException("Application wasn't supposed to be here.");
        }
​
        /// <summary>
        /// Creates a valid Uri from the indepenent parts
        /// </summary>
        private Uri CreateUri(string protocol, string site, int port,string path,string query)
        {
            var url = "";
            if (site.StartsWith(protocol))
            {
                url += site;
            }
            else
            {
                url += protocol + "://" + site;
            }
​
            if (port > 0)
            {
                url += ":" + port;
            }
​
            url += path;
​
            if (query.StartsWith("?") && !String.IsNullOrWhiteSpace(query))
            {
                url += query;
            }
            else if (!String.IsNullOrWhiteSpace(query))
            {
                url += "?" + query;
            }
            return new Uri(url);
        }
​
        /// <summary>
        /// Return the protocol from the supplied Url
        /// </summary>
        private static string GetProtocol(Uri theUrl)
        {
            if (theUrl.OriginalString.Contains("://"))
            {
                return theUrl.OriginalString.Split(new[] {"://"}, StringSplitOptions.None)[0];
            }
            return null;
        }
​
        /// <summary>
        /// Escapes a string by replacing characters having ASCII <=32, >=127, or % with their UTF-8-escaped codes 
        /// Author: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com
        /// </summary>
        private static String Escape(String url)
        {
            if (url == null)
                return null;
            var sb = new StringBuilder();
            foreach (char charUrl in url)
            {
                if (charUrl == ' ')
                    sb.Append("%20");
                else if (charUrl <= 32 || charUrl >= 127 || charUrl == '%')
                {
                    sb.Append(Uri.EscapeDataString(charUrl.ToString(CultureInfo.CurrentCulture)));
                }
                else
                    sb.Append(charUrl);
            }
            return sb.ToString();
        }
​
        /// <summary>
        /// Parses the query URL into several different url strings, each of which should be looked up 
        /// Authors: Henrik Sjostrand, Netvouz, http://www.netvouz.com/, info@netvouz.com & Dave Shanley <dave@buildabrand.com>
        /// </summary>
        public List<string> GetLookupUrLs(String queryUrl)
        {
            var urls = new List<string>();
            if (queryUrl != null)
            {
                try
                {
                    var canonicalizedUrl = CanonicalizeUrl(queryUrl);
                    if (canonicalizedUrl != null)
                    {
                        // canonicalizeURL & unescapeUTF8 returns null on invalid strings, and then we don't add any URLs to the lookup list which means we consider this URL safe
                        var url = new Uri(canonicalizedUrl);
                        var host = url.Host;
                        var path = url.AbsolutePath;
                        var query = url.Query;
                        query = "?" + query;
​
                        // Generate a list of the hosts to test (exact hostname plus up to four truncated hostnames) 
                        var hosts = new List<String> {host};
                        var hostArray = host.Split(new []{"\\."},StringSplitOptions.None);
                        var sb = new StringBuilder();
                        var start = (hostArray.Length < 6 ? 1 : hostArray.Length - 5);
                        var stop = hostArray.Length;
                        for (var i = start; i < stop - 1; i++)
                        {
                            sb = sb.Clear();
                            for (var j = i; j < stop; j++)
                                sb.Append(hostArray[j] + ".");
                            sb.Remove(sb.Length - 1, 1); // Trim trailing dot
                            hosts.Add(sb.ToString());
                        }
​
                        // Generate a list of paths to test
                        var paths = new List<String> {path + query, path};
                        if (!paths.Contains("/"))
                            paths.Add("/");
​
                        var pathElement = "/";
                        var st = path.Split('/');
                        foreach (var thisToken in st)
                        {
                            pathElement = pathElement + thisToken + (thisToken.IndexOf(".", StringComparison.Ordinal) == -1 ? "/" : "");
                            if (!paths.Contains(pathElement))
                                paths.Add(pathElement);
                        }
​
                        foreach (string selectedHost in hosts)
                        {
                            urls.AddRange(paths.Select(t => selectedHost + t));
                        }
                    }
                }
                catch (Exception e)
                {
                    throw new Exception("Could not generate lookup URLs", e);
                }
            }
            return urls;
        }
    }
    
    [TestFixture]
    public class CanonicalisationTest
    {
        
        //Succeeded Tests
        [Test]
        public void TrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%25%32%35"), "http://host/%25");
        }
​
​
        [Test]
        public void DoubleTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%25%32%35%25%32%35"), "http://host/%25%25");
        }
​
        [Test]
        public void NoSlashtrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%2525252525252525"), "http://host/%25");
        }
​
        [Test]
        public void MiddleUrlTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://host/asdf%25%32%35asd"), "http://host/asdf%25asd");
        }
​
        [Test]
        public void PercentageRecognitionTestTest()
        {
            Assert.AreEqual(Canonicalize("http://host/%%%25%32%35asd%%"), "http://host/%25%25%25asd%25%25");
        }
​
        [Test]
        public void BasicTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/"), "http://www.google.com/");
        }
​
        [Test]
        public void AdvancedIpTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/"),
                "http://168.188.99.26/.secure/www.ebay.com/");
        }
​
        [Test]
        public void BasicIpTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/"),
                    "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
        }
​
        [Test]
        public void IpConversionTest()
        {
            Assert.AreEqual(Canonicalize("http://3279880203/blah"), "http://195.127.0.11/blah");
        }
​
        [Test]
        public void NoPrototcolTest()
        {
            Assert.AreEqual(Canonicalize("www.google.com/"), "http://www.google.com/");
        }
​
        [Test]
        public void NoProtocolAndEndingSlashTest()
        {
            Assert.AreEqual(Canonicalize("www.google.com"), "http://www.google.com/");
        }
​
        [Test]
        public void HashtagTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.evil.com/blah#frag"), "http://www.evil.com/blah");
        }
​
        [Test]
        public void CapitalizationTest()
        {
            Assert.AreEqual(Canonicalize("http://www.GOOgle.com/"), "http://www.google.com/");
        }
​
        [Test]
        public void DotTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com.../"), "http://www.google.com/");
        }
​
        [Test]
        public void BackSlashTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/foo\tbar\rbaz\n2"), "http://www.google.com/foobarbaz2");
        }
​
        [Test]
        public void UrlParameterBasicTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?"), "http://www.google.com/q?");
        }
​
        [Test]
        public void UrlparamaterOneParameterTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?r?"), "http://www.google.com/q?r?");
        }
​
        [Test]
        public void UrlParameterDoubleParameterTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/q?r?s"), "http://www.google.com/q?r?s");
        }
​
        [Test]
        public void AdvancedHashTagTestTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo#bar#baz"), "http://evil.com/foo");
        }
​
        [Test]
        public void SlashTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo);"), "http://evil.com/foo);");
        }
​
        [Test]
        public void ParameterSlashTest()
        {
            Assert.AreEqual(Canonicalize("http://evil.com/foo?bar);"), "http://evil.com/foo?bar);");
        }
​
        [Test]
        public void NotrailingSlashTest()
        {
            Assert.AreEqual(Canonicalize("http://notrailingslash.com"), "http://notrailingslash.com/");
        }
​
        [Test]
        public void PortTest()
        {
            Assert.AreEqual(Canonicalize("http://www.gotaport.com:1234/"), "http://www.gotaport.com:1234/");
        }
​
        [Test]
        public void TrailingSpacesTest()
        {
            Assert.AreEqual(Canonicalize("  http://www.google.com/  "), "http://www.google.com/");
        }
​
        [Test]
        public void HttpsTest()
        {
            Assert.AreEqual(Canonicalize("https://www.securesite.com/"), "https://www.securesite.com/");
        }
        
        [Test]
        public void TwoSlashesTest()
        {
            Assert.AreEqual(Canonicalize("http://host.com//twoslashes?more//slashes") , "http://host.com/twoslashes?more//slashes");
        }
​
        private string Canonicalize(string url)
        {
            var canonizer = Canonicalisation.GetInstance();
            return canonizer.CanonicalizeUrl(url);
        }
​
        //Failed Tests
        [Test]
        public void AdvancedLeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("http://%20leadingspace.com/"), "http://%20leadingspace.com/");
        }
        
        [Test]
        public void BasicLocationTest()
        {
            Assert.AreEqual(Canonicalize("http://host.com/ab%23cd") , "http://host.com/ab%23cd");
        }
        
        [Test]
        public void ConversionTest()
        {
            Assert.AreEqual(
                Canonicalize(
                    "http://host%23.com/%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_55%252B"),
                "http://host%23.com/~a!b@c%23d$e%25f^00&11*22(33)44_55+");
        }
        
        [Test]
        public void NoProtocolLeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("%20leadingspace.com/"), "http://%20leadingspace.com/");
        }
        
        [Test]
        public void UnicodeTest()
        {
            Assert.AreEqual(Canonicalize("http://\x01\x80.com/"), "http://%01%80.com/");
        }
        
        [Test]
        public void DotTrimTest()
        {
            Assert.AreEqual(Canonicalize("http://www.google.com/blah/.."), "http://www.google.com/");
        }
         
        
        [Test]
        public void LeadingSpaceTest()
        {
            Assert.AreEqual(Canonicalize("http:// leadingspace.com/"), "http://%20leadingspace.com/");
        }
    }
​

View IL Code