Imports System.IO

Imports System.IO.Compression

Imports System.Net

Imports System

Imports System.Text

​

Public Class WebRequestHelper

​

    Private _siteCookies As CookieContainer

    Public Property SiteCookies As CookieContainer

        Get

            Return _siteCookies

        End Get

        Set(value As CookieContainer)

            _siteCookies = value

        End Set

    End Property

​

    Private _userAgent As String = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"

    Public Property UserAgent As String

        Get

            Return _userAgent

        End Get

        Set(value As String)

            'HttpContext.Current.Trace.Warn("WebRequestHelper: Setting UserAgent: " & value)

            _userAgent = value

        End Set

    End Property

​

    Private _timeout As Integer = 5000

    Public Property Timeout As Integer

        Get

            Return _timeout

        End Get

        Set(value As Integer)

            If value > 999 Then _timeout = value

        End Set

    End Property

​

    Private _responseUri As Uri = Nothing

    Public ReadOnly Property ResponseUri As Uri

        Get

            Return _responseUri

        End Get

    End Property

​

    Private _statusCode As Integer = -1

    Public ReadOnly Property StatusCode As Integer

        Get

            Return _statusCode

        End Get

    End Property

​

    Dim _statusDescription As String = String.Empty

    Public ReadOnly Property StatusDescription As String

        Get

            Return _statusDescription

        End Get

    End Property

​

    Dim _contentSize As Integer = 0

    Public ReadOnly Property ContentSize As Integer

        Get

            Return _contentSize

        End Get

    End Property

​

    Sub New()

​

        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12

        'ServicePointManager.ServerCertificateValidationCallback = New System.Net.Security.RemoteCertificateValidationCallback(AddressOf AcceptAllCertifications)

​

        _siteCookies = New CookieContainer()

​

    End Sub

​

    Private Function AcceptAllCertifications(ByVal sender As Object, ByVal certification As System.Security.Cryptography.X509Certificates.X509Certificate, ByVal chain As System.Security.Cryptography.X509Certificates.X509Chain, ByVal sslPolicyErrors As System.Net.Security.SslPolicyErrors) As Boolean

        'HttpContext.Current.Trace.Warn("Called AcceptAllCertifications")

        Return True

    End Function

​

    Public Function getSiteHTML(ByVal siteUri As Uri, Optional ByVal PostData As String = Nothing) As String

​

        Dim returnHTML As String = String.Empty

​

        Dim Http As HttpWebRequest = WebRequest.Create(siteUri.ToString)

​

        With Http

​

            .CookieContainer = _siteCookies

​

            .AllowAutoRedirect = True

            .MaximumAutomaticRedirections = 25

​

            .Timeout = _timeout

​

            .ProtocolVersion = HttpVersion.Version11

            .Method = "GET"

            .Accept = "text/html, application/xhtml+xml, image/jxr, */*"

            .Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.5")

            .UserAgent = _userAgent

            .Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate")

            .KeepAlive = True

​

        End With

​

        If Not String.IsNullOrEmpty(PostData) Then

            Http.Method = "POST"

            Dim lbPostBuffer As Byte() = Encoding.Default.GetBytes(PostData)

​

            Http.ContentLength = lbPostBuffer.Length

​

            Using PostStream As Stream = Http.GetRequestStream()

                PostStream.Write(lbPostBuffer, 0, lbPostBuffer.Length)

            End Using

        End If

​

        Try

​

            Using WebResponse As HttpWebResponse = Http.GetResponse

​

                _responseUri = WebResponse.ResponseUri

                _statusCode = WebResponse.StatusCode

                _statusDescription = WebResponse.StatusDescription

​

                'If Not String.IsNullOrWhiteSpace(WebResponse.Headers.Get("Content-Length")) Then

                '    _contentSize = WebResponse.Headers.Get("Content-Length")

                'End If

​

                Dim responseStream As Stream = WebResponse.GetResponseStream()

​

                If (WebResponse.ContentEncoding.ToLower().Contains("gzip")) Then

                    responseStream = New GZipStream(responseStream, CompressionMode.Decompress)

                ElseIf (WebResponse.ContentEncoding.ToLower().Contains("deflate")) Then

                    responseStream = New DeflateStream(responseStream, CompressionMode.Decompress)

                End If

​

                _contentSize = WebResponse.ContentLength

                Dim reader As StreamReader = New StreamReader(responseStream, Encoding.Default)

​

                returnHTML = reader.ReadToEnd()

​

                responseStream.Close()

​

            End Using

​

        Catch ex As WebException

​

            _statusCode = -100

            _statusDescription = "WebException: " & ex.Message

​

        End Try

​

        Return returnHTML

​

    End Function

​

End Class

​

Public Module Module1

    Public Sub Main()

​

        Dim voxiWebRequest As New WebRequestHelper

​

        'Works fine

        Dim siteHTML as String = voxiWebRequest.getSiteHTML(New Uri("https://www.amazon.com/Neon-Athletic-Racerback-Spandex-Workout/dp/B01F6O5PB8"))

​

        Console.WriteLine("https://www.amazon.com/Neon-Athletic-Racerback-Spandex-Workout/dp/B01F6O5PB8")

        Console.WriteLine("Status: " & voxiWebRequest.StatusCode)

        Console.WriteLine("Status Description: " & voxiWebRequest.StatusDescription)

        Console.WriteLine("Content Size: " & siteHTML.Length)

​

        Console.WriteLine("")

        Console.WriteLine("")

​

        'Doesn't work

        Dim site2HTML as String = string.Empty

        Try

            site2HTML = voxiWebRequest.getSiteHTML(New Uri("https://www.homedepot.com/p/Sun-Joe-1-2-in-Dia-x-100-ft-Heavy-Duty-Spiral-Constructed-Stainless-Steel-Metal-Garden-Hose-AJSGH100/303010442"))

        Catch ex As WebException

            Console.WriteLine("WebException: " & ex.Message)

        End Try

​

        Console.WriteLine("https://www.homedepot.com/p/Sun-Joe-1-2-in-Dia-x-100-ft-Heavy-Duty-Spiral-Constructed-Stainless-Steel-Metal-Garden-Hose-AJSGH100/303010442")

        Console.WriteLine("Status: " & voxiWebRequest.StatusCode)

        Console.WriteLine("Status Description: " & voxiWebRequest.StatusDescription)

        Console.WriteLine("Content Size: " & site2HTML.Length)

​

    End Sub

End Module