using ScrapySharp.Extensions;
using ScrapySharp.Network;
using System.Collections.Generic;
namespace VandalMessagesScraper
static ScrapingBrowser _browser = new ScrapingBrowser();
static void Main(string[] args)
var url = "https://vandal.elespanol.com/foro/mensaje/970857/-playstation-5-rumores-y-filtraciones-/";
for (int i = 1; i <= 3258; i++)
var messages = ProcessPage(url + i);
static HtmlNode GetHtml(string url)
_browser.Encoding = Encoding.GetEncoding("iso-8859-1");
WebPage webpage = _browser.NavigateToPage(new Uri(url));
static List<MessageData> ProcessPage(string url)
var nodeMessages = html.CssSelect(".mt1 .unpost");
var processedMessages = new List<MessageData>();
foreach (HtmlNode node in nodeMessages)
processedMessages.Add(ProcessMessage(node));
return processedMessages;
static MessageData ProcessMessage(HtmlNode message)
var unknownUser = message.CssSelect(".mensaje_usuario_login").Count() == 0;
? message.CssSelect(".zonausuario_post").First().InnerText.Trim()
: message.CssSelect(".mensaje_usuario_login").First().InnerText.Trim();
var text = HttpUtility.HtmlDecode(message.CssSelect(".zonamensaje_post").CssSelect(".mensaje_texto").First().InnerText);
return new MessageData() { User = user.ToString(), Text = text.ToString() };
static void SavePage(List<MessageData> messages, int numPage)
var page = new StringBuilder();
foreach (MessageData message in messages)
var newLine = string.Format("{0};{1};{2}", numPage, message.User, message.Text);
page.AppendLine(newLine);
Console.WriteLine(newLine);
var fileName = "Page_" + numPage.ToString("D5"); ;