web crawler / spider a website

Question

I want to know , How to make web crawler / spider a website in asp.net ?

Dipankar Biswas · Answer

spider a website Web spider code public class WebSpider { const int LIMIT = 10; string[] invalidTypes = { ".zip", ".doc", ".css", ".pdf", ".xls", ".txt", ".js", ".ico" }; public List Links; public bool IsRunning { get; set; } public WebSpider() { this.Links = new List (); } public void Execute(string url) { this.Links.Clear(); this.Links.Add(new Link() { Status = HttpStatusCode.OK, NavigateUrl = url }); this.IsRunning = true; WaitCallback item = delegate(object state) { this.FindLinks((UrlState)state); }; ThreadPool.QueueUserWorkItem(item, new UrlState() { Url = url, Level = 0 }); } public void FindLinks(UrlState state) { try { string html = new WebClient().DownloadString(state.Url); MatchCollection matches = Regex.Matches(html, "href[ ]*=[ ]*['|"][^"'\r\n]*['|"]"); foreach (Match match in matches) { string value = match.Value; value = Regex.Replace(value, "(href[ ]*=[ ]*')|(href[ ]*=[ ]*")", string.Empty); if (value.EndsWith(""") || value.EndsWith("'")) value = value.Remove(value.Length - 1, 1); if (!Regex.Match(value, @"$(.*)$").Success) { if (!value.Contains("http:")) { Uri baseUri = new Uri(state.Url); Uri absoluteUri = new Uri(baseUri, value); value = absoluteUri.ToString(); } if (this.Links.Exists(x => x.NavigateUrl.Equals(value))) continue; try { bool validLink = true; foreach (string invalidType in invalidTypes) { string v = value.ToLower(); if (v.EndsWith(invalidType) || v.Contains(string.Format("{0}?", invalidType))) { validLink = false; break; } } if (validLink) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(value); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); this.Links.Add(new Link() { Status = response.StatusCode, NavigateUrl = value }); if (response.StatusCode == HttpStatusCode.OK && state.Level { WaitCallback item = delegate(object s) { FindLinks((UrlState)s); }; ThreadPool.QueueUserWorkItem( item, new UrlState() { Url = value, Level = state.Level + 1 }); } } } catch { this.Links.Add(new Link() { Status = HttpStatusCode.ExpectationFailed, NavigateUrl = value }); } } } } catch { /// /// If downloading times out, just ignore... /// } } } public class Link : HyperLink { public HttpStatusCode Status { get; set; } } public class UrlState { public string Url { get; set; } public int Level { get; set; } public UrlState() { this.Level = 0; this.Url = string.Empty; } } Page code public partial class Test : Page { private string script = @"setTimeout(""__doPostBack('{0}','')"", 5000);"; private WebSpider WebSpider { get { return (WebSpider)(Session["webSpider"] ?? (Session["webSpider"] = new WebSpider())); } } protected override void OnLoad(EventArgs e) { base.OnLoad(e); if (!this.WebSpider.IsRunning) { string url = "http://www.charith.gunasekara.web-sphere.co.uk"; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK) this.WebSpider.Execute(url); } if (this.WebSpider.IsRunning) { this.lblStatus.Text = "Processing..."; ScriptManager.RegisterStartupScript(this, this.GetType(), this.GetType().Name, string.Format(script, this.btnUpdateResults.ClientID), true); } this.grvLinks.DataSource = this.WebSpider.Links; this.grvLinks.DataBind(); } } Markup Listing all the links for site: http://www.charith.gunasekara.web-sphere.co.uk Width the maximum depth of 10

Waqeeh Ul Hasan · Answer

Would you please send me rar file of this ?

Riddhi Valecha · Answer

Hi... I tried this code in VS2010 web application. But, it did not work. Can you please share the working code if you have??

web crawler / spider a website

Answers (3)