Hi
i intend to develop an application .. i just want to know your view how you start the thing ..
My original requirement ... any ideas will be duly appreciated ty ..
Write an application which accepts requests to scrape web URLs and store the results.
The application should do the following :
1. Have an HTTP API to accept requests to fetch a particular URL
2. Assign a request ID to the request, and the response of this API should return the request ID
3. Process requests in batches in the background
4. The response of each request (the headers, the body, the time of request, the response size, errors if any) should be stored, which can be retrieved later using a status API
You are free to use any language, platform or database. Your answer will be judged on application and API design, clean code, maintainability, performance and efficiency.
My best wishes
Ty for your help

VulpesPosted Jan 10, 2014, 9:59 AM
http://www.blackbeltcoder.com/Articles/strings/parsing-html-tags-in-c
SUNIL GUTTAPosted Jan 10, 2014, 9:20 AM
Jaganathan BantheswaranPosted Jan 10, 2014, 7:56 AM
SUNIL GUTTAPosted Jan 10, 2014, 6:50 AM
Here some code : throwing compile time error "TEXTPARSER " . reference not added like that ..can you please make this code wok . textparserin which namespace or which assembly i need to import .. TY
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace SoftCircuits.Parsing
{
public class HtmlTag
{
public string Name { get; set; }
public Dictionary
public bool TrailingSlash { get; set; }
public bool HasAttribute(string name)
{
return Attributes.ContainsKey(name);
}
};
public class HtmlParser : TextParser
{
public HtmlParser()
{
}
public HtmlParser(string html) : base(html)
{
}
public bool ParseNext(string name, out HtmlTag tag)
{
tag = null;
if (String.IsNullOrEmpty(name))
return false;
MoveTo('<');
while (!EndOfText)
{
MoveAhead();
char c = Peek();
if (c == '!' && Peek(1) == '-' && Peek(2) == '-')
{
const string endComment = "-->";
MoveTo(endComment);
MoveAhead(endComment.Length);
}
else if (c == '/')
{
MoveTo('>');
MoveAhead();
}
else
{
bool result, inScript;
result = ParseTag(name, ref tag, out inScript);
if (inScript)
MovePastScript();
if (result)
return true;
}
MoveTo('<');
}
return false;
}
protected bool ParseTag(string reqName, ref HtmlTag tag, out bool inScript)
{
bool doctype, requested;
doctype = inScript = requested = false;
string name = ParseTagName();
if (String.Compare(name, "!DOCTYPE", true) == 0)
doctype = true;
else if (String.Compare(name, "script", true) == 0)
inScript = true;
if (reqName == "*" || String.Compare(name, reqName, true) == 0)
{
requested = true;
tag = new HtmlTag();
tag.Name = name;
tag.Attributes = new Dictionary
}
MovePastWhitespace();
while (Peek() != '>' && Peek() != NullChar)
{
if (Peek() == '/')
{
if (requested)
tag.TrailingSlash = true;
MoveAhead();
MovePastWhitespace();
inScript = false;
}
else
{
name = (!doctype) ? ParseAttributeName() : ParseAttributeValue();
MovePastWhitespace();
string value = String.Empty;
if (Peek() == '=')
{
MoveAhead();
MovePastWhitespace();
value = ParseAttributeValue();
MovePastWhitespace();
}
if (requested)
{
if (tag.Attributes.ContainsKey(name))
tag.Attributes.Remove(name);
tag.Attributes.Add(name, value);
}
}
}
MoveAhead();
return requested;
}
protected string ParseTagName()
{
int start = Position;
while (!EndOfText && !Char.IsWhiteSpace(Peek()) && Peek() != '>')
MoveAhead();
return Substring(start, Position);
}
protected string ParseAttributeName()
{
int start = Position;
while (!EndOfText && !Char.IsWhiteSpace(Peek()) && Peek() != '>' && Peek() != '=')
MoveAhead();
return Substring(start, Position);
}
protected string ParseAttributeValue()
{
int start, end;
char c = Peek();
if (c == '"' || c == '\'')
{
MoveAhead();
start = Position;
MoveTo(new char[] { c, '\r', '\n' });
end = Position;
if (Peek() == c)
MoveAhead();
}
else
{
start = Position;
while (!EndOfText && !Char.IsWhiteSpace(c) && c != '>')
{
MoveAhead();
c = Peek();
}
end = Position;
}
return Substring(start, end);
}
protected void MovePastScript()
{
const string endScript = "
while (!EndOfText)
{
MoveTo(endScript, true);
MoveAhead(endScript.Length);
if (Peek() == '>' || Char.IsWhiteSpace(Peek()))
{
MoveTo('>');
MoveAhead();
break;
}
}
}
}
}
Dhirendra MisraPosted Jan 10, 2014, 5:48 AM
Refer this one:
http://www.dotnetperls.com/scraping-html
http://www.codeproject.com/Tips/509473/Simple-Web-Scraper
Jaganathan BantheswaranPosted Jan 10, 2014, 3:51 AM
http://blogs.msdn.com/b/webdev/archive/2013/11/01/introducing-batch-support-in-web-api-and-web-api-odata.aspx
Above is the nice example for WebAPI request batching. Almost you 60% of work is covered.