Post
Ask Question

Automate Downloads From Federal Reserve Sites

Recently, one of my friends, who works for an insurance company, buzzed me with this strange scenario he was encountering with the federal reserve site. There are a bunch of data files that are freely available on the site and the only catch is that they had to click on the "Accept Terms and Conditions" button for downloading.
 
After a few attempts, I found a way to use C# to download the file.

Get the Session Details

The below code first tries to hit the file that generates the session id and fetches the session id for that particular download call. 
  1. static string GetJSession()  
  2.         {  
  3.             string jSession = string.Empty;  
  4.   
  5.             HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create("https://www.frbservices.org/EPaymentsDirectory/fpddir.txt?AgreementSessionObject=Agree/EPaymentsDirectory/fpddir.txt?dataCaptureActive=true");  
  6.             httpWebRequest.CookieContainer = new CookieContainer();  
  7.             HttpWebResponse response = (HttpWebResponse)httpWebRequest.GetResponse();  
  8.   
  9.             Uri uri = new Uri("https://www.frbservices.org");  
  10.   
  11.             return httpWebRequest.CookieContainer.GetCookies(uri)[0]?.Value;  
  12.   
  13.   
  14.         }  

Authenticate the Session 

After fetching the session, it needs to authenticate the session cookie. In a nutshell, the below code is performing the same action as clicking on the "Accept Terms and Condition" button. 
  1. static async Task<string> GetResponse(string JsessionID)  
  2.         {  
  3.             CookieContainer cookies = new CookieContainer();  
  4.             cookies.Add(new Cookie("JSESSIONID", JsessionID, "/", "frbservices.org"));  
  5.             HttpClientHandler handler = new HttpClientHandler();  
  6.             handler.CookieContainer = cookies;  
  7.   
  8.             var client = new HttpClient(handler);  
  9.   
  10.             var requestContent = new FormUrlEncodedContent(new[] {  
  11.                                 //new KeyValuePair<string, string>("Referer", "https://www.frbservices.org/EPaymentsDirectory/agreement.html"),  
  12.                                 new KeyValuePair<string, string>("agreementValue", "Agree")  
  13.             });  
  14.             client.DefaultRequestHeaders.Add("Referer", "https://www.frbservices.org/EPaymentsDirectory/agreement.html");  
  15.   
  16.             HttpResponseMessage response1 = await client.PostAsync("https://frbservices.org/EPaymentsDirectory/submitAgreement", requestContent);  
  17.             Uri uri = new Uri("https://frbservices.org");  
  18.   
  19.             HttpContent responseContent = response1.Content;  
  20.   
  21.             // Get the stream of the content.  
  22.             using (var reader = new StreamReader(await responseContent.ReadAsStreamAsync()))  
  23.             {  
  24.                 // Write the output.  
  25.                 Console.WriteLine(await reader.ReadToEndAsync());  
  26.                 //foreach (Cookie cook in cookies.GetCookies(uri))  
  27.                 //      GetResponse(cook.Value).Wait();  
  28.             }  
  29.             return string.Empty;   
  30.         }  

Download the file 

The below code will download the file and store it in the same folder as exe. 
  1. static void  ExecuteRequest(string jSession)  
  2.         {  
  3.             HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create("https://www.frbservices.org/EPaymentsDirectory/fpddir.txt?AgreementSessionObject=Agree/EPaymentsDirectory/fpddir.txt?dataCaptureActive=true");  
  4.             httpWebRequest.CookieContainer = new CookieContainer();  
  5.   
  6.             //string JSessionIDCookie = "0000DAF3YUdufGWtXOSfuApYRdY:195lt2pqn";  
  7.             httpWebRequest.CookieContainer.Add(new Cookie("JSESSIONID", jSession, "/", "frbservices.org"));  
  8.             //httpWebRequest.CookieContainer.Add(new Cookie("WT_FPC", "id=192.168.18.36-898276336.30653634:lv=1521154903463:ss=1521154842571", "/", "frbservices.org"));  
  9.             httpWebRequest.CookieContainer.Add(new Cookie("abaDataCaptureCookie", "abaDataCaptureCookie", "/", "frbservices.org"));  
  10.   
  11.             HttpWebResponse response = (HttpWebResponse)httpWebRequest.GetResponse();  
  12.             using (Stream s = response.GetResponseStream())  
  13.             {  
  14.                 FileStream os = new FileStream("fpdir.txt", FileMode.OpenOrCreate, FileAccess.Write);  
  15.                 byte[] buff = new byte[102400];  
  16.                 int c = 0;  
  17.                 while ((c = s.Read(buff, 0, 10400)) > 0)  
  18.                 {  
  19.                     os.Write(buff, 0, c);  
  20.                     os.Flush();  
  21.                 }  
  22.                 os.Close();  
  23.                 s.Close();  
  24.             }  
Main Method Code 
  1. tatic void Main(string[] args)  
  2.         {  
  3.   
  4.             // Get the JSession Cookie.   
  5.             string jSession = GetJSession();  
  6.   
  7.             // Authenticate the Cookie  
  8.             GetResponse(jSession).Wait();  
  9.   
  10.             //Download the File.   
  11.             ExecuteRequest(jSession);  
  12.   
  13.               
  14.   
  15.             //string JSessionIDCookie = response1.Cookies["JSESSIONID"]?.Value;   
  16.   
  17.               
  18.         }  
Hope this helps someone. 
 
Happy Coding.