
This blog is a short write-up about the sentiment analysis workflow I developed during the first Covid lockdown. You can find the dashboard website I created here and a blog I had written in my institute here.
Basically it's an Azure and c# solution. The main components involve Azure streaming analytics, Azure Event Hubs, and Azure Blob Store.
The code for the sample app can be found here. This sample app analyzes real-time Twitter feeds using sentiment140 (http://www.sentiment140.com/) to determine whether the tweeted text has a positive, negative, or neutral reaction to the current situation. The keywords used for picking the tweets are “lifting lockdown” and “lifting restrictions” in the same order of the words.
The feed from Twitter API is fed to Azure Event hub for handling huge stream of data, and then this stream of data is fed to azure streaming analytics which runs a tumbling window aggregation function for window width of 1hr, to aggregate the sentiment scores. This aggregated data is then stored in Azure blob store and visualized on this website.
The Tweeter reader is the main component which reads the tweets based on keywords and sends them to eventhub to start the sentiment analysis workflow.
public static IEnumerable < TweetData > StreamStatuses(TwitterConfig config) {
var streamReader = ReadTweets(config);
int count = 0;
while (true) {
string line = null;
try {
line = streamReader.ReadLine();
} catch (Exception) {}
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"")) {
var result = JsonConvert.DeserializeObject < TweetData > (line);
result.RawJson = line;
yield
return result;
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null) {
streamReader = ReadTweets(config);
}
count++;
if (count == 500) {
count = 0;
System.Threading.Thread.Sleep(1000);
}
}
}
static TextReader ReadTweets(TwitterConfig config) {
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds).ToString();
System.Net.ServicePointManager.SecurityProtocol |= System.Net.SecurityProtocolType.Tls12;
var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";
// create oauth signature
var baseString = string.Format("oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" + "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}", config.OAuthConsumerKey, oauth_nonce, oauth_signature_method, oauth_timestamp, config.OAuthToken, oauth_version, Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret), "&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using(var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey))) {
oauth_signature = Convert.ToBase64String(hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
}
// create the request header
var authHeader = string.Format("OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " + "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " + "oauth_token=\"{4}\", oauth_signature=\"{5}\", " + "oauth_version=\"{6}\"", Uri.EscapeDataString(oauth_nonce), Uri.EscapeDataString(oauth_signature_method), Uri.EscapeDataString(oauth_timestamp), Uri.EscapeDataString(config.OAuthConsumerKey), Uri.EscapeDataString(config.OAuthToken), Uri.EscapeDataString(oauth_signature), Uri.EscapeDataString(oauth_version));
// make the request
ServicePointManager.Expect100Continue = false;
var postBody = "track=" + config.Keywords;
resource_url += "?" + postBody;
HttpWebRequest request = (HttpWebRequest) WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var tresponse = request.GetResponseAsync();
if (tresponse.Wait(5000)) return new StreamReader(tresponse.Result.GetResponseStream());
else {
request.Abort();
return StreamReader.Null;
}
}
Join the conversation! Your thoughts help the community grow.