Mercurial > public > tweet-analysis
changeset 8:6541622b6127
add tweet analysis method
| author | Dennis Concepcion Martin <dennisconcepcionmartin@gmail.com> |
|---|---|
| date | Fri, 17 Sep 2021 21:10:02 +0200 |
| parents | 1b1296559c31 |
| children | c16b47541947 |
| files | .aws-sam/build.toml README.md dependencies/python/aws_controller.py dependencies/python/event_controller.py dependencies/python/secrets_controller.py dependencies/python/url_controller.py src/handlers/sentiment.py src/handlers/tweet_sentiment_handler.py src/requirements.txt template.yaml tests/unit/test_event_controller.py tests/unit/test_url_controller.py |
| diffstat | 12 files changed, 178 insertions(+), 130 deletions(-) [+] |
line wrap: on
line diff
--- a/.aws-sam/build.toml Fri Sep 17 17:42:54 2021 +0200 +++ b/.aws-sam/build.toml Fri Sep 17 21:10:02 2021 +0200 @@ -1,11 +1,5 @@ # This file is auto generated by SAM CLI build command [function_build_definitions] -[function_build_definitions.631c5dbb-3ad7-4033-889a-2b55c4ab69a0] -codeuri = "/Users/dennis/Developer/tweet-analysis/handlers" -runtime = "python3.9" -source_md5 = "" -packagetype = "Zip" -functions = ["GetTweetSentimentFunction"] [layer_build_definitions]
--- a/README.md Fri Sep 17 17:42:54 2021 +0200 +++ b/README.md Fri Sep 17 21:10:02 2021 +0200 @@ -1,10 +1,11 @@ # tweet-analysis +Serverless application to fetch & analyse tweets using AWS Comprehend. ## Structure - src - Code for the application's Lambda function. - events - Invocation events that you can use to invoke the function. -- tests - Unit tests for the application code. +- tests - Unit and integration tests for the application code. - template.yaml - A template that defines the application's AWS resources. The application uses several AWS resources, including Lambda functions and an API Gateway API.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dependencies/python/aws_controller.py Fri Sep 17 21:10:02 2021 +0200 @@ -0,0 +1,71 @@ +import boto3 +import base64 +import json +from botocore.exceptions import ClientError + + +class AwsSecretsManager: + @staticmethod + def get_secret(secret_name, region_name='eu-west-2'): + """ + Get Secret Keys from AWS Secrets Manager + :return: + """ + + # Create a Secrets Manager client + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + except ClientError as e: + if e.response['Error']['Code'] == 'DecryptionFailureException': + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + raise e + elif e.response['Error']['Code'] == 'InternalServiceErrorException': + # An error occurred on the server side. + raise e + elif e.response['Error']['Code'] == 'InvalidParameterException': + # You provided an invalid value for a parameter. + raise e + elif e.response['Error']['Code'] == 'InvalidRequestException': + # You provided a parameter value that is not valid for the current state of the resource. + raise e + elif e.response['Error']['Code'] == 'ResourceNotFoundException': + # AWS Can't find the resource that you asked for. + raise e + else: + # Decrypts secret using the associated KMS CMK. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if 'SecretString' in get_secret_value_response: + secret = get_secret_value_response['SecretString'] + else: + secret = base64.b64decode(get_secret_value_response['SecretBinary']) + + return json.loads(secret) + + +class AwsComprehend: + + @staticmethod + def get_sentiment(tweets): + """ + :param tweets: list (string), required + :return: dict + """ + # Create a Comprehend client + session = boto3.session.Session() + comprehend = session.client( + service_name='comprehend', + region_name='eu-west-2' + ) + + response = comprehend.batch_detect_sentiment( + TextList=tweets, + LanguageCode='en' + ) + + return response
--- a/dependencies/python/event_controller.py Fri Sep 17 17:42:54 2021 +0200 +++ b/dependencies/python/event_controller.py Fri Sep 17 21:10:02 2021 +0200 @@ -1,24 +1,27 @@ -def unwrap_sentiment_string_parameters(event): - """ - Unwrap string parameters from /sentiment api call - :param event: dict, required - API Gateway Lambda Proxy Input Format - :return: - """ +class SentimentFunctionEvent: - twitter_user = 'Twitter' - number_of_tweets = '100' + @staticmethod + def unwrap_parameters(event): + """ + Unwrap string parameters from /sentiment api call + :param event: dict, required + API Gateway Lambda Proxy Input Format + :return: + """ + + twitter_user = 'Twitter' + number_of_tweets = '100' - query_string_parameters = event['queryStringParameters'] - if event['queryStringParameters'] is not None: - if 'twitterUser' in query_string_parameters: - twitter_user = query_string_parameters['twitterUser'] - if not twitter_user: - twitter_user = 'Twitter' + query_string_parameters = event['queryStringParameters'] + if event['queryStringParameters'] is not None: + if 'twitterUser' in query_string_parameters: + twitter_user = query_string_parameters['twitterUser'] + if not twitter_user: + twitter_user = 'Twitter' - if 'numberOfTweets' in query_string_parameters: - number_of_tweets = query_string_parameters['numberOfTweets'] - if not number_of_tweets: - number_of_tweets = '100' + if 'numberOfTweets' in query_string_parameters: + number_of_tweets = query_string_parameters['numberOfTweets'] + if not number_of_tweets: + number_of_tweets = '100' - return twitter_user, number_of_tweets + return twitter_user, number_of_tweets
--- a/dependencies/python/secrets_controller.py Fri Sep 17 17:42:54 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,46 +0,0 @@ -import boto3 -import base64 -import json -from botocore.exceptions import ClientError - - -def get_secret(secret_name, region_name="eu-west-2"): - """ - Get Secret Keys from AWS Secrets Manager - :return: - """ - - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name=region_name - ) - - try: - get_secret_value_response = client.get_secret_value(SecretId=secret_name) - except ClientError as e: - if e.response['Error']['Code'] == 'DecryptionFailureException': - # Secrets Manager can't decrypt the protected secret text using the provided KMS key. - raise e - elif e.response['Error']['Code'] == 'InternalServiceErrorException': - # An error occurred on the server side. - raise e - elif e.response['Error']['Code'] == 'InvalidParameterException': - # You provided an invalid value for a parameter. - raise e - elif e.response['Error']['Code'] == 'InvalidRequestException': - # You provided a parameter value that is not valid for the current state of the resource. - raise e - elif e.response['Error']['Code'] == 'ResourceNotFoundException': - # AWS Can't find the resource that you asked for. - raise e - else: - # Decrypts secret using the associated KMS CMK. - # Depending on whether the secret is a string or binary, one of these fields will be populated. - if 'SecretString' in get_secret_value_response: - secret = get_secret_value_response['SecretString'] - else: - secret = base64.b64decode(get_secret_value_response['SecretBinary']) - - return json.loads(secret)
--- a/dependencies/python/url_controller.py Fri Sep 17 17:42:54 2021 +0200 +++ b/dependencies/python/url_controller.py Fri Sep 17 21:10:02 2021 +0200 @@ -1,13 +1,15 @@ -def create_twitter_url(twitter_user, number_of_tweets): - """ - Create url to fetch `max_results` of tweets from `user` - :param twitter_user: string, required - :param number_of_tweets: int, required - :return: string url - """ +class TwitterApi: - formatted_max_results = 'max_results={}'.format(number_of_tweets) - formatted_user = 'query=from:{}'.format(twitter_user) - url = "https://api.twitter.com/2/tweets/search/recent?{}&{}".format(formatted_max_results, formatted_user) + @staticmethod + def create_sentiment_url(twitter_user, number_of_tweets): + """ + Create url to fetch `max_results` of tweets from `user` + :param twitter_user: string, required + :param number_of_tweets: int, required + :return: string url + """ - return url + query = 'query=from:{}'.format(twitter_user) + url = 'https://api.twitter.com/2/tweets/search/recent?max_results={}&{}'.format(number_of_tweets, query) + + return url
--- a/src/handlers/sentiment.py Fri Sep 17 17:42:54 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -import requests - -# noinspection PyUnresolvedReferences -from secrets_controller import get_secret -# noinspection PyUnresolvedReferences -from event_controller import unwrap_sentiment_string_parameters -# noinspection PyUnresolvedReferences -from url_controller import create_twitter_url - - -def get_tweet_sentiment(event, context): - """ - :param event: dict, required - API Gateway Lambda Proxy Input Format - :param context: object, required - Lambda Context runtime methods and attributes - :return: dict - API Gateway Lambda Proxy Output Format - """ - - # Unwrap query string parameters - twitter_user, number_of_tweets = unwrap_sentiment_string_parameters(event) - - # URL creation & authentication - twitter_url = create_twitter_url(twitter_user, number_of_tweets) - twitter_key = get_secret(secret_name='tweet-analysis-keys') - twitter_header = {"Authorization": "Bearer {}".format(twitter_key['BEARER'])} - - # Request tweets to Twitter - twitter_response = requests.request("GET", twitter_url, headers=twitter_header) - - # Analyse tweets with AWS Comprehend - - return { - "statusCode": 200, - "body": { - "tweets": twitter_response.json() - } - }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/handlers/tweet_sentiment_handler.py Fri Sep 17 21:10:02 2021 +0200 @@ -0,0 +1,47 @@ +import json +import requests + +# noinspection PyUnresolvedReferences +from aws_controller import AwsSecretsManager, AwsComprehend +# noinspection PyUnresolvedReferences +from event_controller import SentimentFunctionEvent +# noinspection PyUnresolvedReferences +from url_controller import TwitterApi + + +# noinspection PyUnusedLocal +def get_tweet_sentiment(event, context): + """ + :param event: dict, required + API Gateway Lambda Proxy Input Format + :param context: object, required + Lambda Context runtime methods and attributes + :return: dict + API Gateway Lambda Proxy Output Format + """ + + # Unwrap query string parameters + twitter_user, number_of_tweets = SentimentFunctionEvent.unwrap_parameters(event) + + # URL creation & authentication + twitter_url = TwitterApi.create_sentiment_url(twitter_user, number_of_tweets) + twitter_key = AwsSecretsManager.get_secret(secret_name='tweet-analysis-keys') + twitter_header = {"Authorization": "Bearer {}".format(twitter_key['BEARER'])} + + # Request tweets to Twitter + twitter_response = requests.request("GET", twitter_url, headers=twitter_header) + twitter_json_response = twitter_response.json() + + tweets = [] + for tweet in twitter_json_response['data']['tweets']: + tweets.append(tweet['text']) + + # Analyse tweets with AWS Comprehend + result = AwsComprehend.get_sentiment(tweets) + + return { + "statusCode": 200, + "body": { + "tweets": json.dumps(result) + } + }
--- a/src/requirements.txt Fri Sep 17 17:42:54 2021 +0200 +++ b/src/requirements.txt Fri Sep 17 21:10:02 2021 +0200 @@ -1,1 +1,2 @@ -requests \ No newline at end of file +requests +boto3 \ No newline at end of file
--- a/template.yaml Fri Sep 17 17:42:54 2021 +0200 +++ b/template.yaml Fri Sep 17 21:10:02 2021 +0200 @@ -152,13 +152,13 @@ ## # Define lambda functions - GetTweetSentimentFunction: + TweetSentimentHandler: Type: AWS::Serverless::Function Properties: FunctionName: GetTweetSentimentFunction Description: Fetch tweets and analyse sentiment using AWS Comprehend CodeUri: src/ - Handler: handlers/sentiment.get_tweet_sentiment + Handler: handlers/tweet_sentiment_handler.get_tweet_sentiment Runtime: python3.9 Policies: - AWSSecretsManagerGetSecretValuePolicy:
--- a/tests/unit/test_event_controller.py Fri Sep 17 17:42:54 2021 +0200 +++ b/tests/unit/test_event_controller.py Fri Sep 17 21:10:02 2021 +0200 @@ -1,8 +1,8 @@ from unittest import TestCase -from dependencies.python.event_controller import * +from dependencies.python.event_controller import SentimentFunctionEvent -class TestUnwrapStringParameters(TestCase): +class TestSentimentFunctionEvent(TestCase): @staticmethod def create_event(query_string_parameter): @@ -16,7 +16,7 @@ return event - def test_unwrap_sentiment_string_parameters(self): + def test_unwrap_parameters(self): test_cases = { '1': None, '2': {'twitterUser': ''}, @@ -37,7 +37,7 @@ for test_number in test_cases: event = self.create_event(test_cases[test_number]) - twitter_user, number_of_tweets = unwrap_sentiment_string_parameters(event) + twitter_user, number_of_tweets = SentimentFunctionEvent.unwrap_parameters(event) expected_twitter_user = expected_results[test_number]['twitterUser'] expected_number_of_tweets = expected_results[test_number]['numberOfTweets']
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/unit/test_url_controller.py Fri Sep 17 21:10:02 2021 +0200 @@ -0,0 +1,14 @@ +from unittest import TestCase +from dependencies.python.url_controller import TwitterApi + + +class TestTwitterApi(TestCase): + def test_create_twitter_url(self): + twitter_user = 'Twitter' + number_of_tweets = '50' + url = TwitterApi.create_sentiment_url(twitter_user, number_of_tweets) + expected_url = 'https://api.twitter.com/2/tweets/search/recent?max_results={}&query=from:{}'.format( + number_of_tweets, twitter_user + ) + + self.assertEqual(url, expected_url)