From d57bca01664cb9ab20213ba1f4cf82fdb672b08c Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 29 Dec 2020 16:33:37 -0700 Subject: [PATCH] Beginning of transition to single-table database schema This is aiming for a one-big-table approach, (which is the DynamoDB best practice), instead of having one table for hunts and then a separate per-hunt table for puzzles. This new approach will be more flexible as we start adding database items for rounds, etc. In the old approach, rounds would likely have become yet another hunt-specific table, and the number of per-hunt tables would have started to become rather messy. With this new schema, rounds can instead be just another item, where hunts, rounds, and puzzles can all be intermingled as items in the same table. To get this all to work, we are creating secondary indexes for querying items from the database via attributes other than the primary key, (for example a "channel_id" index). As of this commit, the "New Hunt" button works in that it: * Creates the turbot DynamoDB table if it doesn't exist * Inserts a new item into the table for the hunt itself (active=False) * Marks the sheet_url of the item as "pending" * Creates a new channel for the hunt * Responds to the channel_created event * Creates a Google sheet for the hunt * Stores the sheet_url in the database * Marks the hunt as active=True in the database * Reports to the Slack channel that things are working So that much is working, but everything else has yet to be converted to the new schema. So things that don't yet work include at least: * The turbot "Home" view doesn't yet display anything * The /puzzle command doesn't work yet --- TODO | 5 +++ turbot/events.py | 60 +++++++++----------------------- turbot/interaction.py | 63 ++++++++++++++++++++-------------- turbot_lambda/turbot_lambda.py | 1 + 4 files changed, 60 insertions(+), 69 deletions(-) diff --git a/TODO b/TODO index 56fe242..2d2e728 100644 --- a/TODO +++ b/TODO @@ -1,3 +1,8 @@ +Recently-proposed ideas (not yet prioritized) +--------------------------------------------- +• Set-up an announcements Slack channel that's auto fed by email to a + particular email address. + Low-hanging fruit ----------------- diff --git a/turbot/events.py b/turbot/events.py index 0dabb71..86a3c83 100644 --- a/turbot/events.py +++ b/turbot/events.py @@ -1,9 +1,10 @@ from turbot.blocks import ( section_block, text_block, button_block, actions_block, divider_block ) -import turbot.sheets import turbot.slack +from turbot.sheets import sheets_create, sheets_create_for_puzzle from turbot.slack import slack_send_message, slack_channel_members +from boto3.dynamodb.conditions import Key TURBOT_USER_ID = 'U01B9QM4P9R' @@ -74,9 +75,9 @@ def home(turb, user_id): The return value is a dictionary suitable to be published to the Slack views_publish API.""" - # Behave cleanly if there is no hunts table at all yet. + # Behave cleanly if there is no "turbot" table at all yet. try: - response = turb.db.Table("hunts").scan() + response = turb.table.scan() hunts = response['Items'] except Exception: hunts = [] @@ -114,37 +115,32 @@ def app_home_opened(turb, event): events['app_home_opened'] = app_home_opened def hunt_channel_created(turb, channel_name, channel_id): - """Creates sheet and a DynamoDB table for a newly-created hunt channel""" + """Creates a Google sheet for a newly-created hunt channel""" # First see if we can find an entry for this hunt in the database. # If not, simply return an error and let Slack retry - hunts_table = turb.db.Table("hunts") - response = hunts_table.get_item( - Key={'channel_id': channel_id}, - ConsistentRead=True + response = turb.table.query( + IndexName='channel_id_index', + KeyConditionExpression=Key("channel_id").eq(channel_id) ) - if 'Item' not in response: + if 'Items' not in response: print("Warning: Cannot find channel_id {} in hunts table. " .format(channel_id) + "Letting Slack retry this event") return lambda_error - item = response['Item'] + item = response['Items'][0] if 'sheet_url' in item: print("Info: channel_id {} already has sheet_url {}. Exiting." .format(channel_id, item['sheet_url'])) return lambda_success - # Remove any None items from our item before updating - if not item['url']: - del item['url'] - # Before launching into sheet creation, indicate that we're doing this # in the database. This way, if we take too long to create the sheet # and Slack retries the event, that next event will see this 'pending' # string and cleanly return (eliminating all future retries). item['sheet_url'] = 'pending' - hunts_table.put_item(Item=item) + turb.table.put_item(Item=item) # Also, let the channel users know what we are up to slack_send_message( @@ -153,37 +149,19 @@ def hunt_channel_created(turb, channel_name, channel_id): + "Please wait a minute or two while I create some backend resources.") # Create a sheet for the hunt - sheet = turbot.sheets.sheets_create(turb, item['name']) + sheet = sheets_create(turb, item['name']) # Update the database with the URL of the sheet item['sheet_url'] = sheet['url'] - hunts_table.put_item(Item=item) + turb.table.put_item(Item=item) # Message the channel with the URL of the sheet slack_send_message(turb.slack_client, channel_id, "Sheet created for this hunt: {}".format(sheet['url'])) - # Create a database table for this hunt's puzzles - table = turb.db.create_table( - TableName=channel_name, - KeySchema=[ - {'AttributeName': 'channel_id', 'KeyType': 'HASH'} - ], - AttributeDefinitions=[ - {'AttributeName': 'channel_id', 'AttributeType': 'S'} - ], - ProvisionedThroughput={ - 'ReadCapacityUnits': 5, - 'WriteCapacityUnits': 5 - } - ) - - # Wait until the table exists - table.meta.client.get_waiter('table_exists').wait(TableName=channel_name) - # Mark the hunt as active in the database item['active'] = True - hunts_table.put_item(Item=item) + turb.table.put_item(Item=item) # Message the hunt channel that the database is ready slack_send_message( @@ -220,8 +198,7 @@ def puzzle_channel_created(turb, puzzle_channel_name, puzzle_channel_id): # First see if we can find an entry for this puzzle in the database. # If not, simply return an error and let Slack retry - puzzle_table = turb.db.Table(hunt_id) - response = puzzle_table.get_item( + response = turb.table.get_item( Key={'channel_id': puzzle_channel_id}, ConsistentRead=True ) @@ -238,10 +215,6 @@ def puzzle_channel_created(turb, puzzle_channel_name, puzzle_channel_id): .format(puzzle_channel_id, item['sheet_url'])) return lambda_success - # Remove any None items from our item before updating - if not item['url']: - del item['url'] - # Before launching into sheet creation, indicate that we're doing this # in the database. This way, if we take too long to create the sheet # and Slack retries the event, that next event will see this 'pending' @@ -251,7 +224,7 @@ def puzzle_channel_created(turb, puzzle_channel_name, puzzle_channel_id): puzzle_table.put_item(Item=item) # Create a sheet for the puzzle - sheet = turbot.sheets.sheets_create_for_puzzle(turb, item) + sheet = sheets_create_for_puzzle(turb, item) # Update the database with the URL of the sheet item['sheet_url'] = sheet['url'] @@ -343,7 +316,6 @@ def puzzle_channel_created(turb, puzzle_channel_name, puzzle_channel_id): return lambda_success def channel_created(turb, event): - print("In channel_created with event: {}".format(str(event))) channel = event['channel'] channel_id = channel['id'] diff --git a/turbot/interaction.py b/turbot/interaction.py index a929ac5..5549aa8 100644 --- a/turbot/interaction.py +++ b/turbot/interaction.py @@ -95,32 +95,48 @@ def new_hunt_submission(turb, payload, metadata): "Hunt ID can only contain lowercase letters, " + "numbers, and underscores") - # Check to see if the hunts table exists - hunts_table = turb.db.Table("hunts") - + # Check to see if the turbot table exists try: - exists = hunts_table.table_status in ("CREATING", "UPDATING", - "ACTIVE") + exists = turb.table.table_status in ("CREATING", "UPDATING", + "ACTIVE") except ClientError: exists = False - # Create the hunts table if necessary. + # Create the turbot table if necessary. if not exists: - hunts_table = turb.db.create_table( - TableName='hunts', + turb.table = turb.db.create_table( + TableName='turbot', KeySchema=[ - {'AttributeName': 'channel_id', 'KeyType': 'HASH'}, + {'AttributeName': 'PK', 'KeyType': 'HASH'}, + {'AttributeName': 'SK', 'KeyType': 'RANGE'}, ], AttributeDefinitions=[ + {'AttributeName': 'PK', 'AttributeType': 'S'}, + {'AttributeName': 'SK', 'AttributeType': 'S'}, {'AttributeName': 'channel_id', 'AttributeType': 'S'}, ], ProvisionedThroughput={ 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 - } + }, + GlobalSecondaryIndexes=[ + { + 'IndexName': 'channel_id_index', + 'KeySchema': [ + {'AttributeName': 'channel_id', 'KeyType': 'HASH'} + ], + 'Projection': { + 'ProjectionType': 'ALL' + }, + 'ProvisionedThroughput': { + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + } + ] ) return submission_error("hunt_id", - "Still bootstrapping hunts table. Try again.") + "Still bootstrapping turbot table. Try again.") # Create a channel for the hunt try: @@ -135,12 +151,13 @@ def new_hunt_submission(turb, payload, metadata): # Insert the newly-created hunt into the database # (leaving it as non-active for now until the channel-created handler # finishes fixing it up with a sheet and a companion table) - hunts_table.put_item( + turb.table.put_item( Item={ - 'channel_id': channel_id, + "PK": "hunt-{}".format(hunt_id), + "SK": "hunt-{}".format(hunt_id), + "channel_id": channel_id, "active": False, "name": name, - "hunt_id": hunt_id, "url": url } ) @@ -242,7 +259,7 @@ def channel_is_hunt(turb, channel_id): Returns a dict (filled with database entries) if there is a hunt for this channel, otherwise returns None.""" - return get_table_item(turb, "hunts", 'channel_id', channel_id) + return get_table_item(turb, "channel_id_index", 'channel_id', channel_id) def find_hunt_for_hunt_id(turb, hunt_id): """Given a hunt ID find the database for for that hunt @@ -252,18 +269,14 @@ def find_hunt_for_hunt_id(turb, hunt_id): (channel_id, active, hunt_id, name, url, sheet_url, etc.). """ - hunts_table = turb.db.Table("hunts") + turbot_table = turb.db.Table("turbot") - response = hunts_table.scan( - FilterExpression='hunt_id = :hunt_id', - ExpressionAttributeValues={':hunt_id': hunt_id} - ) + response = turbot_table.get_item(Key={'PK': 'hunt-{}'.format(hunt_id)}) - if 'Items' in response and len(response['Items']): - item = response['Items'][0] - return item - - return None + if 'Item' in response: + return response['Item'] + else: + return None def find_hunt_for_channel(turb, channel_id, channel_name): """Given a channel ID/name find the id/name of the hunt for this channel diff --git a/turbot_lambda/turbot_lambda.py b/turbot_lambda/turbot_lambda.py index b2acf41..aa88339 100644 --- a/turbot_lambda/turbot_lambda.py +++ b/turbot_lambda/turbot_lambda.py @@ -66,6 +66,7 @@ db = boto3.resource('dynamodb') turb = SimpleNamespace() turb.slack_client = slack_client turb.db = db +turb.table = db.Table("turbot") turb.sheets = sheets turb.permissions = permissions -- 2.43.0