From fe23a237c517948eb5c5558d70b052a4c0f83cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Forr=C3=B3?= Date: Sun, 6 Sep 2020 14:57:58 +0200 Subject: [PATCH] Add YouTube webhook and react to notifications --- bot.py | 77 +++++++++++++++++++++++++++++++++++++---------------- endpoint.py | 47 +++++++++++++++++++++++--------- youtube.py | 49 +++++++++++++++++++++------------- 3 files changed, 119 insertions(+), 54 deletions(-) diff --git a/bot.py b/bot.py index d477cf2..c8b7986 100644 --- a/bot.py +++ b/bot.py @@ -36,7 +36,8 @@ config.read('settings.cfg') storage_path = config.get('General', 'storage_path') token = config.get('Discord', 'token') -channel_id = config.getint('Discord', 'channel_id') +twitter_channel_id = config.getint('Discord', 'twitter_channel_id') +youtube_channel_id = config.getint('Discord', 'youtube_channel_id') consumer_key = config.get('Twitter', 'consumer_key') consumer_secret = config.get('Twitter', 'consumer_secret') @@ -53,7 +54,21 @@ class Bot(discord.Client): self.youtube = YouTube(yt_api_key, yt_channel_id) super().__init__() - def get_api(self, user_id): + async def process_event(self, service, data): + action = getattr(self, 'process_{0}_event'.format(service)) + await action(data) + + async def on_message(self, message): + for pattern in self.commands: + m = pattern.match(message.content) + if not m: + continue + d = m.groupdict() + command = d.pop('command') + action = getattr(self, 'perform_{0}'.format(command)) + await action(message, **d) + + def get_twitter_api(self, user_id): handler = tweepy.OAuthHandler(consumer_key, consumer_secret) with open(pathlib.Path(storage_path, 'tokens.json'), 'r') as f: tokens = json.load(f) @@ -63,7 +78,7 @@ class Bot(discord.Client): handler.set_access_token(*access_token) return tweepy.API(handler) - def make_embed(self, tweet): + def make_twitter_embed(self, tweet): tweet_url = '{0}{1}'.format(TWITTER_STATUS_URL, tweet.get('id_str')) author_url = '{0}{1}'.format(TWITTER_USER_URL, tweet.get('user', {}).get('id_str')) author_handle = '@{0}'.format(tweet.get('user', {}).get('screen_name')) @@ -84,7 +99,7 @@ class Bot(discord.Client): ) return embed - async def process_event(self, data): + async def process_twitter_event(self, data): tweets = data.get('tweet_create_events', []) if not tweets: return @@ -94,24 +109,45 @@ class Bot(discord.Client): user_id = data.get('for_user_id') if not user_id: return - api = self.get_api(user_id) + api = self.get_twitter_api(user_id) if not api: return await self.wait_until_ready() - channel = self.get_channel(channel_id) + channel = self.get_channel(twitter_channel_id) if channel: for tweet in tweets: - await channel.send(embed=self.make_embed(tweet)) + await channel.send(embed=self.make_twitter_embed(tweet)) - async def on_message(self, message): - for pattern in self.commands: - m = pattern.match(message.content) - if not m: - continue - d = m.groupdict() - command = d.pop('command') - action = getattr(self, 'perform_{0}'.format(command)) - await action(message, **d) + async def process_youtube_event(self, data): + entry = data.get('feed', {}).get('entry', {}) + if entry.get('yt:channelId') != yt_channel_id: + return + video_id = entry.get('yt:videoId') + if not video_id: + return + try: + video = self.youtube.get_video(video_id) + except YouTubeError as e: + return + channel = self.get_channel(youtube_channel_id) + if channel: + # TODO: setup timer for livestreams + await channel.send('{title}\n{link}'.format(**video)) + + def make_youtube_embed(self, playlist): + embed = discord.Embed( + title=playlist.get('title'), + description=playlist.get('description'), + url=playlist.get('link'), + color=YOUTUBE_COLOR + ) + embed.set_thumbnail(url=playlist.get('thumbnail_url')) + embed.set_author( + name=self.youtube.channel.get('title'), + url=self.youtube.channel.get('link'), + icon_url=self.youtube.channel.get('thumbnail_url') + ) + return embed async def perform_reaction(self, message, query, **kwargs): try: @@ -123,12 +159,7 @@ class Bot(discord.Client): await message.channel.send('Sorry {0}, nothing found'.format(message.author.mention)) return if result.get('kind') == 'playlist': - embed = discord.Embed(title=result.get('title'), url=result.get('link'), - description=result.get('description'), color=YOUTUBE_COLOR) - embed.set_thumbnail(url=result.get('thumbnail_url')) - embed.set_author(name=self.youtube.channel.get('title'), url=self.youtube.channel.get('link'), - icon_url=self.youtube.channel.get('thumbnail_url')) - await message.channel.send(embed=embed) + await message.channel.send(embed=self.make_youtube_embed(result)) else: await message.channel.send('{title}\n{link}'.format(**result)) @@ -144,7 +175,7 @@ def main(): await asyncio.sleep(TIMEOUT) else: try: - await bot.process_event(data) + await bot.process_event(*data) except Exception: queue.put(data) queue.task_done() diff --git a/endpoint.py b/endpoint.py index da1aa61..2143da0 100644 --- a/endpoint.py +++ b/endpoint.py @@ -9,6 +9,7 @@ import pathlib import flask import persistqueue import tweepy +import xmltodict from twitivity import Twitivity, TwitivityError @@ -35,8 +36,10 @@ callback_url = config.get('Twitter', 'callback_url') webhook_url = config.get('Twitter', 'webhook_url') environment_name = config.get('Twitter', 'environment_name') +yt_channel_id = config.get('YouTube', 'channel_id') -def update_tokens(user_id, access_token): + +def update_twitter_tokens(user_id, access_token): path = pathlib.Path(storage_path, 'tokens.json') path.touch(exist_ok=True) with open(path, 'r+') as f: @@ -50,13 +53,7 @@ def update_tokens(user_id, access_token): f.truncate() -def push_event(data): - queue = persistqueue.SQLiteQueue(storage_path) - queue.put(data) - queue.task_done() - - -def setup_webhook(api): +def setup_twitter_webhook(api): twitivity = Twitivity(api, environment_name) try: webhook_id = twitivity.check_webhook() @@ -79,6 +76,12 @@ def setup_webhook(api): return True +def push_event(data): + queue = persistqueue.SQLiteQueue(storage_path) + queue.put(data) + queue.task_done() + + @app.route('/twitter/auth') def twitter_auth(): handler = tweepy.OAuthHandler(consumer_key, consumer_secret, callback_url) @@ -101,8 +104,8 @@ def twitter_callback(): user = api.verify_credentials(include_entities=False, skip_status=True, include_email=False) if not user: return 'Authentication failure', 500 - update_tokens(user.id_str, [handler.access_token, handler.access_token_secret]) - if not setup_webhook(api): + update_twitter_tokens(user.id_str, [handler.access_token, handler.access_token_secret]) + if not setup_twitter_webhook(api): return 'Webhook setup failure', 500 return 'Success', 200 @@ -130,8 +133,28 @@ def twitter_webhook_post(): return False if not verify(): return 'Invalid request signature', 403 - push_event(flask.request.get_json()) - return '', 200 + push_event(('twitter', flask.request.get_json())) + return '', 204 + + +@app.route('/youtube/webhook', methods=['GET']) +def youtube_webhook_get(): + mode = flask.request.args.get('hub.mode') + topic = flask.request.args.get('hub.topic') + challenge = flask.request.args.get('hub.challenge') + if (topic != 'https://www.youtube.com/xml/feeds/videos.xml?channel_id={0}'.format(yt_channel_id) or + mode not in ('subscribe', 'unsubscribe') or not challenge): + return 'Invalid request', 403 + return challenge, 200 + + +@app.route('/youtube/webhook', methods=['POST']) +def youtube_webhook_post(): + data = flask.request.get_data() + if not data: + return '', 204 + push_event(('youtube', xmltodict.parse(data))) + return '', 204 if __name__ == '__main__': diff --git a/youtube.py b/youtube.py index 0b743ac..4f04ac2 100644 --- a/youtube.py +++ b/youtube.py @@ -1,5 +1,6 @@ import html +import dateutil.parser import fuzzywuzzy.fuzz import fuzzywuzzy.process import googleapiclient @@ -19,26 +20,31 @@ class YouTube(object): if key in thumbnails: return thumbnails[key].get('url') + def process_item(self, item): + id = item.get('id', '') + kind = item.get('kind', 'youtube#').split('youtube#')[1] + if kind == 'searchResult': + id = item.get('id', {}).get('videoId', '') + kind = item.get('id', {}).get('kind', 'youtube#').split('youtube#')[1] + if kind == 'playlist': + link = '{0}/view_play_list?p={1}'.format(BASE_URL, id) + else: + link = '{0}/watch?v={1}'.format(BASE_URL, id) + scheduled_start = item.get('liveStreamingDetails', {}).get('scheduledStartTime') + if scheduled_start: + scheduled_start = dateutil.parser.parse(scheduled_start) + return dict( + kind=kind, + link=link, + title=html.unescape(item.get('snippet', {}).get('title', '')), + description=html.unescape(item.get('snippet', {}).get('description', '')), + thumbnail_url=self.get_thumbnail_url(item.get('snippet', {}).get('thumbnails', {})), + live_broadcast=item.get('snippet', {}).get('liveBroadcastContent', 'none'), + scheduled_start=scheduled_start + ) + def process_items(self, items): - result = [] - for item in items: - id = item.get('id', '') - kind = item.get('kind', 'youtube#').split('youtube#')[1] - if kind == 'searchResult': - id = item.get('id', {}).get('videoId', '') - kind = item.get('id', {}).get('kind', 'youtube#').split('youtube#')[1] - if kind == 'playlist': - link = '{0}/view_play_list?p={1}'.format(BASE_URL, id) - else: - link = '{0}/watch?v={1}'.format(BASE_URL, id) - result.append(dict( - kind=kind, - link=link, - title=html.unescape(item.get('snippet', {}).get('title', '')), - description=html.unescape(item.get('snippet', {}).get('description', '')), - thumbnail_url=self.get_thumbnail_url(item.get('snippet', {}).get('thumbnails', {})) - )) - return result + return [self.process_item(i) for i in items] def get_channel(self, channel_id): r = self.client.channels().list(id=channel_id, maxResults=1, part='id,snippet').execute() @@ -50,6 +56,11 @@ class YouTube(object): thumbnail_url=self.get_thumbnail_url(channel.get('snippet', {}).get('thumbnails', {})) ) + def get_video(self, video_id): + r = self.client.videos().list(id=video_id, maxResults=1, part='id,snippet,liveStreamingDetails').execute() + video = r.get('items', [{}]).pop() + return self.process_item(video) + def get_playlists(self): token = '' result = []