import re import dateutil.parser import flask import requests import sqlalchemy from db import db, Quote TWITCH_API_URL = 'https://api.twitch.tv/v5' app = flask.Flask(__name__) app.config.from_envvar('SETTINGS') app.config['DEBUG'] = True db.init_app(app) app.app_context().push() def get_videos(user_id, client_id): def request(offset, limit): url = '{0}/channels/{1}/videos'.format(TWITCH_API_URL, user_id) params = dict(client_id=client_id, offset=offset, limit=limit) r = requests.get(url, params=params) r.raise_for_status() return r.json() result = [] data = request(0, 1) total = data.get('_total', 0) limit = 100 for offset in range(0, total, limit): data = request(offset, limit) for vid in data.get('videos', []): result.append(dict( id=int(vid['_id'].lstrip('v')), title=vid['title'], game=vid['game'], date=dateutil.parser.parse(vid['recorded_at']).date())) return result def get_comments(video_id, client_id): def request(cursor): url = '{0}/videos/{1}/comments'.format(TWITCH_API_URL, video_id) params = dict(client_id=client_id, cursor=cursor) r = requests.get(url, params=params) r.raise_for_status() return r.json() result = [] cursor = '' while True: data = request(cursor) for comment in data.get('comments', []): result.append(comment['message']['body']) cursor = data.get('_next') if not cursor: break return result def main(): videos = get_videos(app.config['TWITCH_USER_ID'], app.config['TWITCH_CLIENT_ID']) last_id = 0 last = db.session.query(Quote).order_by(Quote.date.desc()).first() if last: videos = [v for v in videos if v['date'] >= last.date] last_id = last.id QUOTE = re.compile(r'^\!quote\s+add\s+(?P")?(?P.+)(?(q)")$') quotes = [] for video in videos: app.logger.info('Processing video %d (%s)', video['id'], video['title']) comments = get_comments(video['id'], app.config['TWITCH_CLIENT_ID']) for comment in comments: m = QUOTE.match(comment) if m: quotes.append(dict( text=m.group('text'), video_id=video['id'], game=video['game'], date=video['date'])) id = last_id + 1 for quote in sorted(quotes, key=lambda q: q['date']): now = sqlalchemy.func.now() q = db.session.query(Quote).filter(Quote.text == quote['text']).first() if not q: q = Quote(id=id, created_at=now) id += 1 q.video_id = quote['video_id'] q.date = quote['date'] q.game = quote['game'] q.text = quote['text'] q.updated_at = now app.logger.info('Adding quote %d (%s)', q.id, q.text) db.session.add(q) db.session.commit() if __name__ == '__main__': main()