Add script for Twitch synchronization

api/sync_with_twitch.py

@@ -0,0 +1,101 @@
+import re
+
+import dateutil.parser
+import flask
+import requests
+import sqlalchemy
+
+from db import db, Quote
+
+
+TWITCH_API_URL = 'https://api.twitch.tv/v5'
+
+
+app = flask.Flask(__name__)
+app.config.from_envvar('SETTINGS')
+app.config['DEBUG'] = True
+
+db.init_app(app)
+app.app_context().push()
+
+
+def get_videos(user_id, client_id):
+    def request(offset, limit):
+        url = '{0}/channels/{1}/videos'.format(TWITCH_API_URL, user_id)
+        params = dict(client_id=client_id, offset=offset, limit=limit)
+        r = requests.get(url, params=params)
+        r.raise_for_status()
+        return r.json()
+    result = []
+    data = request(0, 1)
+    total = data.get('_total', 0)
+    limit = 100
+    for offset in range(0, total, limit):
+        data = request(offset, limit)
+        for vid in data.get('videos', []):
+            result.append(dict(
+                id=int(vid['_id'].lstrip('v')),
+                title=vid['title'],
+                game=vid['game'],
+                date=dateutil.parser.parse(vid['recorded_at']).date()))
+    return result
+
+
+def get_comments(video_id, client_id):
+    def request(cursor):
+        url = '{0}/videos/{1}/comments'.format(TWITCH_API_URL, video_id)
+        params = dict(client_id=client_id, cursor=cursor)
+        r = requests.get(url, params=params)
+        r.raise_for_status()
+        return r.json()
+    result = []
+    cursor = ''
+    while True:
+        data = request(cursor)
+        for comment in data.get('comments', []):
+            result.append(comment['message']['body'])
+        cursor = data.get('_next')
+        if not cursor:
+            break
+    return result
+
+
+def main():
+    videos = get_videos(app.config['TWITCH_USER_ID'], app.config['TWITCH_CLIENT_ID'])
+    last_id = 0
+    last = db.session.query(Quote).order_by(Quote.date.desc()).first()
+    if last:
+        videos = [v for v in videos if v['date'] >= last.date]
+        last_id = last.id
+    QUOTE = re.compile(r'^\!quote\s+add\s+(.+)$')
+    quotes = []
+    for video in videos:
+        app.logger.info('Processing video %d (%s)', video['id'], video['title'])
+        comments = get_comments(video['id'], app.config['TWITCH_CLIENT_ID'])
+        for comment in comments:
+            m = QUOTE.match(comment)
+            if m:
+                quotes.append(dict(
+                    text=m.group(1),
+                    video_id=video['id'],
+                    game=video['game'],
+                    date=video['date']))
+    id = last_id + 1
+    for quote in sorted(quotes, key=lambda q: q['date']):
+        now = sqlalchemy.func.now()
+        q = db.session.query(Quote).filter(Quote.text == quote['text']).first()
+        if not q:
+            q = Quote(id=id, created_at=now)
+            id += 1
+        q.video_id = quote['video_id']
+        q.date = quote['date']
+        q.game = quote['game']
+        q.text = quote['text']
+        q.updated_at = now
+        app.logger.info('Adding quote %d (%s)', q.id, q.text)
+        db.session.add(q)
+    db.session.commit()
+
+
+if __name__ == '__main__':
+    main()