ladylilia.com/api/sync_with_twitch.py

import re

import dateutil.parser
import flask
import requests
import sqlalchemy

from db import db, Quote


TWITCH_API_URL = 'https://api.twitch.tv/v5'


app = flask.Flask(__name__)
app.config.from_envvar('SETTINGS')
app.config['DEBUG'] = True

db.init_app(app)
app.app_context().push()


def get_videos(user_id, client_id):
    def request(offset, limit):
        url = '{0}/channels/{1}/videos'.format(TWITCH_API_URL, user_id)
        params = dict(client_id=client_id, offset=offset, limit=limit)
        r = requests.get(url, params=params)
        r.raise_for_status()
        return r.json()
    result = []
    data = request(0, 1)
    total = data.get('_total', 0)
    limit = 100
    for offset in range(0, total, limit):
        data = request(offset, limit)
        for vid in data.get('videos', []):
            result.append(dict(
                id=int(vid['_id'].lstrip('v')),
                title=vid['title'],
                game=vid['game'],
                date=dateutil.parser.parse(vid['recorded_at']).date()))
    return result


def get_comments(video_id, client_id):
    def request(cursor):
        url = '{0}/videos/{1}/comments'.format(TWITCH_API_URL, video_id)
        params = dict(client_id=client_id, cursor=cursor)
        r = requests.get(url, params=params)
        r.raise_for_status()
        return r.json()
    result = []
    cursor = ''
    while True:
        data = request(cursor)
        for comment in data.get('comments', []):
            result.append(comment['message']['body'])
        cursor = data.get('_next')
        if not cursor:
            break
    return result


def main():
    videos = get_videos(app.config['TWITCH_USER_ID'], app.config['TWITCH_CLIENT_ID'])
    last_id = 0
    last = db.session.query(Quote).order_by(Quote.date.desc()).first()
    if last:
        videos = [v for v in videos if v['date'] >= last.date]
        last_id = last.id
    QUOTE = re.compile(r'^\!quote\s+add\s+(?P<q>")?(?P<text>.+)(?(q)")$')
    quotes = []
    for video in videos:
        app.logger.info('Processing video %d (%s)', video['id'], video['title'])
        comments = get_comments(video['id'], app.config['TWITCH_CLIENT_ID'])
        for comment in comments:
            m = QUOTE.match(comment)
            if m:
                quotes.append(dict(
                    text=m.group('text'),
                    video_id=video['id'],
                    game=video['game'],
                    date=video['date']))
    id = last_id + 1
    for quote in sorted(quotes, key=lambda q: q['date']):
        now = sqlalchemy.func.now()
        q = db.session.query(Quote).filter(Quote.text == quote['text']).first()
        if not q:
            q = Quote(id=id, created_at=now)
            id += 1
        q.video_id = quote['video_id']
        q.date = quote['date']
        q.game = quote['game']
        q.text = quote['text']
        q.updated_at = now
        app.logger.info('Adding quote %d (%s)', q.id, q.text)
        db.session.add(q)
    db.session.commit()


if __name__ == '__main__':
    main()