From 45449a6111377a5bc2d3e26b73fe2f087cd027d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Forr=C3=B3?= Date: Sat, 7 Jul 2018 11:29:25 +0200 Subject: [PATCH] Rename twitch-logs-api and synchronize clips --- .../src/app/services/comments.service.ts | 6 +- .../src/app/services/images.service.ts | 2 +- docker-compose.yaml.example | 17 +++--- nginx/nginx.conf | 6 +- .../.gitignore | 0 .../Dockerfile | 0 {twitch-logs-api => twitch-cache-api}/app.py | 58 ++++++++++++++++++- {twitch-logs-api => twitch-cache-api}/db.py | 19 ++++++ .../requirements.txt | 0 {twitch-logs-api => twitch-cache-api}/sync.py | 33 +++++++++-- .../twitch.py | 24 ++++++++ 11 files changed, 145 insertions(+), 20 deletions(-) rename {twitch-logs-api => twitch-cache-api}/.gitignore (100%) rename {twitch-logs-api => twitch-cache-api}/Dockerfile (100%) rename {twitch-logs-api => twitch-cache-api}/app.py (82%) rename {twitch-logs-api => twitch-cache-api}/db.py (73%) rename {twitch-logs-api => twitch-cache-api}/requirements.txt (100%) rename {twitch-logs-api => twitch-cache-api}/sync.py (75%) rename {twitch-logs-api => twitch-cache-api}/twitch.py (77%) diff --git a/apps/twitch-logs/src/app/services/comments.service.ts b/apps/twitch-logs/src/app/services/comments.service.ts index 0d7dcd7..833a336 100644 --- a/apps/twitch-logs/src/app/services/comments.service.ts +++ b/apps/twitch-logs/src/app/services/comments.service.ts @@ -22,7 +22,7 @@ export class CommentsService { filter = '', sortBy = 'recorded_at', sortOrder = 'desc', pageNumber = 0, pageSize = 20): Observable { - return this.http.get('/twitch-logs/api/videos', { + return this.http.get('/twitch-cache/api/videos', { observe: 'response', params: new HttpParams() .set('filter', filter) @@ -42,7 +42,7 @@ export class CommentsService { filter = '', sortBy = 'offset', sortOrder = 'asc', pageNumber = 0, pageSize = 20): Observable { - return this.http.get('/twitch-logs/api/videos/' + videoID + '/comments', { + return this.http.get('/twitch-cache/api/videos/' + videoID + '/comments', { observe: 'response', params: new HttpParams() .set('filter', filter) @@ -62,7 +62,7 @@ export class CommentsService { commenter = '', term = '', sortBy = 'video_recorded_at', sortOrder = 'desc', pageNumber = 0, pageSize = 20): Observable { - return this.http.get('/twitch-logs/api/search', { + return this.http.get('/twitch-cache/api/search', { observe: 'response', params: new HttpParams() .set('commenter', commenter) diff --git a/apps/twitch-logs/src/app/services/images.service.ts b/apps/twitch-logs/src/app/services/images.service.ts index ba15d89..fa3b3bd 100644 --- a/apps/twitch-logs/src/app/services/images.service.ts +++ b/apps/twitch-logs/src/app/services/images.service.ts @@ -43,7 +43,7 @@ export class ImagesService { } getEmotes(): Observable { - return this.http.get('/twitch-logs/api/emotes').pipe( + return this.http.get('/twitch-cache/api/emotes').pipe( map((res: any[]) => { return res.map((emote: any) => ({ id: emote.id, diff --git a/docker-compose.yaml.example b/docker-compose.yaml.example index 3a0e074..26b2f37 100644 --- a/docker-compose.yaml.example +++ b/docker-compose.yaml.example @@ -11,7 +11,7 @@ services: - 127.0.0.1:8080:80 depends_on: - quotes-api - - twitch-logs-api + - twitch-cache-api - cms # Quotes API service with /data/quotes mounted as database storage @@ -27,19 +27,20 @@ services: expose: - 5000 - # Twitch logs API service with /data/twitch-logs mounted as database storage - # TWITCH_CLIENT_ID, TWITCH_OAUTH_TOKEN and TWITCH_CHANNEL_ID are needed for - # Twitch API access and synchronization - twitch-logs-api: + # Twitch cache API service with /data/twitch-cache mounted as database storage + # TWITCH_CLIENT_ID, TWITCH_OAUTH_TOKEN, TWITCH_CHANNEL_ID and TWITCH_CHANNEL_NAME + # are needed for Twitch API access and synchronization + twitch-cache-api: build: - context: ./twitch-logs-api + context: ./twitch-cache-api volumes: - - /data/twitch-logs:/twitch-logs + - /data/twitch-cache:/twitch-cache environment: - - SQLALCHEMY_DATABASE_URI=sqlite:////twitch-logs/twitch-logs.db + - SQLALCHEMY_DATABASE_URI=sqlite:////twitch-cache/twitch-cache.db - TWITCH_CLIENT_ID=__TWITCH_CLIENT_ID__ - TWITCH_OAUTH_TOKEN=__TWITCH_OAUTH_TOKEN__ - TWITCH_CHANNEL_ID=__TWITCH_CHANNEL_ID__ + - TWITCH_CHANNEL_NAME=__TWITCH_CHANNEL_NAME__ expose: - 5000 diff --git a/nginx/nginx.conf b/nginx/nginx.conf index a72bc1b..e37d4ec 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -69,13 +69,13 @@ http { proxy_pass http://quotes-api:5000/; } - location ^~ /twitch-logs/api/ { - rewrite ^/twitch-logs/api(/.*)$ $1 break; + location ^~ /twitch-cache/api/ { + rewrite ^/twitch-cache/api(/.*)$ $1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_http_version 1.1; tcp_nodelay on; - proxy_pass http://twitch-logs-api:5000/; + proxy_pass http://twitch-cache-api:5000/; } location ~ /\.ht { diff --git a/twitch-logs-api/.gitignore b/twitch-cache-api/.gitignore similarity index 100% rename from twitch-logs-api/.gitignore rename to twitch-cache-api/.gitignore diff --git a/twitch-logs-api/Dockerfile b/twitch-cache-api/Dockerfile similarity index 100% rename from twitch-logs-api/Dockerfile rename to twitch-cache-api/Dockerfile diff --git a/twitch-logs-api/app.py b/twitch-cache-api/app.py similarity index 82% rename from twitch-logs-api/app.py rename to twitch-cache-api/app.py index 28b12b0..d21a2a6 100644 --- a/twitch-logs-api/app.py +++ b/twitch-cache-api/app.py @@ -9,7 +9,7 @@ import flask_restful.reqparse import sqlalchemy import sqlalchemy.engine -from db import db, Video, Comment, Association, Emote +from db import db, Video, Comment, Association, Emote, Clip app = flask.Flask(__name__) @@ -90,6 +90,23 @@ emote_fields = { 'code': flask_restful.fields.String(), } +clip_fields = { + 'slug': flask_restful.fields.String(), + 'video_id': flask_restful.fields.Integer(), + 'video_offset': flask_restful.fields.Float(), + 'title': flask_restful.fields.String(), + 'game': flask_restful.fields.String(), + 'duration': flask_restful.fields.Float(), + 'curator_id': flask_restful.fields.Integer(), + 'curator_name': flask_restful.fields.String(), + 'curator_display_name': flask_restful.fields.String(), + 'curator_logo': flask_restful.fields.String(), + 'thumbnail_tiny': flask_restful.fields.String(), + 'thumbnail_small': flask_restful.fields.String(), + 'thumbnail_medium': flask_restful.fields.String(), + 'created_at': flask_restful.fields.DateTime(dt_format='iso8601'), +} + filter_parser = flask_restful.reqparse.RequestParser() filter_parser.add_argument('filter', type=str) @@ -251,6 +268,43 @@ class EmotesResource(flask_restful.Resource): return emotes, 200 +class ClipResource(flask_restful.Resource): + @flask_restful.marshal_with(clip_fields) + def get(self, slug): + q = db.session.query(Clip).filter(Clip.slug == slug) + clip = q.first() + if not clip: + flask_restful.abort(404, message='Clip {0} does not exist'.format(slug)) + return slug, 200 + + +class ClipsResource(flask_restful.Resource): + @flask_restful.marshal_with(clip_fields) + def get(self): + args = filter_parser.parse_args() + q = db.session.query(Clip) + if args['filter']: + q = q.filter(Clip.title.ilike('%{}%'.format(args['filter']))) + count = q.count() + if args['sort_order'] == 'random': + q = q.order_by(sqlalchemy.func.random()) + elif args['sort_by']: + col = getattr(Clip, args['sort_by'], None) + if col: + if args['sort_order']: + order_by = getattr(col, args['sort_order'], None) + if order_by: + q = q.order_by(order_by()) + else: + q = q.order_by(col) + if args['page_size']: + q = q.limit(args['page_size']) + if args['page_number'] and args['page_size']: + q = q.offset(args['page_number'] * args['page_size']) + clips = q.all() + return clips, 200, {'X-Total-Count': count} + + api.add_resource(VideoResource, '/videos/') api.add_resource(VideosResource, '/videos') api.add_resource(CommentResource, '/videos//comments/') @@ -258,6 +312,8 @@ api.add_resource(CommentsResource, '/videos//comments') api.add_resource(SearchResource, '/search') api.add_resource(EmoteResource, '/emotes/') api.add_resource(EmotesResource, '/emotes') +api.add_resource(ClipResource, '/clips/') +api.add_resource(ClipsResource, '/clips') if __name__ == '__main__': diff --git a/twitch-logs-api/db.py b/twitch-cache-api/db.py similarity index 73% rename from twitch-logs-api/db.py rename to twitch-cache-api/db.py index d5978eb..3bcaec3 100644 --- a/twitch-logs-api/db.py +++ b/twitch-cache-api/db.py @@ -53,3 +53,22 @@ class Emote(db.Model): id = db.Column(db.Integer, primary_key=True) code = db.Column(db.String) + + +class Clip(db.Model): + __tablename__ = 'clips' + + slug = db.Column(db.String, primary_key=True) + video_id = db.Column(db.Integer) + video_offset = db.Column(db.Float) + title = db.Column(db.String) + game = db.Column(db.String) + duration = db.Column(db.Float) + curator_id = db.Column(db.Integer) + curator_name = db.Column(db.String) + curator_display_name = db.Column(db.String) + curator_logo = db.Column(db.String) + thumbnail_tiny = db.Column(db.String) + thumbnail_small = db.Column(db.String) + thumbnail_medium = db.Column(db.String) + created_at = db.Column(db.DateTime) diff --git a/twitch-logs-api/requirements.txt b/twitch-cache-api/requirements.txt similarity index 100% rename from twitch-logs-api/requirements.txt rename to twitch-cache-api/requirements.txt diff --git a/twitch-logs-api/sync.py b/twitch-cache-api/sync.py similarity index 75% rename from twitch-logs-api/sync.py rename to twitch-cache-api/sync.py index 174fd43..b9400a5 100644 --- a/twitch-logs-api/sync.py +++ b/twitch-cache-api/sync.py @@ -3,7 +3,7 @@ import os import flask_restful.inputs -from db import Video, Comment, Association, Emote +from db import Video, Comment, Association, Emote, Clip from twitch import Twitch @@ -36,9 +36,10 @@ class Sync(object): app.logger.info('Starting synchronization') with app.app_context(): twitch = Twitch(os.getenv('TWITCH_CLIENT_ID'), os.getenv('TWITCH_OAUTH_TOKEN')) - channel = os.getenv('TWITCH_CHANNEL_ID') + channel_id = os.getenv('TWITCH_CHANNEL_ID') + channel_name = os.getenv('TWITCH_CHANNEL_NAME') updated = [] - for vid in twitch.fetch_videos(channel): + for vid in twitch.fetch_videos(channel_id): id = cls._get(vid, '_id', default='').lstrip('v') if not id: continue @@ -102,7 +103,7 @@ class Sync(object): video.associations.append(assoc) db.session.add(video) db.session.commit() - for em in twitch.fetch_emotes(channel): + for em in twitch.fetch_emotes(channel_id): id = cls._get(em, 'id') if not id: continue @@ -113,4 +114,28 @@ class Sync(object): emote.code = cls._get(em, 'code') db.session.add(emote) db.session.commit() + for clp in twitch.fetch_clips(channel_name): + slug = cls._get(clp, 'slug') + if not slug: + continue + q = db.session.query(Clip).filter(Clip.slug == slug) + clip = q.first() + if not clip: + clip = Clip(slug=slug) + clip.code = cls._get(clp, 'code') + clip.video_id = cls._get(clp, 'vod', 'id') + clip.video_offset = cls._get(clp, 'vod', 'offset') + clip.title = cls._get(clp, 'title') + clip.game = cls._get(clp, 'game') + clip.duration = cls._get(clp, 'duration') + clip.curator_id = cls._get(clp, 'curator', 'id') + clip.curator_name = cls._get(clp, 'curator', 'name') + clip.curator_display_name = cls._get(clp, 'curator', 'display_name') + clip.curator_logo = cls._get(clp, 'curator', 'logo') + clip.thumbnail_tiny = cls._get(clp, 'thumbnails', 'tiny') + clip.thumbnail_small = cls._get(clp, 'thumbnails', 'small') + clip.thumbnail_medium = cls._get(clp, 'thumbnails', 'medium') + clip.created_at = cls._to_datetime(cls._get(clp, 'created_at')) + db.session.add(clip) + db.session.commit() app.logger.info('Synchronization completed') diff --git a/twitch-logs-api/twitch.py b/twitch-cache-api/twitch.py similarity index 77% rename from twitch-logs-api/twitch.py rename to twitch-cache-api/twitch.py index 6ab956f..55fd0d0 100644 --- a/twitch-logs-api/twitch.py +++ b/twitch-cache-api/twitch.py @@ -71,3 +71,27 @@ class Twitch(object): for val in data.get('emoticon_sets', {}).values(): result.extend(val) return result + + def fetch_clips(self, channel_name): + if not channel_name: + return [] + session = FuturesSession() + def get_clips(cursor): + url = 'https://api.twitch.tv/v5/clips/top' + params = dict( + client_id=self.client_id, + channel=channel_name, + period='all', + limit=100, + cursor=cursor) + return session.get(url, params=params) + cursor = '' + result = [] + while True: + request = get_clips(cursor) + data = request.result().json() + result.extend(data.get('clips', [])) + cursor = data.get('_cursor') + if not cursor: + break + return result