Rename twitch-logs-api and synchronize clips

apps/twitch-logs/src/app/services/comments.service.ts

@@ -22,7 +22,7 @@ export class CommentsService {
         filter = '', sortBy = 'recorded_at', sortOrder = 'desc',
         pageNumber = 0, pageSize = 20): Observable<any> {
 
-        return this.http.get('/twitch-logs/api/videos', {
+        return this.http.get('/twitch-cache/api/videos', {
             observe: 'response',
             params: new HttpParams()
                 .set('filter', filter)
@@ -42,7 +42,7 @@ export class CommentsService {
         filter = '', sortBy = 'offset', sortOrder = 'asc',
         pageNumber = 0, pageSize = 20): Observable<any> {
 
-        return this.http.get('/twitch-logs/api/videos/' + videoID + '/comments', {
+        return this.http.get('/twitch-cache/api/videos/' + videoID + '/comments', {
             observe: 'response',
             params: new HttpParams()
                 .set('filter', filter)
@@ -62,7 +62,7 @@ export class CommentsService {
         commenter = '', term = '', sortBy = 'video_recorded_at', sortOrder = 'desc',
         pageNumber = 0, pageSize = 20): Observable<any> {
 
-        return this.http.get('/twitch-logs/api/search', {
+        return this.http.get('/twitch-cache/api/search', {
             observe: 'response',
             params: new HttpParams()
                 .set('commenter', commenter)

apps/twitch-logs/src/app/services/images.service.ts

@@ -43,7 +43,7 @@ export class ImagesService {
     }
 
     getEmotes(): Observable<any> {
-        return this.http.get('/twitch-logs/api/emotes').pipe(
+        return this.http.get('/twitch-cache/api/emotes').pipe(
             map((res: any[]) => {
                 return res.map((emote: any) => ({
                     id: emote.id,

docker-compose.yaml.example

@@ -11,7 +11,7 @@ services:
             - 127.0.0.1:8080:80
         depends_on:
             - quotes-api
-            - twitch-logs-api
+            - twitch-cache-api
             - cms
 
     # Quotes API service with /data/quotes mounted as database storage
@@ -27,19 +27,20 @@ services:
         expose:
             - 5000
 
-    # Twitch logs API service with /data/twitch-logs mounted as database storage
+    # Twitch cache API service with /data/twitch-cache mounted as database storage
-    # TWITCH_CLIENT_ID, TWITCH_OAUTH_TOKEN and TWITCH_CHANNEL_ID are needed for
+    # TWITCH_CLIENT_ID, TWITCH_OAUTH_TOKEN, TWITCH_CHANNEL_ID and TWITCH_CHANNEL_NAME
-    # Twitch API access and synchronization
+    # are needed for Twitch API access and synchronization
-    twitch-logs-api:
+    twitch-cache-api:
         build:
-            context: ./twitch-logs-api
+            context: ./twitch-cache-api
         volumes:
-            - /data/twitch-logs:/twitch-logs
+            - /data/twitch-cache:/twitch-cache
         environment:
-            - SQLALCHEMY_DATABASE_URI=sqlite:////twitch-logs/twitch-logs.db
+            - SQLALCHEMY_DATABASE_URI=sqlite:////twitch-cache/twitch-cache.db
             - TWITCH_CLIENT_ID=__TWITCH_CLIENT_ID__
             - TWITCH_OAUTH_TOKEN=__TWITCH_OAUTH_TOKEN__
             - TWITCH_CHANNEL_ID=__TWITCH_CHANNEL_ID__
+            - TWITCH_CHANNEL_NAME=__TWITCH_CHANNEL_NAME__
         expose:
             - 5000
 

nginx/nginx.conf

@@ -69,13 +69,13 @@ http {
             proxy_pass          http://quotes-api:5000/;
         }
 
-        location ^~ /twitch-logs/api/ {
+        location ^~ /twitch-cache/api/ {
-            rewrite             ^/twitch-logs/api(/.*)$ $1 break;
+            rewrite             ^/twitch-cache/api(/.*)$ $1 break;
             proxy_set_header    Host        $host;
             proxy_set_header    X-Real-IP   $remote_addr;
             proxy_http_version  1.1;
             tcp_nodelay         on;
-            proxy_pass          http://twitch-logs-api:5000/;
+            proxy_pass          http://twitch-cache-api:5000/;
         }
 
         location ~ /\.ht {

twitch-cache-api/app.py

@@ -9,7 +9,7 @@ import flask_restful.reqparse
 import sqlalchemy
 import sqlalchemy.engine
 
-from db import db, Video, Comment, Association, Emote
+from db import db, Video, Comment, Association, Emote, Clip
 
 
 app = flask.Flask(__name__)
@@ -90,6 +90,23 @@ emote_fields = {
     'code': flask_restful.fields.String(),
 }
 
+clip_fields = {
+    'slug': flask_restful.fields.String(),
+    'video_id': flask_restful.fields.Integer(),
+    'video_offset': flask_restful.fields.Float(),
+    'title': flask_restful.fields.String(),
+    'game': flask_restful.fields.String(),
+    'duration': flask_restful.fields.Float(),
+    'curator_id': flask_restful.fields.Integer(),
+    'curator_name': flask_restful.fields.String(),
+    'curator_display_name': flask_restful.fields.String(),
+    'curator_logo': flask_restful.fields.String(),
+    'thumbnail_tiny': flask_restful.fields.String(),
+    'thumbnail_small': flask_restful.fields.String(),
+    'thumbnail_medium': flask_restful.fields.String(),
+    'created_at': flask_restful.fields.DateTime(dt_format='iso8601'),
+}
+
 
 filter_parser = flask_restful.reqparse.RequestParser()
 filter_parser.add_argument('filter', type=str)
@@ -251,6 +268,43 @@ class EmotesResource(flask_restful.Resource):
         return emotes, 200
 
 
+class ClipResource(flask_restful.Resource):
+    @flask_restful.marshal_with(clip_fields)
+    def get(self, slug):
+        q = db.session.query(Clip).filter(Clip.slug == slug)
+        clip = q.first()
+        if not clip:
+            flask_restful.abort(404, message='Clip {0} does not exist'.format(slug))
+        return slug, 200
+
+
+class ClipsResource(flask_restful.Resource):
+    @flask_restful.marshal_with(clip_fields)
+    def get(self):
+        args = filter_parser.parse_args()
+        q = db.session.query(Clip)
+        if args['filter']:
+            q = q.filter(Clip.title.ilike('%{}%'.format(args['filter'])))
+        count = q.count()
+        if args['sort_order'] == 'random':
+            q = q.order_by(sqlalchemy.func.random())
+        elif args['sort_by']:
+            col = getattr(Clip, args['sort_by'], None)
+            if col:
+                if args['sort_order']:
+                    order_by = getattr(col, args['sort_order'], None)
+                    if order_by:
+                        q = q.order_by(order_by())
+                else:
+                    q = q.order_by(col)
+        if args['page_size']:
+            q = q.limit(args['page_size'])
+        if args['page_number'] and args['page_size']:
+            q = q.offset(args['page_number'] * args['page_size'])
+        clips = q.all()
+        return clips, 200, {'X-Total-Count': count}
+
+
 api.add_resource(VideoResource, '/videos/<int:id>')
 api.add_resource(VideosResource, '/videos')
 api.add_resource(CommentResource, '/videos/<int:video_id>/comments/<string:comment_id>')
@@ -258,6 +312,8 @@ api.add_resource(CommentsResource, '/videos/<int:id>/comments')
 api.add_resource(SearchResource, '/search')
 api.add_resource(EmoteResource, '/emotes/<int:id>')
 api.add_resource(EmotesResource, '/emotes')
+api.add_resource(ClipResource, '/clips/<string:slug>')
+api.add_resource(ClipsResource, '/clips')
 
 
 if __name__ == '__main__':

twitch-cache-api/db.py

@@ -53,3 +53,22 @@ class Emote(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     code = db.Column(db.String)
+
+
+class Clip(db.Model):
+    __tablename__ = 'clips'
+
+    slug = db.Column(db.String, primary_key=True)
+    video_id = db.Column(db.Integer)
+    video_offset = db.Column(db.Float)
+    title = db.Column(db.String)
+    game = db.Column(db.String)
+    duration = db.Column(db.Float)
+    curator_id = db.Column(db.Integer)
+    curator_name = db.Column(db.String)
+    curator_display_name = db.Column(db.String)
+    curator_logo = db.Column(db.String)
+    thumbnail_tiny = db.Column(db.String)
+    thumbnail_small = db.Column(db.String)
+    thumbnail_medium = db.Column(db.String)
+    created_at = db.Column(db.DateTime)

twitch-cache-api/sync.py

@@ -3,7 +3,7 @@ import os
 
 import flask_restful.inputs
 
-from db import Video, Comment, Association, Emote
+from db import Video, Comment, Association, Emote, Clip
 from twitch import Twitch
 
 
@@ -36,9 +36,10 @@ class Sync(object):
         app.logger.info('Starting synchronization')
         with app.app_context():
             twitch = Twitch(os.getenv('TWITCH_CLIENT_ID'), os.getenv('TWITCH_OAUTH_TOKEN'))
-            channel = os.getenv('TWITCH_CHANNEL_ID')
+            channel_id = os.getenv('TWITCH_CHANNEL_ID')
+            channel_name = os.getenv('TWITCH_CHANNEL_NAME')
             updated = []
-            for vid in twitch.fetch_videos(channel):
+            for vid in twitch.fetch_videos(channel_id):
                 id = cls._get(vid, '_id', default='').lstrip('v')
                 if not id:
                     continue
@@ -102,7 +103,7 @@ class Sync(object):
                 video.associations.append(assoc)
                 db.session.add(video)
             db.session.commit()
-            for em in twitch.fetch_emotes(channel):
+            for em in twitch.fetch_emotes(channel_id):
                 id = cls._get(em, 'id')
                 if not id:
                     continue
@@ -113,4 +114,28 @@ class Sync(object):
                 emote.code = cls._get(em, 'code')
                 db.session.add(emote)
             db.session.commit()
+            for clp in twitch.fetch_clips(channel_name):
+                slug = cls._get(clp, 'slug')
+                if not slug:
+                    continue
+                q = db.session.query(Clip).filter(Clip.slug == slug)
+                clip = q.first()
+                if not clip:
+                    clip = Clip(slug=slug)
+                clip.code = cls._get(clp, 'code')
+                clip.video_id = cls._get(clp, 'vod', 'id')
+                clip.video_offset = cls._get(clp, 'vod', 'offset')
+                clip.title = cls._get(clp, 'title')
+                clip.game = cls._get(clp, 'game')
+                clip.duration = cls._get(clp, 'duration')
+                clip.curator_id = cls._get(clp, 'curator', 'id')
+                clip.curator_name = cls._get(clp, 'curator', 'name')
+                clip.curator_display_name = cls._get(clp, 'curator', 'display_name')
+                clip.curator_logo = cls._get(clp, 'curator', 'logo')
+                clip.thumbnail_tiny = cls._get(clp, 'thumbnails', 'tiny')
+                clip.thumbnail_small = cls._get(clp, 'thumbnails', 'small')
+                clip.thumbnail_medium = cls._get(clp, 'thumbnails', 'medium')
+                clip.created_at = cls._to_datetime(cls._get(clp, 'created_at'))
+                db.session.add(clip)
+            db.session.commit()
         app.logger.info('Synchronization completed')

twitch-cache-api/twitch.py

@@ -71,3 +71,27 @@ class Twitch(object):
         for val in data.get('emoticon_sets', {}).values():
             result.extend(val)
         return result
+
+    def fetch_clips(self, channel_name):
+        if not channel_name:
+            return []
+        session = FuturesSession()
+        def get_clips(cursor):
+            url = 'https://api.twitch.tv/v5/clips/top'
+            params = dict(
+                client_id=self.client_id,
+                channel=channel_name,
+                period='all',
+                limit=100,
+                cursor=cursor)
+            return session.get(url, params=params)
+        cursor = ''
+        result = []
+        while True:
+            request = get_clips(cursor)
+            data = request.result().json()
+            result.extend(data.get('clips', []))
+            cursor = data.get('_cursor')
+            if not cursor:
+                break
+        return result