Separate synchronization of videos, emotes and clips

twitch-cache-api/app.py

@@ -19,13 +19,25 @@ app.config.update(
     SQLALCHEMY_TRACK_MODIFICATIONS=False,
     SQLALCHEMY_DATABASE_URI=os.getenv('SQLALCHEMY_DATABASE_URI'),
     SCHEDULER_TIMEZONE='UTC',
-    SCHEDULER_JOBS=[dict(
+    SCHEDULER_JOBS=[
-        id='sync',
+        dict(id='sync_videos',
-        func='sync:Sync.perform',
+             func='sync:Sync.sync_videos',
              args=(app, db),
              max_instances=1,
              trigger='interval',
-        seconds=300)])
+             seconds=600),
+        dict(id='sync_emotes',
+             func='sync:Sync.sync_emotes',
+             args=(app, db),
+             max_instances=1,
+             trigger='interval',
+             seconds=1800),
+        dict(id='sync_clips',
+             func='sync:Sync.sync_clips',
+             args=(app, db),
+             max_instances=1,
+             trigger='interval',
+             seconds=120)])
 
 if app.config.get('SQLALCHEMY_DATABASE_URI', '').startswith('sqlite://'):
     @sqlalchemy.event.listens_for(sqlalchemy.engine.Engine, 'connect')

twitch-cache-api/sync.py

@@ -32,12 +32,11 @@ class Sync(object):
         return result.astimezone(tz=datetime.timezone.utc).replace(tzinfo=None)
 
     @classmethod
-    def perform(cls, app, db):
+    def sync_videos(cls, app, db):
-        app.logger.info('Starting synchronization')
+        app.logger.info('Starting synchronization of videos')
         with app.app_context():
             twitch = Twitch(os.getenv('TWITCH_CLIENT_ID'), os.getenv('TWITCH_OAUTH_TOKEN'))
             channel_id = os.getenv('TWITCH_CHANNEL_ID')
-            channel_name = os.getenv('TWITCH_CHANNEL_NAME')
             updated = []
             for vid in twitch.fetch_videos(channel_id):
                 id = cls._get(vid, '_id', default='').lstrip('v')
@@ -103,6 +102,14 @@ class Sync(object):
                 video.associations.append(assoc)
                 db.session.add(video)
             db.session.commit()
+        app.logger.info('Synchronization of videos completed')
+
+    @classmethod
+    def sync_emotes(cls, app, db):
+        app.logger.info('Starting synchronization of emotes')
+        with app.app_context():
+            twitch = Twitch(os.getenv('TWITCH_CLIENT_ID'), os.getenv('TWITCH_OAUTH_TOKEN'))
+            channel_id = os.getenv('TWITCH_CHANNEL_ID')
             for em in twitch.fetch_emotes(channel_id):
                 id = cls._get(em, 'id')
                 if not id:
@@ -114,6 +121,14 @@ class Sync(object):
                 emote.code = cls._get(em, 'code')
                 db.session.add(emote)
             db.session.commit()
+        app.logger.info('Synchronization of emotes completed')
+
+    @classmethod
+    def sync_clips(cls, app, db):
+        app.logger.info('Starting synchronization of clips')
+        with app.app_context():
+            twitch = Twitch(os.getenv('TWITCH_CLIENT_ID'), os.getenv('TWITCH_OAUTH_TOKEN'))
+            channel_name = os.getenv('TWITCH_CHANNEL_NAME')
             for clp in twitch.fetch_clips(channel_name):
                 slug = cls._get(clp, 'slug')
                 if not slug:
@@ -138,4 +153,4 @@ class Sync(object):
                 clip.created_at = cls._to_datetime(cls._get(clp, 'created_at'))
                 db.session.add(clip)
             db.session.commit()
-        app.logger.info('Synchronization completed')
+        app.logger.info('Synchronization of clips completed')