from requests_futures.sessions import FuturesSession class Twitch(object): def __init__(self, client_id): self.client_id = client_id def fetch_videos(self, channel_id): if not channel_id: return [] session = FuturesSession() def get_videos(offset, limit): url = 'https://api.twitch.tv/v5/channels/{0}/videos'.format(channel_id) params = dict( client_id=self.client_id, offset=offset, limit=limit) return session.get(url, params=params) request = get_videos(0, 1) total = request.result().json().get('_total') if not total: return [] requests = [] limit = 100 for offset in range(0, total, limit): requests.append(get_videos(offset, limit)) result = [] for request in requests: result.extend(request.result().json().get('videos', [])) return result def fetch_comments(self, videos): if not videos: return [] session = FuturesSession(max_workers=len(videos)) def get_comments(video_id, cursor): url = 'https://api.twitch.tv/v5/videos/{0}/comments'.format(video_id) params = dict( client_id=self.client_id, cursor=cursor) return session.get(url, params=params) pairs = [(video_id, '') for video_id in videos] result = [] while pairs: requests = [] for video_id, cursor in pairs: requests.append(get_comments(video_id, cursor)) videos, _ = zip(*pairs) pairs = [] for video_id, request in zip(videos, requests): data = request.result().json() result.extend(data.get('comments', [])) cursor = data.get('_next') if cursor: pairs.append((video_id, cursor)) return result