From cbb5c654c5dbead469d2650ac97f6ea3cf4eae21 Mon Sep 17 00:00:00 2001
From: Laszlo Zeke <kkocii@gmail.com>
Date: Sat, 21 Jan 2017 20:13:40 +0100
Subject: [PATCH] Support for v5 twitch api

-Using v5 Twitch API
-Caching for userids
-Using the largest thumbnail available
-Adding description if available
-Changed feed date to creation time
-Changed feed title to use display name
-Some extra error handling + cleanup
---
 TwitchRSS/app.yaml     |   7 ++--
 TwitchRSS/favicon.ico  | Bin 0 -> 7150 bytes
 TwitchRSS/twitchrss.py |  92 ++++++++++++++++++++++++++++-------------
 3 files changed, 68 insertions(+), 31 deletions(-)
 create mode 100644 TwitchRSS/favicon.ico

diff --git a/TwitchRSS/app.yaml b/TwitchRSS/app.yaml
index 96c0da9..14c59dd 100644
--- a/TwitchRSS/app.yaml
+++ b/TwitchRSS/app.yaml
@@ -1,9 +1,10 @@
-application: twitchrss
-version: a002
 runtime: python27
-api_version: 1
 threadsafe: true
 
 handlers:
+- url: /favicon\.ico
+  static_files: favicon.ico
+  upload: favicon\.ico
+
 - url: /.*
   script: twitchrss.app
\ No newline at end of file
diff --git a/TwitchRSS/favicon.ico b/TwitchRSS/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..c8d051647d35afbc001cd091aba44bab09828857
GIT binary patch
literal 7150
zcmeHLX;f548ojMSp^eHAMJAfqpdbp00wSUYrB(J#Ku}<iMFeDb0mNN#iCdyZjTto_
z$5DeB5op#9JIam`+@b~rQD$6nGDeLg;mub~1KrTlWd2RYI_LJQs(bHO8@~6I01y44
zt`2$%5b_RS20$QSZB?9A1&&kS<jK<Z5McgbK%X>ZAxFE99suby2DL-dmi{<g)B!=4
z@KB1C^wcD<ep#u)x>Prqo?-$=XeE>oLJ3*~rE#R1AbS2rU@;h2y@n7>c#qH{4)s|R
zq6p!Hj|ndbZ3H&{4FZet3*i#Mo4_Y@#-aLULdSTr&t&@)Z5@Y;+A+je6MiMUB79Al
zP3TWxak2=^_kh60AdIbXPSri1P`pq*w$N?3f2P4G>SudVOJLe3gcSref{4KScCxnB
zxqkeh%UdP|i|3Xez7pviD7MKuS~hpq#(Xn2mNwxGfz5@jlkH1~e+$*&e9;)Om0Rt6
zvS+r0h%G){lr^&<K7ZQovt`Eoc(EaiF_dtgz{cE8kn_V02)h&$=q@sO;+d{r<|>-f
zv@CmK>FF|4trM4wnV*4;rO=;H{}r}1JyY`$J7p)8SnmlaSbQNN&-4JtW%k=zI8CX4
zq1YF3)w}1pWtca6iuCWhq)%+{5siOvxZG}Dbe6ty|2Mu^TleKqzx+ifBE-{*1M)o&
zM2OA5&A#S0>U6D_Qa}3xpLt#%%SvK|tMg|^CsuE=ip{r5c+|Xt-~4D@Z~k1`lUM>f
z*A?~`wPosE{Zb9ncHckN{=H!5$NsIR5>oyPcocF#!3N;oXj=F{0*~zg6#<|{FAyy)
zd5ogAY+>&i0bw?Qy^kbC_C3Sm5om6(Cs?1?g2`De*jy06@|-&CQ$ECu^CKm;^-ZtA
zaU256cECMN7a7;1u<K?r9McSOynHTkr@%c3#SN|v^`u)`x&|I;V{yLL4YAdq!X;H7
zUsgJh?i#pfNyu((L&nb>on<t<)Af*h)dfDqtC3zGOqv?prx>sRf-CmHr!W{EId<?B
z>Ep8sCq!NjhF`ufwiTM<_+@io4#*?IhO5Wmm^vP(s$8l43s%=2g=fxuIHeiDo4j_Z
zhFF$kjF_wcfV0R9Nkx|6UL3-%yO-gbWsRt8JzTlrikiAP_<@c8R{{LX_hWaF1;r~S
zT^<%>e1x@mCU`|_ZEaqTGtV0F_ovMWEeV5LssM}g{jfr8hW+I>xOLqgO}9O8w50)`
zx7>s8H(m(M(nWri5lU-}aO%l@oP6Abz`_u?WXyzHnl>WIyFfGvi*s!enll+Oxf4-$
z%^DT2U!dewD;B4ZK}wY+ve{W&T7$Iu0C25e;XunXeA4_+_~b1@Q04?weiw*CKXGu+
z3qs~qFARzTqHbPC#Pu31$Z<eq_5|FxAB*%Ct?<tnhk`msh-z)%kU9?BYasdgbNJ^6
zA}reo*Bj$;teRdvZ_6KY2UMwaE{ru4lujN{z{>cUP=P|muUOGd&?827US^b@l(a`W
zk2mtt7`^b4Y4pak5Z(Ch9kTqEEdg7Twr$_Bvzv!CX|`(;*RET?Vadi#n`KQ>E%7n2
zD^{*r9VduS7~4}od}!F{<>3*Lno;~{YIPe!RrN1^OZ)=@gO(zAS;#^Kyw{?|1L%Xm
z<MF(Ge0u~?Rdsf8bv3bC;5O9VW1*)4UUlAl>!CKbcJ>aAPDFQ`0P7hybJj<ev*%a|
z=PKfL`DWA1rwc5G3TCMDr}Xse@{LW{H;d+II#m&`H%@>21Ovl~nv;wsE8taCKlpI8
zK%1|lIfg%0;r!kkHe7QAU+d5Bj~u0tSK|+QcQAj*Q1w4)^x!4i&FjbO|BlLlfjr4P
zyPW`=Ic+cdeG}i)@yGAcrzO%A2}=l_d5W6MeTeNPf#oTzq`c&i`x}RZD@m}uppDpU
z12p~X7<_gBZIpi;_^uRAsis(zu8XSco<I^1d*=b-?*9tkk_6bLjE4F~Nb;0TS9j4j
z&j21G1H{*5;sSlkJ@W#vDn}Q$8=`TrDiTM3;1EuENMHtiGrtc75`i_ff5Vw44fLIM
zhxLUIB)Q1XjsD1KI12Bg7|dA*gqLl_&U<xm%dmpoh0!=tW()4+7f8Ny1ScNe#RAHw
z*NF|#{AfQ;rU8xgO|N3#^~$5zTr>yUD_oG6tB22vbSeLc0N0j|yTq@kK1{jB7#z7|
zh4|{vaH`Ut_*0bYIADE&Iqb9Su(yT7!Kaj8l_la%V*=vDUiiA)lyv__*v&f#zx@cF
z1z}j7V}i!V2T@pS4$G2rNW6UoPPyK=aoZnXG?rpvh9ypyPJqElAmVB|<u?Hc&zgja
zhE<e19R~MgJw&fL?ESs~&A04u>PHS+Z)KyjY7F+=&4*XkG)eBYF@Fjk-*H7;Z3-%X
zdWguXL%3LDi@Ikb+MmgI)Up-{HAk?o*cjngl95>-M02gcX<DyK$xh_e`%(K_Y;1gq
z%}sQUg`sGEyccCH$yk!1htx_l6hF;DQ5%Qyo8FXX>41B#gYqA<(em^Y+-%-Ve#+_I
zmjBruU~f$-UGr~mEBo!zQ%#vzE<STAH^{~BgQL41LM3DLFen@3_Zk%Gdt13;eJ_qT
z8RU2OjVc*JcB^E%d#QFgdJ)M{9S*t9&W6n2Uss>R(%F=&5GB`cZ!m*`Zt7ATN{L=U
zEJg7(HFoXZv(r$;WADD6bXN??6nMM^ZtojSjoY!)P$NM5<2O&B^V~OW-eTmjHEGyr
zZ1dc%co|)bCq6XP5d!b|>(*-uH>hpw#UP=0yy?T@5$TzrwnkGWaqVnrr=m@iP{Y(H
zTh<h=T;;fIwaWash$%ts4h04Y(O)fm5d8<NU<4w7NMTfT^czQz5TWK_B!>!%r(t9?
za{2P!^}%Y(28{rOB-eycI4dNicl~rVA739E%wkEqMnK?<pryU*iKYBqWtNk1KSLFP
z_gtyE=Q($t7ag#>p2tGt>GM?H^&Hm+J>~U!0w-rR3r(SmnrnxFZGe1fXUvCL*yuuN
z3)s=G%}3VKBa_#%mQ+7yo(v_NuUu~>(F-*+*a^&?tt|d@3(W~5Fk`0P@JSj2*gd3u
zT$w?o)sB_%3@5USgt3XK##FOu9lgEi$BflwR}l3H8UqFz^g&M-4U8N`ml73$woV`P
p?A|e8pyZM=ct{`g{Umo0RmrWTFZzy<a?hUYecSVP2mTLt;6KMuACmw8

literal 0
HcmV?d00001

diff --git a/TwitchRSS/twitchrss.py b/TwitchRSS/twitchrss.py
index 08c6ec0..fc130bd 100644
--- a/TwitchRSS/twitchrss.py
+++ b/TwitchRSS/twitchrss.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2016 Laszlo Zeke
+# Copyright 2017, 2016 Laszlo Zeke
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,6 +24,13 @@ from feedformatter import Feed
 from google.appengine.api import memcache
 
 TWITCH_CLIENT_ID = 'Insert_key_here'
+VODCACHE_PREFIX = 'vodcache'
+USERIDCACHE_PREFIX = 'userid'
+VOD_URL_TEMPLATE = 'https://api.twitch.tv/kraken/channels/%s/videos?broadcast_type=archive,highlight,upload'
+USERID_URL_TEMPLATE = 'https://api.twitch.tv/kraken/users?login=%s'
+VODCACHE_LIFETIME = 120
+USERIDCACHE_LIFETIME = 0  # No expire
+
 
 class MainPage(webapp2.RequestHandler):
     def get(self):
@@ -51,22 +58,33 @@ class MainPage(webapp2.RequestHandler):
 
 class RSSVoDServer(webapp2.RequestHandler):
     def get(self, channel):
-        channel_json = self.lookup_cache(channel)
-        if channel_json == '':
-            channel_json = self.fetch_json(channel)
-            if channel_json == '':
-                self.abort(404)
-            else:
-                self.store_cache(channel, channel_json)
-
+        userid_json = self.fetch_userid(channel)
+        (channel_display_name, channel_id) = self.extract_userid(json.loads(userid_json))
+        channel_json = self.fetch_vods(channel_id)
         decoded_json = json.loads(channel_json)
-        rss_data = self.construct_rss(channel, decoded_json)
+        rss_data = self.construct_rss(channel, decoded_json, channel_display_name)
         self.response.headers['Content-Type'] = 'application/xhtml+xml'
         self.response.write(rss_data)
 
+    def fetch_userid(self, channel_name):
+        return self.fetch_or_cache_object(channel_name, USERIDCACHE_PREFIX, USERID_URL_TEMPLATE, USERIDCACHE_LIFETIME)
+
+    def fetch_vods(self, channel_id):
+        return self.fetch_or_cache_object(channel_id, VODCACHE_PREFIX, VOD_URL_TEMPLATE, VODCACHE_LIFETIME)
+
+    def fetch_or_cache_object(self, channel, key_prefix, url_template, cache_time):
+        json_data = self.lookup_cache(channel, key_prefix)
+        if json_data == '':
+            json_data = self.fetch_json(channel, url_template)
+            if json_data == '':
+                self.abort(404)
+            else:
+                self.store_cache(channel, json_data, key_prefix, cache_time)
+        return json_data
+
     @staticmethod
-    def lookup_cache(channel_name):
-        cached_data = memcache.get('vodcache:%s' % channel_name)
+    def lookup_cache(channel_name, key_prefix):
+        cached_data = memcache.get('%s:v5:%s' % (key_prefix, channel_name))
         if cached_data is not None:
             logging.debug('Cache hit for %s' % channel_name)
             return cached_data
@@ -75,43 +93,58 @@ class RSSVoDServer(webapp2.RequestHandler):
             return ''
 
     @staticmethod
-    def store_cache(channel_name, data):
+    def store_cache(channel_name, data, key_prefix, cache_lifetime):
         try:
             logging.debug('Cached data for %s' % channel_name)
-            memcache.set('vodcache:%s' % channel_name, data, 120)
-        except:
+            memcache.set('%s:v5:%s' % (key_prefix, channel_name), data, cache_lifetime)
+        except BaseException as e:
+            logging.warning('Memcache exception: %s' % e)
             return
 
     @staticmethod
-    def fetch_json(channel):
-        url = 'https://api.twitch.tv/kraken/channels/%s/videos?broadcasts=true' % channel
+    def fetch_json(id, url_template):
+        url = url_template % id
         headers = {
-            'Accept': 'application/vnd.twitchtv.v3+json',
+            'Accept': 'application/vnd.twitchtv.v5+json',
             'Client-ID': TWITCH_CLIENT_ID
         }
         request = urllib2.Request(url, headers=headers)
         try:
             result = urllib2.urlopen(request)
-            logging.debug('Fetch from twitch for %s with code %s' % (channel, result.getcode()))
+            logging.debug('Fetch from twitch for %s with code %s' % (id, result.getcode()))
             return result.read()
-        except urllib2.URLError, e:
+        except urllib2.URLError as e:
+            logging.warning("Fetch exception caught: %s" % e)
             return ''
 
-    def construct_rss(self, channel_name, vods_info):
+    def extract_userid(self, user_info):
+        userlist = user_info.get('users')
+        if userlist is None or len(userlist) < 1:
+            logging.info('No such user found.')
+            self.abort(404)
+        # Get the first id in the list
+        userid = userlist[0].get('_id')
+        username = userlist[0].get('display_name')
+        if username and userid:
+            return username, userid
+        else:
+            logging.warning('Userid is not found in %s' % user_info)
+            self.abort(404)
+
+    def construct_rss(self, channel_name, vods_info, display_name):
         feed = Feed()
 
         # Set the feed/channel level properties
-        feed.feed["title"] = "%s's Twitch video RSS" % channel_name
+        feed.feed["title"] = "%s's Twitch video RSS" % display_name
         feed.feed["link"] = "https://twitchrss.appspot.com/"
         feed.feed["author"] = "Twitch RSS Gen"
-        feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % channel_name
+        feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % display_name
 
         # Create an item
         try:
             if vods_info['videos'] is not None:
                 for vod in vods_info['videos']:
                     item = {}
-                    link = ""
                     if vod["status"] == "recording":
                         link = "http://www.twitch.tv/%s" % channel_name
                         item["title"] = "%s - LIVE" % vod['title']
@@ -119,15 +152,18 @@ class RSSVoDServer(webapp2.RequestHandler):
                         link = vod['url']
                         item["title"] = vod['title']
                     item["link"] = link
-                    item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview'])
-                    d = datetime.datetime.strptime(vod['recorded_at'], '%Y-%m-%dT%H:%M:%SZ')
+                    item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview']['large'])
+                    if vod.get('description_html'):
+                        item["description"] += "<br/>" + vod['description_html']
+                    d = datetime.datetime.strptime(vod['created_at'], '%Y-%m-%dT%H:%M:%SZ')
                     item["pubDate"] = d.timetuple()
                     item["guid"] = vod['_id']
-                    if vod["status"] == "recording": # To show a different news item when live is over
+                    if vod["status"] == "recording":  # To show a different news item when recording is over
                         item["guid"] += "_live"
                     item["ttl"] = '10'
                     feed.items.append(item)
-        except KeyError:
+        except KeyError as e:
+            logging.warning('Issue with json: %s\nException: %s' % (vods_info, e))
             self.abort(404)
 
         return feed.format_rss2_string()