Migrate application to python3
Notables: - Python 3 migration - The client id is now set via environment variable instead of hardcoding it in a py file - Using Flask instead of web2 - Migrate away from Memcache (Google depeciated it in py3) to an in-memory data structure - Moved landing page to static serving
This commit is contained in:
Laszlo Zeke
parent
fc67ccba75
commit
5fccffc2f0
|
|
@ -7,6 +7,7 @@ __pycache__/
|
|||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
venv/
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
## Twitch RSS Webapp for Google App Engine
|
||||
This project is a very small web application for serving RSS feed for broadcasts
|
||||
in Twitch. It fetches data from [Twitch API](https://github.com/justintv/twitch-api) and caches in Memcache.
|
||||
in Twitch. It fetches data from [Twitch API](https://dev.twitch.tv/docs) and caches in Memcache.
|
||||
The engine is webapp2.
|
||||
|
||||
A running version can be tried out at:
|
||||
A running version can be tried out at:
|
||||
https://twitchrss.appspot.com/vod/twitch
|
||||
|
||||
There is also a VOD only endpoint if you don't want to see ongoing streams which are known to break some readers:
|
||||
https://twitchrss.appspot.com/vodonly/twitch
|
||||
|
||||
### Deployment
|
||||
First you should set your own Twitch API client ID in the app.yaml.
|
||||
See how to deploy on [Google App Engine](https://cloud.google.com/appengine/docs/python/gettingstartedpython27/introduction).
|
||||
|
||||
### Other things
|
||||
|
|
|
|||
|
|
@ -1,15 +1,21 @@
|
|||
runtime: python27
|
||||
threadsafe: true
|
||||
runtime: python38
|
||||
|
||||
entrypoint: gunicorn -b :$PORT twitchrss:app
|
||||
|
||||
env_variables:
|
||||
TWITCH_CLIENT_ID: __INSERT_TWITCH_CLIENT_ID_HERE__
|
||||
|
||||
handlers:
|
||||
- url: /favicon\.ico
|
||||
static_files: favicon.ico
|
||||
upload: favicon\.ico
|
||||
|
||||
- url: /.*
|
||||
script: twitchrss.app
|
||||
- url: /
|
||||
static_files: index.html
|
||||
upload: index\.html
|
||||
|
||||
- url: /.+
|
||||
script: auto
|
||||
|
||||
automatic_scaling:
|
||||
max_instances: 1
|
||||
max_idle_instances: 1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
#
|
||||
# Copyright 2017, 2016 Laszlo Zeke
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
TWITCH_CLIENT_ID = 'Insert_key_here'
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
"""`appengine_config` gets loaded when starting a new application instance."""
|
||||
import vendor
|
||||
# insert `lib` as a site directory so our `main` module can load
|
||||
# third-party libraries, and override built-ins with newer
|
||||
# versions.
|
||||
vendor.add('lib')
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# Feedformatter
|
||||
# Copyright (c) 2008, Luke Maurits <luke@maurits.id.au>
|
||||
# Copyright (c) 2015, Laszlo Zeke
|
||||
# Copyright (c) 2020, Laszlo Zeke
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -28,7 +28,7 @@
|
|||
|
||||
__version__ = "0.4"
|
||||
|
||||
from cStringIO import StringIO
|
||||
from io import StringIO
|
||||
|
||||
# This "staircase" of import attempts is ugly. If there's a nicer way to do
|
||||
# this, please let me know!
|
||||
|
|
@ -75,7 +75,7 @@ _rss2_channel_mappings = (
|
|||
(("title",), "title"),
|
||||
(("link", "url"), "link"),
|
||||
(("description", "desc", "summary"), "description"),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("rss2",x)),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("rss2",x)),
|
||||
(("category",), "category"),
|
||||
(("language",), "language"),
|
||||
(("copyright",), "copyright"),
|
||||
|
|
@ -91,9 +91,9 @@ _rss2_item_mappings = (
|
|||
(("link", "url"), "link"),
|
||||
(("description", "desc", "summary"), "description"),
|
||||
(("guid", "id"), "guid"),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("rss2",x)),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("rss2",x)),
|
||||
(("category",), "category"),
|
||||
(("author",), "author", lambda(x): _rssify_author(x))
|
||||
(("author",), "author", lambda x: _rssify_author(x))
|
||||
)
|
||||
|
||||
# Atom 1.0 ----------
|
||||
|
|
@ -102,19 +102,19 @@ _atom_feed_mappings = (
|
|||
(("title",), "title"),
|
||||
(("link", "url"), "id"),
|
||||
(("description", "desc", "summary"), "subtitle"),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("atom",x)),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("atom",x)),
|
||||
(("category",), "category"),
|
||||
(("author",), "author", lambda(x): _atomise_author(x))
|
||||
(("author",), "author", lambda x: _atomise_author(x))
|
||||
)
|
||||
|
||||
_atom_item_mappings = (
|
||||
(("title",), "title"),
|
||||
(("link", "url"), "id"),
|
||||
(("link", "url"), "link", lambda(x): _atomise_link(x)),
|
||||
(("link", "url"), "link", lambda x: _atomise_link(x)),
|
||||
(("description", "desc", "summary"), "summary"),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("atom",x)),
|
||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("atom",x)),
|
||||
(("category",), "category"),
|
||||
(("author",), "author", lambda(x): _atomise_author(x))
|
||||
(("author",), "author", lambda x: _atomise_author(x))
|
||||
)
|
||||
|
||||
def _get_tz_offset():
|
||||
|
|
@ -146,20 +146,20 @@ def _convert_datetime(time):
|
|||
elif type(time) is int or type(time) is float:
|
||||
# Assume this is a seconds-since-epoch time
|
||||
return localtime(time)
|
||||
elif type(time) is str:
|
||||
elif type(time) is str:
|
||||
if time.isalnum():
|
||||
# String is alphanumeric - a time stamp?
|
||||
try:
|
||||
return strptime(time, "%a, %d %b %Y %H:%M:%S %Z")
|
||||
except ValueError:
|
||||
raise Exception("Unrecongised time format!")
|
||||
raise Exception("Unrecongised time format!")
|
||||
else:
|
||||
# Maybe this is a string of an epoch time?
|
||||
try:
|
||||
return localtime(float(time))
|
||||
except ValueError:
|
||||
# Guess not.
|
||||
raise Exception("Unrecongised time format!")
|
||||
raise Exception("Unrecongised time format!")
|
||||
else:
|
||||
# No idea what this is. Give up!
|
||||
raise Exception("Unrecongised time format!")
|
||||
|
|
@ -171,7 +171,7 @@ def _format_datetime(feed_type, time):
|
|||
used in a validly formatted feed of type feed_type. Raise an
|
||||
Exception if this cannot be done.
|
||||
"""
|
||||
|
||||
|
||||
# First, convert time into a time structure
|
||||
time = _convert_datetime(time)
|
||||
|
||||
|
|
@ -187,7 +187,7 @@ def _atomise_link(link):
|
|||
return dict
|
||||
else:
|
||||
return {"href" : link}
|
||||
|
||||
|
||||
def _atomise_author(author):
|
||||
|
||||
"""
|
||||
|
|
@ -214,7 +214,7 @@ def _rssify_author(author):
|
|||
Convert author from whatever it is to a plain old email string for
|
||||
use in an RSS 2.0 feed.
|
||||
"""
|
||||
|
||||
|
||||
if type(author) is dict:
|
||||
try:
|
||||
return author["email"]
|
||||
|
|
@ -241,8 +241,8 @@ def _add_subelems(root_element, mappings, dictionary):
|
|||
elif len(mapping) == 3:
|
||||
value = mapping[2](dictionary[key])
|
||||
_add_subelem(root_element, mapping[1], value)
|
||||
break
|
||||
|
||||
break
|
||||
|
||||
def _add_subelem(root_element, name, value):
|
||||
|
||||
if value is None:
|
||||
|
|
@ -282,7 +282,7 @@ def _stringify(tree, pretty):
|
|||
class Feed:
|
||||
|
||||
### INTERNAL METHODS ------------------------------
|
||||
|
||||
|
||||
def __init__(self, feed=None, items=None):
|
||||
|
||||
if feed:
|
||||
|
|
@ -296,7 +296,7 @@ class Feed:
|
|||
self.entries = self.items
|
||||
|
||||
### RSS 1.0 STUFF ------------------------------
|
||||
|
||||
|
||||
def validate_rss1(self):
|
||||
|
||||
"""Raise an InvalidFeedException if the feed cannot be validly
|
||||
|
|
@ -325,14 +325,14 @@ class Feed:
|
|||
if "link" not in item:
|
||||
raise InvalidFeedException("Each item element in an RSS 1.0 "
|
||||
"feed must contain a link subelement")
|
||||
|
||||
|
||||
def format_rss1_string(self, validate=True, pretty=False):
|
||||
|
||||
"""Format the feed as RSS 1.0 and return the result as a string."""
|
||||
|
||||
if validate:
|
||||
self.validate_rss1()
|
||||
RSS1root = ET.Element( 'rdf:RDF',
|
||||
RSS1root = ET.Element( 'rdf:RDF',
|
||||
{"xmlns:rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"xmlns" : "http://purl.org/rss/1.0/"} )
|
||||
RSS1channel = ET.SubElement(RSS1root, 'channel',
|
||||
|
|
@ -394,7 +394,7 @@ class Feed:
|
|||
RSS2root = ET.Element( 'rss', {'version':'2.0'} )
|
||||
RSS2channel = ET.SubElement( RSS2root, 'channel' )
|
||||
_add_subelems(RSS2channel, _rss2_channel_mappings, self.feed)
|
||||
for item in self.items:
|
||||
for item in self.items:
|
||||
RSS2item = ET.SubElement ( RSS2channel, 'item' )
|
||||
_add_subelems(RSS2item, _rss2_item_mappings, item)
|
||||
return _stringify(RSS2root, pretty=pretty)
|
||||
|
|
@ -472,11 +472,11 @@ def main():
|
|||
item["guid"] = "1234567890"
|
||||
feed.items.append(item)
|
||||
print("---- RSS 1.0 ----")
|
||||
print feed.format_rss1_string(pretty=True)
|
||||
print(feed.format_rss1_string(pretty=True))
|
||||
print("---- RSS 2.0 ----")
|
||||
print feed.format_rss2_string(pretty=True)
|
||||
print(feed.format_rss2_string(pretty=True))
|
||||
print("---- Atom 1.0 ----")
|
||||
print feed.format_atom_string(pretty=True)
|
||||
print(feed.format_atom_string(pretty=True))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Twitch stream RSS generator</title>
|
||||
</head>
|
||||
<body>
|
||||
<p style="font-family: helvetica; font-size:20pt; padding: 20px;">
|
||||
Twitch stream RSS generator
|
||||
</p>
|
||||
<p style="font-family: helvetica; font-size:12pt; padding: 20px;">
|
||||
You can get RSS of broadcasts by subscribing to https://twitchrss.appspot.com/vod/<channel name><br/>
|
||||
For example: <a href="https://twitchrss.appspot.com/vod/riotgames">https://twitchrss.appspot.com/vod/riotgames</a><br/><br/>
|
||||
You can use the /vodonly handle to get only vods without ongoing streams.
|
||||
Not endorsed by Twitch.tv, just a fun project.<br/>
|
||||
<a href="https://github.com/lzeke0/TwitchRSS">Project home</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
appdirs==1.4.3
|
||||
CacheControl==0.12.6
|
||||
certifi==2019.11.28
|
||||
chardet==3.0.4
|
||||
click==7.1.2
|
||||
colorama==0.4.3
|
||||
contextlib2==0.6.0
|
||||
distlib==0.3.0
|
||||
distro==1.4.0
|
||||
expiringdict==1.2.1
|
||||
Flask==1.1.2
|
||||
gunicorn==20.0.4
|
||||
html5lib==1.0.1
|
||||
idna==2.8
|
||||
ipaddr==2.2.0
|
||||
itsdangerous==1.1.0
|
||||
Jinja2==2.11.2
|
||||
lockfile==0.12.2
|
||||
MarkupSafe==1.1.1
|
||||
msgpack==0.6.2
|
||||
packaging==20.3
|
||||
pep517==0.8.2
|
||||
progress==1.5
|
||||
pyparsing==2.4.6
|
||||
pytoml==0.1.21
|
||||
requests==2.22.0
|
||||
retrying==1.3.3
|
||||
six==1.14.0
|
||||
urllib3==1.25.8
|
||||
webencodings==0.5.1
|
||||
Werkzeug==1.0.1
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#
|
||||
# Copyright 2017, 2016 Laszlo Zeke
|
||||
# Copyright 2020 Laszlo Zeke
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -14,192 +14,166 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
import webapp2
|
||||
from webapp2 import Route
|
||||
import urllib2
|
||||
from flask import abort, Flask
|
||||
import urllib
|
||||
import json
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
from os import environ
|
||||
from feedformatter import Feed
|
||||
from google.appengine.api import memcache
|
||||
from app_id import TWITCH_CLIENT_ID
|
||||
from StringIO import StringIO
|
||||
from expiringdict import ExpiringDict
|
||||
from io import BytesIO
|
||||
import gzip
|
||||
|
||||
|
||||
VODCACHE_PREFIX = 'vodcache'
|
||||
USERIDCACHE_PREFIX = 'userid'
|
||||
VOD_URL_TEMPLATE = 'https://api.twitch.tv/kraken/channels/%s/videos?broadcast_type=archive,highlight,upload&limit=10'
|
||||
USERID_URL_TEMPLATE = 'https://api.twitch.tv/kraken/users?login=%s'
|
||||
VODCACHE_LIFETIME = 600
|
||||
USERIDCACHE_LIFETIME = 0 # No expire
|
||||
USERIDCACHE_LIFETIME = 24 * 60 * 60
|
||||
CHANNEL_FILTER = re.compile("^[a-zA-Z0-9_]{2,25}$")
|
||||
TWITCH_CLIENT_ID = environ.get("TWITCH_CLIENT_ID")
|
||||
logging.basicConfig(level=logging.DEBUG if environ.get('DEBUG') else logging.INFO)
|
||||
|
||||
if not TWITCH_CLIENT_ID:
|
||||
raise Exception("Twitch API client id is not set.")
|
||||
|
||||
|
||||
class MainPage(webapp2.RequestHandler):
|
||||
def get(self):
|
||||
self.response.headers['Content-Type'] = 'text/html'
|
||||
html_resp = """
|
||||
<html>
|
||||
<head>
|
||||
<title>Twitch stream RSS generator</title>
|
||||
</head>
|
||||
<body>
|
||||
<p style="font-family: helvetica; font-size:20pt; padding: 20px;">
|
||||
Twitch stream RSS generator
|
||||
</p>
|
||||
<p style="font-family: helvetica; font-size:12pt; padding: 20px;">
|
||||
You can get RSS of broadcasts by subscribing to https://twitchrss.appspot.com/vod/<channel name><br/>
|
||||
For example: <a href="https://twitchrss.appspot.com/vod/riotgames">https://twitchrss.appspot.com/vod/riotgames</a><br/><br/>
|
||||
You can use the /vodonly handle to get only vods without ongoing streams.
|
||||
Not endorsed by Twitch.tv, just a fun project.<br/>
|
||||
<a href="https://github.com/lzeke0/TwitchRSS">Project home</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
self.response.write(html_resp)
|
||||
app = Flask(__name__)
|
||||
|
||||
vodcache = ExpiringDict(max_len=200, max_age_seconds=VODCACHE_LIFETIME)
|
||||
useridcache = ExpiringDict(max_len=1000, max_age_seconds=USERIDCACHE_LIFETIME)
|
||||
|
||||
|
||||
class RSSVoDServer(webapp2.RequestHandler):
|
||||
def get(self, channel):
|
||||
self._get_inner(channel)
|
||||
@app.route('/vod/<string:channel>', methods=['GET', 'HEAD'])
|
||||
def vod(channel):
|
||||
if CHANNEL_FILTER.match(channel):
|
||||
return get_inner(channel)
|
||||
else:
|
||||
abort(404)
|
||||
|
||||
def _get_inner(self, channel, add_live=True):
|
||||
userid_json = self.fetch_userid(channel)
|
||||
(channel_display_name, channel_id) = self.extract_userid(json.loads(userid_json))
|
||||
channel_json = self.fetch_vods(channel_id)
|
||||
decoded_json = json.loads(channel_json)
|
||||
rss_data = self.construct_rss(channel, decoded_json, channel_display_name, add_live)
|
||||
self.response.headers['Content-Type'] = 'application/rss+xml'
|
||||
self.response.write(rss_data)
|
||||
|
||||
def head(self,channel):
|
||||
self.get(channel)
|
||||
@app.route('/vodonly/<string:channel>', methods=['GET', 'HEAD'])
|
||||
def vodonly(channel):
|
||||
if CHANNEL_FILTER.match(channel):
|
||||
return get_inner(channel, add_live=False)
|
||||
else:
|
||||
abort(404)
|
||||
|
||||
def fetch_userid(self, channel_name):
|
||||
return self.fetch_or_cache_object(channel_name, USERIDCACHE_PREFIX, USERID_URL_TEMPLATE, USERIDCACHE_LIFETIME)
|
||||
|
||||
def fetch_vods(self, channel_id):
|
||||
return self.fetch_or_cache_object(channel_id, VODCACHE_PREFIX, VOD_URL_TEMPLATE, VODCACHE_LIFETIME)
|
||||
def get_inner(channel, add_live=True):
|
||||
userid_json = fetch_userid(channel)
|
||||
(channel_display_name, channel_id) = extract_userid(json.loads(userid_json))
|
||||
channel_json = fetch_vods(channel_id)
|
||||
decoded_json = json.loads(channel_json)
|
||||
rss_data = construct_rss(channel, decoded_json, channel_display_name, add_live)
|
||||
headers = {'Content-Type': 'application/rss+xml'}
|
||||
return rss_data, headers
|
||||
|
||||
def fetch_or_cache_object(self, channel, key_prefix, url_template, cache_time):
|
||||
json_data = self.lookup_cache(channel, key_prefix)
|
||||
|
||||
def fetch_userid(channel_name):
|
||||
return fetch_or_cache_object(channel_name, useridcache, USERID_URL_TEMPLATE)
|
||||
|
||||
|
||||
def fetch_vods(channel_id):
|
||||
return fetch_or_cache_object(channel_id, vodcache, VOD_URL_TEMPLATE)
|
||||
|
||||
|
||||
def fetch_or_cache_object(key, cachedict, url_template):
|
||||
json_data = cachedict.get(key)
|
||||
if not json_data:
|
||||
json_data = fetch_json(key, url_template)
|
||||
if not json_data:
|
||||
json_data = self.fetch_json(channel, url_template)
|
||||
if not json_data:
|
||||
self.abort(404)
|
||||
else:
|
||||
self.store_cache(channel, json_data, key_prefix, cache_time)
|
||||
return json_data
|
||||
|
||||
@staticmethod
|
||||
def lookup_cache(channel_name, key_prefix):
|
||||
cached_data = memcache.get('%s:v5:%s' % (key_prefix, channel_name))
|
||||
if cached_data is not None:
|
||||
logging.debug('Cache hit for %s' % channel_name)
|
||||
return cached_data
|
||||
abort(404)
|
||||
else:
|
||||
logging.debug('Cache miss for %s' % channel_name)
|
||||
return ''
|
||||
cachedict[key] = json_data
|
||||
return json_data
|
||||
|
||||
@staticmethod
|
||||
def store_cache(channel_name, data, key_prefix, cache_lifetime):
|
||||
|
||||
def fetch_json(id, url_template):
|
||||
url = url_template % id
|
||||
headers = {
|
||||
'Accept': 'application/vnd.twitchtv.v5+json',
|
||||
'Client-ID': TWITCH_CLIENT_ID,
|
||||
'Accept-Encoding': 'gzip'
|
||||
}
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
retries = 0
|
||||
while retries < 3:
|
||||
try:
|
||||
logging.debug('Cached data for %s' % channel_name)
|
||||
memcache.set('%s:v5:%s' % (key_prefix, channel_name), data, cache_lifetime)
|
||||
except BaseException as e:
|
||||
logging.warning('Memcache exception: %s' % e)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def fetch_json(id, url_template):
|
||||
url = url_template % id
|
||||
headers = {
|
||||
'Accept': 'application/vnd.twitchtv.v5+json',
|
||||
'Client-ID': TWITCH_CLIENT_ID,
|
||||
'Accept-Encoding': 'gzip'
|
||||
}
|
||||
request = urllib2.Request(url, headers=headers)
|
||||
retries = 0
|
||||
while retries < 3:
|
||||
try:
|
||||
result = urllib2.urlopen(request, timeout=3)
|
||||
logging.debug('Fetch from twitch for %s with code %s' % (id, result.getcode()))
|
||||
if result.info().get('Content-Encoding') == 'gzip':
|
||||
logging.debug('Fetched gzip content')
|
||||
buf = StringIO(result.read())
|
||||
f = gzip.GzipFile(fileobj=buf)
|
||||
return f.read()
|
||||
return result.read()
|
||||
except BaseException as e:
|
||||
logging.warning("Fetch exception caught: %s" % e)
|
||||
retries += 1
|
||||
return ''
|
||||
|
||||
def extract_userid(self, user_info):
|
||||
userlist = user_info.get('users')
|
||||
if not userlist:
|
||||
logging.info('No such user found.')
|
||||
self.abort(404)
|
||||
# Get the first id in the list
|
||||
userid = userlist[0].get('_id')
|
||||
username = userlist[0].get('display_name')
|
||||
if username and userid:
|
||||
return username, userid
|
||||
else:
|
||||
logging.warning('Userid is not found in %s' % user_info)
|
||||
self.abort(404)
|
||||
|
||||
def construct_rss(self, channel_name, vods_info, display_name, add_live=True):
|
||||
feed = Feed()
|
||||
|
||||
# Set the feed/channel level properties
|
||||
feed.feed["title"] = "%s's Twitch video RSS" % display_name
|
||||
feed.feed["link"] = "https://twitchrss.appspot.com/"
|
||||
feed.feed["author"] = "Twitch RSS Gen"
|
||||
feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % display_name
|
||||
feed.feed["ttl"] = '10'
|
||||
|
||||
# Create an item
|
||||
try:
|
||||
if vods_info['videos']:
|
||||
for vod in vods_info['videos']:
|
||||
item = {}
|
||||
if vod["status"] == "recording":
|
||||
if not add_live:
|
||||
continue
|
||||
link = "http://www.twitch.tv/%s" % channel_name
|
||||
item["title"] = "%s - LIVE" % vod['title']
|
||||
item["category"] = "live"
|
||||
else:
|
||||
link = vod['url']
|
||||
item["title"] = vod['title']
|
||||
item["category"] = vod['broadcast_type']
|
||||
item["link"] = link
|
||||
item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview']['large'])
|
||||
if vod.get('game'):
|
||||
item["description"] += "<br/>" + vod['game']
|
||||
if vod.get('description_html'):
|
||||
item["description"] += "<br/>" + vod['description_html']
|
||||
d = datetime.datetime.strptime(vod['created_at'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
item["pubDate"] = d.timetuple()
|
||||
item["guid"] = vod['_id']
|
||||
if vod["status"] == "recording": # To show a different news item when recording is over
|
||||
item["guid"] += "_live"
|
||||
feed.items.append(item)
|
||||
except KeyError as e:
|
||||
logging.warning('Issue with json: %s\nException: %s' % (vods_info, e))
|
||||
self.abort(404)
|
||||
|
||||
return feed.format_rss2_string()
|
||||
|
||||
class RSSVoDServerOnlyVoD(RSSVoDServer):
|
||||
def get(self, channel):
|
||||
self._get_inner(channel, add_live=False)
|
||||
result = urllib.request.urlopen(request, timeout=3)
|
||||
logging.debug('Fetch from twitch for %s with code %s' % (id, result.getcode()))
|
||||
if result.info().get('Content-Encoding') == 'gzip':
|
||||
logging.debug('Fetched gzip content')
|
||||
buf = BytesIO(result.read())
|
||||
f = gzip.GzipFile(fileobj=buf)
|
||||
return f.read()
|
||||
return result.read()
|
||||
except Exception as e:
|
||||
logging.warning("Fetch exception caught: %s" % e)
|
||||
retries += 1
|
||||
return None
|
||||
|
||||
|
||||
app = webapp2.WSGIApplication([
|
||||
Route('/', MainPage),
|
||||
Route('/vod/<channel:[a-zA-Z0-9_]{2,25}>', RSSVoDServer),
|
||||
Route('/vodonly/<channel:[a-zA-Z0-9_]{2,25}>', RSSVoDServerOnlyVoD)
|
||||
], debug=False)
|
||||
def extract_userid(user_info):
|
||||
userlist = user_info.get('users')
|
||||
if not userlist:
|
||||
logging.info('No such user found.')
|
||||
abort(404)
|
||||
# Get the first id in the list
|
||||
userid = userlist[0].get('_id')
|
||||
username = userlist[0].get('display_name')
|
||||
if username and userid:
|
||||
return username, userid
|
||||
else:
|
||||
logging.warning('Userid is not found in %s' % user_info)
|
||||
abort(404)
|
||||
|
||||
|
||||
def construct_rss(channel_name, vods_info, display_name, add_live=True):
|
||||
feed = Feed()
|
||||
|
||||
# Set the feed/channel level properties
|
||||
feed.feed["title"] = "%s's Twitch video RSS" % display_name
|
||||
feed.feed["link"] = "https://twitchrss.appspot.com/"
|
||||
feed.feed["author"] = "Twitch RSS Generated"
|
||||
feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % display_name
|
||||
feed.feed["ttl"] = '10'
|
||||
|
||||
# Create an item
|
||||
try:
|
||||
if vods_info['videos']:
|
||||
for vod in vods_info['videos']:
|
||||
item = {}
|
||||
if vod["status"] == "recording":
|
||||
if not add_live:
|
||||
continue
|
||||
link = "http://www.twitch.tv/%s" % channel_name
|
||||
item["title"] = "%s - LIVE" % vod['title']
|
||||
item["category"] = "live"
|
||||
else:
|
||||
link = vod['url']
|
||||
item["title"] = vod['title']
|
||||
item["category"] = vod['broadcast_type']
|
||||
item["link"] = link
|
||||
item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview']['large'])
|
||||
if vod.get('game'):
|
||||
item["description"] += "<br/>" + vod['game']
|
||||
if vod.get('description_html'):
|
||||
item["description"] += "<br/>" + vod['description_html']
|
||||
d = datetime.datetime.strptime(vod['created_at'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
item["pubDate"] = d.timetuple()
|
||||
item["guid"] = vod['_id']
|
||||
if vod["status"] == "recording": # To show a different news item when recording is over
|
||||
item["guid"] += "_live"
|
||||
feed.items.append(item)
|
||||
except KeyError as e:
|
||||
logging.warning('Issue with json: %s\nException: %s' % (vods_info, e))
|
||||
abort(404)
|
||||
|
||||
return feed.format_rss2_string()
|
||||
|
||||
|
||||
# For debug
|
||||
if __name__ == "__main__":
|
||||
app.run(host='127.0.0.1', port=8080, debug=True)
|
||||
|
|
|
|||
|
|
@ -1,71 +0,0 @@
|
|||
#
|
||||
# Copyright 2014 Jon Wayne Parrott, [proppy], Michael R. Bernstein
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Notes:
|
||||
# - Imported from https://github.com/jonparrott/Darth-Vendor/.
|
||||
# - Added license header.
|
||||
# - Renamed `darth.vendor` to `vendor.add` to match upcoming SDK interface.
|
||||
# - Renamed `position` param to `index` to match upcoming SDK interface.
|
||||
# - Removed funny arworks docstring.
|
||||
|
||||
import site
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
|
||||
def add(folder, index=1):
|
||||
"""
|
||||
Adds the given folder to the python path. Supports namespaced packages.
|
||||
By default, packages in the given folder take precedence over site-packages
|
||||
and any previous path manipulations.
|
||||
|
||||
Args:
|
||||
folder: Path to the folder containing packages, relative to ``os.getcwd()``
|
||||
position: Where in ``sys.path`` to insert the vendor packages. By default
|
||||
this is set to 1. It is inadvisable to set it to 0 as it will override
|
||||
any modules in the current working directory.
|
||||
"""
|
||||
|
||||
# Check if the path contains a virtualenv.
|
||||
site_dir = os.path.join(folder, 'lib', 'python' + sys.version[:3], 'site-packages')
|
||||
if os.path.exists(site_dir):
|
||||
folder = site_dir
|
||||
# Otherwise it's just a normal path, make it absolute.
|
||||
else:
|
||||
folder = os.path.join(os.path.dirname(__file__), folder)
|
||||
|
||||
# Use site.addsitedir() because it appropriately reads .pth
|
||||
# files for namespaced packages. Unfortunately, there's not an
|
||||
# option to choose where addsitedir() puts its paths in sys.path
|
||||
# so we have to do a little bit of magic to make it play along.
|
||||
|
||||
# We're going to grab the current sys.path and split it up into
|
||||
# the first entry and then the rest. Essentially turning
|
||||
# ['.', '/site-packages/x', 'site-packages/y']
|
||||
# into
|
||||
# ['.'] and ['/site-packages/x', 'site-packages/y']
|
||||
# The reason for this is we want '.' to remain at the top of the
|
||||
# list but we want our vendor files to override everything else.
|
||||
sys.path, remainder = sys.path[:1], sys.path[1:]
|
||||
|
||||
# Now we call addsitedir which will append our vendor directories
|
||||
# to sys.path (which was truncated by the last step.)
|
||||
site.addsitedir(folder)
|
||||
|
||||
# Finally, we'll add the paths we removed back.
|
||||
# The final product is something like this:
|
||||
# ['.', '/vendor-folder', /site-packages/x', 'site-packages/y']
|
||||
sys.path.extend(remainder)
|
||||
Loading…
Reference in New Issue