Migrate application to python3
Notables: - Python 3 migration - The client id is now set via environment variable instead of hardcoding it in a py file - Using Flask instead of web2 - Migrate away from Memcache (Google depeciated it in py3) to an in-memory data structure - Moved landing page to static serving
This commit is contained in:
parent
fc67ccba75
commit
5fccffc2f0
|
@ -7,6 +7,7 @@ __pycache__/
|
||||||
|
|
||||||
# Distribution / packaging
|
# Distribution / packaging
|
||||||
.Python
|
.Python
|
||||||
|
venv/
|
||||||
env/
|
env/
|
||||||
build/
|
build/
|
||||||
develop-eggs/
|
develop-eggs/
|
||||||
|
|
|
@ -1,15 +1,16 @@
|
||||||
## Twitch RSS Webapp for Google App Engine
|
## Twitch RSS Webapp for Google App Engine
|
||||||
This project is a very small web application for serving RSS feed for broadcasts
|
This project is a very small web application for serving RSS feed for broadcasts
|
||||||
in Twitch. It fetches data from [Twitch API](https://github.com/justintv/twitch-api) and caches in Memcache.
|
in Twitch. It fetches data from [Twitch API](https://dev.twitch.tv/docs) and caches in Memcache.
|
||||||
The engine is webapp2.
|
The engine is webapp2.
|
||||||
|
|
||||||
A running version can be tried out at:
|
A running version can be tried out at:
|
||||||
https://twitchrss.appspot.com/vod/twitch
|
https://twitchrss.appspot.com/vod/twitch
|
||||||
|
|
||||||
There is also a VOD only endpoint if you don't want to see ongoing streams which are known to break some readers:
|
There is also a VOD only endpoint if you don't want to see ongoing streams which are known to break some readers:
|
||||||
https://twitchrss.appspot.com/vodonly/twitch
|
https://twitchrss.appspot.com/vodonly/twitch
|
||||||
|
|
||||||
### Deployment
|
### Deployment
|
||||||
|
First you should set your own Twitch API client ID in the app.yaml.
|
||||||
See how to deploy on [Google App Engine](https://cloud.google.com/appengine/docs/python/gettingstartedpython27/introduction).
|
See how to deploy on [Google App Engine](https://cloud.google.com/appengine/docs/python/gettingstartedpython27/introduction).
|
||||||
|
|
||||||
### Other things
|
### Other things
|
||||||
|
|
|
@ -1,15 +1,21 @@
|
||||||
runtime: python27
|
runtime: python38
|
||||||
threadsafe: true
|
|
||||||
|
entrypoint: gunicorn -b :$PORT twitchrss:app
|
||||||
|
|
||||||
|
env_variables:
|
||||||
|
TWITCH_CLIENT_ID: __INSERT_TWITCH_CLIENT_ID_HERE__
|
||||||
|
|
||||||
handlers:
|
handlers:
|
||||||
- url: /favicon\.ico
|
- url: /favicon\.ico
|
||||||
static_files: favicon.ico
|
static_files: favicon.ico
|
||||||
upload: favicon\.ico
|
upload: favicon\.ico
|
||||||
|
|
||||||
- url: /.*
|
- url: /
|
||||||
script: twitchrss.app
|
static_files: index.html
|
||||||
|
upload: index\.html
|
||||||
|
|
||||||
|
- url: /.+
|
||||||
|
script: auto
|
||||||
|
|
||||||
automatic_scaling:
|
automatic_scaling:
|
||||||
max_instances: 1
|
max_instances: 1
|
||||||
max_idle_instances: 1
|
|
||||||
|
|
||||||
|
|
|
@ -1,17 +0,0 @@
|
||||||
#
|
|
||||||
# Copyright 2017, 2016 Laszlo Zeke
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
TWITCH_CLIENT_ID = 'Insert_key_here'
|
|
|
@ -1,6 +0,0 @@
|
||||||
"""`appengine_config` gets loaded when starting a new application instance."""
|
|
||||||
import vendor
|
|
||||||
# insert `lib` as a site directory so our `main` module can load
|
|
||||||
# third-party libraries, and override built-ins with newer
|
|
||||||
# versions.
|
|
||||||
vendor.add('lib')
|
|
|
@ -1,6 +1,6 @@
|
||||||
# Feedformatter
|
# Feedformatter
|
||||||
# Copyright (c) 2008, Luke Maurits <luke@maurits.id.au>
|
# Copyright (c) 2008, Luke Maurits <luke@maurits.id.au>
|
||||||
# Copyright (c) 2015, Laszlo Zeke
|
# Copyright (c) 2020, Laszlo Zeke
|
||||||
# All rights reserved.
|
# All rights reserved.
|
||||||
#
|
#
|
||||||
# Redistribution and use in source and binary forms, with or without
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
@ -28,7 +28,7 @@
|
||||||
|
|
||||||
__version__ = "0.4"
|
__version__ = "0.4"
|
||||||
|
|
||||||
from cStringIO import StringIO
|
from io import StringIO
|
||||||
|
|
||||||
# This "staircase" of import attempts is ugly. If there's a nicer way to do
|
# This "staircase" of import attempts is ugly. If there's a nicer way to do
|
||||||
# this, please let me know!
|
# this, please let me know!
|
||||||
|
@ -75,7 +75,7 @@ _rss2_channel_mappings = (
|
||||||
(("title",), "title"),
|
(("title",), "title"),
|
||||||
(("link", "url"), "link"),
|
(("link", "url"), "link"),
|
||||||
(("description", "desc", "summary"), "description"),
|
(("description", "desc", "summary"), "description"),
|
||||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("rss2",x)),
|
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("rss2",x)),
|
||||||
(("category",), "category"),
|
(("category",), "category"),
|
||||||
(("language",), "language"),
|
(("language",), "language"),
|
||||||
(("copyright",), "copyright"),
|
(("copyright",), "copyright"),
|
||||||
|
@ -91,9 +91,9 @@ _rss2_item_mappings = (
|
||||||
(("link", "url"), "link"),
|
(("link", "url"), "link"),
|
||||||
(("description", "desc", "summary"), "description"),
|
(("description", "desc", "summary"), "description"),
|
||||||
(("guid", "id"), "guid"),
|
(("guid", "id"), "guid"),
|
||||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("rss2",x)),
|
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("rss2",x)),
|
||||||
(("category",), "category"),
|
(("category",), "category"),
|
||||||
(("author",), "author", lambda(x): _rssify_author(x))
|
(("author",), "author", lambda x: _rssify_author(x))
|
||||||
)
|
)
|
||||||
|
|
||||||
# Atom 1.0 ----------
|
# Atom 1.0 ----------
|
||||||
|
@ -102,19 +102,19 @@ _atom_feed_mappings = (
|
||||||
(("title",), "title"),
|
(("title",), "title"),
|
||||||
(("link", "url"), "id"),
|
(("link", "url"), "id"),
|
||||||
(("description", "desc", "summary"), "subtitle"),
|
(("description", "desc", "summary"), "subtitle"),
|
||||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("atom",x)),
|
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("atom",x)),
|
||||||
(("category",), "category"),
|
(("category",), "category"),
|
||||||
(("author",), "author", lambda(x): _atomise_author(x))
|
(("author",), "author", lambda x: _atomise_author(x))
|
||||||
)
|
)
|
||||||
|
|
||||||
_atom_item_mappings = (
|
_atom_item_mappings = (
|
||||||
(("title",), "title"),
|
(("title",), "title"),
|
||||||
(("link", "url"), "id"),
|
(("link", "url"), "id"),
|
||||||
(("link", "url"), "link", lambda(x): _atomise_link(x)),
|
(("link", "url"), "link", lambda x: _atomise_link(x)),
|
||||||
(("description", "desc", "summary"), "summary"),
|
(("description", "desc", "summary"), "summary"),
|
||||||
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda(x): _format_datetime("atom",x)),
|
(("pubDate", "pubdate", "date", "published", "updated"), "pubDate", lambda x: _format_datetime("atom",x)),
|
||||||
(("category",), "category"),
|
(("category",), "category"),
|
||||||
(("author",), "author", lambda(x): _atomise_author(x))
|
(("author",), "author", lambda x: _atomise_author(x))
|
||||||
)
|
)
|
||||||
|
|
||||||
def _get_tz_offset():
|
def _get_tz_offset():
|
||||||
|
@ -146,20 +146,20 @@ def _convert_datetime(time):
|
||||||
elif type(time) is int or type(time) is float:
|
elif type(time) is int or type(time) is float:
|
||||||
# Assume this is a seconds-since-epoch time
|
# Assume this is a seconds-since-epoch time
|
||||||
return localtime(time)
|
return localtime(time)
|
||||||
elif type(time) is str:
|
elif type(time) is str:
|
||||||
if time.isalnum():
|
if time.isalnum():
|
||||||
# String is alphanumeric - a time stamp?
|
# String is alphanumeric - a time stamp?
|
||||||
try:
|
try:
|
||||||
return strptime(time, "%a, %d %b %Y %H:%M:%S %Z")
|
return strptime(time, "%a, %d %b %Y %H:%M:%S %Z")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise Exception("Unrecongised time format!")
|
raise Exception("Unrecongised time format!")
|
||||||
else:
|
else:
|
||||||
# Maybe this is a string of an epoch time?
|
# Maybe this is a string of an epoch time?
|
||||||
try:
|
try:
|
||||||
return localtime(float(time))
|
return localtime(float(time))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
# Guess not.
|
# Guess not.
|
||||||
raise Exception("Unrecongised time format!")
|
raise Exception("Unrecongised time format!")
|
||||||
else:
|
else:
|
||||||
# No idea what this is. Give up!
|
# No idea what this is. Give up!
|
||||||
raise Exception("Unrecongised time format!")
|
raise Exception("Unrecongised time format!")
|
||||||
|
@ -171,7 +171,7 @@ def _format_datetime(feed_type, time):
|
||||||
used in a validly formatted feed of type feed_type. Raise an
|
used in a validly formatted feed of type feed_type. Raise an
|
||||||
Exception if this cannot be done.
|
Exception if this cannot be done.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# First, convert time into a time structure
|
# First, convert time into a time structure
|
||||||
time = _convert_datetime(time)
|
time = _convert_datetime(time)
|
||||||
|
|
||||||
|
@ -187,7 +187,7 @@ def _atomise_link(link):
|
||||||
return dict
|
return dict
|
||||||
else:
|
else:
|
||||||
return {"href" : link}
|
return {"href" : link}
|
||||||
|
|
||||||
def _atomise_author(author):
|
def _atomise_author(author):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
@ -214,7 +214,7 @@ def _rssify_author(author):
|
||||||
Convert author from whatever it is to a plain old email string for
|
Convert author from whatever it is to a plain old email string for
|
||||||
use in an RSS 2.0 feed.
|
use in an RSS 2.0 feed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if type(author) is dict:
|
if type(author) is dict:
|
||||||
try:
|
try:
|
||||||
return author["email"]
|
return author["email"]
|
||||||
|
@ -241,8 +241,8 @@ def _add_subelems(root_element, mappings, dictionary):
|
||||||
elif len(mapping) == 3:
|
elif len(mapping) == 3:
|
||||||
value = mapping[2](dictionary[key])
|
value = mapping[2](dictionary[key])
|
||||||
_add_subelem(root_element, mapping[1], value)
|
_add_subelem(root_element, mapping[1], value)
|
||||||
break
|
break
|
||||||
|
|
||||||
def _add_subelem(root_element, name, value):
|
def _add_subelem(root_element, name, value):
|
||||||
|
|
||||||
if value is None:
|
if value is None:
|
||||||
|
@ -282,7 +282,7 @@ def _stringify(tree, pretty):
|
||||||
class Feed:
|
class Feed:
|
||||||
|
|
||||||
### INTERNAL METHODS ------------------------------
|
### INTERNAL METHODS ------------------------------
|
||||||
|
|
||||||
def __init__(self, feed=None, items=None):
|
def __init__(self, feed=None, items=None):
|
||||||
|
|
||||||
if feed:
|
if feed:
|
||||||
|
@ -296,7 +296,7 @@ class Feed:
|
||||||
self.entries = self.items
|
self.entries = self.items
|
||||||
|
|
||||||
### RSS 1.0 STUFF ------------------------------
|
### RSS 1.0 STUFF ------------------------------
|
||||||
|
|
||||||
def validate_rss1(self):
|
def validate_rss1(self):
|
||||||
|
|
||||||
"""Raise an InvalidFeedException if the feed cannot be validly
|
"""Raise an InvalidFeedException if the feed cannot be validly
|
||||||
|
@ -325,14 +325,14 @@ class Feed:
|
||||||
if "link" not in item:
|
if "link" not in item:
|
||||||
raise InvalidFeedException("Each item element in an RSS 1.0 "
|
raise InvalidFeedException("Each item element in an RSS 1.0 "
|
||||||
"feed must contain a link subelement")
|
"feed must contain a link subelement")
|
||||||
|
|
||||||
def format_rss1_string(self, validate=True, pretty=False):
|
def format_rss1_string(self, validate=True, pretty=False):
|
||||||
|
|
||||||
"""Format the feed as RSS 1.0 and return the result as a string."""
|
"""Format the feed as RSS 1.0 and return the result as a string."""
|
||||||
|
|
||||||
if validate:
|
if validate:
|
||||||
self.validate_rss1()
|
self.validate_rss1()
|
||||||
RSS1root = ET.Element( 'rdf:RDF',
|
RSS1root = ET.Element( 'rdf:RDF',
|
||||||
{"xmlns:rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
{"xmlns:rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||||
"xmlns" : "http://purl.org/rss/1.0/"} )
|
"xmlns" : "http://purl.org/rss/1.0/"} )
|
||||||
RSS1channel = ET.SubElement(RSS1root, 'channel',
|
RSS1channel = ET.SubElement(RSS1root, 'channel',
|
||||||
|
@ -394,7 +394,7 @@ class Feed:
|
||||||
RSS2root = ET.Element( 'rss', {'version':'2.0'} )
|
RSS2root = ET.Element( 'rss', {'version':'2.0'} )
|
||||||
RSS2channel = ET.SubElement( RSS2root, 'channel' )
|
RSS2channel = ET.SubElement( RSS2root, 'channel' )
|
||||||
_add_subelems(RSS2channel, _rss2_channel_mappings, self.feed)
|
_add_subelems(RSS2channel, _rss2_channel_mappings, self.feed)
|
||||||
for item in self.items:
|
for item in self.items:
|
||||||
RSS2item = ET.SubElement ( RSS2channel, 'item' )
|
RSS2item = ET.SubElement ( RSS2channel, 'item' )
|
||||||
_add_subelems(RSS2item, _rss2_item_mappings, item)
|
_add_subelems(RSS2item, _rss2_item_mappings, item)
|
||||||
return _stringify(RSS2root, pretty=pretty)
|
return _stringify(RSS2root, pretty=pretty)
|
||||||
|
@ -472,11 +472,11 @@ def main():
|
||||||
item["guid"] = "1234567890"
|
item["guid"] = "1234567890"
|
||||||
feed.items.append(item)
|
feed.items.append(item)
|
||||||
print("---- RSS 1.0 ----")
|
print("---- RSS 1.0 ----")
|
||||||
print feed.format_rss1_string(pretty=True)
|
print(feed.format_rss1_string(pretty=True))
|
||||||
print("---- RSS 2.0 ----")
|
print("---- RSS 2.0 ----")
|
||||||
print feed.format_rss2_string(pretty=True)
|
print(feed.format_rss2_string(pretty=True))
|
||||||
print("---- Atom 1.0 ----")
|
print("---- Atom 1.0 ----")
|
||||||
print feed.format_atom_string(pretty=True)
|
print(feed.format_atom_string(pretty=True))
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
|
@ -0,0 +1,17 @@
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Twitch stream RSS generator</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<p style="font-family: helvetica; font-size:20pt; padding: 20px;">
|
||||||
|
Twitch stream RSS generator
|
||||||
|
</p>
|
||||||
|
<p style="font-family: helvetica; font-size:12pt; padding: 20px;">
|
||||||
|
You can get RSS of broadcasts by subscribing to https://twitchrss.appspot.com/vod/<channel name><br/>
|
||||||
|
For example: <a href="https://twitchrss.appspot.com/vod/riotgames">https://twitchrss.appspot.com/vod/riotgames</a><br/><br/>
|
||||||
|
You can use the /vodonly handle to get only vods without ongoing streams.
|
||||||
|
Not endorsed by Twitch.tv, just a fun project.<br/>
|
||||||
|
<a href="https://github.com/lzeke0/TwitchRSS">Project home</a>
|
||||||
|
</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
|
@ -0,0 +1,31 @@
|
||||||
|
appdirs==1.4.3
|
||||||
|
CacheControl==0.12.6
|
||||||
|
certifi==2019.11.28
|
||||||
|
chardet==3.0.4
|
||||||
|
click==7.1.2
|
||||||
|
colorama==0.4.3
|
||||||
|
contextlib2==0.6.0
|
||||||
|
distlib==0.3.0
|
||||||
|
distro==1.4.0
|
||||||
|
expiringdict==1.2.1
|
||||||
|
Flask==1.1.2
|
||||||
|
gunicorn==20.0.4
|
||||||
|
html5lib==1.0.1
|
||||||
|
idna==2.8
|
||||||
|
ipaddr==2.2.0
|
||||||
|
itsdangerous==1.1.0
|
||||||
|
Jinja2==2.11.2
|
||||||
|
lockfile==0.12.2
|
||||||
|
MarkupSafe==1.1.1
|
||||||
|
msgpack==0.6.2
|
||||||
|
packaging==20.3
|
||||||
|
pep517==0.8.2
|
||||||
|
progress==1.5
|
||||||
|
pyparsing==2.4.6
|
||||||
|
pytoml==0.1.21
|
||||||
|
requests==2.22.0
|
||||||
|
retrying==1.3.3
|
||||||
|
six==1.14.0
|
||||||
|
urllib3==1.25.8
|
||||||
|
webencodings==0.5.1
|
||||||
|
Werkzeug==1.0.1
|
|
@ -1,5 +1,5 @@
|
||||||
#
|
#
|
||||||
# Copyright 2017, 2016 Laszlo Zeke
|
# Copyright 2020 Laszlo Zeke
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
|
@ -14,192 +14,166 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
import webapp2
|
from flask import abort, Flask
|
||||||
from webapp2 import Route
|
import urllib
|
||||||
import urllib2
|
|
||||||
import json
|
import json
|
||||||
import datetime
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
|
from os import environ
|
||||||
from feedformatter import Feed
|
from feedformatter import Feed
|
||||||
from google.appengine.api import memcache
|
from expiringdict import ExpiringDict
|
||||||
from app_id import TWITCH_CLIENT_ID
|
from io import BytesIO
|
||||||
from StringIO import StringIO
|
|
||||||
import gzip
|
import gzip
|
||||||
|
|
||||||
|
|
||||||
VODCACHE_PREFIX = 'vodcache'
|
|
||||||
USERIDCACHE_PREFIX = 'userid'
|
|
||||||
VOD_URL_TEMPLATE = 'https://api.twitch.tv/kraken/channels/%s/videos?broadcast_type=archive,highlight,upload&limit=10'
|
VOD_URL_TEMPLATE = 'https://api.twitch.tv/kraken/channels/%s/videos?broadcast_type=archive,highlight,upload&limit=10'
|
||||||
USERID_URL_TEMPLATE = 'https://api.twitch.tv/kraken/users?login=%s'
|
USERID_URL_TEMPLATE = 'https://api.twitch.tv/kraken/users?login=%s'
|
||||||
VODCACHE_LIFETIME = 600
|
VODCACHE_LIFETIME = 600
|
||||||
USERIDCACHE_LIFETIME = 0 # No expire
|
USERIDCACHE_LIFETIME = 24 * 60 * 60
|
||||||
|
CHANNEL_FILTER = re.compile("^[a-zA-Z0-9_]{2,25}$")
|
||||||
|
TWITCH_CLIENT_ID = environ.get("TWITCH_CLIENT_ID")
|
||||||
|
logging.basicConfig(level=logging.DEBUG if environ.get('DEBUG') else logging.INFO)
|
||||||
|
|
||||||
|
if not TWITCH_CLIENT_ID:
|
||||||
|
raise Exception("Twitch API client id is not set.")
|
||||||
|
|
||||||
|
|
||||||
class MainPage(webapp2.RequestHandler):
|
app = Flask(__name__)
|
||||||
def get(self):
|
|
||||||
self.response.headers['Content-Type'] = 'text/html'
|
vodcache = ExpiringDict(max_len=200, max_age_seconds=VODCACHE_LIFETIME)
|
||||||
html_resp = """
|
useridcache = ExpiringDict(max_len=1000, max_age_seconds=USERIDCACHE_LIFETIME)
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Twitch stream RSS generator</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<p style="font-family: helvetica; font-size:20pt; padding: 20px;">
|
|
||||||
Twitch stream RSS generator
|
|
||||||
</p>
|
|
||||||
<p style="font-family: helvetica; font-size:12pt; padding: 20px;">
|
|
||||||
You can get RSS of broadcasts by subscribing to https://twitchrss.appspot.com/vod/<channel name><br/>
|
|
||||||
For example: <a href="https://twitchrss.appspot.com/vod/riotgames">https://twitchrss.appspot.com/vod/riotgames</a><br/><br/>
|
|
||||||
You can use the /vodonly handle to get only vods without ongoing streams.
|
|
||||||
Not endorsed by Twitch.tv, just a fun project.<br/>
|
|
||||||
<a href="https://github.com/lzeke0/TwitchRSS">Project home</a>
|
|
||||||
</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
self.response.write(html_resp)
|
|
||||||
|
|
||||||
|
|
||||||
class RSSVoDServer(webapp2.RequestHandler):
|
@app.route('/vod/<string:channel>', methods=['GET', 'HEAD'])
|
||||||
def get(self, channel):
|
def vod(channel):
|
||||||
self._get_inner(channel)
|
if CHANNEL_FILTER.match(channel):
|
||||||
|
return get_inner(channel)
|
||||||
|
else:
|
||||||
|
abort(404)
|
||||||
|
|
||||||
def _get_inner(self, channel, add_live=True):
|
|
||||||
userid_json = self.fetch_userid(channel)
|
|
||||||
(channel_display_name, channel_id) = self.extract_userid(json.loads(userid_json))
|
|
||||||
channel_json = self.fetch_vods(channel_id)
|
|
||||||
decoded_json = json.loads(channel_json)
|
|
||||||
rss_data = self.construct_rss(channel, decoded_json, channel_display_name, add_live)
|
|
||||||
self.response.headers['Content-Type'] = 'application/rss+xml'
|
|
||||||
self.response.write(rss_data)
|
|
||||||
|
|
||||||
def head(self,channel):
|
@app.route('/vodonly/<string:channel>', methods=['GET', 'HEAD'])
|
||||||
self.get(channel)
|
def vodonly(channel):
|
||||||
|
if CHANNEL_FILTER.match(channel):
|
||||||
|
return get_inner(channel, add_live=False)
|
||||||
|
else:
|
||||||
|
abort(404)
|
||||||
|
|
||||||
def fetch_userid(self, channel_name):
|
|
||||||
return self.fetch_or_cache_object(channel_name, USERIDCACHE_PREFIX, USERID_URL_TEMPLATE, USERIDCACHE_LIFETIME)
|
|
||||||
|
|
||||||
def fetch_vods(self, channel_id):
|
def get_inner(channel, add_live=True):
|
||||||
return self.fetch_or_cache_object(channel_id, VODCACHE_PREFIX, VOD_URL_TEMPLATE, VODCACHE_LIFETIME)
|
userid_json = fetch_userid(channel)
|
||||||
|
(channel_display_name, channel_id) = extract_userid(json.loads(userid_json))
|
||||||
|
channel_json = fetch_vods(channel_id)
|
||||||
|
decoded_json = json.loads(channel_json)
|
||||||
|
rss_data = construct_rss(channel, decoded_json, channel_display_name, add_live)
|
||||||
|
headers = {'Content-Type': 'application/rss+xml'}
|
||||||
|
return rss_data, headers
|
||||||
|
|
||||||
def fetch_or_cache_object(self, channel, key_prefix, url_template, cache_time):
|
|
||||||
json_data = self.lookup_cache(channel, key_prefix)
|
def fetch_userid(channel_name):
|
||||||
|
return fetch_or_cache_object(channel_name, useridcache, USERID_URL_TEMPLATE)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_vods(channel_id):
|
||||||
|
return fetch_or_cache_object(channel_id, vodcache, VOD_URL_TEMPLATE)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_or_cache_object(key, cachedict, url_template):
|
||||||
|
json_data = cachedict.get(key)
|
||||||
|
if not json_data:
|
||||||
|
json_data = fetch_json(key, url_template)
|
||||||
if not json_data:
|
if not json_data:
|
||||||
json_data = self.fetch_json(channel, url_template)
|
abort(404)
|
||||||
if not json_data:
|
|
||||||
self.abort(404)
|
|
||||||
else:
|
|
||||||
self.store_cache(channel, json_data, key_prefix, cache_time)
|
|
||||||
return json_data
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def lookup_cache(channel_name, key_prefix):
|
|
||||||
cached_data = memcache.get('%s:v5:%s' % (key_prefix, channel_name))
|
|
||||||
if cached_data is not None:
|
|
||||||
logging.debug('Cache hit for %s' % channel_name)
|
|
||||||
return cached_data
|
|
||||||
else:
|
else:
|
||||||
logging.debug('Cache miss for %s' % channel_name)
|
cachedict[key] = json_data
|
||||||
return ''
|
return json_data
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def store_cache(channel_name, data, key_prefix, cache_lifetime):
|
def fetch_json(id, url_template):
|
||||||
|
url = url_template % id
|
||||||
|
headers = {
|
||||||
|
'Accept': 'application/vnd.twitchtv.v5+json',
|
||||||
|
'Client-ID': TWITCH_CLIENT_ID,
|
||||||
|
'Accept-Encoding': 'gzip'
|
||||||
|
}
|
||||||
|
request = urllib.request.Request(url, headers=headers)
|
||||||
|
retries = 0
|
||||||
|
while retries < 3:
|
||||||
try:
|
try:
|
||||||
logging.debug('Cached data for %s' % channel_name)
|
result = urllib.request.urlopen(request, timeout=3)
|
||||||
memcache.set('%s:v5:%s' % (key_prefix, channel_name), data, cache_lifetime)
|
logging.debug('Fetch from twitch for %s with code %s' % (id, result.getcode()))
|
||||||
except BaseException as e:
|
if result.info().get('Content-Encoding') == 'gzip':
|
||||||
logging.warning('Memcache exception: %s' % e)
|
logging.debug('Fetched gzip content')
|
||||||
return
|
buf = BytesIO(result.read())
|
||||||
|
f = gzip.GzipFile(fileobj=buf)
|
||||||
@staticmethod
|
return f.read()
|
||||||
def fetch_json(id, url_template):
|
return result.read()
|
||||||
url = url_template % id
|
except Exception as e:
|
||||||
headers = {
|
logging.warning("Fetch exception caught: %s" % e)
|
||||||
'Accept': 'application/vnd.twitchtv.v5+json',
|
retries += 1
|
||||||
'Client-ID': TWITCH_CLIENT_ID,
|
return None
|
||||||
'Accept-Encoding': 'gzip'
|
|
||||||
}
|
|
||||||
request = urllib2.Request(url, headers=headers)
|
|
||||||
retries = 0
|
|
||||||
while retries < 3:
|
|
||||||
try:
|
|
||||||
result = urllib2.urlopen(request, timeout=3)
|
|
||||||
logging.debug('Fetch from twitch for %s with code %s' % (id, result.getcode()))
|
|
||||||
if result.info().get('Content-Encoding') == 'gzip':
|
|
||||||
logging.debug('Fetched gzip content')
|
|
||||||
buf = StringIO(result.read())
|
|
||||||
f = gzip.GzipFile(fileobj=buf)
|
|
||||||
return f.read()
|
|
||||||
return result.read()
|
|
||||||
except BaseException as e:
|
|
||||||
logging.warning("Fetch exception caught: %s" % e)
|
|
||||||
retries += 1
|
|
||||||
return ''
|
|
||||||
|
|
||||||
def extract_userid(self, user_info):
|
|
||||||
userlist = user_info.get('users')
|
|
||||||
if not userlist:
|
|
||||||
logging.info('No such user found.')
|
|
||||||
self.abort(404)
|
|
||||||
# Get the first id in the list
|
|
||||||
userid = userlist[0].get('_id')
|
|
||||||
username = userlist[0].get('display_name')
|
|
||||||
if username and userid:
|
|
||||||
return username, userid
|
|
||||||
else:
|
|
||||||
logging.warning('Userid is not found in %s' % user_info)
|
|
||||||
self.abort(404)
|
|
||||||
|
|
||||||
def construct_rss(self, channel_name, vods_info, display_name, add_live=True):
|
|
||||||
feed = Feed()
|
|
||||||
|
|
||||||
# Set the feed/channel level properties
|
|
||||||
feed.feed["title"] = "%s's Twitch video RSS" % display_name
|
|
||||||
feed.feed["link"] = "https://twitchrss.appspot.com/"
|
|
||||||
feed.feed["author"] = "Twitch RSS Gen"
|
|
||||||
feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % display_name
|
|
||||||
feed.feed["ttl"] = '10'
|
|
||||||
|
|
||||||
# Create an item
|
|
||||||
try:
|
|
||||||
if vods_info['videos']:
|
|
||||||
for vod in vods_info['videos']:
|
|
||||||
item = {}
|
|
||||||
if vod["status"] == "recording":
|
|
||||||
if not add_live:
|
|
||||||
continue
|
|
||||||
link = "http://www.twitch.tv/%s" % channel_name
|
|
||||||
item["title"] = "%s - LIVE" % vod['title']
|
|
||||||
item["category"] = "live"
|
|
||||||
else:
|
|
||||||
link = vod['url']
|
|
||||||
item["title"] = vod['title']
|
|
||||||
item["category"] = vod['broadcast_type']
|
|
||||||
item["link"] = link
|
|
||||||
item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview']['large'])
|
|
||||||
if vod.get('game'):
|
|
||||||
item["description"] += "<br/>" + vod['game']
|
|
||||||
if vod.get('description_html'):
|
|
||||||
item["description"] += "<br/>" + vod['description_html']
|
|
||||||
d = datetime.datetime.strptime(vod['created_at'], '%Y-%m-%dT%H:%M:%SZ')
|
|
||||||
item["pubDate"] = d.timetuple()
|
|
||||||
item["guid"] = vod['_id']
|
|
||||||
if vod["status"] == "recording": # To show a different news item when recording is over
|
|
||||||
item["guid"] += "_live"
|
|
||||||
feed.items.append(item)
|
|
||||||
except KeyError as e:
|
|
||||||
logging.warning('Issue with json: %s\nException: %s' % (vods_info, e))
|
|
||||||
self.abort(404)
|
|
||||||
|
|
||||||
return feed.format_rss2_string()
|
|
||||||
|
|
||||||
class RSSVoDServerOnlyVoD(RSSVoDServer):
|
|
||||||
def get(self, channel):
|
|
||||||
self._get_inner(channel, add_live=False)
|
|
||||||
|
|
||||||
|
|
||||||
app = webapp2.WSGIApplication([
|
def extract_userid(user_info):
|
||||||
Route('/', MainPage),
|
userlist = user_info.get('users')
|
||||||
Route('/vod/<channel:[a-zA-Z0-9_]{2,25}>', RSSVoDServer),
|
if not userlist:
|
||||||
Route('/vodonly/<channel:[a-zA-Z0-9_]{2,25}>', RSSVoDServerOnlyVoD)
|
logging.info('No such user found.')
|
||||||
], debug=False)
|
abort(404)
|
||||||
|
# Get the first id in the list
|
||||||
|
userid = userlist[0].get('_id')
|
||||||
|
username = userlist[0].get('display_name')
|
||||||
|
if username and userid:
|
||||||
|
return username, userid
|
||||||
|
else:
|
||||||
|
logging.warning('Userid is not found in %s' % user_info)
|
||||||
|
abort(404)
|
||||||
|
|
||||||
|
|
||||||
|
def construct_rss(channel_name, vods_info, display_name, add_live=True):
|
||||||
|
feed = Feed()
|
||||||
|
|
||||||
|
# Set the feed/channel level properties
|
||||||
|
feed.feed["title"] = "%s's Twitch video RSS" % display_name
|
||||||
|
feed.feed["link"] = "https://twitchrss.appspot.com/"
|
||||||
|
feed.feed["author"] = "Twitch RSS Generated"
|
||||||
|
feed.feed["description"] = "The RSS Feed of %s's videos on Twitch" % display_name
|
||||||
|
feed.feed["ttl"] = '10'
|
||||||
|
|
||||||
|
# Create an item
|
||||||
|
try:
|
||||||
|
if vods_info['videos']:
|
||||||
|
for vod in vods_info['videos']:
|
||||||
|
item = {}
|
||||||
|
if vod["status"] == "recording":
|
||||||
|
if not add_live:
|
||||||
|
continue
|
||||||
|
link = "http://www.twitch.tv/%s" % channel_name
|
||||||
|
item["title"] = "%s - LIVE" % vod['title']
|
||||||
|
item["category"] = "live"
|
||||||
|
else:
|
||||||
|
link = vod['url']
|
||||||
|
item["title"] = vod['title']
|
||||||
|
item["category"] = vod['broadcast_type']
|
||||||
|
item["link"] = link
|
||||||
|
item["description"] = "<a href=\"%s\"><img src=\"%s\" /></a>" % (link, vod['preview']['large'])
|
||||||
|
if vod.get('game'):
|
||||||
|
item["description"] += "<br/>" + vod['game']
|
||||||
|
if vod.get('description_html'):
|
||||||
|
item["description"] += "<br/>" + vod['description_html']
|
||||||
|
d = datetime.datetime.strptime(vod['created_at'], '%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
item["pubDate"] = d.timetuple()
|
||||||
|
item["guid"] = vod['_id']
|
||||||
|
if vod["status"] == "recording": # To show a different news item when recording is over
|
||||||
|
item["guid"] += "_live"
|
||||||
|
feed.items.append(item)
|
||||||
|
except KeyError as e:
|
||||||
|
logging.warning('Issue with json: %s\nException: %s' % (vods_info, e))
|
||||||
|
abort(404)
|
||||||
|
|
||||||
|
return feed.format_rss2_string()
|
||||||
|
|
||||||
|
|
||||||
|
# For debug
|
||||||
|
if __name__ == "__main__":
|
||||||
|
app.run(host='127.0.0.1', port=8080, debug=True)
|
||||||
|
|
|
@ -1,71 +0,0 @@
|
||||||
#
|
|
||||||
# Copyright 2014 Jon Wayne Parrott, [proppy], Michael R. Bernstein
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
# Notes:
|
|
||||||
# - Imported from https://github.com/jonparrott/Darth-Vendor/.
|
|
||||||
# - Added license header.
|
|
||||||
# - Renamed `darth.vendor` to `vendor.add` to match upcoming SDK interface.
|
|
||||||
# - Renamed `position` param to `index` to match upcoming SDK interface.
|
|
||||||
# - Removed funny arworks docstring.
|
|
||||||
|
|
||||||
import site
|
|
||||||
import os.path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
def add(folder, index=1):
|
|
||||||
"""
|
|
||||||
Adds the given folder to the python path. Supports namespaced packages.
|
|
||||||
By default, packages in the given folder take precedence over site-packages
|
|
||||||
and any previous path manipulations.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
folder: Path to the folder containing packages, relative to ``os.getcwd()``
|
|
||||||
position: Where in ``sys.path`` to insert the vendor packages. By default
|
|
||||||
this is set to 1. It is inadvisable to set it to 0 as it will override
|
|
||||||
any modules in the current working directory.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Check if the path contains a virtualenv.
|
|
||||||
site_dir = os.path.join(folder, 'lib', 'python' + sys.version[:3], 'site-packages')
|
|
||||||
if os.path.exists(site_dir):
|
|
||||||
folder = site_dir
|
|
||||||
# Otherwise it's just a normal path, make it absolute.
|
|
||||||
else:
|
|
||||||
folder = os.path.join(os.path.dirname(__file__), folder)
|
|
||||||
|
|
||||||
# Use site.addsitedir() because it appropriately reads .pth
|
|
||||||
# files for namespaced packages. Unfortunately, there's not an
|
|
||||||
# option to choose where addsitedir() puts its paths in sys.path
|
|
||||||
# so we have to do a little bit of magic to make it play along.
|
|
||||||
|
|
||||||
# We're going to grab the current sys.path and split it up into
|
|
||||||
# the first entry and then the rest. Essentially turning
|
|
||||||
# ['.', '/site-packages/x', 'site-packages/y']
|
|
||||||
# into
|
|
||||||
# ['.'] and ['/site-packages/x', 'site-packages/y']
|
|
||||||
# The reason for this is we want '.' to remain at the top of the
|
|
||||||
# list but we want our vendor files to override everything else.
|
|
||||||
sys.path, remainder = sys.path[:1], sys.path[1:]
|
|
||||||
|
|
||||||
# Now we call addsitedir which will append our vendor directories
|
|
||||||
# to sys.path (which was truncated by the last step.)
|
|
||||||
site.addsitedir(folder)
|
|
||||||
|
|
||||||
# Finally, we'll add the paths we removed back.
|
|
||||||
# The final product is something like this:
|
|
||||||
# ['.', '/vendor-folder', /site-packages/x', 'site-packages/y']
|
|
||||||
sys.path.extend(remainder)
|
|
Loading…
Reference in New Issue