From 527077fab4f0a7d8e08d1af144b29bb955923042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matt=C3=A9o=20Delabre?= Date: Mon, 13 Sep 2021 00:00:24 +0200 Subject: [PATCH] Add youtube endpoint --- feedleware/__init__.py | 3 +- feedleware/twitch/__init__.py | 2 +- feedleware/twitch/feed.py | 10 +-- feedleware/twitch/twitch.py | 43 +++------- feedleware/util.py | 33 ++++++++ feedleware/youtube/__init__.py | 21 +++++ feedleware/youtube/feed.py | 47 +++++++++++ feedleware/youtube/youtube.py | 141 +++++++++++++++++++++++++++++++++ 8 files changed, 260 insertions(+), 40 deletions(-) create mode 100644 feedleware/util.py create mode 100644 feedleware/youtube/__init__.py create mode 100644 feedleware/youtube/feed.py create mode 100644 feedleware/youtube/youtube.py diff --git a/feedleware/__init__.py b/feedleware/__init__.py index 70c120f..10172f6 100644 --- a/feedleware/__init__.py +++ b/feedleware/__init__.py @@ -3,12 +3,13 @@ import logging import sys from os import environ from flask import Flask -from . import twitch +from . import twitch, youtube logger = logging.getLogger(__name__) blueprints = { "twitch": twitch, + "youtube": youtube, } diff --git a/feedleware/twitch/__init__.py b/feedleware/twitch/__init__.py index 55241b9..f2c6cac 100644 --- a/feedleware/twitch/__init__.py +++ b/feedleware/twitch/__init__.py @@ -9,7 +9,7 @@ def create_blueprint(config): twitch = Blueprint("twitch", __name__) @twitch.route("/", methods=["GET", "HEAD"]) - def vod(login: str): + def get(login: str): try: return ( construct_rss(client, login), diff --git a/feedleware/twitch/feed.py b/feedleware/twitch/feed.py index d4b3009..9b4327b 100644 --- a/feedleware/twitch/feed.py +++ b/feedleware/twitch/feed.py @@ -1,12 +1,8 @@ -from datetime import datetime from ..feedformatter import Feed +from ..util import parse_iso from .twitch import APIClient -def parse_iso(iso: str) -> datetime: - return datetime.fromisoformat(iso.replace("Z", "+00:00")) - - def construct_rss(client: APIClient, login: str) -> str: """ Build a RSS stream for a Twitch user. @@ -28,9 +24,9 @@ def construct_rss(client: APIClient, login: str) -> str: # Set the feed/channel level properties feed.feed["title"] = user_info["display_name"] feed.feed["link"] = user_url - feed.feed["author"] = "Twitch RSS Generated" + feed.feed["author"] = "Feedleware" feed.feed["description"] = user_info["description"] - feed.feed["ttl"] = '10' + feed.feed["ttl"] = "10" if stream is not None: item = {} diff --git a/feedleware/twitch/twitch.py b/feedleware/twitch/twitch.py index d1b460b..af4e596 100644 --- a/feedleware/twitch/twitch.py +++ b/feedleware/twitch/twitch.py @@ -8,6 +8,7 @@ import urllib import urllib.request from typing import Any, Dict, Optional from cachetools import cached, TTLCache +from ..util import send_with_retry HTTPRequest = urllib.request.Request @@ -43,40 +44,17 @@ class APIClient: """ self.client_id: str = client_id self.secret: str = secret - self.retries: int = 3 + self.retries = retries self.oauth_token: str = "" self.oauth_expire_epoch: int = 0 - def _send_with_retry(self, request: HTTPRequest) -> HTTPResponse: - """ - Send an HTTP request and retry in case of failure. - - The number of retries is configured by `self.retries`. - - :param request: request to try sending - :returns: response sent by the server - :throws HTTPException: if the number of retries is exceeded - """ - retries = self.retries - last_err = HTTPException() - - while retries: - try: - return urllib.request.urlopen(request, timeout=3) - except HTTPException as err: - logger.warning("HTTP error: %s", err) - retries -= 1 - last_err = err - - raise last_err - - def authorize(self) -> bool: """ Refresh the current OAuth app access token if needed. - This uses the Twitch OAuth client credentials flow. See: + This uses the Twitch OAuth client credentials flow. See + for details. :returns: true if the OAuth app access token was refreshed, false if the existing one was still valid @@ -99,7 +77,7 @@ class APIClient: method="POST", ) - http_response = self._send_with_retry(request) + http_response = send_with_retry(request, self.retries) response = json.loads(http_response.read().decode("utf-8")) self.oauth_token = response["access_token"] @@ -138,7 +116,7 @@ class APIClient: method=method, ) - http_response = self._send_with_retry(request) + http_response = send_with_retry(request, self.retries) if http_response.info().get("Content-Encoding") == "gzip": return json.loads(gzip.decompress(http_response.read())) @@ -150,6 +128,8 @@ class APIClient: """ Get information about a user. + See for details. + :param login: user login :returns: user information :throws HTTPException: if the query fails @@ -163,13 +143,10 @@ class APIClient: method="GET", data={"login": login}, ) - assert type(response) == dict if "data" not in response or not response["data"]: raise NoSuchUser(f"User '{login}' does not exist") - assert type(response["data"]) == list - assert type(response["data"][0]) == dict return response["data"][0] @cached(cache=TTLCache(maxsize=1000, ttl=10 * 60)) @@ -177,6 +154,8 @@ class APIClient: """ Get the list of videos from a channel. + See for details. + :param channel_id: channel ID :returns: list of videos :throws HTTPException: if the query fails @@ -198,6 +177,8 @@ class APIClient: """ Get the information about the stream currently active on a channel. + See for details. + :param channel_id: channel ID :returns: stream information or None :throws HTTPException: if the query fails diff --git a/feedleware/util.py b/feedleware/util.py new file mode 100644 index 0000000..96823a4 --- /dev/null +++ b/feedleware/util.py @@ -0,0 +1,33 @@ +from datetime import datetime +import http +import urllib +import urllib.request + +HTTPRequest = urllib.request.Request +HTTPResponse = http.client.HTTPResponse +HTTPException = http.client.HTTPException + + +def parse_iso(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +def send_with_retry(request: HTTPRequest, retries: int) -> HTTPResponse: + """ + Send an HTTP request and retry in case of failure. + + :param request: request to try sending + :param retries: number of retries + :returns: response sent by the server + :throws HTTPException: if the number of retries is exceeded + """ + last_err = HTTPException() + + while retries: + try: + return urllib.request.urlopen(request, timeout=3) + except HTTPException as err: + retries -= 1 + last_err = err + + raise last_err diff --git a/feedleware/youtube/__init__.py b/feedleware/youtube/__init__.py new file mode 100644 index 0000000..1bd99ea --- /dev/null +++ b/feedleware/youtube/__init__.py @@ -0,0 +1,21 @@ +from flask import abort, Blueprint +from .youtube import APIClient, NoSuchChannel +from .feed import construct_rss + + +def create_blueprint(config): + """Create a YouTube endpoint blueprint.""" + client = APIClient(config["key"]) + twitch = Blueprint("youtube", __name__) + + @twitch.route("/", methods=["GET", "HEAD"]) + def vod(login: str): + try: + return ( + construct_rss(client, login), + {"Content-Type": "application/rss+xml"}, + ) + except NoSuchChannel: + abort(404) + + return twitch diff --git a/feedleware/youtube/feed.py b/feedleware/youtube/feed.py new file mode 100644 index 0000000..cae5862 --- /dev/null +++ b/feedleware/youtube/feed.py @@ -0,0 +1,47 @@ +from ..feedformatter import Feed +from ..util import parse_iso +from .youtube import APIClient + + +def construct_rss(client: APIClient, channel_id: str) -> str: + """ + Build a RSS stream for a YouTube channel. + + :param client: YouTube API client + :param channel_id: channel ID + :returns: RSS stream + :raises HTTPException: if one of the requests fail + :raises NoSuchChannel: if the channel does not exist + """ + channel_info = client.channel(channel_id) + videos = client.playlist(channel_info["playlist"]) + + feed = Feed() + channel_url = f"https://www.youtube.com/channel/{channel_id}" + + # Set the feed/channel level properties + feed.feed["title"] = channel_info["title"] + feed.feed["link"] = channel_url + feed.feed["author"] = "Feedleware" + feed.feed["description"] = channel_info["description"] + feed.feed["ttl"] = "30" + + for video in videos: + item = {} + + video_id = video["resourceId"]["videoId"] + link = f"https://www.youtube.com/watch?v={video_id}" + thumbnail = video["thumbnails"]["standard"]["url"] + + item["guid"] = video["resourceId"]["videoId"] + item["title"] = video.get("title", "Untitled Video") + item["link"] = link + item["description"] = ( + f'

' + + video["description"] + ) + item["pubDate"] = parse_iso(video["publishedAt"]).timetuple() + + feed.items.append(item) + + return feed.format_rss2_string() diff --git a/feedleware/youtube/youtube.py b/feedleware/youtube/youtube.py new file mode 100644 index 0000000..7c5ee78 --- /dev/null +++ b/feedleware/youtube/youtube.py @@ -0,0 +1,141 @@ +import gzip +import http +import json +import logging +import urllib +import urllib.request +from typing import Any, Iterable, Optional, Tuple +from cachetools import cached, TTLCache +from ..util import send_with_retry + + +HTTPRequest = urllib.request.Request +HTTPResponse = http.client.HTTPResponse +HTTPException = http.client.HTTPException +logger = logging.getLogger(__name__) + + +class NoSuchChannel(Exception): + """Raised when an unknown channel is queried.""" + + +class APIClient: + """Client for the YouTube Data API.""" + + def __init__(self, key: str = "", retries: int = 3): + """ + Create a YouTube Data API client. + + See for details. + + :param key: YouTube API key + :param retries: number of times to retry each request in case of failure + """ + self.key = key + self.retries = retries + + def _query( + self, + url: str, + method: str = "GET", + data: Iterable[Tuple[str, str]] = [] + ) -> Any: + """ + Low-level method to query the API. + + :param url: URL to query + :param method: HTTP method to use + :param data: payload dictionary to send + :returns: JSON data + :throws HTTPException: if the query fails + """ + logger.debug("Querying %s %s %s", method, url, data) + + headers = { + "Accept": "application/json", + "Accept-Encoding": "gzip", + } + + payload = ( + *data, + ("key", self.key), + ) + + request = HTTPRequest( + url=f"{url}?{urllib.parse.urlencode(payload)}", + headers=headers, + method=method, + ) + + http_response = send_with_retry(request, self.retries) + + if http_response.info().get("Content-Encoding") == "gzip": + return json.loads(gzip.decompress(http_response.read())) + else: + return json.loads(http_response.read()) + + @cached(cache=TTLCache(maxsize=1000, ttl=24 * 60 * 60)) + def channel(self, channel_id: str) -> Any: + """ + Get information about a channel. + + See + for details. + + :param channel_id: channel ID + :returns: channel information + :throws HTTPException: if the query fails + :throws NoSuchChannel: if the channel doesn’t exist + """ + response = self._query( + url="https://youtube.googleapis.com/youtube/v3/channels", + method="GET", + data=( + ("part", "id"), + ("part", "snippet"), + ("part", "contentDetails"), + ("id", channel_id), + ("maxResults", 1), + ) + ) + + if response["pageInfo"]["totalResults"] == 0: + raise NoSuchChannel(f"Channel '{channel_id}' does not exist") + + data = response["items"][0] + + return { + "id": data["id"], + "playlist": data["contentDetails"]["relatedPlaylists"]["uploads"], + **response["items"][0]["snippet"], + } + + @cached(cache=TTLCache(maxsize=1000, ttl=30 * 60)) + def playlist(self, playlist_id: str) -> Any: + """ + Get the latest videos from a playlist. + + See + for details. + + :param playlist_id: channel ID + :returns: list of latest videos + :throws HTTPException: if the query fails + """ + response = self._query( + url="https://youtube.googleapis.com/youtube/v3/playlistItems", + method="GET", + data=( + ("part", "snippet"), + ("part", "status"), + ("playlistId", playlist_id), + ("maxResults", 50), + ) + ) + + return [ + item["snippet"] + for item in response["items"] + if item["status"]["privacyStatus"] == "public" + and item["snippet"]["resourceId"]["kind"] == "youtube#video" + ]