From bcb8143f1d8890095a307c63e16c8461da72591d Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 19 Mar 2023 01:40:08 +0000 Subject: [PATCH 1/6] [extractor/sbs] Overhaul extractor for new APIs --- youtube_dl/extractor/sbs.py | 237 ++++++++++++++++++++++++++++++++++-- 1 file changed, 227 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 0a806ee4e..e0af123e7 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -1,33 +1,65 @@ # coding: utf-8 from __future__ import unicode_literals +import sys + from .common import InfoExtractor +from ..compat import ( + compat_HTTPError, + compat_kwargs, + compat_str, +) from ..utils import ( - smuggle_url, + error_to_compat_str, ExtractorError, + float_or_none, + GeoRestrictedError, + int_or_none, + parse_duration, + parse_iso8601, + smuggle_url, + traverse_obj, + update_url_query, + url_or_none, + variadic, ) class SBSIE(InfoExtractor): IE_DESC = 'sbs.com.au' - _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand(?:/video/(?:single/)?|.*?\bplay=|/watch/)|news/(?:embeds/)?video/)(?P[0-9]+)' + _VALID_URL = r'''(?x) + https?://(?:www\.)?sbs\.com\.au/(?: + ondemand(?: + /video/(?:single/)?| + /movie/[^/]+/| + /(?:tv|news)-series/(?:[^/]+/){3}| + .*?\bplay=|/watch/ + )|news/(?:embeds/)?video/ + )(?P[0-9]+)''' + _EMBED_REGEX = [r'''(?x)] + (?: + ]+?src= + ) + (["\'])(?Phttps?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1'''] _TESTS = [{ # Original URL is handled by the generic IE which finds the iframe: # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', - 'md5': '3150cf278965eeabb5b4cea1c963fe0a', + 'md5': 'e49d0290cb4f40d893b8dfe760dce6b0', 'info_dict': { - 'id': '_rFBPRPO4pMR', + 'id': '320403011771', # '_rFBPRPO4pMR', 'ext': 'mp4', 'title': 'Dingo Conservation (The Feed)', 'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5', - 'thumbnail': r're:http://.*\.jpg', + 'thumbnail': r're:https?://.*\.jpg', 'duration': 308, 'timestamp': 1408613220, 'upload_date': '20140821', 'uploader': 'SBSC', }, + 'expected_warnings': ['Unable to download JSON metadata'], }, { 'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed', 'only_matching': True, @@ -46,9 +78,191 @@ class SBSIE(InfoExtractor): }, { 'url': 'https://www.sbs.com.au/ondemand/watch/1698704451971', 'only_matching': True, + }, { + 'url': 'https://www.sbs.com.au/ondemand/movie/coherence/1469404227931', + 'only_matching': True, + }, { + 'note': 'Live stream', + 'url': 'https://www.sbs.com.au/ondemand/video/1726824003663/sbs-24x7-live-stream-nsw', + 'only_matching': True, + }, { + 'url': 'https://www.sbs.com.au/ondemand/news-series/dateline/dateline-2022/dateline-s2022-ep26/2072245827515', + 'only_matching': True, + }, { + 'url': 'https://www.sbs.com.au/ondemand/tv-series/the-handmaids-tale/season-5/the-handmaids-tale-s5-ep1/2065631811776', + 'only_matching': True, }] + def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True): + if errnote is False: + return False + if errnote is None: + errnote = 'Unable to download webpage' + + errmsg = '%s: %s' % (errnote, error_to_compat_str(err)) + if fatal: + raise ExtractorError(errmsg, sys.exc_info()[2], cause=err, video_id=video_id) + else: + self._downloader.report_warning(errmsg) + return False + + def _download_webpage_handle(self, url, video_id, *args, **kwargs): + # note, errnote, fatal, encoding, data, headers={}, query, expected_status + # specialised to detect geo-block + + errnote = args[2] if len(args) > 2 else kwargs.get('errnote') + fatal = args[3] if len(args) > 3 else kwargs.get('fatal') + exp = args[7] if len(args) > 7 else kwargs.get('expected_status') + + exp = variadic(exp or [], allowed_types=(compat_str, )) + if 403 not in exp and '403' not in exp: + exp = list(exp) + exp.append(403) + else: + exp = None + + if exp: + if len(args) > 7: + args = list(args) + args[7] = exp + else: + kwargs['expected_status'] = exp + kwargs = compat_kwargs(kwargs) + + ret = super(SBSIE, self)._download_webpage_handle(url, video_id, *args, **kwargs) + if ret is False: + return ret + webpage, urlh = ret + + if urlh.getcode() == 403: + if urlh.headers.get('x-error-reason') == 'geo-blocked': + countries = ['AU'] + if fatal: + self.raise_geo_restricted(countries=countries) + err = GeoRestrictedError( + 'This Australian content is not available from your location due to geo restriction', + countries=countries) + else: + err = compat_HTTPError(urlh.geturl(), 403, 'HTTP Error 403: Forbidden', urlh.headers, urlh) + ret = self.__handle_request_webpage_error(err, video_id, errnote, fatal) + + return ret + + def _extract_m3u8_formats(self, m3u8_url, video_id, *args, **kwargs): + # ext, entry_protocol, preference, m3u8_id, note, errnote, fatal, + # live, data, headers, query + entry_protocol = args[1] if len(args) > 1 else kwargs.get('entry_protocol') + if not entry_protocol: + entry_protocol = 'm3u8_native' + if len(args) > 1: + args = list(args) + args[1] = entry_protocol + else: + kwargs['entry_protocol'] = entry_protocol + kwargs = compat_kwargs(kwargs) + + return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs) + + _AUS_TV_PARENTAL_GUIDELINES = { + 'P': 0, + 'C': 7, + 'G': 0, + 'PG': 0, + 'M': 14, + 'MA15+': 15, + 'MAV15+': 15, + 'R18+': 18, + } + _PLAYER_API = 'https://www.sbs.com.au/api/v3' + _CATALOGUE_API = 'https://catalogue.pr.sbsod.com/' + + def _call_api(self, video_id, path, query=None, data=None, headers=None, fatal=True): + return self._download_json(update_url_query( + self._CATALOGUE_API + path, query), + video_id, headers=headers, fatal=fatal) or {} + + def _get_smil_url(self, video_id): + return update_url_query( + self._PLAYER_API + 'video_smil', {'id': video_id}) + + def _get_player_data(self, video_id, headers=None, fatal=False): + return self._download_json(update_url_query( + self._PLAYER_API + 'video_stream', {'id': video_id, 'context': 'tv'}), + video_id, headers=headers, fatal=fatal) or {} + def _real_extract(self, url): + video_id = self._match_id(url) + # get media links directly though later metadata may contain contentUrl + smil_url = self._get_smil_url(video_id) + formats = self._extract_smil_formats(smil_url, video_id, fatal=False) or [] + self._sort_formats(formats) + + # try for metadata from the same source + player_data = self._get_player_data(video_id, fatal=False) + media = traverse_obj(player_data, 'video_object', expected_type=dict) or {} + # get, or add, metadata from catalogue + media.update(self._call_api(video_id, 'mpx-media/' + video_id, fatal=not media)) + + # utils candidate for use with traverse_obj() + def txt_or_none(s): + return (s.strip() or None) if isinstance(s, compat_str) else None + + # expected_type fn for thumbs + def xlate_thumb(t): + u = url_or_none(t.get('contentUrl')) + return u and { + 'id': t.get('name'), + 'url': u, + 'width': int_or_none(t.get('width')), + 'height': int_or_none(t.get('height')), + } + + # may be numeric or timecoded + def really_parse_duration(d): + result = float_or_none(d) + if result is None: + result = parse_duration(d) + return result + + def traverse_media(*args, **kwargs): + if 'expected_type' not in kwargs: + kwargs['expected_type'] = txt_or_none + kwargs = compat_kwargs(kwargs) + return traverse_obj(media, *args, **kwargs) + + return { + 'id': video_id, + 'title': traverse_media(('displayTitles', Ellipsis, 'title'), + get_all=False) or media['title'], + 'formats': formats, + 'description': traverse_media('description'), + 'categories': traverse_media( + ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name')), + 'tags': traverse_media( + (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis)), + 'age_limit': self._AUS_TV_PARENTAL_GUIDELINES.get(traverse_media( + 'classificationID', 'contentRating', default='').upper()), + 'thumbnails': traverse_media(('thumbnails', Ellipsis), + expected_type=xlate_thumb), + 'duration': traverse_media('duration', + expected_type=really_parse_duration), + 'series': traverse_media(('partOfSeries', 'name'), 'seriesTitle'), + 'series_id': traverse_media(('partOfSeries', 'uuid'), 'seriesID'), + 'season_number': traverse_media( + (('partOfSeries', None), 'seasonNumber'), + expected_type=int_or_none, get_all=False), + 'episode_number': traverse_media('episodeNumber', + expected_type=int_or_none), + 'release_year': traverse_media('releaseYear', + expected_type=int_or_none), + 'timestamp': traverse_media( + 'datePublished', ('publication', 'startDate'), + expected_type=parse_iso8601), + 'channel': traverse_media(('taxonomy', 'channel', 'name')), + 'uploader': 'SBSC', + } + + def _old_real_extract(self, url): video_id = self._match_id(url) player_params = self._download_json( 'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id) @@ -66,13 +280,16 @@ class SBSIE(InfoExtractor): error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '') raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True) - urls = player_params['releaseUrls'] - theplatform_url = (urls.get('progressive') or urls.get('html') - or urls.get('standard') or player_params['relatedItemsURL']) + media_url = traverse_obj( + player_params, ('releaseUrls', ('progressive', 'html', 'standard', 'htmlandroid')), + expected_type=url_or_none) + if not media_url: + raise ExtractorError('No', expected=True) return { '_type': 'url_transparent', - 'ie_key': 'ThePlatform', + # 'ie_key': 'ThePlatform', 'id': video_id, - 'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}), + 'url': smuggle_url(self._proto_relative_url(media_url), {'force_smil_url': True}), + 'is_live': player_params.get('streamType') == 'live', } From d1dbd37b094e539797e27aed4f004dc2e57baff2 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 19 Mar 2023 12:14:07 +0000 Subject: [PATCH 2/6] [SBS] Updates for review and rethink --- youtube_dl/extractor/sbs.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index e0af123e7..95e9ec035 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -44,12 +44,13 @@ class SBSIE(InfoExtractor): (["\'])(?Phttps?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1'''] _TESTS = [{ - # Original URL is handled by the generic IE which finds the iframe: + # Exceptional unrestricted show for testing, thanks SBS, + # from an iframe of this page, handled by the generic IE, now 404: # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', 'md5': 'e49d0290cb4f40d893b8dfe760dce6b0', 'info_dict': { - 'id': '320403011771', # '_rFBPRPO4pMR', + 'id': '320403011771', # formerly '_rFBPRPO4pMR', no longer found 'ext': 'mp4', 'title': 'Dingo Conservation (The Feed)', 'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5', @@ -107,13 +108,14 @@ class SBSIE(InfoExtractor): return False def _download_webpage_handle(self, url, video_id, *args, **kwargs): - # note, errnote, fatal, encoding, data, headers={}, query, expected_status + # note, errnote, fatal, encoding, data, headers, query, expected_status # specialised to detect geo-block errnote = args[2] if len(args) > 2 else kwargs.get('errnote') fatal = args[3] if len(args) > 3 else kwargs.get('fatal') exp = args[7] if len(args) > 7 else kwargs.get('expected_status') + # add 403 to expected codes for interception exp = variadic(exp or [], allowed_types=(compat_str, )) if 403 not in exp and '403' not in exp: exp = list(exp) @@ -145,6 +147,9 @@ class SBSIE(InfoExtractor): else: err = compat_HTTPError(urlh.geturl(), 403, 'HTTP Error 403: Forbidden', urlh.headers, urlh) ret = self.__handle_request_webpage_error(err, video_id, errnote, fatal) + if exp: + # caller doesn't expect 403 + return False return ret @@ -163,15 +168,16 @@ class SBSIE(InfoExtractor): return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs) - _AUS_TV_PARENTAL_GUIDELINES = { + AUS_TV_PARENTAL_GUIDELINES = { 'P': 0, 'C': 7, 'G': 0, 'PG': 0, - 'M': 14, + 'M': 15, 'MA15+': 15, - 'MAV15+': 15, + 'AV15+': 15, 'R18+': 18, + 'NC': 0, # not classified (unofficial, used by SBS) } _PLAYER_API = 'https://www.sbs.com.au/api/v3' _CATALOGUE_API = 'https://catalogue.pr.sbsod.com/' @@ -240,7 +246,7 @@ class SBSIE(InfoExtractor): ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name')), 'tags': traverse_media( (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis)), - 'age_limit': self._AUS_TV_PARENTAL_GUIDELINES.get(traverse_media( + 'age_limit': self.AUS_TV_PARENTAL_GUIDELINES.get(traverse_media( 'classificationID', 'contentRating', default='').upper()), 'thumbnails': traverse_media(('thumbnails', Ellipsis), expected_type=xlate_thumb), @@ -262,6 +268,7 @@ class SBSIE(InfoExtractor): 'uploader': 'SBSC', } + # just come behind the shed with me, mate def _old_real_extract(self, url): video_id = self._match_id(url) player_params = self._download_json( From b028b2fa27482f3ccb2017031b61a8e9c031c036 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 1 May 2023 15:14:38 +0000 Subject: [PATCH 3/6] Align PR with merged yt-dlp code --- youtube_dl/extractor/sbs.py | 194 ++++++++++++------------------------ 1 file changed, 63 insertions(+), 131 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 95e9ec035..e8ddc7b32 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -1,27 +1,21 @@ # coding: utf-8 from __future__ import unicode_literals -import sys - from .common import InfoExtractor from ..compat import ( - compat_HTTPError, compat_kwargs, compat_str, ) from ..utils import ( - error_to_compat_str, - ExtractorError, + HEADRequest, float_or_none, - GeoRestrictedError, int_or_none, + merge_dicts, parse_duration, parse_iso8601, - smuggle_url, traverse_obj, update_url_query, url_or_none, - variadic, ) @@ -31,7 +25,7 @@ class SBSIE(InfoExtractor): https?://(?:www\.)?sbs\.com\.au/(?: ondemand(?: /video/(?:single/)?| - /movie/[^/]+/| + /(?:movie|tv-program)/[^/]+/| /(?:tv|news)-series/(?:[^/]+/){3}| .*?\bplay=|/watch/ )|news/(?:embeds/)?video/ @@ -59,6 +53,8 @@ class SBSIE(InfoExtractor): 'timestamp': 1408613220, 'upload_date': '20140821', 'uploader': 'SBSC', + 'tags': None, + 'categories': None, }, 'expected_warnings': ['Unable to download JSON metadata'], }, { @@ -92,67 +88,11 @@ class SBSIE(InfoExtractor): }, { 'url': 'https://www.sbs.com.au/ondemand/tv-series/the-handmaids-tale/season-5/the-handmaids-tale-s5-ep1/2065631811776', 'only_matching': True, + }, { + 'url': 'https://www.sbs.com.au/ondemand/tv-program/autun-romes-forgotten-sister/2116212803602', + 'only_matching': True, }] - def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True): - if errnote is False: - return False - if errnote is None: - errnote = 'Unable to download webpage' - - errmsg = '%s: %s' % (errnote, error_to_compat_str(err)) - if fatal: - raise ExtractorError(errmsg, sys.exc_info()[2], cause=err, video_id=video_id) - else: - self._downloader.report_warning(errmsg) - return False - - def _download_webpage_handle(self, url, video_id, *args, **kwargs): - # note, errnote, fatal, encoding, data, headers, query, expected_status - # specialised to detect geo-block - - errnote = args[2] if len(args) > 2 else kwargs.get('errnote') - fatal = args[3] if len(args) > 3 else kwargs.get('fatal') - exp = args[7] if len(args) > 7 else kwargs.get('expected_status') - - # add 403 to expected codes for interception - exp = variadic(exp or [], allowed_types=(compat_str, )) - if 403 not in exp and '403' not in exp: - exp = list(exp) - exp.append(403) - else: - exp = None - - if exp: - if len(args) > 7: - args = list(args) - args[7] = exp - else: - kwargs['expected_status'] = exp - kwargs = compat_kwargs(kwargs) - - ret = super(SBSIE, self)._download_webpage_handle(url, video_id, *args, **kwargs) - if ret is False: - return ret - webpage, urlh = ret - - if urlh.getcode() == 403: - if urlh.headers.get('x-error-reason') == 'geo-blocked': - countries = ['AU'] - if fatal: - self.raise_geo_restricted(countries=countries) - err = GeoRestrictedError( - 'This Australian content is not available from your location due to geo restriction', - countries=countries) - else: - err = compat_HTTPError(urlh.geturl(), 403, 'HTTP Error 403: Forbidden', urlh.headers, urlh) - ret = self.__handle_request_webpage_error(err, video_id, errnote, fatal) - if exp: - # caller doesn't expect 403 - return False - - return ret - def _extract_m3u8_formats(self, m3u8_url, video_id, *args, **kwargs): # ext, entry_protocol, preference, m3u8_id, note, errnote, fatal, # live, data, headers, query @@ -168,24 +108,28 @@ class SBSIE(InfoExtractor): return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs) + _GEO_COUNTRIES = ['AU'] + # naming for exportability AUS_TV_PARENTAL_GUIDELINES = { 'P': 0, 'C': 7, 'G': 0, 'PG': 0, - 'M': 15, + 'M': 14, 'MA15+': 15, 'AV15+': 15, + 'MAV15+': 15, 'R18+': 18, 'NC': 0, # not classified (unofficial, used by SBS) } _PLAYER_API = 'https://www.sbs.com.au/api/v3' _CATALOGUE_API = 'https://catalogue.pr.sbsod.com/' + _VOD_BASE_URL = 'https://sbs-vod-prod-01.akamaized.net/' def _call_api(self, video_id, path, query=None, data=None, headers=None, fatal=True): return self._download_json(update_url_query( self._CATALOGUE_API + path, query), - video_id, headers=headers, fatal=fatal) or {} + video_id, headers=headers or {}, fatal=fatal) or {} def _get_smil_url(self, video_id): return update_url_query( @@ -194,13 +138,23 @@ class SBSIE(InfoExtractor): def _get_player_data(self, video_id, headers=None, fatal=False): return self._download_json(update_url_query( self._PLAYER_API + 'video_stream', {'id': video_id, 'context': 'tv'}), - video_id, headers=headers, fatal=fatal) or {} + video_id, headers=headers or {}, fatal=fatal) or {} def _real_extract(self, url): video_id = self._match_id(url) # get media links directly though later metadata may contain contentUrl smil_url = self._get_smil_url(video_id) formats = self._extract_smil_formats(smil_url, video_id, fatal=False) or [] + + if not formats: + urlh = self._request_webpage( + HEADRequest(self._VOD_BASE_URL), video_id, + note='Checking geo-restriction', fatal=False, expected_status=403) + if urlh: + error_reasons = urlh.headers.get_all('x-error-reason') or [] + if 'geo-blocked' in error_reasons: + self.raise_geo_restricted(countries=self._GEO_COUNTRIES) + self._sort_formats(formats) # try for metadata from the same source @@ -231,72 +185,50 @@ class SBSIE(InfoExtractor): return result def traverse_media(*args, **kwargs): + nkwargs = None if 'expected_type' not in kwargs: kwargs['expected_type'] = txt_or_none - kwargs = compat_kwargs(kwargs) + nkwargs = kwargs + if 'get_all' not in kwargs: + kwargs['get_all'] = False + nkwargs = kwargs + if nkwargs: + kwargs = compat_kwargs(nkwargs) return traverse_obj(media, *args, **kwargs) - return { + # For named episodes, use the catalogue's title to set episode, rather than generic 'Episode N'. + if traverse_media('partOfSeries', expected_type=dict): + media['epName'] = traverse_media('title') + + return merge_dicts(*reversed(({ 'id': video_id, - 'title': traverse_media(('displayTitles', Ellipsis, 'title'), - get_all=False) or media['title'], - 'formats': formats, - 'description': traverse_media('description'), - 'categories': traverse_media( - ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name')), - 'tags': traverse_media( - (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis)), + }, dict((k, traverse_media(v)) for k, v in { + 'title': 'name', + 'description': 'description', + 'channel': ('taxonomy', 'channel', 'name'), + 'series': ((('partOfSeries', 'name'), 'seriesTitle')), + 'series_id': ((('partOfSeries', 'uuid'), 'seriesID')), + 'episode': 'epName', + }.items()), { + 'season_number': traverse_media((('partOfSeries', None), 'seasonNumber'), expected_type=int_or_none), + 'episode_number': traverse_media('episodeNumber', expected_type=int_or_none), + 'timestamp': traverse_media('datePublished', ('publication', 'startDate'), + expected_type=parse_iso8601), + 'release_year': traverse_media('releaseYear', expected_type=int_or_none), + 'duration': traverse_media('duration', expected_type=really_parse_duration), + 'is_live': traverse_media('liveStream', expected_type=bool), 'age_limit': self.AUS_TV_PARENTAL_GUIDELINES.get(traverse_media( 'classificationID', 'contentRating', default='').upper()), + 'categories': traverse_media( + ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name'), + get_all=True) or None, + 'tags': traverse_media( + (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis), + get_all=True) or None, 'thumbnails': traverse_media(('thumbnails', Ellipsis), - expected_type=xlate_thumb), - 'duration': traverse_media('duration', - expected_type=really_parse_duration), - 'series': traverse_media(('partOfSeries', 'name'), 'seriesTitle'), - 'series_id': traverse_media(('partOfSeries', 'uuid'), 'seriesID'), - 'season_number': traverse_media( - (('partOfSeries', None), 'seasonNumber'), - expected_type=int_or_none, get_all=False), - 'episode_number': traverse_media('episodeNumber', - expected_type=int_or_none), - 'release_year': traverse_media('releaseYear', - expected_type=int_or_none), - 'timestamp': traverse_media( - 'datePublished', ('publication', 'startDate'), - expected_type=parse_iso8601), - 'channel': traverse_media(('taxonomy', 'channel', 'name')), + expected_type=xlate_thumb, get_all=True), + 'formats': formats, + # TODO: _extract_smil_formats_and_subtitles() + # 'subtitles': subtitles, 'uploader': 'SBSC', - } - - # just come behind the shed with me, mate - def _old_real_extract(self, url): - video_id = self._match_id(url) - player_params = self._download_json( - 'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id) - - error = player_params.get('error') - if error: - error_message = 'Sorry, The video you are looking for does not exist.' - video_data = error.get('results') or {} - error_code = error.get('errorCode') - if error_code == 'ComingSoon': - error_message = '%s is not yet available.' % video_data.get('title', '') - elif error_code in ('Forbidden', 'intranetAccessOnly'): - error_message = 'Sorry, This video cannot be accessed via this website' - elif error_code == 'Expired': - error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '') - raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True) - - media_url = traverse_obj( - player_params, ('releaseUrls', ('progressive', 'html', 'standard', 'htmlandroid')), - expected_type=url_or_none) - if not media_url: - raise ExtractorError('No', expected=True) - - return { - '_type': 'url_transparent', - # 'ie_key': 'ThePlatform', - 'id': video_id, - 'url': smuggle_url(self._proto_relative_url(media_url), {'force_smil_url': True}), - 'is_live': player_params.get('streamType') == 'live', - } + }))) From ab5617be9e6311cace5c00dbe15f7ed3a1f69d87 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 1 May 2023 16:22:29 +0100 Subject: [PATCH 4/6] Update youtube_dl/extractor/sbs.py --- youtube_dl/extractor/sbs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index e8ddc7b32..67bb10b59 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -40,7 +40,8 @@ class SBSIE(InfoExtractor): _TESTS = [{ # Exceptional unrestricted show for testing, thanks SBS, # from an iframe of this page, handled by the generic IE, now 404: - # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation + # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation, but replaced by + # https://www.sbs.com.au/programs/video/320403011771/Dingo-Conservation-The-Feed 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', 'md5': 'e49d0290cb4f40d893b8dfe760dce6b0', 'info_dict': { From de91fe794dfa134ebd43f911624d42ff3426aae2 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 8 Oct 2024 23:51:02 +0100 Subject: [PATCH 5/6] Update 2023 draft to current API version, etc --- youtube_dl/extractor/sbs.py | 102 +++++++++++++++--------------------- 1 file changed, 42 insertions(+), 60 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index 67bb10b59..a319abdd0 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -13,6 +13,7 @@ from ..utils import ( merge_dicts, parse_duration, parse_iso8601, + T, traverse_obj, update_url_query, url_or_none, @@ -35,7 +36,7 @@ class SBSIE(InfoExtractor): ]+?src= ) - (["\'])(?Phttps?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1'''] + ("|\')(?Phttps?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1'''] _TESTS = [{ # Exceptional unrestricted show for testing, thanks SBS, @@ -94,18 +95,14 @@ class SBSIE(InfoExtractor): 'only_matching': True, }] + # change default entry_protocol kwarg for _extract_smil_formats() + # TODO: ..._and_subtitles() def _extract_m3u8_formats(self, m3u8_url, video_id, *args, **kwargs): - # ext, entry_protocol, preference, m3u8_id, note, errnote, fatal, - # live, data, headers, query - entry_protocol = args[1] if len(args) > 1 else kwargs.get('entry_protocol') - if not entry_protocol: - entry_protocol = 'm3u8_native' - if len(args) > 1: - args = list(args) - args[1] = entry_protocol - else: - kwargs['entry_protocol'] = entry_protocol - kwargs = compat_kwargs(kwargs) + # ext, entry_protocol, ... + entry_protocol = kwargs.get('entry_protocol') + if not entry_protocol and len(args) <= 1: + kwargs['entry_protocol'] = 'm3u8_native' + kwargs = compat_kwargs(kwargs) return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs) @@ -144,8 +141,8 @@ class SBSIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) # get media links directly though later metadata may contain contentUrl - smil_url = self._get_smil_url(video_id) - formats = self._extract_smil_formats(smil_url, video_id, fatal=False) or [] + formats, subtitles = self._extract_smil_formats( # self._extract_smil_formats_and_subtitles( + self._get_smil_url(video_id), video_id, fatal=False), {} if not formats: urlh = self._request_webpage( @@ -160,16 +157,16 @@ class SBSIE(InfoExtractor): # try for metadata from the same source player_data = self._get_player_data(video_id, fatal=False) - media = traverse_obj(player_data, 'video_object', expected_type=dict) or {} + media = traverse_obj(player_data, 'video_object', T(dict)) or {} + # get, or add, metadata from catalogue media.update(self._call_api(video_id, 'mpx-media/' + video_id, fatal=not media)) - # utils candidate for use with traverse_obj() def txt_or_none(s): return (s.strip() or None) if isinstance(s, compat_str) else None # expected_type fn for thumbs - def xlate_thumb(t): + def mk_thumb(t): u = url_or_none(t.get('contentUrl')) return u and { 'id': t.get('name'), @@ -185,51 +182,36 @@ class SBSIE(InfoExtractor): result = parse_duration(d) return result - def traverse_media(*args, **kwargs): - nkwargs = None - if 'expected_type' not in kwargs: - kwargs['expected_type'] = txt_or_none - nkwargs = kwargs - if 'get_all' not in kwargs: - kwargs['get_all'] = False - nkwargs = kwargs - if nkwargs: - kwargs = compat_kwargs(nkwargs) - return traverse_obj(media, *args, **kwargs) - # For named episodes, use the catalogue's title to set episode, rather than generic 'Episode N'. - if traverse_media('partOfSeries', expected_type=dict): - media['epName'] = traverse_media('title') + if traverse_obj(media, ('partOfSeries', T(dict))): + media['epName'] = traverse_obj(media, 'title') - return merge_dicts(*reversed(({ + str = txt_or_none # instant compat + return merge_dicts({ 'id': video_id, - }, dict((k, traverse_media(v)) for k, v in { - 'title': 'name', - 'description': 'description', - 'channel': ('taxonomy', 'channel', 'name'), - 'series': ((('partOfSeries', 'name'), 'seriesTitle')), - 'series_id': ((('partOfSeries', 'uuid'), 'seriesID')), - 'episode': 'epName', - }.items()), { - 'season_number': traverse_media((('partOfSeries', None), 'seasonNumber'), expected_type=int_or_none), - 'episode_number': traverse_media('episodeNumber', expected_type=int_or_none), - 'timestamp': traverse_media('datePublished', ('publication', 'startDate'), - expected_type=parse_iso8601), - 'release_year': traverse_media('releaseYear', expected_type=int_or_none), - 'duration': traverse_media('duration', expected_type=really_parse_duration), - 'is_live': traverse_media('liveStream', expected_type=bool), - 'age_limit': self.AUS_TV_PARENTAL_GUIDELINES.get(traverse_media( - 'classificationID', 'contentRating', default='').upper()), - 'categories': traverse_media( - ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name'), - get_all=True) or None, - 'tags': traverse_media( - (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis), - get_all=True) or None, - 'thumbnails': traverse_media(('thumbnails', Ellipsis), - expected_type=xlate_thumb, get_all=True), + }, traverse_obj(media, { + 'title': ('name', T(str)), + 'description': ('description', T(str)), + 'channel': ('taxonomy', 'channel', 'name', T(str)), + 'series': ((('partOfSeries', 'name'), 'seriesTitle'), T(str)), + 'series_id': ((('partOfSeries', 'uuid'), 'seriesID'), T(str)), + 'season_number': (('partOfSeries', None), 'seasonNumber', T(int_or_none)), + 'episode': ('epName', T(str)), + 'episode_number': ('episodeNumber', T(int_or_none)), + 'timestamp': ('datePublished', ('publication', 'startDate'), T(parse_iso8601)), + 'release_year': ('releaseYear', T(int_or_none)), + 'duration': ('duration', T(really_parse_duration)), + 'is_live': ('liveStream', T(bool)), + 'age_limit': ('classificationID', 'contentRating', + T(lambda x: self.AUS_TV_PARENTAL_GUIDELINES.get(x, '').upper() or None)), # dict.get is unhashable in py3.7 + }, get_all=False), traverse_obj(media, { + 'categories': (('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), + 'name', T(str))), + 'tags': (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), + Ellipsis, T(str)), + 'thumbnails': ('thumbnails', lambda _, v: v['contentUrl'], T(mk_thumb)), + }), { 'formats': formats, - # TODO: _extract_smil_formats_and_subtitles() - # 'subtitles': subtitles, + 'subtitles': subtitles, 'uploader': 'SBSC', - }))) + }, rev=True) From 80d03d7e69c6462ad54d51f58a858b1af0ba18a6 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 9 Oct 2024 01:47:18 +0100 Subject: [PATCH 6/6] Fix/improve timestamp, categories, tags --- youtube_dl/extractor/sbs.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py index a319abdd0..28333eaa1 100644 --- a/youtube_dl/extractor/sbs.py +++ b/youtube_dl/extractor/sbs.py @@ -55,8 +55,8 @@ class SBSIE(InfoExtractor): 'timestamp': 1408613220, 'upload_date': '20140821', 'uploader': 'SBSC', - 'tags': None, - 'categories': None, + 'tags': 'mincount:10', + 'categories': 'count:2', }, 'expected_warnings': ['Unable to download JSON metadata'], }, { @@ -198,17 +198,22 @@ class SBSIE(InfoExtractor): 'season_number': (('partOfSeries', None), 'seasonNumber', T(int_or_none)), 'episode': ('epName', T(str)), 'episode_number': ('episodeNumber', T(int_or_none)), - 'timestamp': ('datePublished', ('publication', 'startDate'), T(parse_iso8601)), + 'timestamp': (('datePublished', ('publication', 'startDate')), T(parse_iso8601)), 'release_year': ('releaseYear', T(int_or_none)), 'duration': ('duration', T(really_parse_duration)), 'is_live': ('liveStream', T(bool)), 'age_limit': ('classificationID', 'contentRating', T(lambda x: self.AUS_TV_PARENTAL_GUIDELINES.get(x, '').upper() or None)), # dict.get is unhashable in py3.7 }, get_all=False), traverse_obj(media, { - 'categories': (('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), - 'name', T(str))), - 'tags': (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), - Ellipsis, T(str)), + 'categories': ((('genres', Ellipsis), + ('taxonomy', ((('genre', 'subgenre'), Ellipsis, 'name'), 'useType'))), + T(str)), + 'tags': ((((('keywords',), + ('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice'))), + Ellipsis), + ('taxonomy', ('era', 'location', 'section', 'subject', 'theme'), + Ellipsis, 'name')), + T(str)), 'thumbnails': ('thumbnails', lambda _, v: v['contentUrl'], T(mk_thumb)), }), { 'formats': formats,