From d93e536ffd34063f55df2346bbaa797ab1e867b5 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 31 Jan 2024 15:07:49 +0100 Subject: [PATCH] feat: hooked into ytdl to sign the function --- .idea/modules.xml | 1 + .idea/music-downloader.iml | 1 + .idea/vcs.xml | 1 + src/music_kraken/__main__.py | 7 +- src/music_kraken/connection/connection.py | 1 - .../pages/youtube_music/youtube_music.py | 138 ++++++++++++------ .../config/config_files/youtube_config.py | 5 +- src/music_kraken/utils/shared.py | 3 + 8 files changed, 108 insertions(+), 49 deletions(-) diff --git a/.idea/modules.xml b/.idea/modules.xml index 56fe61a..05a1d7b 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -5,6 +5,7 @@ + \ No newline at end of file diff --git a/.idea/music-downloader.iml b/.idea/music-downloader.iml index 27eed28..721bcfd 100644 --- a/.idea/music-downloader.iml +++ b/.idea/music-downloader.iml @@ -10,5 +10,6 @@ + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 35eb1dd..09dbadc 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,5 +2,6 @@ + \ No newline at end of file diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index ee4b3fb..b90757b 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -104,8 +104,11 @@ def cli(): if arguments.r: import os - if os.path.exists(shared.CONFIG_FILE): - os.remove(shared.CONFIG_FILE) + + for file in shared.CONFIG_DIRECTORY.iterdir(): + if file.is_file(): + print(f"Deleting {file}....") + file.unlink() read_config() exit() diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index d46a99a..9a66a12 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -232,7 +232,6 @@ class Connection: sleep_after_404=sleep_after_404, is_heartbeat=is_heartbeat, name=name, - user_agent=main_settings["user_agent"], **kwargs ) diff --git a/src/music_kraken/pages/youtube_music/youtube_music.py b/src/music_kraken/pages/youtube_music/youtube_music.py index 9e987bb..f0b84c8 100644 --- a/src/music_kraken/pages/youtube_music/youtube_music.py +++ b/src/music_kraken/pages/youtube_music/youtube_music.py @@ -1,5 +1,7 @@ +from __future__ import unicode_literals, annotations + from typing import Dict, List, Optional, Set, Type -from urllib.parse import urlparse, urlunparse, quote, parse_qs +from urllib.parse import urlparse, urlunparse, quote, parse_qs, urlencode import logging import random import json @@ -7,6 +9,7 @@ from dataclasses import dataclass import re from functools import lru_cache +import youtube_dl from youtube_dl.jsinterp import JSInterpreter from youtube_dl.extractor.youtube import YoutubeIE @@ -17,7 +20,6 @@ from ...utils.functions import get_current_millis from .yt_utils.jsinterp import JSInterpreter - if DEBUG: from ...utils.debug_utils import dump_to_file @@ -104,8 +106,6 @@ class YouTubeMusicCredentials: player_url: str - - @property def player_id(self): @lru_cache(128) @@ -128,15 +128,34 @@ class YouTubeMusicCredentials: return _extract_player_info(self.player_url) +class MusicKrakenYoutubeIE(YoutubeIE): + def __init__(self, *args, main_instance: YoutubeMusic, **kwargs): + self.main_instance = main_instance + super().__init__(*args, **kwargs) + + +class MusicKrakenYoutubeDL(youtube_dl.YoutubeDL): + def __init__(self, main_instance: YoutubeMusic, ydl_opts: dict, **kwargs): + self.main_instance = main_instance + super().__init__(ydl_opts or {}, **kwargs) + super().__enter__() + + def __del__(self): + super().__exit__(None, None, None) + + + class YoutubeMusic(SuperYouTube): # CHANGE SOURCE_TYPE = SourcePages.YOUTUBE_MUSIC LOGGER = logging_settings["youtube_music_logger"] - def __init__(self, *args, **kwargs): - self.connection: YoutubeMusicConnection = YoutubeMusicConnection(logger=self.LOGGER, - accept_language="en-US,en;q=0.5") + def __init__(self, *args, ydl_opts: dict = None, **kwargs): + self.connection: YoutubeMusicConnection = YoutubeMusicConnection( + logger=self.LOGGER, + accept_language="en-US,en;q=0.5" + ) self.credentials: YouTubeMusicCredentials = YouTubeMusicCredentials( api_key=youtube_settings["youtube_music_api_key"], ctoken="", @@ -149,7 +168,17 @@ class YoutubeMusic(SuperYouTube): if self.credentials.api_key == "" or DEBUG_YOUTUBE_INITIALIZING: self._fetch_from_main_page() - SuperYouTube.__init__(self,*args, **kwargs) + SuperYouTube.__init__(self, *args, **kwargs) + + self.download_connection: Connection = Connection( + host="https://music.youtube.com/", + logger=self.LOGGER, + sleep_after_404=youtube_settings["sleep_after_youtube_403"] + ) + + # https://github.com/ytdl-org/youtube-dl/blob/master/README.md#embedding-youtube-dl + self.ydl = MusicKrakenYoutubeDL(self, ydl_opts) + self.yt_ie = MusicKrakenYoutubeIE(downloader=self.ydl, main_instance=self) def _fetch_from_main_page(self): """ @@ -283,7 +312,6 @@ class YoutubeMusic(SuperYouTube): default='{}' )) or {} - def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: return super().get_source_type(source) @@ -454,15 +482,19 @@ class YoutubeMusic(SuperYouTube): r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\(' ), - code, group='sig') + code, group='sig') jsi = JSInterpreter(code) initial_function = jsi.extract_function(funcname) + return lambda s: initial_function([s]) - def _decrypt_signature(self, s): - signing_func = self._extract_signature_function(player_url=youtube_settings["player_url"]) - print(signing_func) + def _decrypt_signature(self, video_id, s): + print(youtube_settings["player_url"]) + res = self._extract_signature_function(player_url=youtube_settings["player_url"]) + test_string = ''.join(map(str, range(len(s)))) + cache_spec = [ord(c) for c in res(test_string)] + signing_func = lambda _s: ''.join(_s[i] for i in cache_spec) return signing_func(s) def _parse_adaptive_formats(self, data: list, video_id) -> dict: @@ -475,16 +507,45 @@ class YoutubeMusic(SuperYouTube): if not fmt_url: sc = parse_qs(possible_format["signatureCipher"]) print(sc["s"][0]) - signature = self._decrypt_signature(sc['s'][0]) + signature = self._decrypt_signature(video_id, sc['s'][0]) print(signature) - sp = sc.get("sp", ["sig"])[0] + tmp = sc.get("sp", ["sig"]) + sig_key = "signature" if len(tmp) <= 0 else tmp[-1] + fmt_url = sc.get("url", [None])[0] - fmt_url += '&' + sp + '=' + signature + ftm_parsed = urlparse(fmt_url) + q = parse_qs(ftm_parsed.query) + q[sig_key] = [signature] + print(json.dumps(q, indent=4)) + print(sig_key) + query_str = urlencode(q) + print(query_str) + + fmt_url = urlunparse(( + ftm_parsed.scheme, + ftm_parsed.netloc, + ftm_parsed.path, + ftm_parsed.params, + query_str, + ftm_parsed.fragment + )) + + """ + if not isinstance(url, tuple): + url = compat_urllib_parse.urlparse(url) + query = kwargs.pop('query_update', None) + if query: + qs = compat_parse_qs(url.query) + qs.update(query) + kwargs['query'] = compat_urllib_parse_urlencode(qs, True) + kwargs = compat_kwargs(kwargs) + return compat_urllib_parse.urlunparse(url._replace(**kwargs)) + """ return { - "bitrate": fmt.get("bitrate"), + "bitrate": fmt.get("bitrate"), "url": fmt_url } @@ -512,32 +573,16 @@ class YoutubeMusic(SuperYouTube): return parse_format(best_format) def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: - """ - curl 'https://music.youtube.com/youtubei/v1/player?key=AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30&prettyPrint=false' - --compressed -X POST - -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0' - -H 'Accept: */*' - -H 'Accept-Language: en-US,en;q=0.5' - -H 'Accept-Encoding: gzip, deflate, br' - -H 'Content-Type: application/json' - -H 'Referer: https://music.youtube.com/' - -H 'X-Goog-Visitor-Id: CgtHdmkzbGhaMDltVSj4j5mtBjIKCgJERRIEEgAgOA%3D%3D' - -H 'X-Youtube-Bootstrap-Logged-In: false' - -H 'X-Youtube-Client-Name: 67' -H 'X-Youtube-Client-Version: 1.20240103.01.00' - -H 'Origin: https://music.youtube.com' - -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Connection: keep-alive' -H 'Alt-Used: music.youtube.com' - -H 'Cookie: SOCS=CAISNQgREitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjQwMTA5LjA1X3AwGgJlbiACGgYIgI6XrQY; YSC=r46McyPx8dE; VISITOR_PRIVACY_METADATA=CgJERRIEEgAgOA%3D%3D; CONSENT=PENDING+663; VISITOR_INFO1_LIVE=Gvi3lhZ09mU; _gcl_au=1.1.396177275.1705396217; ST-1hw5vco=csn=MC4xNTI3OTkwMzQyOTc1MzQ2&itct=CNgDEMn0AhgDIhMItMS6_cfhgwMVDMtCBR1u5wb6' -H 'TE: trailers' - --data-raw '{ - "videoId":"QeQrfsqPMCs", - "context":{"client":{"hl":"en","gl":"DE","remoteHost":"129.143.170.58","deviceMake":"","deviceModel":"","visitorData":"CgtHdmkzbGhaMDltVSj4j5mtBjIKCgJERRIEEgAgOA%3D%3D","userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0,gzip(gfe)","clientName":"WEB_REMIX","clientVersion":"1.20240103.01.00","osName":"X11","osVersion":"","originalUrl":"https://music.youtube.com/?cbrd=1","platform":"DESKTOP","clientFormFactor":"UNKNOWN_FORM_FACTOR","configInfo":{"appInstallData":"CPiPma0GEL2ZsAUQqJqwBRCmgbAFEP24_RIQjaKwBRDNlbAFENWIsAUQmaSwBRD6p7AFEL75rwUQmvCvBRDT4a8FEL2KsAUQrtT-EhC36v4SENnJrwUQnouwBRDJ968FEJP8rwUQuIuuBRDM364FEIiHsAUQ0I2wBRDnuq8FEPOhsAUQ2piwBRDMrv4SEIjjrwUQooGwBRDuorAFEM6osAUQ6-j-EhC3nbAFEKXC_hIQ9fmvBRDh8q8FEJmUsAUQt--vBRD8hbAFEKigsAUQrLevBRC_o7AFEOuTrgUQqfevBRDd6P4SEJj8_hIQ6YywBRC9tq4FEOupsAUQ5LP-EhDfhP8SEOrDrwUQqKGwBRC8-a8FEPKYsAU%3D"},"browserName":"Firefox","browserVersion":"121.0","acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8","deviceExperimentId":"ChxOek15TkRZeU1EazNOalE0TXpVNU1EQXhOZz09EPiPma0GGPiPma0G","screenWidthPoints":780,"screenHeightPoints":638,"screenPixelDensity":2,"screenDensityFloat":2,"utcOffsetMinutes":60,"userInterfaceTheme":"USER_INTERFACE_THEME_DARK","timeZone":"Europe/Berlin","playerType":"UNIPLAYER","tvAppInfo":{"livingRoomAppMode":"LIVING_ROOM_APP_MODE_UNSPECIFIED"},"clientScreen":"WATCH_FULL_SCREEN"},"user":{"lockedSafetyMode":false},"request":{"useSsl":true,"internalExperimentFlags":[],"consistencyTokenJars":[]},"clientScreenNonce":"MC4xNTI3OTkwMzQyOTc1MzQ2","adSignalsInfo":{"params":[{"key":"dt","value":"1705396224619"},{"key":"flash","value":"0"},{"key":"frm","value":"0"},{"key":"u_tz","value":"60"},{"key":"u_his","value":"5"},{"key":"u_h","value":"800"},{"key":"u_w","value":"1280"},{"key":"u_ah","value":"769"},{"key":"u_aw","value":"1280"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"638"},{"key":"biw","value":"780"},{"key":"brdim","value":"0,31,0,31,1280,31,1280,769,780,638"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}]},"clickTracking":{"clickTrackingParams":"CNgDEMn0AhgDIhMItMS6_cfhgwMVDMtCBR1u5wb6"}},"playbackContext":{"contentPlaybackContext":{"html5Preference":"HTML5_PREF_WANTS","lactMilliseconds":"22","referer":"https://music.youtube.com/","signatureTimestamp":19732,"autoCaptionsDefaultOn":false,"mdxContext":{}}},"cpn":"Aqv99K7Z_3tj9ACA","playlistId":"RDAMVMQeQrfsqPMCs","captionParams":{},"serviceIntegrityDimensions":{"poToken":"MnQLhidwfIVPEAu-woG_SQU69mfPclEz7kVUmC1dNP8EQN7NNyVdF3KcVIuKRKrcXlwOXEQg3hc5qXSBbbQU_M7lxx9zgQMelv9iZwWfWlLyI9RoZXB1wipAYHWNzxu7rMqDwRn5M6WS4RRIeHcld9P_YZRYdg=="} - }' - :param source: - :param stop_at_level: - :return: - """ - song = Song(source_list=[ - source - ]) + # implement the functionality yt_dl provides + ydl_res = self.yt_ie._real_extract(source.url) + print(ydl_res) + + source.audio_url = ydl_res.get("formats")[0].get("url") + song = Song( + title=ydl_res.get("title"), + source_list=[source], + ) + return song parsed_url = urlparse(source.url) video_id = parse_qs(parsed_url.query)['v'] @@ -575,4 +620,9 @@ class YoutubeMusic(SuperYouTube): return super().download_song_to_target(source, target) print(source.audio_url) - return self.download_connection.stream_into(source.audio_url, target, description=desc, raw_url=True) + return self.download_connection.stream_into(source.audio_url, target, description=desc, headers={ + "Host": "rr1---sn-cxaf0x-nugl.googlevideo.com" + }) + + def __del__(self): + self.ydl.__exit__() diff --git a/src/music_kraken/utils/config/config_files/youtube_config.py b/src/music_kraken/utils/config/config_files/youtube_config.py index 3c525d5..4610f18 100644 --- a/src/music_kraken/utils/config/config_files/youtube_config.py +++ b/src/music_kraken/utils/config/config_files/youtube_config.py @@ -30,12 +30,13 @@ Dw. if it is empty, Rachel will fetch it automatically for you <333 Attribute(name="youtube_music_clean_data", default_value=True, description="If set to true, it exclusively fetches artists/albums/songs, not things like user channels etc."), UrlAttribute(name="youtube_url", default_value=[ "https://www.youtube.com/", - "https://www.youtu.be/" + "https://www.youtu.be/", + "https://music.youtube.com/", ], description="""This is used to detect, if an url is from youtube, or any alternativ frontend. If any instance seems to be missing, run music kraken with the -f flag."""), Attribute(name="use_sponsor_block", default_value=True, description="Use sponsor block to remove adds or simmilar from the youtube videos."), - Attribute(name="player_url", default_value="/s/player/80b90bfd/player_ias.vflset/en_US/base.js", description=""" + Attribute(name="player_url", default_value="https://music.youtube.com/s/player/80b90bfd/player_ias.vflset/en_US/base.js", description=""" This is needed to fetch videos without invidious """), Attribute(name="youtube_music_consent_cookies", default_value={ diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index d1645f8..31cadec 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -1,5 +1,6 @@ import random +from .path_manager import LOCATIONS from .config import main_settings DEBUG = True @@ -15,6 +16,8 @@ def get_random_message() -> str: return random.choice(main_settings['happy_messages']) +CONFIG_DIRECTORY = LOCATIONS.CONFIG_DIRECTORY + HIGHEST_ID = 2 ** main_settings['id_bits'] HELP_MESSAGE = """to search: