musicdl 2.1.11__py3-none-any.whl → 2.7.3__py3-none-any.whl

musicdl/modules/sources/yinyuedao.py

@@ -0,0 +1,144 @@
+'''
+Function:
+    Implementation of YinyuedaoMusicClient: https://1mp3.top/
+Author:
+    Zhenchao Jin
+WeChat Official Account (微信公众号):
+    Charles的皮卡丘
+'''
+import json_repair
+from bs4 import BeautifulSoup
+from .base import BaseMusicClient
+from rich.progress import Progress
+from ..utils import legalizestring, usesearchheaderscookies, safeextractfromdict, SongInfo, QuarkParser
+
+
+'''YinyuedaoMusicClient'''
+class YinyuedaoMusicClient(BaseMusicClient):
+    source = 'YinyuedaoMusicClient'
+    def __init__(self, **kwargs):
+        super(YinyuedaoMusicClient, self).__init__(**kwargs)
+        if not self.quark_parser_config.get('cookies'): self.logger_handle.warning(f'{self.source}.__init__ >>> "quark_parser_config" is not configured, so song downloads are restricted and only mp3 files can be downloaded.')
+        self.default_search_headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-encoding": "gzip, deflate, br, zstd",
+            "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+            "priority": "u=0, i",
+            "referer": "https://1mp3.top/",
+            "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": "\"Windows\"",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "same-origin",
+            "sec-fetch-user": "?1",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+        }
+        self.default_download_headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
+        }
+        self.default_headers = self.default_search_headers
+        self._initsession()
+    '''_constructsearchurls'''
+    def _constructsearchurls(self, keyword: str, rule: dict = None, request_overrides: dict = None):
+        # init
+        rule, request_overrides = rule or {}, request_overrides or {}
+        # construct search urls
+        search_urls = [f'https://1mp3.top/?key={keyword}&search=1']
+        self.search_size_per_page = self.search_size_per_source
+        # return
+        return search_urls
+    '''_parsesearchresultsfromhtml'''
+    def _parsesearchresultsfromhtml(self, html_text: str):
+        soup = BeautifulSoup(html_text, "lxml")
+        search_results = []
+        for li in soup.select("#musicList > li"):
+            music_id_attr = li.get("data-music-id")
+            music_title_attr = li.get("data-music-title")
+            music_singer_attr = li.get("data-music-singer")
+            music_cover_attr = li.get("data-music-cover")
+            a_download = li.select_one("a.download-btn")
+            music_json_str = a_download.get("data-music")
+            music_data = json_repair.loads(music_json_str)
+            search_results.append({
+                "id_attr": music_id_attr, "title_attr": music_title_attr, "singer_attr": music_singer_attr, "cover_attr": music_cover_attr,
+                "id": music_data.get("id"), "title": music_data.get("title"), "singer": music_data.get("singer"), "picurl": music_data.get("picurl"),
+                "create_time": music_data.get("create_time"), "mtype": music_data.get("mtype"), "downlist": music_data.get("downlist", []),
+                "ktmdownlist": music_data.get("ktmdownlist", []),
+            })
+        return search_results
+    '''_search'''
+    @usesearchheaderscookies
+    def _search(self, keyword: str = '', search_url: str = '', request_overrides: dict = None, song_infos: list = [], progress: Progress = None, progress_id: int = 0):
+        # init
+        request_overrides = request_overrides or {}
+        # successful
+        try:
+            # --search results
+            resp = self.get(search_url, **request_overrides)
+            resp.raise_for_status()
+            search_results = self._parsesearchresultsfromhtml(resp.text)
+            for search_result in search_results:
+                # --download results
+                if not isinstance(search_result, dict) or ('id' not in search_result):
+                    continue
+                song_info = SongInfo(source=self.source)
+                # ----parse from quark links
+                if self.quark_parser_config.get('cookies'):
+                    quark_download_urls = [*search_result.get('downlist', []), *search_result.get('ktmdownlist', [])]
+                    for quark_download_url in quark_download_urls:
+                        song_fmt = safeextractfromdict(quark_download_url, ['format'], '')
+                        if not song_fmt or song_fmt.lower() in ['mp3']: continue
+                        song_info = SongInfo(source=self.source)
+                        try:
+                            quark_wav_download_url = quark_download_url['url']
+                            download_result, download_url = QuarkParser.parsefromurl(quark_wav_download_url, **self.quark_parser_config)
+                            if not download_url: continue
+                            download_url_status = self.quark_audio_link_tester.test(download_url, request_overrides)
+                            download_url_status['probe_status'] = self.quark_audio_link_tester.probe(download_url, request_overrides)
+                            ext = download_url_status['probe_status']['ext']
+                            if ext == 'NULL': ext = 'flac'
+                            song_info.update(dict(
+                                download_url=download_url, download_url_status=download_url_status, raw_data={'search': search_result, 'download': download_result},
+                                default_download_headers=self.quark_default_download_headers, ext=ext, file_size=download_url_status['probe_status']['file_size']
+                            ))
+                            if song_info.with_valid_download_url: break
+                        except:
+                            continue
+                # ----parse from play url
+                if not song_info.with_valid_download_url:
+                    song_info = SongInfo(source=self.source)
+                    try:
+                        resp = self.get(f'https://1mp3.top/include/geturl.php?id={search_result["id"]}', **request_overrides)
+                        resp.raise_for_status()
+                        download_url = resp.text.strip()
+                        download_url_status = self.audio_link_tester.test(download_url, request_overrides)
+                        download_url_status['probe_status'] = self.audio_link_tester.probe(download_url, request_overrides)
+                        ext = download_url_status['probe_status']['ext']
+                        if ext == 'NULL': download_url.split('.')[-1].split('?')[0] or 'mp3'
+                        song_info.update(dict(
+                            download_url=download_url, download_url_status=download_url_status, raw_data={'search': search_result, 'download': {}},
+                            ext=ext, file_size=download_url_status['probe_status']['file_size']
+                        ))
+                    except:
+                        continue
+                if not song_info.with_valid_download_url: continue
+                # ----parse more infos
+                lyric_result, lyric = dict(), 'NULL'
+                song_info.raw_data['lyric'] = lyric_result
+                song_info.update(dict(
+                    lyric=lyric, duration='-:-:-', song_name=legalizestring(search_result.get('title', 'NULL'), replace_null_string='NULL'), 
+                    singers=legalizestring(search_result.get('singer', 'NULL'), replace_null_string='NULL'), album='NULL', identifier=search_result['id'],
+                ))
+                # --append to song_infos
+                song_infos.append(song_info)
+                # --judgement for search_size
+                if self.strict_limit_search_size_per_page and len(song_infos) >= self.search_size_per_page: break
+            # --update progress
+            progress.update(progress_id, description=f"{self.source}.search >>> {search_url} (Success)")
+        # failure
+        except Exception as err:
+            progress.update(progress_id, description=f"{self.source}.search >>> {search_url} (Error: {err})")
+        # return
+        return song_infos

musicdl/modules/sources/youtube.py

@@ -0,0 +1,238 @@
+'''
+Function:
+    Implementation of YouTubeMusicClient: https://music.youtube.com/
+Author:
+    Zhenchao Jin
+WeChat Official Account (微信公众号):
+    Charles的皮卡丘
+'''
+import copy
+import random
+from ytmusicapi import YTMusic
+from .base import BaseMusicClient
+from rich.progress import Progress
+from ..utils.youtubeutils import YouTube
+from ..utils import legalizestring, resp2json, usesearchheaderscookies, byte2mb, seconds2hms, usedownloadheaderscookies, touchdir, SongInfo, SongInfoUtils
+
+
+'''constants'''
+REPAIDAPI_KEYS = [
+    "1556f6ccb2msh64b807485156b33p12a66djsn7e197bc23f19", "323be663ccmsh35173cb3c5403c2p10f08fjsnad5cf7b49fa9", "7ca8a11abdmsh7eb89d767db710cp191ce7jsnf0274562b5b9", "14a93bf7a5msh51f11db7d121aeap1c4625jsn5cfbb89ff05b", 
+    "6111b62d2dmshd13d20b55abbbe8p1f8551jsn2e425d987b94", "fcdf2c5ba1msh397175bf89fe87ep19fe51jsn76bc6eddb936", "ab47a3c2camsh4097dc8ff0fb89fp1d95e6jsna047ccc76ab6", "3b6e8de0bfmshf1110629d4a95b0p11b000jsn264f29acc2b0",
+    "9203689df7msh9f35abfe7467ce8p12ca71jsn625743a17f66", "ef908b5eeamsh50984dd5b890d6ep1ab791jsn2fb248ebf19a",
+]
+
+
+'''YouTubeMusicClient'''
+class YouTubeMusicClient(BaseMusicClient):
+    source = 'YouTubeMusicClient'
+    def __init__(self, **kwargs):
+        super(YouTubeMusicClient, self).__init__(**kwargs)
+        self.default_search_headers = {}
+        self.default_download_headers = {}
+        self.default_headers = self.default_search_headers
+        self._initsession()
+    '''_download'''
+    @usedownloadheaderscookies
+    def _download(self, song_info: SongInfo, request_overrides: dict = None, downloaded_song_infos: list = [], progress: Progress = None, song_progress_id: int = 0):
+        if isinstance(song_info.download_url, str): return super()._download(song_info=song_info, request_overrides=request_overrides, downloaded_song_infos=downloaded_song_infos, progress=progress, song_progress_id=song_progress_id)
+        request_overrides = request_overrides or {}
+        try:
+            touchdir(song_info.work_dir)
+            total_size, chunk_size, downloaded_size = int(song_info.download_url.filesize), song_info.get('chunk_size', 1024 * 1024), 0
+            progress.update(song_progress_id, total=total_size)
+            with open(song_info.save_path, "wb") as fp:
+                for chunk in song_info.download_url.iterchunks(chunk_size=chunk_size):
+                    if not chunk: continue
+                    fp.write(chunk)
+                    downloaded_size = downloaded_size + len(chunk)
+                    if total_size > 0:
+                        downloading_text = "%0.2fMB/%0.2fMB" % (downloaded_size / 1024 / 1024, total_size / 1024 / 1024)
+                    else:
+                        progress.update(song_progress_id, total=downloaded_size)
+                        downloading_text = "%0.2fMB/%0.2fMB" % (downloaded_size / 1024 / 1024, downloaded_size / 1024 / 1024)
+                    progress.advance(song_progress_id, len(chunk))
+                    progress.update(song_progress_id, description=f"{self.source}.download >>> {song_info['song_name']} (Downloading: {downloading_text})")
+            progress.update(song_progress_id, description=f"{self.source}.download >>> {song_info['song_name']} (Success)")
+            downloaded_song_infos.append(SongInfoUtils.fillsongtechinfo(copy.deepcopy(song_info), logger_handle=self.logger_handle, disable_print=self.disable_print))
+        except Exception as err:
+            progress.update(song_progress_id, description=f"{self.source}.download >>> {song_info['song_name']} (Error: {err})")
+        return downloaded_song_infos
+    '''_searchwithytsearchanddownloadmp3rapidapi'''
+    def _searchwithytsearchanddownloadmp3rapidapi(self, keyword: str, request_overrides: dict = None):
+        # init
+        request_overrides = request_overrides or {}
+        default_headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+            "X-Rapidapi-Host": "youtube-music-api3.p.rapidapi.com", "X-Rapidapi-Key": random.choice(REPAIDAPI_KEYS),
+            "Referer": "https://music-download-lake.vercel.app/", "Origin": "https://music-download-lake.vercel.app",
+            "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+        }
+        # search
+        params = {'q': keyword, 'type': 'song', 'limit': self.search_size_per_source}
+        for repaidapi_key in REPAIDAPI_KEYS:
+            try:
+                default_headers['X-Rapidapi-Key'] = repaidapi_key
+                resp = self.get(f'https://youtube-music-api3.p.rapidapi.com/search', params=params, headers=default_headers, **request_overrides)
+                resp.raise_for_status()
+                search_results = resp2json(resp=resp)['result']
+                assert len(search_results) > 0
+                break
+            except:
+                continue
+        # return
+        return search_results
+    '''_parsewithytsearchanddownloadmp3rapidapi'''
+    def _parsewithytsearchanddownloadmp3rapidapi(self, search_result: dict, request_overrides: dict = None):
+        # init
+        request_overrides, vid = request_overrides or {}, search_result['videoId']
+        default_headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+            "X-Rapidapi-Host": "yt-search-and-download-mp3.p.rapidapi.com", "X-Rapidapi-Key": random.choice(REPAIDAPI_KEYS),
+            "Referer": "https://music-download-lake.vercel.app/", "Origin": "https://music-download-lake.vercel.app",
+            "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+        }
+        # parse
+        for repaidapi_key in REPAIDAPI_KEYS:
+            try:
+                self.default_headers['X-Rapidapi-Key'] = repaidapi_key
+                resp = self.get(f'https://yt-search-and-download-mp3.p.rapidapi.com/mp3?url=https://www.youtube.com/watch?v={vid}', headers=default_headers, **request_overrides)
+                resp.raise_for_status()
+                download_result = resp2json(resp=resp)
+            except:
+                continue
+            if not download_result.get('download'): continue
+            song_info = SongInfo(
+                source=self.source, download_url=download_result['download'], download_url_status=self.audio_link_tester.test(download_result['download'], request_overrides), 
+                ext='mp3', raw_data={'search': search_result, 'download': download_result}, duration=search_result.get('duration')
+            )
+            song_info.download_url_status['probe_status'] = self.audio_link_tester.probe(song_info.download_url, request_overrides)
+            ext, file_size = song_info.download_url_status['probe_status']['ext'], song_info.download_url_status['probe_status']['file_size']
+            if file_size and file_size != 'NULL': song_info.file_size = file_size
+            if not song_info.file_size: song_info.file_size = 'NULL'
+            if ext and ext != 'NULL': song_info.ext = ext
+            if song_info.with_valid_download_url: break
+        # return
+        return song_info
+    '''_parsvidewithmp3youtube'''
+    def _parsvidewithmp3youtube(self, search_result: dict, request_overrides: dict = None):
+        # init
+        request_overrides, vid = request_overrides or {}, search_result['videoId']
+        request_headers = {"Content-Type": "application/json", "Origin": "https://iframe.y2meta-uk.com", "Accept": "*/*"}
+        download_headers = {"Content-Type": "application/json", "Origin": "https://iframe.y2meta-uk.com", "Accept": "*/*", "Referer": "https://iframe.y2meta-uk.com/"}
+        request_overrides_test = copy.deepcopy(request_overrides)
+        request_overrides_test['headers'] = download_headers
+        # get key
+        resp = self.get('https://api.mp3youtube.cc/v2/sanity/key', headers=request_headers, **request_overrides)
+        resp.raise_for_status()
+        # parse
+        request_headers.update(dict(key=resp2json(resp)['key']))
+        for quality in ['320', '256', '192', '128', '64']:
+            audio_payload = {"link": f"https://youtu.be/{vid}", "format": "mp3", "audioBitrate": quality, "videoQuality": "720", "vCodec": "h264"}
+            try:
+                resp = self.post('https://api.mp3youtube.cc/v2/converter', json=audio_payload, headers=request_headers, **request_overrides)
+                resp.raise_for_status()
+            except:
+                continue
+            download_result = resp2json(resp=resp)
+            if not download_result.get('url'): continue
+            song_info = SongInfo(
+                source=self.source, default_download_headers=download_headers, download_url=download_result['url'],
+                download_url_status=self.audio_link_tester.test(download_result['url'], request_overrides_test), ext='mp3',
+                raw_data={'search': search_result, 'download': download_result}, duration=search_result.get('duration')
+            )
+            song_info.download_url_status['probe_status'] = self.audio_link_tester.probe(song_info.download_url, request_overrides_test)
+            ext, file_size = song_info.download_url_status['probe_status']['ext'], song_info.download_url_status['probe_status']['file_size']
+            if file_size and file_size != 'NULL': song_info.file_size = file_size
+            if not song_info.file_size: song_info.file_size = 'NULL'
+            if ext and ext != 'NULL': song_info.ext = ext
+            if song_info.with_valid_download_url: break
+        # return
+        return song_info
+    '''_constructsearchurls'''
+    def _constructsearchurls(self, keyword: str, rule: dict = None, request_overrides: dict = None):
+        # init
+        rule, request_overrides = rule or {}, request_overrides or {}
+        # adapt ytmusicapi to conduct music file search
+        proxies = None
+        if self.proxied_session_client is not None: proxies = self.proxied_session_client.getrandomproxy()
+        ytmusic_api = YTMusic(
+            auth=rule.get('auth', None), user=rule.get('user', None), requests_session=None, proxies=request_overrides.get('proxies', None) or proxies,
+            language=rule.get('language', 'en'), location=rule.get('location', ''), oauth_credentials=rule.get('oauth_credentials', ''), 
+        )
+        # search rules
+        search_rule = {
+            'query': keyword, 'filter': rule.get('filter', None), 'scope': rule.get('scope', None), 'limit': self.search_size_per_source,
+            'ignore_spelling': rule.get('ignore_spelling', False)
+        }
+        search_urls = [{'ytmusic_api': ytmusic_api, 'search_rule': search_rule}]
+        self.search_size_per_page = self.search_size_per_source
+        # return
+        return search_urls
+    '''_search'''
+    @usesearchheaderscookies
+    def _search(self, keyword: str = '', search_url: dict = {}, request_overrides: dict = None, song_infos: list = [], progress: Progress = None, progress_id: int = 0):
+        # init
+        request_overrides = request_overrides or {}
+        search_meta = copy.deepcopy(search_url)
+        ytmusic_api, search_rule = search_meta['ytmusic_api'], search_meta['search_rule']
+        # successful
+        try:
+            # --search results, if search scale not very large, try rapid api first
+            if self.search_size_per_source <= 10:
+                try:
+                    search_results = self._searchwithytsearchanddownloadmp3rapidapi(keyword=keyword, request_overrides=request_overrides)
+                    if len(search_results) == 0: raise
+                except:
+                    search_results = ytmusic_api.search(**search_rule)
+                    search_results = [s for s in search_results if s['resultType'] in ['song']]
+            else:
+                search_results = ytmusic_api.search(**search_rule)
+                search_results = [s for s in search_results if s['resultType'] in ['song']]
+            for search_result in search_results:
+                # --download results
+                if not isinstance(search_result, dict) or 'videoId' not in search_result:
+                    continue
+                song_info, ump_success = SongInfo(source=self.source), False
+                # ----download with UMP
+                try:
+                    download_url = YouTube(video_id=search_result['videoId']).streams.getaudioonly()
+                    song_info = SongInfo(
+                        source=self.source, download_url=download_url, download_url_status={'ok': True}, file_size=byte2mb(download_url.filesize),
+                        file_size_bytes=download_url.filesize, duration_s=int(download_url.durationMs) // 1000, duration=seconds2hms(int(download_url.durationMs) // 1000),
+                        ext='mp3', raw_data={'search': search_result, 'download': {}},
+                    )
+                    if song_info.file_size_bytes < 100: raise
+                    ump_success = True
+                except:
+                    ump_success = False
+                    pass
+                # ----parse with _parsvidewithmp3youtube
+                if not ump_success:
+                    song_info = self._parsvidewithmp3youtube(search_result=search_result, request_overrides=request_overrides)
+                # ----parse with _parsewithytsearchanddownloadmp3rapidapi
+                if (not ump_success) and (not song_info.with_valid_download_url):
+                    song_info = self._parsewithytsearchanddownloadmp3rapidapi(search_result=search_result, request_overrides=request_overrides)
+                # ----parse more information
+                format_duration = lambda d: "{:02}:{:02}:{:02}".format(*([0] * (3 - len(d.split(":"))) + list(map(int, d.split(":")))))
+                song_info.duration = format_duration(song_info.duration)
+                singers = ', '.join([singer.get('name', 'NULL') for singer in search_result.get('artists', [])]) if search_result.get('artists') else search_result.get('author', 'NULL')
+                song_info.update(dict(
+                    singers=legalizestring(singers, replace_null_string='NULL'), song_name=legalizestring(search_result.get('title', 'NULL'), replace_null_string='NULL'), 
+                    album=legalizestring(search_result.get('album', 'NULL'), replace_null_string='NULL'), identifier=search_result['videoId'],
+                ))
+                # --lyric results
+                lyric_result, lyric = dict(), 'NULL'
+                song_info.raw_data['lyric'] = lyric_result
+                song_info.lyric = lyric
+                # --append to song_infos
+                song_infos.append(song_info)
+                # --judgement for search_size
+                if self.strict_limit_search_size_per_page and len(song_infos) >= self.search_size_per_page: break
+            # --update progress
+            progress.update(progress_id, description=f"{self.source}.search >>> {search_url} (Success)")
+        # failure
+        except Exception as err:
+            progress.update(progress_id, description=f"{self.source}.search >>> {search_url} (Error: {err})")
+        # return
+        return song_infos

musicdl/modules/utils/__init__.py

@@ -1,4 +1,12 @@
-'''import all'''
-from .downloader import Downloader
-from .logger import Logger, printTable
-from .misc import checkDir, seconds2hms, loadConfig, filterBadCharacter
+'''initialize'''
+from .data import SongInfo
+from .lyric import WhisperLRC
+from .quarkparser import QuarkParser
+from .songinfoutils import SongInfoUtils
+from .modulebuilder import BaseModuleBuilder
+from .logger import LoggerHandle, colorize, printtable, printfullline, smarttrunctable
+from .misc import (
+    AudioLinkTester, legalizestring, touchdir, seconds2hms, byte2mb, cachecookies, resp2json, isvalidresp, safeextractfromdict, replacefile,
+    usedownloadheaderscookies, useparseheaderscookies, usesearchheaderscookies, cookies2dict, cookies2string, estimatedurationwithfilesizebr,
+    estimatedurationwithfilelink
+)