misato 0.7.9__tar.gz

misato-0.7.9/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 MiyukiQAQ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

misato-0.7.9/MANIFEST.in

@@ -0,0 +1 @@
+

misato-0.7.9/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: misato
+Version: 0.7.9
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: curl_cffi
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python

misato-0.7.9/README.md

@@ -0,0 +1,126 @@
+## ⭐️ misato
+
+A tool for downloading videos from the "MissAV" website.
+
+## ⚙️ Installation
+
+To install misato from the Python Package Index (PyPI) run:
+
+```
+pip install misato
+```
+
+To upgrade misato from the Python Package Index (PyPI) run:
+
+```
+pip install --upgrade misato
+```
+
+## 📷 Snapshot
+
+![snapshot.png](resources/readme_pics/snapshot.png)
+
+## 📖 Instructions
+
+```
+[root@misato ~]# misato -h
+usage: main.py [-h] [-auto  [...]] [-auth  [...]] [-limit] [-search] [-file] [-proxy] [-ffmpeg] [-cover] [-ffcover] [-noban] [-title] [-quality] [-retry] [-delay] [-timeout]
+
+A tool for downloading videos from the "MissAV" website.
+
+Main Options:
+Use the -auto   option to specify the video or playlist URLs to download. can be mixed.
+Use the -auth   option to specify the username and password to download the videos collected by the account.
+Use the -search option to search for movie by serial number and download it.
+Use the -file   option to download video or playlist URLs in the file. ( Each line is a URL )
+
+Additional Options:
+Use the -limit   option to limit the number of downloads. 
+Use the -proxy   option to configure http proxy server ip and port.
+Use the -ffmpeg  option to get the best video quality. ( Recommend! )
+Use the -cover   option to save the cover when downloading the video
+Use the -ffcover option to set the cover as the video preview (ffmpeg required)
+Use the -noban   option to turn off the misato banner when downloading the video
+Use the -title   option to use the full title as the movie file name
+Use the -quality option to specify the movie resolution (360, 480, 720, 1080...)
+Use the -retry   option to specify the number of retries for downloading segments
+Use the -delay   option to specify the delay before retry ( seconds )
+Use the -timeout option to specify the timeout for segment download ( seconds )
+
+options:
+  -h, --help     show this help message and exit
+  -auto  [ ...]  Multiple movie and playlist URLs can be mixed. separate with spaces
+  -auth  [ ...]  Username and password, separate with space
+  -limit         Limit the number of downloads
+  -search        Movie serial number
+  -file          File path
+  -proxy         HTTP(S) proxy
+  -ffmpeg        Enable ffmpeg processing
+  -cover         Download video cover
+  -ffcover       Set cover as preview (ffmpeg required)
+  -noban         Do not display the banner
+  -title         Full title as file name
+  -quality       Specify the movie resolution
+  -retry         Number of retries for downloading segments
+  -delay         Delay in seconds before retry
+  -timeout       Timeout in seconds for segment download
+
+Examples:
+  misato -auto "https://missav.ai/sw-950" "https://missav.ai/dm132/actresses/JULIA"
+  misato -auto "https://missav.ai/dm132/actresses/JULIA" -limit 20 -ffcover
+  misato -auto "https://missav.ai/sw-950" "https://missav.ai/dandy-917"
+  misato -auto "https://missav.ai/sw-950" -proxy localhost:7890
+  misato -auth misato@gmail.com miyukiQAQ -ffmpeg -noban -limit 20
+  misato -file /home/misato/url.txt -ffmpeg -title -limit 20
+  misato -search sw-950 -ffcover -quality 720
+```
+
+## 💬 The ```-auto``` option
+
+- Use the -auto option to download movies from a playlist.
+- This playlist can be a public playlist created by your own account, or any playlist displayed based on search results or tag filters.
+- **You should wrap the playlist URL with " " when you use the -auto option.**
+
+Command Examples:
+- ```misato -auto "https://missav.ai/search/JULIA?filters=uncensored-leak&sort=saved" -limit 50 -ffmpeg```
+- ```misato -auto "https://missav.ai/search/JULIA?filters=individual&sort=views" -limit 20 -ffmpeg```
+- ```misato -auto "https://missav.ai/dm132/actresses/JULIA" -limit 20 -ffmpeg```
+- ```misato -auto "https://missav.ai/playlists/ewzoukev" -limit 20 -ffmpeg```
+- ```misato -auto "https://missav.ai/dm444/en/labels/WANZ" -limit 20 -ffmpeg```
+- ```misato -auto "https://missav.ai/dm21/en/makers/Takara%20Visual" -limit 20 -ffmpeg```
+- ```misato -auto "https://missav.ai/dm1/en/genres/4K" -limit 20 -ffmpeg```
+
+## 💡 Precautions
+
+- If you are from an ancient oriental country, you will most likely need a proxy.
+- Use ffmpeg to synthesize videos for the best experience.
+
+## 👀 About FFmpeg
+
+1. If you want misato to use ffmpeg to process the video, use the -ffmpeg option.
+2. Please check whether the ffmpeg command is valid before using the -ffmpeg option. (e.g. ```ffmpeg -version```)
+3. To install FFmpeg, please refer to https://ffmpeg.org/
+
+## 📄 Disclaimer
+
+This project is licensed under the [MIT License](LICENSE). The following additional disclaimers and notices apply:
+
+### 1. Legal Compliance
+- This software is provided solely for **communication, research, learning, and personal use**.  
+- Users are responsible for ensuring that their use of this software complies with all applicable laws and regulations in their jurisdiction.  
+- The software must not be used for any unlawful, unethical, or unauthorized purposes, including but not limited to violating third-party rights or legal restrictions.
+
+### 2. No Warranty
+As stated in the MIT License:  
+> "THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT."
+
+### 3. Limitation of Liability
+- The author(s) shall not be held liable for any claims, damages, or other liabilities arising from or in connection with the use or performance of this software.  
+- Users bear all risks and responsibilities for the use of this software, including but not limited to data loss, system damage, or legal consequences.
+
+### 4. Third-Party Dependencies
+- This project may include or depend on third-party libraries or tools. Users are responsible for reviewing and complying with the licenses and terms of these dependencies.
+
+### 5. Security and Privacy
+- This software may interact with user systems, networks, or data. Users should implement appropriate security measures to protect sensitive information and infrastructure.  
+- The authors are not responsible for any security vulnerabilities or data breaches resulting from the use of this software.

misato-0.7.9/misato/config.py

@@ -0,0 +1,20 @@
+RECORD_FILE = 'downloaded_urls_misato.txt'
+FFMPEG_INPUT_FILE = 'ffmpeg_input_misato.txt'
+TMP_HTML_FILE = 'tmp_movie_misato.html'
+MOVIE_SAVE_PATH_ROOT = 'movies_folder_misato'
+COVER_URL_PREFIX = 'https://fourhoi.com/'
+VIDEO_M3U8_PREFIX = 'https://surrit.com/'
+VIDEO_PLAYLIST_SUFFIX = '/playlist.m3u8'
+HREF_REGEX_MOVIE_COLLECTION = r'<a class="text-secondary group-hover:text-primary" href="([^"]+)" alt="'
+HREF_REGEX_PUBLIC_PLAYLIST = r'<a href="([^"]+)" alt="'
+HREF_REGEX_NEXT_PAGE = r'<a href="([^"]+)" rel="next"'
+MATCH_UUID_PATTERN = r'm3u8\|([a-f0-9\|]+)\|com\|surrit\|https\|video'
+MATCH_TITLE_PATTERN = r'<title>([^"]+)</title>'
+RESOLUTION_PATTERN = r'RESOLUTION=(\d+)x(\d+)'
+MAGIC_NUMBER = 114514
+RETRY = 5
+DELAY = 2
+TIMEOUT = 10
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+}

misato-0.7.9/misato/ffmpeg_processor.py

@@ -0,0 +1,24 @@
+import subprocess
+from typing import Optional
+from misato.config import FFMPEG_INPUT_FILE
+from misato.logger import logger
+
+
+class FFmpegProcessor:
+    @staticmethod
+    def create_video_from_segments(segment_files: list[str], output_file: str, cover_file: Optional[str] = None) -> None:
+        with open(FFMPEG_INPUT_FILE, 'w') as f:
+            for file in segment_files:
+                f.write(f"file '{file}'\n")
+        ffmpeg_command = ['ffmpeg', '-y', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', FFMPEG_INPUT_FILE]
+        if cover_file:
+            ffmpeg_command.extend(['-i', cover_file, '-map', '0', '-map', '1', '-c', 'copy', '-disposition:v:1', 'attached_pic'])
+        else:
+            ffmpeg_command.extend(['-c', 'copy'])
+        ffmpeg_command.append(output_file)
+        try:
+            subprocess.run(ffmpeg_command, check=True, stdout=subprocess.DEVNULL)
+            logger.info("FFmpeg execution completed.")
+        except subprocess.CalledProcessError as e:
+            logger.error(f"FFmpeg execution failed: {e}")
+            raise

misato-0.7.9/misato/http_client.py

@@ -0,0 +1,29 @@
+from typing import Optional
+import time
+from curl_cffi import requests
+from misato.config import HEADERS, RETRY, DELAY, TIMEOUT
+from misato.logger import logger
+
+
+class HttpClient:
+    def get(self, url: str, cookies: Optional[dict] = None, retries: int = RETRY, delay: int = DELAY, timeout: int = TIMEOUT) -> Optional[bytes]:
+        for attempt in range(retries):
+            try:
+                response = requests.get(url=url, headers=HEADERS, cookies=cookies, timeout=timeout, verify=False)
+                return response.content
+            except Exception as e:
+                logger.error(f"Failed to fetch data (attempt {attempt + 1}/{retries}): {e} url is: {url}")
+                time.sleep(delay)
+        logger.error(f"Max retries reached. Failed to fetch data. url is: {url}")
+        return None
+
+    def post(self, url: str, data: dict, cookies: Optional[dict] = None, retries: int = RETRY, delay: int = DELAY, timeout: int = TIMEOUT) -> Optional[requests.Response]:
+        for attempt in range(retries):
+            try:
+                response = requests.post(url=url, data=data, headers=HEADERS, cookies=cookies, timeout=timeout, verify=False)
+                return response
+            except Exception as e:
+                logger.error(f"Failed to post data (attempt {attempt + 1}/{retries}): {e} url is: {url}")
+                time.sleep(delay)
+        logger.error(f"Max retries reached. Failed to post data. url is: {url}")
+        return None

misato-0.7.9/misato/logger.py

@@ -0,0 +1,17 @@
+import logging
+
+logger = logging.getLogger('misato-logger')
+logger.setLevel(logging.DEBUG)
+
+file_handler = logging.FileHandler('misato.log')
+file_handler.setLevel(logging.DEBUG)
+
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('misato - %(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)

misato-0.7.9/misato/main.py

@@ -0,0 +1,188 @@
+import argparse
+import os
+import subprocess
+from misato.logger import logger
+from misato.config import MOVIE_SAVE_PATH_ROOT, RECORD_FILE, MAGIC_NUMBER
+from misato.http_client import HttpClient
+from misato.url_sources import SingleUrlSource, PlaylistSource, AuthSource, SearchSource, FileSource, AutoUrlSource
+from misato.video_downloader import VideoDownloader
+from misato.utils import delete_all_subfolders, ThreadSafeCounter
+
+banner = """
+ ██████   ██████  ███                        █████       ███ 
+░░██████ ██████  ░░░                        ░░███       ░░░  
+ ░███░█████░███  ████  █████ ████ █████ ████ ░███ █████ ████ 
+ ░███░░███ ░███ ░░███ ░░███ ░███ ░░███ ░███  ░███░░███ ░░███ 
+ ░███ ░░░  ░███  ░███  ░███ ░███  ░███ ░███  ░██████░   ░███ 
+ ░███      ░███  ░███  ░███ ░███  ░███ ░███  ░███░░███  ░███ 
+ █████     █████ █████ ░░███████  ░░████████ ████ █████ █████
+░░░░░     ░░░░░ ░░░░░   ░░░░░███   ░░░░░░░░ ░░░░ ░░░░░ ░░░░░ 
+                        ███ ░███                             
+                       ░░██████                              
+                        ░░░░░░                               
+"""
+
+
+class DownloadTracker:
+    def __init__(self, record_file: str):
+        self.record_file = record_file
+        self.downloaded_urls = set()
+        if os.path.exists(record_file):
+            with open(record_file, 'r', encoding='utf-8') as f:
+                self.downloaded_urls.update(line.strip() for line in f)
+
+    def is_downloaded(self, url: str) -> bool:
+        return url in self.downloaded_urls
+
+    def record_download(self, url: str) -> None:
+        self.downloaded_urls.add(url)
+        with open(self.record_file, 'a', encoding='utf-8') as f:
+            f.write(url + '\n')
+
+
+def check_ffmpeg_command(ffmpeg: bool) -> bool:
+    if not ffmpeg:
+        return True
+    try:
+        subprocess.run(['ffmpeg', '-version'], check=True, stdout=subprocess.DEVNULL)
+        return True
+    except Exception:
+        return False
+
+
+def validate_args(args):
+    params = [args.auth, args.search, args.file, args.auto]
+    if sum(param is not None for param in params) != 1:
+        logger.error("Exactly one of -auto, -auth, -search, -file must be specified.")
+        exit(MAGIC_NUMBER)
+    if args.auth and len(args.auth) != 2:
+        logger.error("Auth requires username and password.")
+        exit(MAGIC_NUMBER)
+    if not check_ffmpeg_command(args.ffmpeg) or not check_ffmpeg_command(args.ffcover):
+        logger.error("FFmpeg command status error.")
+        exit(MAGIC_NUMBER)
+    for opt in ['limit', 'quality', 'retry', 'delay', 'timeout']:
+        value = getattr(args, opt)
+        if value and (not value.isdigit() or int(value) <= 0):
+            logger.error(f"The -{opt} option must be a positive integer.")
+            exit(MAGIC_NUMBER)
+    if args.file and (not os.path.isfile(args.file) or os.path.getsize(args.file) == 0):
+        logger.error("The -file option must be a valid non-empty file.")
+        exit(MAGIC_NUMBER)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='A tool for downloading videos from the "MissAV" website.\n'
+                    '\n'
+                    'Main Options:\n'
+                    'Use the -auto   option to specify the video or playlist URLs to download. can be mixed.\n'
+                    'Use the -auth   option to specify the username and password to download the videos collected by the account.\n'
+                    'Use the -search option to search for movie by serial number and download it.\n'
+                    'Use the -file   option to download video or playlist URLs in the file. ( Each line is a URL )\n'
+                    '\n'
+                    'Additional Options:\n'
+                    'Use the -limit   option to limit the number of downloads. \n'
+                    'Use the -proxy   option to configure http proxy server ip and port.\n'
+                    'Use the -ffmpeg  option to get the best video quality. ( Recommend! )\n'
+                    'Use the -cover   option to save the cover when downloading the video\n'
+                    'Use the -ffcover option to set the cover as the video preview (ffmpeg required)\n'
+                    'Use the -noban   option to turn off the misato banner when downloading the video\n'
+                    'Use the -title   option to use the full title as the movie file name\n'
+                    'Use the -quality option to specify the movie resolution (360, 480, 720, 1080...)\n'
+                    'Use the -retry   option to specify the number of retries for downloading segments\n'
+                    'Use the -delay   option to specify the delay before retry ( seconds )\n'
+                    'Use the -timeout option to specify the timeout for segment download ( seconds )\n',
+        epilog='Examples:\n'
+               '  misato -auto "https://missav.ai/sw-950" "https://missav.ai/dm132/actresses/JULIA"\n'
+               '  misato -auto "https://missav.ai/dm132/actresses/JULIA" -limit 20 -ffcover\n'
+               '  misato -auto "https://missav.ai/sw-950" "https://missav.ai/dandy-917"\n'
+               '  misato -auto "https://missav.ai/sw-950" -proxy localhost:7890\n'
+               '  misato -auth misato@gmail.com misatoQAQ -ffmpeg -noban -limit 20\n'
+               '  misato -file /home/misato/url.txt -ffmpeg -title -limit 20\n'
+               '  misato -search sw-950 -ffcover -quality 720\n',
+        formatter_class=argparse.RawTextHelpFormatter
+    )
+    parser.add_argument('-auto', nargs='+', metavar='', help='Multiple movie and playlist URLs can be mixed. separate with spaces')
+    parser.add_argument('-auth', nargs='+', metavar='', help='Username and password, separate with space')
+    parser.add_argument('-limit', type=str, metavar='', help='Limit the number of downloads')
+    parser.add_argument('-search', type=str, metavar='', help='Movie serial number')
+    parser.add_argument('-file', type=str, metavar='', help='File path')
+    parser.add_argument('-proxy', type=str, metavar='', help='HTTP(S) proxy')
+    parser.add_argument('-ffmpeg', action='store_true', help='Enable ffmpeg processing')
+    parser.add_argument('-cover', action='store_true', help='Download video cover')
+    parser.add_argument('-ffcover', action='store_true', help='Set cover as preview (ffmpeg required)')
+    parser.add_argument('-noban', action='store_true', help='Do not display the banner')
+    parser.add_argument('-title', action='store_true', help='Full title as file name')
+    parser.add_argument('-quality', type=str, metavar='', help='Specify the movie resolution')
+    parser.add_argument('-retry', type=str, metavar='', help='Number of retries for downloading segments')
+    parser.add_argument('-delay', type=str, metavar='', help='Delay in seconds before retry')
+    parser.add_argument('-timeout', type=str, metavar='', help='Timeout in seconds for segment download')
+
+    args = parser.parse_args()
+    logger.info(str(args))
+    validate_args(args)
+
+    if not args.noban:
+        print(banner)
+
+    if args.ffcover:
+        args.ffmpeg = True
+        args.cover = True
+
+    if args.proxy:
+        logger.info("Network proxy enabled.")
+        os.environ["http_proxy"] = f"http://{args.proxy}"
+        os.environ["https_proxy"] = f"http://{args.proxy}"
+
+    http_client = HttpClient()
+    movie_counter = ThreadSafeCounter()
+    source = (
+        AutoUrlSource(movie_counter=movie_counter, auto_urls=args.auto, limit=args.limit) if args.auto else
+        AuthSource(movie_counter=movie_counter, username=args.auth[0], password=args.auth[1], limit=args.limit) if args.auth else
+        SearchSource(movie_counter=movie_counter, key=args.search) if args.search else
+        FileSource(movie_counter=movie_counter, file_path=args.file, limit=args.limit) if args.file else None
+    )
+    if not source:
+        logger.error("No source specified.")
+        exit(MAGIC_NUMBER)
+
+    movie_urls = source.get_urls()
+    if not movie_urls:
+        logger.error("No URLs to download.")
+        exit(MAGIC_NUMBER)
+
+    download_tracker = DownloadTracker(RECORD_FILE)
+    options = {
+        'download_action': True,
+        'write_action': True,
+        'ffmpeg_action': args.ffmpeg,
+        'num_threads': os.cpu_count(),
+        'cover_action': args.cover,
+        'title_action': args.title,
+        'cover_as_preview': args.ffcover,
+        'quality': int(args.quality) if args.quality else None,
+        'retry': int(args.retry) if args.retry else 5,
+        'delay': int(args.delay) if args.delay else 2,
+        'timeout': int(args.timeout) if args.timeout else 10
+    }
+
+    for url in movie_urls:
+        if download_tracker.is_downloaded(url):
+            logger.info(f"{url} already downloaded, skipping.")
+            continue
+        delete_all_subfolders(MOVIE_SAVE_PATH_ROOT)
+        try:
+            logger.info(f"Processing URL: {url}")
+            downloader = VideoDownloader(url, http_client, options)
+            downloader.download()
+            download_tracker.record_download(url)
+            logger.info(f"Processing URL Complete: {url}")
+            print()
+        except Exception as e:
+            logger.error(f"Failed to download {url}: {e}")
+        delete_all_subfolders(MOVIE_SAVE_PATH_ROOT)
+
+
+if __name__ == "__main__":
+    main()

misato-0.7.9/misato/url_sources.py

@@ -0,0 +1,159 @@
+from abc import ABC, abstractmethod
+import re
+from typing import Optional
+from misato.http_client import HttpClient
+from misato.config import HREF_REGEX_PUBLIC_PLAYLIST, HREF_REGEX_NEXT_PAGE, MATCH_UUID_PATTERN
+from misato.logger import logger
+from misato.utils import ThreadSafeCounter
+from enum import Enum
+
+class UrlType(Enum):
+    SINGLE = 1
+    PLAYLIST = 2
+
+class UrlSource(ABC):
+    @abstractmethod
+    def get_urls(self) -> list[str]:
+        pass
+
+    @staticmethod
+    def movie_count_log(movie_counter: ThreadSafeCounter, movie_url: str):
+        logger.info(f"Movie {movie_counter.increment_and_get()} url: {movie_url}")
+
+    @staticmethod
+    def get_urls_from_list(movie_counter: ThreadSafeCounter, play_list_url: str, limit: Optional[str], cookie=None, http_client: HttpClient = None) -> list[str]:
+        movie_url_list = []
+        url = play_list_url
+        while url and (limit is None or movie_counter.get() < limit):
+            html_source = http_client.get(url, cookies=cookie)
+            if html_source is None:
+                break
+            html_source = html_source.decode('utf-8')
+            movie_url_matches = re.findall(HREF_REGEX_PUBLIC_PLAYLIST, html_source)
+            temp_url_list = list(set(movie_url_matches))
+            for movie_url in temp_url_list:
+                movie_url_list.append(movie_url)
+                UrlSource.movie_count_log(movie_counter, movie_url)
+                if limit and movie_counter.get() == limit:
+                    return movie_url_list
+            next_page_matches = re.findall(HREF_REGEX_NEXT_PAGE, html_source)
+            url = next_page_matches[0].replace('&amp;', '&') if next_page_matches else None
+        return movie_url_list
+
+class SingleUrlSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, url: str, limit: Optional[str]):
+        self.movie_counter = movie_counter
+        self.url = url
+        self.limit = int(limit) if limit else None
+
+    def get_urls(self) -> list[str]:
+        if self.limit and self.movie_counter.get() == self.limit:
+            return []
+        else:
+            UrlSource.movie_count_log(self.movie_counter, self.url)
+            return [self.url]
+
+class PlaylistSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, playlist_url: str, limit: Optional[str]):
+        self.movie_counter = movie_counter
+        self.playlist_url = playlist_url
+        self.limit = int(limit) if limit else None
+        self.http_client = HttpClient()
+
+    def get_urls(self) -> list[str]:
+        url = self.playlist_url
+        return UrlSource.get_urls_from_list(movie_counter=self.movie_counter, play_list_url=url, limit=self.limit, cookie=None, http_client=self.http_client)
+
+class AutoUrlSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, auto_urls: list[str], limit: Optional[str]):
+        self.movie_counter = movie_counter
+        self.auto_urls = auto_urls
+        self.limit = int(limit) if limit else None
+        self.http_client = HttpClient()
+
+    def get_urls(self) -> list[str]:
+        movie_url_list = []
+
+        for url in self.auto_urls:
+
+            url_type : UrlType = self._determine_url_type(url)
+            if url_type == UrlType.SINGLE:
+                single_url_source = SingleUrlSource(movie_counter=self.movie_counter, url=url, limit=self.limit)
+                movie_url_list.extend(single_url_source.get_urls())
+            else:
+                playlist_source = PlaylistSource(movie_counter=self.movie_counter, playlist_url=url, limit=self.limit)
+                movie_url_list.extend(playlist_source.get_urls())
+
+        return movie_url_list
+
+    def _determine_url_type(self, url: str) -> Optional[UrlType]:
+        if self._is_movie_url(url):
+            return UrlType.SINGLE
+        else:
+            return UrlType.PLAYLIST
+
+    def _is_movie_url(self, url: str) -> bool:
+        html = self.http_client.get(url)
+        if not html:
+            return False
+        html = html.decode('utf-8')
+        match = re.search(MATCH_UUID_PATTERN, html)
+        if not match:
+            return False
+        return True
+
+class AuthSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, username: str, password: str, limit: Optional[str]):
+        self.movie_counter = movie_counter
+        self.http_client = HttpClient()
+        self.cookie = self._login(username, password)
+        self.limit = int(limit) if limit else None
+
+    def _login(self, username: str, password: str) -> dict:
+        response = self.http_client.post('https://missav.ai/api/login', data={'email': username, 'password': password})
+        if response and response.status_code == 200:
+            cookie_info = response.cookies.get_dict()
+            if "user_uuid" in cookie_info:
+                logger.info(f"User uuid: {cookie_info['user_uuid']}")
+                return cookie_info
+        logger.error("Login failed, check your network connection or account information.")
+        exit(114514)
+
+    def get_urls(self) -> list[str]:
+        url = 'https://missav.ai/saved'
+        return UrlSource.get_urls_from_list(movie_counter=self.movie_counter, play_list_url=url, limit=self.limit, cookie=self.cookie, http_client=self.http_client)
+
+class SearchSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, key: str):
+        self.movie_counter = movie_counter
+        self.key = key
+        self.http_client = HttpClient()
+
+    def get_urls(self) -> list[str]:
+        search_url = f"https://missav.ai/search/{self.key}"
+        search_regex = r'<a href="([^"]+)" alt="' + self.key + '" >'
+        html_source = self.http_client.get(search_url)
+        if html_source is None:
+            logger.error(f"Search failed, key: {self.key}")
+            return []
+        html_source = html_source.decode('utf-8')
+        movie_url_matches = re.findall(search_regex, html_source)
+        temp_url_list = list(set(movie_url_matches))
+        if temp_url_list:
+            logger.info(f"Search {self.key} successfully: {temp_url_list[0]}")
+            UrlSource.movie_count_log(self.movie_counter, temp_url_list[0])
+            return [temp_url_list[0]]
+        logger.error(f"Search failed, key: {self.key}")
+        return []
+
+class FileSource(UrlSource):
+    def __init__(self, movie_counter: ThreadSafeCounter, file_path: str, limit: Optional[str]):
+        self.movie_counter = movie_counter
+        self.file_path = file_path
+        self.limit = int(limit) if limit else None
+
+    def get_urls(self) -> list[str]:
+        with open(self.file_path, 'r', encoding='utf-8') as f:
+            urls = [line.strip() for line in f.readlines() if line.strip()]
+        auto_url_source = AutoUrlSource(movie_counter=self.movie_counter, auto_urls=urls, limit=self.limit)
+        return auto_url_source.get_urls()

misato-0.7.9/misato/utils.py

@@ -0,0 +1,71 @@
+import threading
+import sys
+import os
+import shutil
+from typing import Tuple
+
+
+class ThreadSafeCounter:
+    def __init__(self) -> None:
+        self._count = 0
+        self._lock = threading.Lock()
+
+    def increment_and_get(self) -> int:
+        with self._lock:
+            self._count += 1
+            return self._count
+
+    def get(self) -> int:
+        with self._lock:
+            return self._count
+
+    def reset(self) -> None:
+        with self._lock:
+            self._count = 0
+
+
+def display_progress_bar(max_value: int, file_counter: ThreadSafeCounter) -> None:
+    bar_length = 50
+    current_value = file_counter.increment_and_get()
+    progress = current_value / max_value
+    block = int(round(bar_length * progress))
+    text = f"\rProgress: [{'#' * block + '-' * (bar_length - block)}] {current_value}/{max_value}"
+    sys.stdout.write(text)
+    sys.stdout.flush()
+
+
+def split_integer_into_intervals(integer: int, n: int) -> list[Tuple[int, int]]:
+    interval_size = integer // n
+    remainder = integer % n
+    intervals = [(i * interval_size, (i + 1) * interval_size) for i in range(n)]
+    if remainder:
+        intervals[-1] = (intervals[-1][0], intervals[-1][1] + remainder)
+    return intervals
+
+
+def find_last_non_empty_line(text: str) -> str:
+    lines = text.splitlines()
+    for line in reversed(lines):
+        if line.strip():
+            return line
+    raise Exception("Failed to find the last non-empty line in m3u8 playlist.")
+
+
+def find_closest(arr: list[int], target: int) -> int:
+    closest = arr[0]
+    min_diff = abs(arr[0] - target)
+    for num in arr:
+        diff = abs(num - target)
+        if diff < min_diff:
+            min_diff = diff
+            closest = num
+    return closest
+
+
+def delete_all_subfolders(folder_path: str) -> None:
+    if not os.path.exists(folder_path):
+        return
+    for item in os.listdir(folder_path):
+        item_path = os.path.join(folder_path, item)
+        if os.path.isdir(item_path):
+            shutil.rmtree(item_path)

misato-0.7.9/misato/video_downloader.py

@@ -0,0 +1,160 @@
+import os
+import re
+from typing import Optional, Tuple
+import threading
+from misato.config import MOVIE_SAVE_PATH_ROOT, MATCH_UUID_PATTERN, MATCH_TITLE_PATTERN, COVER_URL_PREFIX, TMP_HTML_FILE, RESOLUTION_PATTERN, VIDEO_M3U8_PREFIX, VIDEO_PLAYLIST_SUFFIX
+from misato.http_client import HttpClient
+from misato.logger import logger
+from misato.utils import ThreadSafeCounter, display_progress_bar, split_integer_into_intervals, find_last_non_empty_line, find_closest
+from misato.ffmpeg_processor import FFmpegProcessor
+
+
+class VideoDownloader:
+    def __init__(self, url: str, http_client: HttpClient, options: dict):
+        self.url = url
+        self.http_client = http_client
+        self.movie_name = url.split('/')[-1]
+        self.movie_folder = os.path.join(MOVIE_SAVE_PATH_ROOT, self.movie_name)
+        self.options = options
+        self.uuid = None
+        self.title = None
+        self.final_file_name = None
+        self.counter = ThreadSafeCounter()
+
+    def _fetch_metadata(self) -> bool:
+        html = self.http_client.get(self.url)
+        if not html:
+            logger.error(f"Failed to fetch HTML for {self.url}")
+            return False
+        html = html.decode('utf-8')
+        with open(TMP_HTML_FILE, 'w', encoding='utf-8') as file:
+            file.write(html)
+        match = re.search(MATCH_UUID_PATTERN, html)
+        if not match:
+            logger.error("Failed to match uuid.")
+            return False
+        result = match.group(1)
+        self.uuid = "-".join(result.split("|")[::-1])
+        logger.info(f"Matching uuid successfully: {self.uuid}")
+        title_match = re.search(MATCH_TITLE_PATTERN, html)
+        if title_match:
+            illegal_chars = '<>:"/\|?* '
+            origin_title = title_match.group(1)
+            safe_title = origin_title
+            for char in illegal_chars:
+                safe_title = safe_title.replace(char, '_')
+            if "uncensored" in self.url:
+                safe_title += "_uncensored"
+            self.title = safe_title
+        return True
+
+    def _download_cover(self) -> None:
+        if not self.options.get('cover_action'):
+            return
+        cover_url = f"{COVER_URL_PREFIX}{self.movie_name}/cover-n.jpg"
+        cover_content = self.http_client.get(cover_url)
+        if cover_content:
+            cover_path = os.path.join(MOVIE_SAVE_PATH_ROOT, f"{self.movie_name}-cover.jpg")
+            with open(cover_path, 'wb') as f:
+                f.write(cover_content)
+        else:
+            logger.error(f"Failed to download cover for {self.movie_name}")
+
+    def _get_final_quality_and_resolution(self, playlist: str) -> Tuple[Optional[str], Optional[str]]:
+        matches = re.findall(RESOLUTION_PATTERN, playlist)
+        quality_map = {height: width for width, height in matches}
+        quality_list = list(quality_map.keys())
+        if not quality_list:
+            logger.error("No resolutions found in playlist.")
+            return None, None
+        quality = self.options.get('quality')
+        if quality is None:
+            final_quality = quality_list[-1] + 'p'
+            resolution_url = find_last_non_empty_line(playlist)
+        else:
+            target = int(quality)
+            closest_height = find_closest([int(h) for h in quality_list], target)
+            final_quality = str(closest_height) + 'p'
+            url_type_x = f"{quality_map[str(closest_height)]}x{closest_height}/video.m3u8"
+            url_type_p = f"{closest_height}p/video.m3u8"
+            resolution_url = url_type_x if url_type_x in playlist else url_type_p if url_type_p in playlist else find_last_non_empty_line(playlist)
+        return final_quality, resolution_url
+
+    def _thread_task(self, start: int, end: int, uuid: str, resolution: str, video_offset_max: int) -> None:
+        for i in range(start, end):
+            url = f"https://surrit.com/{uuid}/{resolution}/video{i}.jpeg"
+            content = self.http_client.get(url, retries=self.options.get('retry', 5), delay=self.options.get('delay', 2), timeout=self.options.get('timeout', 10))
+            if content:
+                file_path = os.path.join(self.movie_folder, f"video{i}.jpeg")
+                with open(file_path, 'wb') as f:
+                    f.write(content)
+                display_progress_bar(video_offset_max + 1, self.counter)
+            else:
+                logger.error(f"Failed to download segment {i} for {self.movie_name}")
+
+    def _download_segments(self, uuid: str, resolution: str, video_offset_max: int) -> None:
+        if not self.options.get('download_action'):
+            return
+        intervals = split_integer_into_intervals(video_offset_max + 1, self.options.get('num_threads', os.cpu_count()))
+        self.counter.reset()
+        threads = []
+        for start, end in intervals:
+            thread = threading.Thread(target=self._thread_task, args=(start, end, uuid, resolution, video_offset_max))
+            threads.append(thread)
+            thread.start()
+        for thread in threads:
+            thread.join()
+        self.counter.reset()
+
+    def _check_integrity(self, video_offset_max: int) -> None:
+        downloaded_files = len([f for f in os.listdir(self.movie_folder) if f.endswith('.jpeg')])
+        total_files = video_offset_max + 1
+        integrity = downloaded_files / total_files
+        print()
+        logger.info(f"File integrity for {self.movie_name}: {integrity:.2%} ({downloaded_files}/{total_files} files)")
+
+    def _assemble_video(self, video_offset_max: int) -> None:
+        if not self.options.get('write_action'):
+            return
+        self.final_file_name = f"{self.movie_name}_{self.final_quality}"
+        output_file = os.path.join(MOVIE_SAVE_PATH_ROOT, f"{self.final_file_name}.mp4")
+        if self.options.get('ffmpeg_action'):
+            segment_files = [os.path.join(self.movie_folder, f"video{i}.jpeg") for i in range(video_offset_max + 1) if os.path.exists(os.path.join(self.movie_folder, f"video{i}.jpeg"))]
+            cover_file = os.path.join(MOVIE_SAVE_PATH_ROOT, f"{self.movie_name}-cover.jpg") if self.options.get('cover_as_preview') and os.path.exists(os.path.join(MOVIE_SAVE_PATH_ROOT, f"{self.movie_name}-cover.jpg")) else None
+            FFmpegProcessor.create_video_from_segments(segment_files, output_file, cover_file)
+        else:
+            with open(output_file, 'wb') as outfile:
+                for i in range(video_offset_max + 1):
+                    file_path = os.path.join(self.movie_folder, f"video{i}.jpeg")
+                    if os.path.exists(file_path):
+                        with open(file_path, 'rb') as infile:
+                            outfile.write(infile.read())
+        if self.options.get('title_action') and self.title:
+            os.rename(output_file, os.path.join(MOVIE_SAVE_PATH_ROOT, f"{self.title}.mp4"))
+
+    def download(self) -> None:
+        if not self._fetch_metadata():
+            return
+        playlist_url = f"{VIDEO_M3U8_PREFIX}{self.uuid}{VIDEO_PLAYLIST_SUFFIX}"
+        playlist = self.http_client.get(playlist_url)
+        if not playlist:
+            logger.error("Failed to fetch playlist.")
+            return
+        playlist = playlist.decode('utf-8')
+        self.final_quality, resolution_url = self._get_final_quality_and_resolution(playlist)
+        if not self.final_quality:
+            return
+        video_m3u8_url = f"{VIDEO_M3U8_PREFIX}{self.uuid}/{resolution_url}"
+        video_m3u8 = self.http_client.get(video_m3u8_url)
+        if not video_m3u8:
+            logger.error("Failed to fetch video m3u8.")
+            return
+        video_m3u8 = video_m3u8.decode('utf-8')
+        video_offset_max_str = video_m3u8.splitlines()[-2]
+        video_offset_max = int(re.search(r'\d+', video_offset_max_str).group(0))
+        if not os.path.exists(self.movie_folder):
+            os.makedirs(self.movie_folder)
+        self._download_cover()
+        self._download_segments(self.uuid, resolution_url.split('/')[0], video_offset_max)
+        self._check_integrity(video_offset_max)
+        self._assemble_video(video_offset_max)

misato-0.7.9/misato.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: misato
+Version: 0.7.9
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: curl_cffi
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python

misato-0.7.9/misato.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.py
+misato/__init__.py
+misato/config.py
+misato/ffmpeg_processor.py
+misato/http_client.py
+misato/logger.py
+misato/main.py
+misato/url_sources.py
+misato/utils.py
+misato/video_downloader.py
+misato.egg-info/PKG-INFO
+misato.egg-info/SOURCES.txt
+misato.egg-info/dependency_links.txt
+misato.egg-info/entry_points.txt
+misato.egg-info/requires.txt
+misato.egg-info/top_level.txt
+test/__init__.py
+test/test.py

misato-0.7.9/misato.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

misato-0.7.9/misato.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+misato = misato.main:main

misato-0.7.9/misato.egg-info/requires.txt

@@ -0,0 +1 @@
+curl_cffi

misato-0.7.9/misato.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+misato
+test

misato-0.7.9/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

misato-0.7.9/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='misato',
+    version='0.7.9',
+    packages=find_packages(),
+    install_requires=[
+        'curl_cffi',
+    ],
+    entry_points={
+        'console_scripts': [
+            'misato=misato.main:main',
+        ],
+    },
+    python_requires='>=3.9',
+)

misato-0.7.9/test/test.py

@@ -0,0 +1,10 @@
+import os
+
+if __name__ == '__main__':
+
+    proxy = "localhost:7890"
+
+    os.environ["http_proxy"] = f"http://{proxy}"
+    os.environ["https_proxy"] = f"http://{proxy}"
+
+    resources.miyuki.download(movie_url="https://missav.com/ja/fc2-ppv-4597386", quality="700", download_action=False, ffmpeg_action=True, retry=10, delay=20, timeout=30, title_action=True)