mangagraph 0.0.1__tar.gz

mangagraph-0.0.1/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.1
+Name: mangagraph
+Version: 0.0.1
+Summary: Async manga parser-converter from mangalib to telegraph pages
+Home-page: https://github.com/damirTAG/mangagraph
+Author: damirTAG
+Author-email: damirtagilbayev17@gmail.com
+Keywords: mangalib,mangalib-parser,manga,telegraph
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Requires-Dist: aiohttp
+Requires-Dist: sqlalchemy
+Requires-Dist: telegraph
+Requires-Dist: asyncio

mangagraph-0.0.1/mangagraph/__init__.py

@@ -0,0 +1,15 @@
+from .parser import Mangagraph
+from .models import Chapter
+from .exceptions import (
+    MangagraphError,
+    InvalidURLException,
+    RequestFailedException
+)
+
+__all__ = [
+    'Mangagraph',
+    'Chapter',
+    'MangagraphError',
+    'InvalidURLException',
+    'RequestFailedException'
+]

mangagraph-0.0.1/mangagraph/cli.py

@@ -0,0 +1,41 @@
+import argparse
+import asyncio
+import logging
+
+from .parser import Mangagraph
+from .exceptions import MangagraphError
+
+def main():
+    parser = argparse.ArgumentParser(description="Mangagraph")
+    parser.add_argument('url', type=str, help='URL of the manga to process')
+    parser.add_argument('--db', type=str, default='manga.db', help='Database file name')
+    parser.add_argument('--mirror', action='store_true', help='Use graph.org as mirror')
+    # parser.add_argument('--log', type=str, default='manga_parser.log', help='Log file name')
+    
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.StreamHandler()
+        ]
+    )
+    logger = logging.getLogger(__name__)
+
+    try:
+        parser_instance = Mangagraph(
+            db_path=args.db,
+            use_mirror=args.mirror
+        )
+        toc_url, mirror_toc_url = asyncio.run(parser_instance.process_manga(args.url))
+        logger.info(f"База данных создана!")
+        logger.info(f"Оглавление: {toc_url}")
+        logger.info(f"Зеркало оглавления: {mirror_toc_url}")
+    except MangagraphError as e:
+        logger.error(f"Parser error: {e}")
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+
+if __name__ == "__main__":
+    main()

mangagraph-0.0.1/mangagraph/exceptions.py

@@ -0,0 +1,14 @@
+class MangagraphError(Exception):
+    pass
+
+class InvalidURLException(MangagraphError):
+    def __init__(self, url, message="Invalid URL provided."):
+        self.url = url
+        self.message = message
+        super().__init__(f"{self.message} URL: {self.url}")
+
+class RequestFailedException(MangagraphError):
+    def __init__(self, url, message="Request to the URL failed."):
+        self.url = url
+        self.message = message
+        super().__init__(f"{self.message} URL: {self.url}")

mangagraph-0.0.1/mangagraph/models.py

@@ -0,0 +1,20 @@
+from sqlalchemy import Column, Integer, String, Text
+from sqlalchemy.ext.declarative import declarative_base
+
+from datetime import datetime
+
+Base = declarative_base()
+
+class Chapter(Base):
+    __tablename__ = 'chapters'
+    
+    id = Column(Integer, primary_key=True)
+    volume = Column(Integer)
+    chapter = Column(Integer)
+    title = Column(String)
+    url = Column(Text)
+    mirror_url = Column(Text)  # Alternative URL if telegra.ph is not accessible
+    created_at = Column(String, default=lambda: datetime.now().isoformat())
+
+    def __repr__(self):
+        return f"<Chapter(volume={self.volume}, chapter={self.chapter}, title={self.title})>"

mangagraph-0.0.1/mangagraph/parser.py

@@ -0,0 +1,358 @@
+"""
+Author: https://github.com/damirtag | https://t.me/damirtag
+GH Repo: https://github.com/damirtag/mangagraph
+
+MIT License
+
+Copyright (c) 2025 Tagilbayev Damir
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import asyncio
+import aiohttp
+import logging
+
+from typing import List, Dict, Any, Tuple
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from .models import Base, Chapter
+from .exceptions import (
+    MangagraphError,
+    RequestFailedException
+)
+from .utils import (
+    MangaLibUrl, 
+    estimate_remaining_time, 
+    extract_slug
+)
+
+from telegraph.aio import Telegraph
+
+class Mangagraph:
+    """
+    Автор: https://github.com/damirtag
+
+    Параметры:
+        db_name (str): Имя базы данных в которой будет хранится Том, глава, название главы, ссылка на телеграф, зеркало, дата создания
+        use_mirror (bool): Использовать зеркало как base url для telegraph, по дефолту False
+    """
+    MAX_CONCURRENT = 3
+    # В 1 мин обрабатывается 12 глав 
+    # что = 12 страницам телеграф в секунду
+    # При учете того что запросы делаются каждые 5 сек
+    CHAPTERS_PER_MINUTE = 12
+
+
+    def __init__(
+        self, 
+        db_name: str = 'manga.db',
+        use_mirror: bool = False
+    ):
+        self.db_name = db_name if db_name.endswith('.db') else db_name + '.db'
+        self.logger = self._setup_logger()
+        self.engine = create_engine(f'sqlite:///{self.db_name}')
+        Base.metadata.create_all(self.engine)
+        self.Session = sessionmaker(bind=self.engine)
+
+        self.domain = 'telegra.ph' if not use_mirror else 'graph.org'
+        self.telegraph = Telegraph(domain=self.domain)
+        
+        self.base_img_url = "https://img33.imgslib.link"
+        self.semaphore = asyncio.Semaphore(self.MAX_CONCURRENT)
+
+        self.processed_count = 0
+        self.total_chapters = 0
+        self.flood_wait_count = 0
+
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        }
+
+    def _setup_logger(self):
+        logger = logging.getLogger('mangagraph')
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter('[mangagraph]: %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        if not logger.handlers:
+            logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+        return logger
+
+    async def _make_request(self, session: aiohttp.ClientSession, url: str, params: Dict = None) -> Dict:
+        async with self.semaphore:
+            for attempt in range(3):
+                try:
+                    async with session.get(url, params=params, headers=self.headers) as response:
+                        response.raise_for_status()
+                        return await response.json()
+                except Exception as e:
+                    if attempt == 2:
+                        raise RequestFailedException(url, str(e))
+                    await asyncio.sleep(2 ** attempt)
+
+    async def _get_manga_name(self, session: aiohttp.ClientSession, slug: str) -> str:
+        url = f"https://api2.mangalib.me/api/manga/{slug}"
+        data = await self._make_request(session, url)
+        rus_name = data['data']['rus_name']
+        if rus_name:
+            return rus_name
+        return data['data']['name']
+
+    async def get_chapters_info(self, session: aiohttp.ClientSession, slug: str) -> List[Dict[str, Any]]:
+        url = f"https://api2.mangalib.me/api/manga/{slug}/chapters"
+        data = await self._make_request(session, url)
+        return data['data']
+
+    async def get_chapter_pages(
+        self, 
+        session: aiohttp.ClientSession, 
+        slug: str, 
+        volume: int, 
+        chapter: int
+    ) -> List[str]:
+        url = f"https://api2.mangalib.me/api/manga/{slug}/chapter"
+        params = {'number': chapter, 'volume': volume}
+        data = await self._make_request(session, url, params)
+        return [f"{self.base_img_url}{page['url']}" for page in data['data']['pages']]
+
+    async def _create_telegraph_page(
+            self, 
+            title: str, 
+            image_urls: List[str],
+            retry_count: int = 3
+        ) -> Tuple[str, str]:
+        html_content = "".join(f'<img src="{url}"/>\n' for url in image_urls)
+        
+        for attempt in range(retry_count):
+            try:
+                response = await self.telegraph.create_page(
+                    title=title,
+                    html_content=html_content,
+                    author_name='Auto-Generated by MGLParser',
+                    author_url='https://t.me/damirtag'
+                )
+                
+                return (
+                    f"https://telegra.ph/{response['path']}", 
+                    f"https://graph.org/{response['path']}"
+                )
+            except Exception as e:
+                if "FLOOD_WAIT" in str(e):
+                    self.flood_wait_count += 1
+                    wait_time = 7
+                    if "FLOOD_WAIT_" in str(e):
+                        try:
+                            wait_time = int(str(e).split("FLOOD_WAIT_")[1])
+                        except:
+                            pass
+                            
+                    self.logger.warning(
+                        f'Flood wait #{self.flood_wait_count} detected, ' 
+                        f'waiting {wait_time} seconds. ' 
+                        f'Total floods: {self.flood_wait_count}'
+                    )
+                    
+                    await asyncio.sleep(wait_time)
+                    await self.telegraph.create_account(
+                        short_name='Damir',
+                        author_name='Auto-Generated by MGLParser',
+                        author_url='https://t.me/damirtag'
+                    )
+                    continue
+                else:
+                    raise MangagraphError(f"Не удалось создать telegraph страницу: {str(e)}")
+
+    async def _construct_chapters_list(
+        self, 
+        title: str, 
+        chapters: List[Tuple[int, int, str, str, str]]
+    ) -> str:
+        """Creates a table of contents page in Telegraph."""
+        content = [
+            {
+                "tag": "p",
+                "children": ["Создано Mangagraph, developer - ", {
+                    "tag": "a", 
+                    "attrs": {"href": "https://t.me/damirtag"}, 
+                    "children": ["@damirtag"]}
+                ]
+            }
+        ]
+        
+        for volume, chapter_num, chapter_title, url, mirror_url in chapters:
+            content.append({
+                "tag": "p",
+                "children": [
+                    {
+                        "tag": "a",
+                        "attrs": {"href": mirror_url},
+                        "children": [f"Volume {volume} Chapter {chapter_num}: {chapter_title}"]
+                    }
+                ]
+            })
+
+        try:
+            response = await self.telegraph.create_page(
+                title=title,
+                author_name='Damir',
+                author_url='https://github.com/damirtag/mangagraph',
+                content=content
+            )
+
+            return response['url']
+        except Exception as e:
+            raise MangagraphError(f'Telegraph says: {str(e)}')
+
+    async def process_manga(self, manga_url: MangaLibUrl):
+        """
+        Параметры:
+            manga_url (str): URL манги, которую нужно обработать.
+
+        Возвращает:
+
+            Кортеж из двух строк:
+
+                - URL оглавления в Telegraph.
+
+                - URL зеркала оглавления в graph.org (если используется).
+
+        Исключения:
+            MangagraphError: Выбрасывается, если возникает ошибка при обработке манги.
+            InvalidURLException: Выбрасывается, если URL манги недействителен.
+            RequestFailedException: Выбрасывается, если запрос к API завершается неудачей.
+        """
+        db_session = self.Session()
+        await self.telegraph.create_account(
+            short_name='Damir',
+            author_name='Создано mangagraph by @damirtag',
+            author_url='https://github.com/damirtag/mangagraph'
+        )
+
+        slug = extract_slug(manga_url)
+        
+        async with aiohttp.ClientSession() as session:
+            try:
+                manga_name = await self._get_manga_name(session, slug)
+                chapters = await self.get_chapters_info(session, slug)
+                
+                self.total_chapters = len(chapters)
+                self.processed_count = 0
+                processed_chapters = []
+
+                first_chapter = chapters[0]
+                pages = await self.get_chapter_pages(
+                    session,
+                    slug,
+                    first_chapter.get('volume'),
+                    first_chapter.get('number')
+                )
+                
+                for chapter_info in chapters:
+                    volume = chapter_info.get('volume')
+                    chapter_num = chapter_info.get('number')
+                    title = (
+                        f"{manga_name} | {chapter_info.get('name')}" 
+                        or 
+                        f"{manga_name} | Том {volume}, Глава {chapter_num}"
+                    )
+
+                    existing_chapter = db_session.query(Chapter).filter_by(
+                        volume=volume, 
+                        chapter=chapter_num
+                    ).first()
+
+                    if existing_chapter:
+                        self.processed_count += 1
+                        processed_chapters.append(
+                            (volume, chapter_num, title, existing_chapter.url, existing_chapter.mirror_url)
+                        )
+                        remaining = self.total_chapters - self.processed_count
+                        est_time = estimate_remaining_time(remaining)
+                        self.logger.info(
+                            f"Глава {chapter_num} уже существует, пропускаем... " 
+                            f"[{self.processed_count}/{self.total_chapters}] " 
+                            f"Примерное время: {est_time}"
+                        )
+                        continue
+
+                    try:
+                        pages = await self.get_chapter_pages(
+                            session,
+                            slug, 
+                            volume, 
+                            chapter_num
+                        )
+                        
+                        url, mirror_url = await self._create_telegraph_page(
+                            title=title,
+                            image_urls=pages
+                        )
+                        
+                        new_chapter = Chapter(
+                            volume=volume,
+                            chapter=chapter_num,
+                            title=title,
+                            url=url,
+                            mirror_url=mirror_url
+                        )
+                        db_session.add(new_chapter)
+                        db_session.commit()
+                        
+                        self.processed_count += 1
+                        processed_chapters.append((volume, chapter_num, title, url, mirror_url))
+                        
+                        remaining = self.total_chapters - self.processed_count
+                        est_time = estimate_remaining_time(remaining)
+                        self.logger.info(
+                            f"Processed chapter: {title} " 
+                            f"[{self.processed_count}/{self.total_chapters}] " 
+                            f"Remaining time: {est_time}"
+                        )
+                        
+                        await asyncio.sleep(5)
+                        
+                    except Exception as e:
+                        self.logger.error(
+                            f"Ошибка обработки главы {chapter_num}: {str(e)}\n" 
+                            f"Всего обработано: {self.processed_count}/{self.total_chapters}"
+                        )
+                        db_session.rollback()
+                        break
+
+                if processed_chapters:
+                    toc_url = await self._construct_chapters_list(
+                        manga_name,
+                        processed_chapters
+                    )
+                    mirror_toc_url = toc_url.replace("telegra.ph", "graph.org")
+
+                    self.logger.info(f"Создано оглавление: {toc_url}")
+                    self.logger.info(f"Зеркало: {mirror_toc_url}")
+                    self.logger.info(f"Всего обработано: {self.processed_count}/{self.total_chapters}")
+
+                    return toc_url, mirror_toc_url
+
+            except (KeyboardInterrupt, SystemExit):
+                self.logger.info('Sayonara!')
+
+            finally:
+                db_session.close()

mangagraph-0.0.1/mangagraph/utils.py

@@ -0,0 +1,50 @@
+
+import re
+from urllib.parse import urlparse
+from .exceptions import InvalidURLException
+
+
+class MangaLibUrl:
+    MANGALIB_URL_PATTERNS = [
+        r"^https://mangalib\.me/ru/manga/[\w\-]+(\?.+)?$",
+        r"^https://mangalib\.me/ru/\d+--[\w\-]+/read/v\d+/c\d+(\?.+)?$"
+    ]
+    
+    def __init__(self, url: str):
+        self.url = url
+        self._validate_url()
+
+    def _validate_url(self):
+        parsed_url = urlparse(self.url)
+        if not (parsed_url.scheme == "https" and parsed_url.netloc == "mangalib.me"):
+            raise ValueError("Неверный адресс. Только 'https://mangalib.me' доступен.")
+        
+        if not any(re.compile(pattern).match(self.url) for pattern in self.MANGALIB_URL_PATTERNS):
+            raise ValueError(
+                "Неверный URL. Ссылка должна быть одного из типов:\n"
+                "- 'https://mangalib.me/ru/manga/{slug_url}'\n"
+                "- 'https://mangalib.me/ru/{slug_url}/read/v{volume}/c{chapter}'"
+            )
+
+    def __str__(self):
+        return self.url
+
+def estimate_remaining_time(remaining_chapters: int, chapters_per_minute: int = 12) -> str:
+    estimated_minutes = (remaining_chapters / chapters_per_minute) * 1.1
+    
+    if estimated_minutes < 1:
+        return "меньше минуты"
+        
+    hours = int(estimated_minutes // 60)
+    minutes = int(estimated_minutes % 60)
+    
+    if hours > 0:
+        return f"{hours} ч {minutes} мин"
+    return f"{minutes} мин"
+
+def extract_slug(url: str) -> str:
+    pattern = r'/manga/(\d+--[^?]+)'
+    match = re.search(pattern, url)
+    if match:
+        return match.group(1)
+    raise InvalidURLException(url, "Invalid MangaLib URL format.")

mangagraph-0.0.1/mangagraph.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.1
+Name: mangagraph
+Version: 0.0.1
+Summary: Async manga parser-converter from mangalib to telegraph pages
+Home-page: https://github.com/damirTAG/mangagraph
+Author: damirTAG
+Author-email: damirtagilbayev17@gmail.com
+Keywords: mangalib,mangalib-parser,manga,telegraph
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Requires-Dist: aiohttp
+Requires-Dist: sqlalchemy
+Requires-Dist: telegraph
+Requires-Dist: asyncio

mangagraph-0.0.1/mangagraph.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+pyproject.toml
+setup.py
+mangagraph/__init__.py
+mangagraph/cli.py
+mangagraph/exceptions.py
+mangagraph/models.py
+mangagraph/parser.py
+mangagraph/utils.py
+mangagraph.egg-info/PKG-INFO
+mangagraph.egg-info/SOURCES.txt
+mangagraph.egg-info/dependency_links.txt
+mangagraph.egg-info/entry_points.txt
+mangagraph.egg-info/requires.txt
+mangagraph.egg-info/top_level.txt
+tests/__init__.py

mangagraph-0.0.1/mangagraph.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

mangagraph-0.0.1/mangagraph.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mangagraph = mangagraph.cli:main

mangagraph-0.0.1/mangagraph.egg-info/requires.txt

@@ -0,0 +1,4 @@
+aiohttp
+sqlalchemy
+telegraph
+asyncio

mangagraph-0.0.1/mangagraph.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+mangagraph
+tests

mangagraph-0.0.1/pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

mangagraph-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mangagraph-0.0.1/setup.py

@@ -0,0 +1,40 @@
+from setuptools import setup, find_packages
+
+import pathlib
+
+LIB_NAME = 'mangagraph'
+
+__version__ = '0.0.1'
+
+setup(
+    name='mangagraph',
+    version=__version__,
+    description='Async manga parser-converter from mangalib to telegraph pages',
+    url='https://github.com/damirTAG/mangagraph',
+    author='damirTAG',
+    author_email='damirtagilbayev17@gmail.com',
+    packages=find_packages(),
+    install_requires=[
+        'aiohttp',
+        'sqlalchemy',
+        'telegraph',
+        'asyncio'
+    ],
+    entry_points={
+        'console_scripts': [
+            'mangagraph= mangagraph.cli:main'
+        ]
+    },
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'Operating System :: OS Independent',
+    ],
+    keywords=[
+        'mangalib',
+        'mangalib-parser',
+        'manga',
+        'telegraph'
+    ],
+    python_requires='>=3.7',
+    include_package_data=False
+)