reycrawler 1.0.7__tar.gz

reycrawler-1.0.7/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

reycrawler-1.0.7/.python-version

@@ -0,0 +1 @@
+3.13

reycrawler-1.0.7/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2025 ReyXBo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

reycrawler-1.0.7/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: reycrawler
+Version: 1.0.7
+Summary: Web crawler method set.
+Project-URL: Homepage, https://www.reyxbo.com
+Project-URL: Repository, https://github.com/reyxbo/reycrawler-py.git
+Author-email: Rey <reyxbo@163.com>
+License: Copyright 2025 ReyXBo
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+License-File: LICENSE
+Keywords: crawl Web,crawler,rey,reyxbo
+Requires-Python: >=3.12
+Requires-Dist: bs4
+Requires-Dist: fake-useragent
+Requires-Dist: reydb
+Requires-Dist: reykit
+Requires-Dist: selenium
+Description-Content-Type: text/markdown
+
+# reycrawler
+
+> Web crawler method set.
+
+## Install
+
+```
+pip install reyfetch
+```

reycrawler-1.0.7/README.md

@@ -0,0 +1,9 @@
+# reycrawler
+
+> Web crawler method set.
+
+## Install
+
+```
+pip install reyfetch
+```

reycrawler-1.0.7/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "reycrawler"
+version = "1.0.7"
+description = "Web crawler method set."
+readme = "README.md"
+requires-python = ">=3.12"
+license = {file = "LICENSE"}
+keywords = ["rey", "reyxbo", "crawler", "crawl Web"]
+authors = [
+    {name = "Rey", email = "reyxbo@163.com"}
+]
+dependencies = [
+    "bs4",
+    "fake_useragent",
+    "reydb",
+    "reykit",
+    "selenium"
+]
+
+[project.urls]
+Homepage = "https://www.reyxbo.com"
+Repository = "https://github.com/reyxbo/reycrawler-py.git"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

reycrawler-1.0.7/src/reycrawler/__init__.py

@@ -0,0 +1,18 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+@Time    : 2023-02-19
+@Author  : Rey
+@Contact : reyxbo@163.com
+@Explain : Web crawler method set.
+
+Modules
+-------
+rall : All import methods.
+rbase : Base methods.
+rbrowser : Browser methods.
+rdouban : Crawl Douban website methods.
+rdouban : Crawl Douban website methods.
+rdouban : Crawl Douban website methods.
+"""

reycrawler-1.0.7/src/reycrawler/rall.py

@@ -0,0 +1,20 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+@Time    : 2022-12-08
+@Author  : Rey
+@Contact : reyxbo@163.com
+@Explain : All import methods.
+"""
+
+# ruff: noqa: F403
+
+from .rbaidu import *
+from .rbase import *
+from .rbrowser import *
+from .rdouban import *
+from .rother import *
+from .rsina import *
+from .rtoutiao import *
+from .rweibo import *

reycrawler-1.0.7/src/reycrawler/rbaidu.py

@@ -0,0 +1,116 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+@Time    : 2024-01-10
+@Author  : Rey
+@Contact : reyxbo@163.com
+@Explain : Baidu website crawling methods.
+"""
+
+from json import loads as json_loads
+from reykit.rnet import request
+from reykit.rre import search
+from reykit.rtime import now
+
+__all__ = (
+    'crawl_baidu_calendar',
+)
+
+def crawl_baidu_calendar(
+    year: int | None = None,
+    month: int | None = None
+) -> list[dict]:
+    """
+    Crawl Baidu website calendar table.
+
+    Parameters
+    ----------
+    year : Given year.
+        - `None`: Now year.
+    month : Given month.
+        - `None`: Now month.
+
+    Returns
+    -------
+    Calendar table.
+    """
+
+    # Get parameter.
+    now_date = now('date')
+    year = year or now_date.year
+    month = month or now_date.month
+    if month == 12:
+        month = 1
+    else:
+        month += 1
+    url = 'https://opendata.baidu.com/data/inner'
+    query = '%s年%s月' % (year, month)
+    params = {
+        'tn': 'reserved_all_res_tn',
+        'type': 'json',
+        'resource_id': '52109',
+        'query': query,
+        'apiType': 'yearMonthData',
+        'cb': 'jsonp_1706670926975_94318'
+    }
+
+    # Request.
+    response = request(url, params)
+
+    # Extract.
+    pattern = '{.+}'
+    text = search(pattern, response.text)
+    data: dict = json_loads(text)
+    table: list[dict] = data['Result'][0]['DisplayData']['resultData']['tplData']['data']['almanac']
+
+    # Convert.
+    week_dict = {
+        '一': 0,
+        '二': 1,
+        '三': 2,
+        '四': 3,
+        '五': 4,
+        '六': 5,
+        '日': 6
+    }
+    table = [
+        {
+            'year': int(row['year']),
+            'month': int(row['month']),
+            'day': int(row['day']),
+            'week': week_dict[row['cnDay']],
+            'work': row.get('status'),
+            'festival': [
+                {
+                    'name': info['name'],
+                    'url': info.get('baikeUrl')
+                }
+                for info in row.get('festivalInfoList', [])
+            ],
+            'animal': row['animal'],
+            'lunar_year': int(row['lunarYear']),
+            'lunar_month': int(row['lunarMonth']),
+            'lunar_day': int(row['lunarDate']),
+            'gz_year': row['gzYear'],
+            'gz_month': row['gzMonth'],
+            'gz_day': row['gzDate'],
+            'suit': row['suit'].split('.'),
+            'avoid': row['avoid'].split('.'),
+            'url': row['yjJumpUrl']
+        }
+        for row in table
+    ]
+    for row in table:
+        week = row['week']
+        work = row['work']
+        match work:
+            case None:
+                is_work_day = week not in (5, 6)
+            case '1':
+                is_work_day = False
+            case '2':
+                is_work_day = True
+        row['work'] = is_work_day
+
+    return table

reycrawler-1.0.7/src/reycrawler/rbase.py

@@ -0,0 +1,24 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+@Time    : 2023-12-29
+@Author  : Rey
+@Contact : reyxbo@163.com
+@Explain : Base methods.
+"""
+
+from fake_useragent import UserAgent
+from reykit.rbase import Base
+
+__all__ = (
+    'CrawlerBase',
+    'ua'
+)
+
+class CrawlerBase(Base):
+    """
+    Crawler base type.
+    """
+
+ua = UserAgent()

reycrawler-1.0.7/src/reycrawler/rbrowser.py

@@ -0,0 +1,371 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+@Time    : 2025-10-24
+@Author  : Rey
+@Contact : reyxbo@163.com
+@Explain : Browser methods.
+"""
+
+from typing import Any, Literal
+from traceback import format_exc
+from enum import StrEnum
+from selenium.webdriver import Edge, Chrome, EdgeOptions, ChromeOptions
+from reydb import rorm, DatabaseEngine
+from reykit.rbase import throw
+from reykit.rnet import join_url
+from reykit.rtime import TimeMark, now, sleep
+
+from .rbase import CrawlerBase
+
+__all__ = (
+    'CrawlBrowserPageStatusEnum',
+    'CrawlerORMTableCrawlBrowserPage',
+    'CrawlerBrowser',
+    'crawl_page',
+    'add_db_crawl_task',
+    'wait_db_crawl_task',
+    'crawl_page_use_db'
+)
+
+class CrawlBrowserPageStatusEnum(CrawlerBase, StrEnum):
+    """
+    Crawl browser page status enumeration type.
+    """
+
+    WAIT = 'wait'
+    'Wait crawl.'
+    SUCCESS = 'success'
+    'Crawl successded.'
+    FAIL = 'fail'
+    'Crawl failed.'
+    CANCEL = 'cancel'
+    'Crawl cancelled.'
+
+class CrawlerORMTableCrawlBrowserPage(CrawlerBase, rorm.Table):
+    """
+    Crawler `crawl_browser_page` table ORM model.
+    """
+
+    __name__ = 'crawl_browser_page'
+    __comment__ = 'Crawl browser page HTML text table.'
+    create_time: rorm.Datetime = rorm.Field(field_default=':time', not_null=True, index_n=True, comment='Record create time.')
+    update_time: rorm.Datetime | None = rorm.Field(field_default=':time', arg_default=now, index_n=True, comment='Record update time.')
+    id: int = rorm.Field(key_auto=True, comment='ID.')
+    url: str = rorm.Field(rorm.types.VARCHAR(8182), not_null=True, comment='Target URL.')
+    html: str | None = rorm.Field(rorm.types.TEXT, comment='Crawled HTML text.')
+    status: str = rorm.Field(rorm.ENUM(CrawlBrowserPageStatusEnum), field_default=CrawlBrowserPageStatusEnum.WAIT, not_null=True, comment='Crawl status.')
+    note: str | None = rorm.Field(rorm.types.VARCHAR(500), comment='Note.')
+
+class CrawlerBrowser(CrawlerBase):
+    """
+    Control browser crawler type.
+    """
+
+    def __init__(
+        self,
+        driver: Literal['edge', 'chrome'] = 'edge',
+        headless: bool = True,
+        echo: bool = False,
+        db_engine: DatabaseEngine | None = None
+    ) -> None:
+        """
+        Build instance attributes.
+
+        Parameters
+        ----------
+        driver : Browser driver type.
+            - `Literal['edge']`: Edge browser.
+            - `Literal['chrome']`: Chrome browser.
+        headless : Whether use headless mode.
+        echo : Whether to print the report.
+        db_engine : Database engine.
+            - `None`: Not use database.
+            - `Database`: Automatic crawl page by database table.
+        """
+
+        # Parameter.
+        self.echo = echo
+        self.db_engine = db_engine
+        match driver:
+            case 'edge':
+                driver_type = Edge
+                driver_option_type = EdgeOptions
+            case 'chrome':
+                driver_type = Chrome
+                driver_option_type = ChromeOptions
+
+        # Build.
+
+        ## Build Database.
+        if self.db_engine is not None:
+            self.build_db()
+
+        ## Driver.
+        options = driver_option_type()
+        if headless:
+            options.add_argument('--headless')
+        self.driver = driver_type(options)
+
+        ## Crawl by database.
+        if self.db_engine is not None:
+            self.__loop_crawl_by_db()
+
+    def build_db(self) -> None:
+        """
+        Check and build database tables.
+        """
+
+        # Check.
+        if self.db_engine is None:
+            throw(ValueError, self.db_engine)
+
+        # Parameter.
+
+        ## Table.
+        tables = [CrawlerORMTableCrawlBrowserPage]
+
+        # Build.
+
+        ## WeChat.
+        self.db_engine.build(tables=tables, skip=True)
+
+    def __loop_crawl_by_db(self) -> None:
+        """
+        Loop crawl by database table.
+        """
+
+        # Echo.
+        if self.echo:
+            print('Start loop crawl by database table.')
+
+        # Loop.
+        while True:
+
+            ## Crawl.
+            self.__crawl_by_db()
+
+            ## Sleep.
+            sleep(1)
+
+    def __crawl_by_db(self) -> None:
+        """
+        Crawl by database table.
+        """
+
+        # Get task.
+        task_table = self.db_engine.execute.select(
+            'crawl_browser_page',
+            ['id', 'url'],
+            f'"status" = \'{CrawlBrowserPageStatusEnum.WAIT}\'',
+            order='"create_time" ASC'
+        )
+
+        # Crawl.
+        for id_, url in task_table:
+            try:
+                self.request(url)
+            except BaseException:
+                exc_text = format_exc()
+                print(exc_text)
+                status = CrawlBrowserPageStatusEnum.FAIL
+            else:
+                status = CrawlBrowserPageStatusEnum.SUCCESS
+
+            ## Database.
+            data = {
+                'id': id_,
+                'update_time': ':NOW()',
+                'html': self.page,
+                'status': status
+            }
+            self.db_engine.execute.update('crawl_browser_page', data)
+
+    def request(
+        self,
+        url: str,
+        params: dict[str, Any] | None = None
+    ) -> None:
+        """
+        Request URL.
+
+        Parameters
+        ----------
+        url : URL.
+        params : URL parameters.
+        """
+
+        # Parameter.
+        params = params or {}
+        url = join_url(url, **params)
+
+        # Echo.
+        if self.echo:
+            print(f'Crawl URL "{url}"')
+
+        # Request.
+        self.driver.get(url)
+
+    @property
+    def page(self) -> str:
+        """
+        Return page elements document.
+
+        Returns
+        -------
+        Page elements document.
+        """
+
+        # Parameter.
+        page_source = self.driver.page_source
+
+        return page_source
+
+    __call__ = request
+
+def crawl_page(
+    url: str,
+    params: dict[str, Any] | None = None
+) -> str:
+    """
+    Crawl page elements document.
+
+    Parameters
+    ----------
+    url : URL.
+    params : URL parameters.
+
+    Returns
+    -------
+    Page elements document.
+    """
+
+    # Parameter.
+    browser = CrawlerBrowser(headless=True)
+
+    # Request.
+    browser.request(url, params)
+
+    # Page.
+    page = browser.page
+
+    return page
+
+def add_db_crawl_task(
+    db_engine: DatabaseEngine,
+    url: str,
+    params: dict[str, Any] | None = None,
+    note: str | None = None
+) -> int:
+    """
+    Add crawl task into database table.
+
+    Parameters
+    ----------
+    db_engine : Database engine.
+    url : Target URL.
+    params : URL parameters.
+    note : Note.
+
+    Returns
+    -------
+    Record ID.
+    """
+
+    # Parameter.
+    params = params or {}
+    url = join_url(url, **params)
+
+    # Inesrt.
+    data = {'url': url, 'note': note}
+    result = db_engine.execute.insert(
+        'crawl_browser_page',
+        data,
+        returning='id'
+    )
+    record_id: int = result.scalar()
+
+    return record_id
+
+def wait_db_crawl_task(
+    db_engine: DatabaseEngine,
+    record_id: int,
+    timeout: int = 60
+) -> str:
+    """
+    Wait crawl task of database table, and get HTML text.
+
+    Parameters
+    ----------
+    db_engine : Database engine.
+    record_id : Record ID.
+    timeout : Timeout seconds.
+
+    Returns
+    -------
+    HTML text.
+    """
+
+    # Loop.
+    tm = TimeMark()
+    tm()
+    while True:
+
+        # Select.
+        result = db_engine.execute.select(
+            'crawl_browser_page',
+            ['html', 'status'],
+            '"id" = :record_id',
+            limit=1,
+            record_id=record_id
+        )
+
+        # Check.
+        if result.empty:
+            throw(AssertionError, record_id)
+
+        # Complete.
+        row = result.to_row()
+        if row['status'] == CrawlBrowserPageStatusEnum.SUCCESS:
+            html = row['html']
+            return html
+
+        # Timeout.
+        tm()
+        if tm.total_spend > timeout:
+            throw(TimeoutError, record_id)
+
+        # Sleep.
+        sleep(1)
+
+def crawl_page_use_db(
+    db_engine: DatabaseEngine,
+    url: str,
+    params: dict[str, Any] | None = None,
+    note: str | None = None,
+    timeout: int = 60
+) -> str:
+    """
+    Add crawl task into database table, wait and get HTML text.
+
+    Parameters
+    ----------
+    db_engine : Database engine.
+    url : Target URL.
+    params : URL parameters.
+    note : Note.
+    timeout : Timeout seconds.
+
+    Returns
+    -------
+    HTML text.
+    """
+
+    # Add.
+    record_id = add_db_crawl_task(db_engine, url, params, note)
+
+    # Wait.
+    html = wait_db_crawl_task(db_engine, record_id, timeout)
+
+    return html