scwrap 0.1.0__tar.gz

scwrap-0.1.0/.gitignore

@@ -0,0 +1,207 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

scwrap-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

scwrap-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nishizawa Takamasa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scwrap-0.1.0/PKG-INFO

@@ -0,0 +1,202 @@
+Metadata-Version: 2.4
+Name: scwrap
+Version: 0.1.0
+Summary: Lightweight scraping helpers: wrapped Page/parser APIs (Patchright, Playwright, selectolax), browser presets, CSV/Parquet and logging utilities.
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: patchright>=1.40
+Requires-Dist: playwright>=1.40
+Requires-Dist: selectolax>=0.3
+Requires-Dist: pandas>=2.0
+Requires-Dist: pyarrow>=14.0
+Requires-Dist: camoufox>=0.4
+Requires-Dist: loguru>=0.7
+
+# scwrap
+
+## Overview - 概要
+
+scwrap is a scraping utility library built on Patchright, Playwright, and selectolax.  
+scwrap は Patchright / Playwright（`Page` API）と selectolax をベースにしたスクレイピングユーティリティライブラリです。**細かい挙動はプリミティブの組み合わせで組み立てる**前提の薄いラッパーです（「よしなに」な自動修復は置かない方針）。
+
+DOM・パーサのラッパーは **`scwrap`**（`wrap_page` / `wrap_parser` などのファクトリー）から、ブラウザ起動は **`scwrap.browser`**、CSV やログなどの周辺は **`scwrap.utils`** から import します。
+
+
+## Requirements - 必要条件
+
+- Python 3.12 or higher（`requires-python` は `pyproject.toml` 参照）
+- 主要依存: patchright, playwright, selectolax, pandas, pyarrow, camoufox, loguru（一覧・下限は `pyproject.toml` の `[project.dependencies]`）
+- `write_parquet` は **pandas + pyarrow**（`pyarrow` は依存に含まれる）。別エンジンに切り替える場合のみ `fastparquet` などが必要になることがあります。
+- ブラウザ: **Patchright / Playwright 用の取得**と、下記のとおり **`patchright_page` は Google Chrome 前提**です。
+
+## Installation - インストール
+
+### pip
+
+```
+pip install scwrap
+```
+
+### uv (推奨)
+
+```
+uv add scwrap
+```
+
+Playwright / Patchright が使うブラウザバイナリは別途取得してください。  
+加えて **`patchright_page()` は `channel='chrome'` で起動するため、マシンに [Google Chrome](https://www.google.com/chrome/) がインストールされている必要があります**（Chromium のみの環境では起動に失敗することがあります）。
+
+### Patchright（Chromium 等）
+
+#### pip
+
+```
+python -m patchright install chromium
+```
+
+#### uv (推奨)
+
+```
+uv run patchright install chromium
+```
+
+### Camoufox（Firefox）
+
+#### pip
+
+```
+camoufox fetch
+```
+
+#### uv (推奨)
+
+```
+uv run camoufox fetch
+```
+
+## メソッド
+
+### `scwrap`（ラッパー）
+
+ブラウザ側は `wrap_page(page)` が起点です。`goto`・`wait`・`css` などはこの戻り値に対して呼びます。要素が複数なら `css(...)` はグループを返し、先頭だけなら `.first`、正規表現で絞り込みは `.grep(pattern)`、相対 URL の解決には `.urls`（単一は `.url`）を使います。テキストや生の要素は `.text` / `.raw` プロパティです。
+
+静的 HTML（selectolax）側は `wrap_parser(parser)` から `css` / `grep` / `text` など（ノードは `wrap_node` 系）。クラス実装は非公開で、**コンストラクトは常にこれらのファクトリー経由**にしてください。
+
+### `scwrap.browser`
+
+- **`patchright_page()`** … コンテキストマネージャ。Patchright で **Google Chrome**（`channel='chrome'`）を起動し、**毎回クリーンな `BrowserContext`** の `Page` を `with` に渡す（永続プロファイルは使わない）。`headless=False`・`no_viewport=True` などは固定。
+
+- **`camoufox_page(locale=...)`** … Camoufox（Firefox）で `Page` を開く。  
+  _例:_ `with camoufox_page(locale='en-US,en') as page:`  
+  デフォルトの `locale` は `'ja-JP,ja'`。`headless=False`・`humanize=True` は固定。
+
+ウィンドウ最大化が必要なら、コードではなく **ブラウザ上で手動**してください（起動引数に依存させない）。
+
+### `scwrap.utils`
+
+`log_to_file`・`from_here`・`parse_html`・`append_csv`・`write_parquet`・`save_html`・`hash_name`・`random_sleep` など（各関数は `scwrap/utils.py` を参照）。`log_to_file` はログファイルの **親ディレクトリが無いと失敗**するので、必要なら先に `Path.mkdir` するか、`save_html` のように親を作る処理を挟んでください。
+
+
+## Basic Usage - 基本的な使い方
+
+```python
+from scwrap import wrap_page
+from scwrap.browser import patchright_page
+from scwrap.utils import log_to_file, append_csv, from_here, random_sleep
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+    p.goto('https://www.foobarbaz1.jp')
+
+    pref_urls = p.css('li.item > ul > li > a').urls
+
+    classroom_urls = []
+    for i, url in enumerate(pref_urls, 1):
+        print(f'pref_urls {i}/{len(pref_urls)}')
+        if not url or not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        classroom_urls.extend(p.css('.school-area h4 a').urls)
+
+    for i, url in enumerate(classroom_urls, 1):
+        print(f'classroom_urls {i}/{len(classroom_urls)}')
+        if not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        append_csv(fh('csv/out.csv'), {
+            'URL': page.url,
+            '教室名': p.css('h1 .text01').first.text,
+            '住所': p.css('.item .mapText').first.text,
+            '電話番号': p.css('.item .phoneNumber').first.text,
+            'HP': p.css('th').grep('ホームページ').first.next('td').css('a').first.url,
+        })
+```
+
+## Save HTML while scraping - スクレイピングしながらHTMLを保存する
+
+```python
+from scwrap import wrap_page
+from scwrap.browser import camoufox_page
+from scwrap.utils import log_to_file, append_csv, from_here, hash_name, random_sleep, save_html
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+with camoufox_page() as page:
+    ctx = {}
+    p = wrap_page(page)
+    p.goto('https://www.foobarbaz1.jp')
+
+    ctx['アイテムURLs'] = p.css('ul.items > li > a').urls
+
+    for i, url in enumerate(ctx['アイテムURLs'], 1):
+        print(f"アイテムURLs {i}/{len(ctx['アイテムURLs'])}")
+        if not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        if p.wait('#logo', timeout=10000).raw is None:
+            continue
+        file_name = f'{hash_name(url)}.html'
+        if not save_html(fh('html') / file_name, page.content()):
+            continue
+        append_csv(fh('outurlhtml.csv'), {
+            'URL': url,
+            'HTML': file_name,
+        })
+```
+
+## Scrape from local HTML files - 保存済みHTMLからスクレイピングしてParquetに出力する
+
+```python
+import pandas as pd
+
+from scwrap import wrap_parser
+from scwrap.utils import log_to_file, from_here, parse_html, write_parquet
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+df = pd.read_csv(fh('outurlhtml.csv'))
+results = []
+for i, (url, path) in enumerate(zip(df['URL'], df['HTML']), 1):
+    print(f'outhtml {i}/{len(df)}')
+    if not (parser := parse_html(fh('html') / path)):
+        continue
+    p = wrap_parser(parser)
+    results.append({
+        'URL': url,
+        '教室名': p.css('h1 .text02').first.text,
+        '住所': p.css('.item .mapText').first.text,
+        '所在地': p.css('dt').grep(r'所在地').first.next('dd').text,
+    })
+write_parquet(fh('outhtml.parquet'), results)
+```
+
+## License - ライセンス
+
+[MIT](./LICENSE)
+

scwrap-0.1.0/README.md

@@ -0,0 +1,186 @@
+# scwrap
+
+## Overview - 概要
+
+scwrap is a scraping utility library built on Patchright, Playwright, and selectolax.  
+scwrap は Patchright / Playwright（`Page` API）と selectolax をベースにしたスクレイピングユーティリティライブラリです。**細かい挙動はプリミティブの組み合わせで組み立てる**前提の薄いラッパーです（「よしなに」な自動修復は置かない方針）。
+
+DOM・パーサのラッパーは **`scwrap`**（`wrap_page` / `wrap_parser` などのファクトリー）から、ブラウザ起動は **`scwrap.browser`**、CSV やログなどの周辺は **`scwrap.utils`** から import します。
+
+
+## Requirements - 必要条件
+
+- Python 3.12 or higher（`requires-python` は `pyproject.toml` 参照）
+- 主要依存: patchright, playwright, selectolax, pandas, pyarrow, camoufox, loguru（一覧・下限は `pyproject.toml` の `[project.dependencies]`）
+- `write_parquet` は **pandas + pyarrow**（`pyarrow` は依存に含まれる）。別エンジンに切り替える場合のみ `fastparquet` などが必要になることがあります。
+- ブラウザ: **Patchright / Playwright 用の取得**と、下記のとおり **`patchright_page` は Google Chrome 前提**です。
+
+## Installation - インストール
+
+### pip
+
+```
+pip install scwrap
+```
+
+### uv (推奨)
+
+```
+uv add scwrap
+```
+
+Playwright / Patchright が使うブラウザバイナリは別途取得してください。  
+加えて **`patchright_page()` は `channel='chrome'` で起動するため、マシンに [Google Chrome](https://www.google.com/chrome/) がインストールされている必要があります**（Chromium のみの環境では起動に失敗することがあります）。
+
+### Patchright（Chromium 等）
+
+#### pip
+
+```
+python -m patchright install chromium
+```
+
+#### uv (推奨)
+
+```
+uv run patchright install chromium
+```
+
+### Camoufox（Firefox）
+
+#### pip
+
+```
+camoufox fetch
+```
+
+#### uv (推奨)
+
+```
+uv run camoufox fetch
+```
+
+## メソッド
+
+### `scwrap`（ラッパー）
+
+ブラウザ側は `wrap_page(page)` が起点です。`goto`・`wait`・`css` などはこの戻り値に対して呼びます。要素が複数なら `css(...)` はグループを返し、先頭だけなら `.first`、正規表現で絞り込みは `.grep(pattern)`、相対 URL の解決には `.urls`（単一は `.url`）を使います。テキストや生の要素は `.text` / `.raw` プロパティです。
+
+静的 HTML（selectolax）側は `wrap_parser(parser)` から `css` / `grep` / `text` など（ノードは `wrap_node` 系）。クラス実装は非公開で、**コンストラクトは常にこれらのファクトリー経由**にしてください。
+
+### `scwrap.browser`
+
+- **`patchright_page()`** … コンテキストマネージャ。Patchright で **Google Chrome**（`channel='chrome'`）を起動し、**毎回クリーンな `BrowserContext`** の `Page` を `with` に渡す（永続プロファイルは使わない）。`headless=False`・`no_viewport=True` などは固定。
+
+- **`camoufox_page(locale=...)`** … Camoufox（Firefox）で `Page` を開く。  
+  _例:_ `with camoufox_page(locale='en-US,en') as page:`  
+  デフォルトの `locale` は `'ja-JP,ja'`。`headless=False`・`humanize=True` は固定。
+
+ウィンドウ最大化が必要なら、コードではなく **ブラウザ上で手動**してください（起動引数に依存させない）。
+
+### `scwrap.utils`
+
+`log_to_file`・`from_here`・`parse_html`・`append_csv`・`write_parquet`・`save_html`・`hash_name`・`random_sleep` など（各関数は `scwrap/utils.py` を参照）。`log_to_file` はログファイルの **親ディレクトリが無いと失敗**するので、必要なら先に `Path.mkdir` するか、`save_html` のように親を作る処理を挟んでください。
+
+
+## Basic Usage - 基本的な使い方
+
+```python
+from scwrap import wrap_page
+from scwrap.browser import patchright_page
+from scwrap.utils import log_to_file, append_csv, from_here, random_sleep
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+    p.goto('https://www.foobarbaz1.jp')
+
+    pref_urls = p.css('li.item > ul > li > a').urls
+
+    classroom_urls = []
+    for i, url in enumerate(pref_urls, 1):
+        print(f'pref_urls {i}/{len(pref_urls)}')
+        if not url or not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        classroom_urls.extend(p.css('.school-area h4 a').urls)
+
+    for i, url in enumerate(classroom_urls, 1):
+        print(f'classroom_urls {i}/{len(classroom_urls)}')
+        if not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        append_csv(fh('csv/out.csv'), {
+            'URL': page.url,
+            '教室名': p.css('h1 .text01').first.text,
+            '住所': p.css('.item .mapText').first.text,
+            '電話番号': p.css('.item .phoneNumber').first.text,
+            'HP': p.css('th').grep('ホームページ').first.next('td').css('a').first.url,
+        })
+```
+
+## Save HTML while scraping - スクレイピングしながらHTMLを保存する
+
+```python
+from scwrap import wrap_page
+from scwrap.browser import camoufox_page
+from scwrap.utils import log_to_file, append_csv, from_here, hash_name, random_sleep, save_html
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+with camoufox_page() as page:
+    ctx = {}
+    p = wrap_page(page)
+    p.goto('https://www.foobarbaz1.jp')
+
+    ctx['アイテムURLs'] = p.css('ul.items > li > a').urls
+
+    for i, url in enumerate(ctx['アイテムURLs'], 1):
+        print(f"アイテムURLs {i}/{len(ctx['アイテムURLs'])}")
+        if not p.goto(url):
+            continue
+        random_sleep(1, 2)
+        if p.wait('#logo', timeout=10000).raw is None:
+            continue
+        file_name = f'{hash_name(url)}.html'
+        if not save_html(fh('html') / file_name, page.content()):
+            continue
+        append_csv(fh('outurlhtml.csv'), {
+            'URL': url,
+            'HTML': file_name,
+        })
+```
+
+## Scrape from local HTML files - 保存済みHTMLからスクレイピングしてParquetに出力する
+
+```python
+import pandas as pd
+
+from scwrap import wrap_parser
+from scwrap.utils import log_to_file, from_here, parse_html, write_parquet
+
+fh = from_here(__file__)
+log_to_file(fh('log/scraping.log'))
+
+df = pd.read_csv(fh('outurlhtml.csv'))
+results = []
+for i, (url, path) in enumerate(zip(df['URL'], df['HTML']), 1):
+    print(f'outhtml {i}/{len(df)}')
+    if not (parser := parse_html(fh('html') / path)):
+        continue
+    p = wrap_parser(parser)
+    results.append({
+        'URL': url,
+        '教室名': p.css('h1 .text02').first.text,
+        '住所': p.css('.item .mapText').first.text,
+        '所在地': p.css('dt').grep(r'所在地').first.next('dd').text,
+    })
+write_parquet(fh('outhtml.parquet'), results)
+```
+
+## License - ライセンス
+
+[MIT](./LICENSE)

scwrap-0.1.0/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from scwrap!")
+
+
+if __name__ == "__main__":
+    main()

scwrap-0.1.0/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "scwrap"
+version = "0.1.0"
+description = "Lightweight scraping helpers: wrapped Page/parser APIs (Patchright, Playwright, selectolax), browser presets, CSV/Parquet and logging utilities."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "patchright>=1.40",
+    "playwright>=1.40",
+    "selectolax>=0.3",
+    "pandas>=2.0",
+    "pyarrow>=14.0",
+    "camoufox>=0.4",
+    "loguru>=0.7",
+]

scwrap-0.1.0/scwrap/__init__.py

@@ -0,0 +1,15 @@
+from .scwrap import (
+    Page,
+    wrap_node,
+    wrap_node_group,
+    wrap_page,
+    wrap_parser,
+)
+
+__all__ = [
+    "Page",
+    "wrap_node",
+    "wrap_node_group",
+    "wrap_page",
+    "wrap_parser",
+]

scwrap-0.1.0/scwrap/browser.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from camoufox.sync_api import Camoufox
+from patchright.sync_api import Page as PatchrightPage, sync_playwright
+from playwright.sync_api import Page as PlaywrightPage
+
+Page = PatchrightPage | PlaywrightPage
+
+
+@contextmanager
+def patchright_page() -> Iterator[Page]:
+    with sync_playwright() as pw:
+        with pw.chromium.launch(
+            channel='chrome',
+            headless=False,
+        ) as browser:
+            with browser.new_context(no_viewport=True) as context:
+                page = context.new_page()
+                yield page
+
+
+@contextmanager
+def camoufox_page(locale: str | list[str] = 'ja-JP,ja') -> Iterator[Page]:
+    with Camoufox(
+        headless=False,
+        humanize=True,
+        locale=locale,
+    ) as browser:
+        page = browser.new_page()
+        yield page

scwrap-0.1.0/scwrap/scwrap.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import random
+import re
+import time
+import unicodedata as ud
+from urllib.parse import urljoin
+
+from loguru import logger
+from patchright.sync_api import Page as PatchrightPage, ElementHandle as PatchrightElementHandle
+from playwright.sync_api import Page as PlaywrightPage, ElementHandle as PlaywrightElementHandle
+from selectolax.lexbor import LexborHTMLParser, LexborNode
+
+
+Page = PatchrightPage | PlaywrightPage
+ElementHandle = PatchrightElementHandle | PlaywrightElementHandle
+
+
+def wrap_page(page: Page) -> _WrappedPage:
+    return _WrappedPage(page)
+
+class _PageScoped:
+    _page: Page
+
+    def wrap_element(self, elem: ElementHandle | None) -> _WrappedElement:
+        return _WrappedElement(self._page, elem)
+
+    def wrap_element_group(self, elems: list[_WrappedElement]) -> _WrappedElementGroup:
+        return _WrappedElementGroup(self._page, elems)
+
+
+def wrap_parser(parser: LexborHTMLParser) -> _WrappedParser:
+    return _WrappedParser(parser)
+
+def wrap_node(node: LexborNode | None) -> _WrappedNode:
+    return _WrappedNode(node)
+
+def wrap_node_group(nodes: list[_WrappedNode]) -> _WrappedNodeGroup:
+    return _WrappedNodeGroup(nodes)
+
+
+class _WrappedPage(_PageScoped):
+    def __init__(self, page: Page) -> None:
+        self._page = page
+
+    def css(self, selector: str) -> _WrappedElementGroup:
+        elems = self._page.query_selector_all(selector)
+        return self.wrap_element_group([self.wrap_element(e) for e in elems])
+
+    def goto(self, url: str | None, try_cnt: int = 3, wait_range: tuple[float, float] = (3, 5)) -> bool:
+        if not url:
+            return False
+        for i in range(try_cnt):
+            try:
+                if self._page.goto(url) is not None:
+                    return True
+                else:
+                    reason = "response is None"
+            except Exception as e:
+                reason = f"{type(e).__name__}: {e}"
+            logger.warning(f"[goto] {url} ({i+1}/{try_cnt}) {reason}")
+            if i + 1 < try_cnt:
+                time.sleep(random.uniform(*wait_range))
+        logger.error(f"[goto] giving up: {url}")
+        return False
+
+    def wait(self, selector: str, timeout: int = 15000) -> _WrappedElement:
+        try:
+            elem = self._page.wait_for_selector(selector, timeout=timeout)
+            return self.wrap_element(elem)
+        except Exception as e:
+            logger.warning(f"[wait] {type(e).__name__}: {e} | selector={selector!r} | url={self._page.url}")
+            return self.wrap_element(None)
+
+
+class _WrappedElement(_PageScoped):
+    def __init__(self, page: Page, elem: ElementHandle | None) -> None:
+        self._page = page
+        self._elem = elem
+
+    @property
+    def raw(self) -> ElementHandle | None:
+        return self._elem
+    
+    def css(self, selector: str) -> _WrappedElementGroup:
+        elems = self._elem.query_selector_all(selector) if self._elem else []
+        return self.wrap_element_group([self.wrap_element(e) for e in elems])
+
+    def next(self, selector: str) -> _WrappedElement:
+        if self._elem is None:
+            return self.wrap_element(None)
+        try:
+            elem = self._elem.evaluate_handle(
+                """(el, sel) => {
+                    let cur = el.nextElementSibling;
+                    while (cur) {
+                        if (cur.matches(sel)) return cur;
+                        cur = cur.nextElementSibling;
+                    }
+                    return null;
+                }""",
+                selector,
+            ).as_element()
+            return self.wrap_element(elem)
+        except Exception as e:
+            logger.error(f"[next] {self._elem} {type(e).__name__}: {e}")
+            return self.wrap_element(None)
+
+    @property
+    def text(self) -> str | None:
+        if self._elem is None:
+            return None
+        if not (text := self._elem.text_content()):
+            return None
+        if not (t := text.strip()):
+            return None
+        return t
+        
+    def attr(self, attr_name: str) -> str | None:
+        if self._elem is None:
+            return None
+        return a.strip() if (a := self._elem.get_attribute(attr_name)) else None
+
+    @property
+    def url(self) -> str | None:
+        if not (href := self.attr('href')):
+            return None
+        if re.search(r'(?i)^(?:#|javascript:|mailto:|tel:|data:)', href):
+            return None
+        return urljoin(self._page.url, href)
+
+class _WrappedElementGroup(_PageScoped):
+    def __init__(self, page: Page, elems: list[_WrappedElement]) -> None:
+        self._page = page
+        self._elems = elems
+
+    @property
+    def raw(self) -> list[_WrappedElement]:
+        return self._elems
+
+    @property
+    def first(self) -> _WrappedElement:
+        return self._elems[0] if self._elems else self.wrap_element(None)
+
+    def grep(self, pattern: str) -> _WrappedElementGroup:
+        prog = re.compile(pattern)
+        filtered = [
+            e for e in self._elems
+            if (t := e.text) and prog.search(ud.normalize('NFKC', t))
+        ]
+        return self.wrap_element_group(filtered)
+
+    @property
+    def texts(self) -> list[str | None]:
+        return [e.text for e in self._elems]
+
+    def attrs(self, attr_name: str) -> list[str | None]:
+        return [e.attr(attr_name) for e in self._elems]
+
+    @property
+    def urls(self) -> list[str | None]:
+        return [e.url for e in self._elems]
+
+
+class _WrappedParser:
+    def __init__(self, parser: LexborHTMLParser) -> None:
+        self._parser = parser
+
+    def css(self, selector: str) -> _WrappedNodeGroup:
+        nodes = self._parser.css(selector)
+        return wrap_node_group([wrap_node(n) for n in nodes])
+
+class _WrappedNode:
+    def __init__(self, node: LexborNode | None) -> None:
+        self._node = node
+
+    @property
+    def raw(self) -> LexborNode | None:
+        return self._node
+
+    def css(self, selector: str) -> _WrappedNodeGroup:
+        nodes = self._node.css(selector) if self._node else []
+        return wrap_node_group([wrap_node(n) for n in nodes])
+
+    def next(self, selector: str) -> _WrappedNode:
+        if self._node is None:
+            return wrap_node(None)
+        cur = self._node.next
+        while cur is not None:
+            if cur.is_element_node and cur.css_matches(selector):
+                return wrap_node(cur)
+            cur = cur.next
+        return wrap_node(None)
+
+    @property
+    def text(self) -> str | None:
+        if self._node is None:
+            return None
+        return t if (t := self._node.text(strip=True)) else None
+
+    def attr(self, attr_name: str) -> str | None:
+        if self._node is None:
+            return None
+        return a.strip() if (a := self._node.attributes.get(attr_name)) else None
+
+class _WrappedNodeGroup:
+    def __init__(self, nodes: list[_WrappedNode]) -> None:
+        self._nodes = nodes
+    
+    @property
+    def raw(self) -> list[_WrappedNode]:
+        return self._nodes
+
+    @property
+    def first(self) -> _WrappedNode:
+        return self._nodes[0] if self._nodes else wrap_node(None)
+
+    def grep(self, pattern: str) -> _WrappedNodeGroup:
+        prog = re.compile(pattern)
+        filtered = [
+            n for n in self._nodes
+            if (t := n.text) and prog.search(ud.normalize('NFKC', t))
+        ]
+        return wrap_node_group(filtered)
+
+    @property
+    def texts(self) -> list[str | None]:
+        return [n.text for n in self._nodes]
+
+    def attrs(self, attr_name: str) -> list[str | None]:
+        return [n.attr(attr_name) for n in self._nodes]
+

scwrap-0.1.0/scwrap/utils.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import hashlib
+import random
+import time
+from pathlib import Path
+from typing import Callable
+
+import pandas as pd
+from loguru import logger
+from selectolax.lexbor import LexborHTMLParser
+
+
+def parse_html(path: Path | str) -> LexborHTMLParser | None:
+    try:
+        return LexborHTMLParser(Path(path).read_text(encoding='utf-8'))
+    except Exception as e:
+        logger.error(f"[parse_html] {path} {type(e).__name__}: {e}")
+        return None
+
+
+def from_here(file: str) -> Callable[[str], Path]:
+    base = Path(file).resolve().parent
+    return lambda path: base / path
+
+
+def random_sleep(a: float, b: float) -> None:
+    time.sleep(random.uniform(a, b))
+
+
+def append_csv(path: Path | str, row: dict) -> None:
+    p = Path(path)
+    try:
+        pd.DataFrame([row]).to_csv(
+            p,
+            mode='a',
+            index=False,
+            header=True if not p.exists() else p.stat().st_size == 0,
+            encoding='utf-8-sig',
+        )
+    except Exception as e:
+        logger.error(f"[append_csv] {path} {row} {type(e).__name__}: {e}")
+
+
+def write_parquet(path: Path | str, rows: list[dict]) -> None:
+    try:
+        pd.DataFrame(rows).to_parquet(
+            Path(path),
+            index=False,
+        )
+    except Exception as e:
+        logger.error(f"[write_parquet] {path} {type(e).__name__}: {e}")
+
+
+def hash_name(key: str) -> str:
+    return hashlib.md5(key.encode()).hexdigest()
+
+
+def save_html(filepath: Path, html: str) -> bool:
+    try:
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        filepath.write_text(html, encoding="utf-8", errors="replace")
+        return True
+    except Exception as e:
+        logger.error(f"[save_html] {filepath} {type(e).__name__}: {e}")
+        return False
+
+
+def log_to_file(path: Path | str) -> None:
+    logger.add(Path(path), level="WARNING", encoding="utf-8")