domx 0.1.1__py3-none-any.whl

domx/__init__.py

@@ -0,0 +1,39 @@
+from .domx import (
+    ElementGrep,
+    ElementHandle,
+    Frame,
+    NodeGrep,
+    Page,
+    Response,
+    WrappedElement,
+    WrappedElementGroup,
+    WrappedFrame,
+    WrappedNode,
+    WrappedNodeGroup,
+    WrappedPage,
+    WrappedParser,
+    wrap_node,
+    wrap_node_group,
+    wrap_page,
+    wrap_parser,
+)
+
+__all__ = [
+    "Page",
+    "ElementHandle",
+    "Frame",
+    "Response",
+    "wrap_page",
+    "wrap_parser",
+    "wrap_node",
+    "wrap_node_group",
+    "WrappedPage",
+    "WrappedFrame",
+    "WrappedElement",
+    "WrappedElementGroup",
+    "ElementGrep",
+    "WrappedParser",
+    "WrappedNode",
+    "WrappedNodeGroup",
+    "NodeGrep",
+]

domx/browser.py

@@ -0,0 +1,39 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from camoufox.sync_api import Camoufox
+from patchright.sync_api import Page as PatchrightPage, sync_playwright
+from playwright.sync_api import Page as PlaywrightPage
+
+Page = PatchrightPage | PlaywrightPage
+
+
+_VIEWPORT_FULL_HD: dict[str, int] = {'width': 1920, 'height': 1080}
+
+
+@contextmanager
+def patchright_page(*, large_viewport: bool = False) -> Iterator[Page]:
+    with sync_playwright() as pw:
+        with pw.chromium.launch(
+            channel='chrome',
+            headless=False,
+        ) as browser:
+            ctx_kw: dict = {}
+            if large_viewport:
+                ctx_kw['viewport'] = _VIEWPORT_FULL_HD
+            with browser.new_context(**ctx_kw) as context:
+                page = context.new_page()
+                yield page
+
+
+@contextmanager
+def camoufox_page(*, large_viewport: bool = False) -> Iterator[Page]:
+    with Camoufox(
+        headless=False,
+        humanize=True,
+    ) as browser:
+        if large_viewport:
+            page = browser.new_page(viewport=_VIEWPORT_FULL_HD)
+        else:
+            page = browser.new_page()
+        yield page

domx/domx.py

@@ -0,0 +1,515 @@
+from __future__ import annotations
+
+import html
+from collections.abc import Iterator
+import random
+import re
+import time
+import unicodedata as ud
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+from urllib.parse import urljoin
+
+from loguru import logger
+from patchright.sync_api import Frame as PatchFrame, Page as PatchPage, ElementHandle as PatchElementHandle, Response as PatchResponse
+from playwright.sync_api import Frame as PlayFrame, Page as PlayPage, ElementHandle as PlayElementHandle, Response as PlayResponse
+from selectolax.lexbor import LexborHTMLParser, LexborNode
+
+
+Page = PatchPage | PlayPage
+ElementHandle = PatchElementHandle | PlayElementHandle
+Response = PatchResponse | PlayResponse
+Frame = PatchFrame | PlayFrame
+
+_DOMX_META_URL = 'domx:url'
+_DOMX_META_SAVED_AT = 'domx:saved_at'
+
+_UNUSABLE_INLINE_URL = re.compile(r'(?i)^(?:#|javascript:|mailto:|tel:|data:)')
+
+_ELEMENT_NEXT = 'nextElementSibling'
+_ELEMENT_PREV = 'previousElementSibling'
+_ELEMENT_PARENT = 'parentElement'
+
+_NODE_NEXT = 'next'
+_NODE_PREV = 'prev'
+_NODE_PARENT = 'parent'
+
+
+def wrap_page(page: Page) -> WrappedPage:
+    return WrappedPage(page)
+
+
+class _PageScoped:
+    _page: Page
+
+    def wrap_element(self, elem: ElementHandle | None) -> WrappedElement:
+        return WrappedElement(self._page, elem)
+
+    def wrap_element_group(self, elems: list[WrappedElement]) -> WrappedElementGroup:
+        return WrappedElementGroup(self._page, elems)
+
+    def wrap_frame(self, frame: Frame | None) -> WrappedFrame:
+        return WrappedFrame(self._page, frame)
+
+
+def wrap_parser(parser: LexborHTMLParser) -> WrappedParser:
+    return WrappedParser(parser)
+
+
+def wrap_node(node: LexborNode | None) -> WrappedNode:
+    return WrappedNode(node)
+
+
+def wrap_node_group(nodes: list[WrappedNode]) -> WrappedNodeGroup:
+    return WrappedNodeGroup(nodes)
+
+
+class WrappedFrame(_PageScoped):
+    def __init__(self, page: Page, frame: Frame | None) -> None:
+        self._page = page
+        self._frame = frame
+
+    def __bool__(self) -> bool:
+        return self._frame is not None
+
+    @property
+    def raw(self) -> Frame | None:
+        return self._frame
+
+    def s(self, selector: str) -> WrappedElement:
+        if self._frame is None:
+            return self.wrap_element(None)
+        elem = self._frame.query_selector(selector)
+        return self.wrap_element(elem)
+
+    def ss(self, selector: str) -> WrappedElementGroup:
+        if self._frame is None:
+            return self.wrap_element_group([])
+        elems = self._frame.query_selector_all(selector)
+        return self.wrap_element_group([self.wrap_element(e) for e in elems])
+
+    def wait(self, selector: str, state: str = 'attached', timeout: int = 15000) -> WrappedElement:
+        if self._frame is None:
+            return self.wrap_element(None)
+        try:
+            elem = self._frame.wait_for_selector(selector, state=state, timeout=timeout)
+            return self.wrap_element(elem)
+        except Exception as e:
+            logger.warning(
+                f'[wait] {type(e).__name__}: {e} | selector={selector!r} | url={self._page.url}'
+            )
+            return self.wrap_element(None)
+
+
+class WrappedPage(_PageScoped):
+    def __init__(self, page: Page) -> None:
+        self._page = page
+
+    @property
+    def raw(self) -> Page:
+        return self._page
+
+    def s(self, selector: str) -> WrappedElement:
+        elem = self._page.query_selector(selector)
+        return self.wrap_element(elem)
+
+    def ss(self, selector: str) -> WrappedElementGroup:
+        elems = self._page.query_selector_all(selector)
+        return self.wrap_element_group([self.wrap_element(e) for e in elems])
+
+    def frame(self, iframe_selector: str) -> WrappedFrame:
+        iframe_elem = self._page.query_selector(iframe_selector)
+        if iframe_elem is None:
+            return self.wrap_frame(None)
+        try:
+            fr = iframe_elem.content_frame()
+            return self.wrap_frame(fr)
+        except Exception as e:
+            logger.error(f'[frame] {type(e).__name__}: {e} | iframe_selector={iframe_selector!r}')
+            return self.wrap_frame(None)
+
+    def goto(
+        self,
+        url: str | None,
+        try_cnt: int = 3,
+        wait_range: tuple[float, float] = (3, 5),
+        sleep_after: tuple[float, float] | None = (1, 2),
+    ) -> Response | None:
+        if not url:
+            return None
+        for i in range(try_cnt):
+            try:
+                response = self._page.goto(url)
+                if response is not None:
+                    if sleep_after is not None:
+                        time.sleep(random.uniform(*sleep_after))
+                    return response
+                reason = 'response is None'
+            except Exception as e:
+                reason = f'{type(e).__name__}: {e}'
+            logger.warning(f'[goto] {url} ({i+1}/{try_cnt}) {reason}')
+            if i + 1 < try_cnt:
+                time.sleep(random.uniform(*wait_range))
+        logger.error(f'[goto] giving up: {url}')
+        return None
+
+    def wait(self, selector: str, state: str = 'attached', timeout: int = 15000) -> WrappedElement:
+        try:
+            elem = self._page.wait_for_selector(selector, state=state, timeout=timeout)
+            return self.wrap_element(elem)
+        except Exception as e:
+            logger.warning(f'[wait] {type(e).__name__}: {e} | selector={selector!r} | url={self._page.url}')
+            return self.wrap_element(None)
+
+    def html(self, with_url: bool = False, with_saved_at: bool = False) -> str:
+        content = self._page.content()
+        metas: list[str] = []
+        if with_url:
+            metas.append(
+                f'<meta name="{_DOMX_META_URL}" content="{html.escape(self._page.url)}">'
+            )
+        if with_saved_at:
+            ts = datetime.now(timezone.utc).isoformat()
+            metas.append(f'<meta name="{_DOMX_META_SAVED_AT}" content="{ts}">')
+        return ''.join(metas) + content
+
+
+class WrappedElement(_PageScoped):
+    def __init__(self, page: Page, elem: ElementHandle | None) -> None:
+        self._page = page
+        self._elem = elem
+
+    def __bool__(self) -> bool:
+        return self._elem is not None
+
+    @property
+    def raw(self) -> ElementHandle | None:
+        return self._elem
+
+    def s(self, selector: str) -> WrappedElement:
+        elem = self._elem.query_selector(selector) if self._elem else None
+        return self.wrap_element(elem)
+
+    def ss(self, selector: str) -> WrappedElementGroup:
+        elems = self._elem.query_selector_all(selector) if self._elem else []
+        return self.wrap_element_group([self.wrap_element(e) for e in elems])
+
+    def frame(self, iframe_selector: str | None = None) -> WrappedFrame:
+        if self._elem is None:
+            return self.wrap_frame(None)
+        try:
+            if iframe_selector is None:
+                fr = self._elem.content_frame()
+            else:
+                iframe_elem = self._elem.query_selector(iframe_selector)
+                if iframe_elem is None:
+                    return self.wrap_frame(None)
+                fr = iframe_elem.content_frame()
+            return self.wrap_frame(fr)
+        except Exception as e:
+            logger.error(
+                f'[frame] {type(e).__name__}: {e} | iframe_selector={iframe_selector!r}'
+            )
+            return self.wrap_frame(None)
+
+    def _walk_relative(self, selector: str, axis: str, label: str) -> WrappedElement:
+        if self._elem is None:
+            return self.wrap_element(None)
+        try:
+            elem = self._elem.evaluate_handle(
+                '''(el, args) => {
+                    const [sel, axis] = args;
+                    let cur = el[axis];
+                    while (cur) {
+                        if (cur.matches(sel)) return cur;
+                        cur = cur[axis];
+                    }
+                    return null;
+                }''',
+                [selector, axis],
+            ).as_element()
+            return self.wrap_element(elem)
+        except Exception as e:
+            logger.error(f'[{label}] {self._elem} {type(e).__name__}: {e}')
+            return self.wrap_element(None)
+
+    def next(self, selector: str) -> WrappedElement:
+        return self._walk_relative(selector, _ELEMENT_NEXT, 'next')
+
+    def prev(self, selector: str) -> WrappedElement:
+        return self._walk_relative(selector, _ELEMENT_PREV, 'prev')
+
+    def parent(self, selector: str) -> WrappedElement:
+        return self._walk_relative(selector, _ELEMENT_PARENT, 'parent')
+
+    @property
+    def text(self) -> str | None:
+        if self._elem is None:
+            return None
+        return text if (text := self._elem.text_content()) else None
+
+    def attr(self, attr_name: str) -> str | None:
+        if self._elem is None:
+            return None
+        return attr if (attr := self._elem.get_attribute(attr_name)) else None
+
+    def _resolved_url_from_attr(self, attr_name: str) -> str | None:
+        if self._elem is None:
+            return None
+        if not (attr := self._elem.get_attribute(attr_name)):
+            return None
+        if not (a := attr.strip()):
+            return None
+        if _UNUSABLE_INLINE_URL.search(a):
+            return None
+        return urljoin(self._page.url, a)
+
+    @property
+    def url(self) -> str | None:
+        return self._resolved_url_from_attr('href')
+
+    @property
+    def src(self) -> str | None:
+        return self._resolved_url_from_attr('src')
+    
+    def scroll_into_view(self) -> None:
+        if self._elem is None:
+            logger.warning('[scroll_into_view] element is None')
+            return
+        try:
+            self._elem.evaluate(
+                '''(el) => el.scrollIntoView({ behavior: "smooth", block: "center", inline: "nearest" });'''
+            )
+            self._elem.wait_for_element_state('stable')
+        except Exception as e:
+            logger.warning(f'[scroll_into_view] {type(e).__name__}: {e} | url={self._page.url!r}')
+
+    def screenshot(
+        self,
+        path: Path,
+        image_type: Literal['png', 'jpeg'] = 'png',
+    ) -> bool:
+        if self._elem is None:
+            logger.warning('[screenshot] element is None')
+            return False
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            self._elem.screenshot(
+                path=path,
+                type=image_type,
+                animations='disabled',
+            )
+            return True
+        except Exception as e:
+            logger.warning(f'[screenshot] {type(e).__name__}: {e} | url={self._page.url!r}')
+            return False
+
+
+class WrappedElementGroup(_PageScoped):
+    def __init__(self, page: Page, elems: list[WrappedElement]) -> None:
+        self._page = page
+        self._elems = elems
+
+    def __iter__(self) -> Iterator[WrappedElement]:
+        return iter(self._elems)
+
+    def __len__(self) -> int:
+        return len(self._elems)
+
+    def __getitem__(self, key: int | slice) -> WrappedElement | WrappedElementGroup:
+        if isinstance(key, slice):
+            return WrappedElementGroup(self._page, self._elems[key])
+        return self._elems[key]
+
+    def __add__(self, other: WrappedElementGroup) -> WrappedElementGroup:
+        if not isinstance(other, WrappedElementGroup):
+            raise TypeError(
+                'WrappedElementGroup 同士のみ + で結合できます '
+                f'（右辺は {type(other).__name__}）'
+            )
+        if self._page is not other._page:
+            raise ValueError('異なる Page に紐づいた WrappedElementGroup は結合できません')
+        return WrappedElementGroup(self._page, self._elems + other._elems)
+
+    @property
+    def raw(self) -> list[WrappedElement]:
+        return self._elems
+
+    @property
+    def re(self) -> ElementGrep:
+        pairs: list[tuple[str, WrappedElement]] = []
+        for e in self._elems:
+            if (t := e.text):
+                pairs.append((ud.normalize('NFKC', t), e))
+        return ElementGrep(self._page, pairs)
+
+    @property
+    def urls(self) -> list[str]:
+        return [u for e in self._elems if (u := e.url)]
+
+
+class ElementGrep(_PageScoped):
+    def __init__(self, page: Page, pairs: list[tuple[str, WrappedElement]]) -> None:
+        self._page = page
+        self._pairs = pairs
+
+    def s(self, pattern: str) -> WrappedElement:
+        try:
+            prog = re.compile(pattern)
+            for text, e in self._pairs:
+                if prog.search(text):
+                    return e
+        except Exception as e:
+            logger.warning(f'[grep] {type(e).__name__}: {e} | pattern={pattern!r}')
+        return self.wrap_element(None)
+
+    def ss(self, pattern: str) -> WrappedElementGroup:
+        try:
+            prog = re.compile(pattern)
+            filtered = [e for text, e in self._pairs if prog.search(text)]
+            return self.wrap_element_group(filtered)
+        except Exception as e:
+            logger.warning(f'[grep] {type(e).__name__}: {e} | pattern={pattern!r}')
+            return self.wrap_element_group([])
+
+
+class WrappedParser:
+    def __init__(self, parser: LexborHTMLParser) -> None:
+        self._parser = parser
+
+    @property
+    def raw(self) -> LexborHTMLParser:
+        return self._parser
+
+    def s(self, selector: str) -> WrappedNode:
+        node = self._parser.css_first(selector)
+        return wrap_node(node)
+
+    def ss(self, selector: str) -> WrappedNodeGroup:
+        nodes = self._parser.css(selector)
+        return wrap_node_group([wrap_node(n) for n in nodes])
+
+    @property
+    def url(self) -> str | None:
+        node = self._parser.css_first(f'meta[name="{_DOMX_META_URL}"]')
+        if node is None:
+            return None
+        return node.attributes.get('content') or None
+
+    @property
+    def saved_at(self) -> str | None:
+        node = self._parser.css_first(f'meta[name="{_DOMX_META_SAVED_AT}"]')
+        if node is None:
+            return None
+        return node.attributes.get('content') or None
+
+
+class WrappedNode:
+    def __init__(self, node: LexborNode | None) -> None:
+        self._node = node
+
+    def __bool__(self) -> bool:
+        return self._node is not None
+
+    @property
+    def raw(self) -> LexborNode | None:
+        return self._node
+
+    def s(self, selector: str) -> WrappedNode:
+        node = self._node.css_first(selector) if self._node else None
+        return wrap_node(node)
+
+    def ss(self, selector: str) -> WrappedNodeGroup:
+        nodes = self._node.css(selector) if self._node else []
+        return wrap_node_group([wrap_node(n) for n in nodes])
+
+    def _walk_relative(self, selector: str, axis: str) -> WrappedNode:
+        if self._node is None:
+            return wrap_node(None)
+        cur = getattr(self._node, axis)
+        while cur is not None:
+            if cur.is_element_node and cur.css_matches(selector):
+                return wrap_node(cur)
+            cur = getattr(cur, axis)
+        return wrap_node(None)
+
+    def next(self, selector: str) -> WrappedNode:
+        return self._walk_relative(selector, _NODE_NEXT)
+
+    def prev(self, selector: str) -> WrappedNode:
+        return self._walk_relative(selector, _NODE_PREV)
+
+    def parent(self, selector: str) -> WrappedNode:
+        return self._walk_relative(selector, _NODE_PARENT)
+
+    @property
+    def text(self) -> str | None:
+        if self._node is None:
+            return None
+        return text if (text := self._node.text()) else None
+
+    def attr(self, attr_name: str) -> str | None:
+        if self._node is None:
+            return None
+        return attr if (attr := self._node.attributes.get(attr_name)) else None
+
+
+class WrappedNodeGroup:
+    def __init__(self, nodes: list[WrappedNode]) -> None:
+        self._nodes = nodes
+
+    def __iter__(self) -> Iterator[WrappedNode]:
+        return iter(self._nodes)
+
+    def __len__(self) -> int:
+        return len(self._nodes)
+
+    def __getitem__(self, key: int | slice) -> WrappedNode | WrappedNodeGroup:
+        if isinstance(key, slice):
+            return WrappedNodeGroup(self._nodes[key])
+        return self._nodes[key]
+
+    def __add__(self, other: WrappedNodeGroup) -> WrappedNodeGroup:
+        if not isinstance(other, WrappedNodeGroup):
+            raise TypeError(
+                'WrappedNodeGroup 同士のみ + で結合できます '
+                f'（右辺は {type(other).__name__}）'
+            )
+        return WrappedNodeGroup(self._nodes + other._nodes)
+
+    @property
+    def raw(self) -> list[WrappedNode]:
+        return self._nodes
+
+    @property
+    def re(self) -> NodeGrep:
+        pairs: list[tuple[str, WrappedNode]] = []
+        for n in self._nodes:
+            if (t := n.text):
+                pairs.append((ud.normalize('NFKC', t), n))
+        return NodeGrep(pairs)
+
+
+class NodeGrep:
+    def __init__(self, pairs: list[tuple[str, WrappedNode]]) -> None:
+        self._pairs = pairs
+
+    def s(self, pattern: str) -> WrappedNode:
+        try:
+            prog = re.compile(pattern)
+            for text, n in self._pairs:
+                if prog.search(text):
+                    return n
+        except Exception as e:
+            logger.warning(f'[grep] {type(e).__name__}: {e} | pattern={pattern!r}')
+        return wrap_node(None)
+
+    def ss(self, pattern: str) -> WrappedNodeGroup:
+        try:
+            prog = re.compile(pattern)
+            filtered = [n for text, n in self._pairs if prog.search(text)]
+            return wrap_node_group(filtered)
+        except Exception as e:
+            logger.warning(f'[grep] {type(e).__name__}: {e} | pattern={pattern!r}')
+            return wrap_node_group([])

domx/utils.py

@@ -0,0 +1,162 @@
+import csv
+import hashlib
+import os
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import Callable, Iterable
+
+from loguru import logger
+from selectolax.lexbor import LexborHTMLParser
+from tqdm import tqdm
+
+
+def _ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def parse_html(path: Path) -> LexborHTMLParser | None:
+    try:
+        return LexborHTMLParser(path.read_bytes())
+    except Exception as e:
+        logger.error(f'[parse_html] {path} {type(e).__name__}: {e}')
+        return None
+
+def from_here(file: str) -> Callable[[str], Path]:
+    base = Path(file).resolve().parent
+    return lambda path: base / path
+
+def append_csv(path: Path, row: dict) -> None:
+    '''``row`` を 1 行だけ CSV に追記する（ファイルが無ければ作成）。
+
+    Excel 互換のため、**ファイル新規作成時のみ先頭に UTF-8 BOM** を書く
+    （``utf-8-sig`` で open）。既存ファイルへの追記では BOM を書かない
+    （中途 BOM は不正になるため）。ファイルが新規 / 空ならヘッダ行を書く。
+    列順は ``row.keys()`` の順で、2 回目以降のキーずれは検知しない
+    （pandas 版と同じ挙動）。
+    '''
+    try:
+        _ensure_parent(path)
+        need_header = not path.exists() or path.stat().st_size == 0
+        encoding = 'utf-8-sig' if need_header else 'utf-8'
+        with open(path, mode='a', newline='', encoding=encoding) as f:
+            w = csv.DictWriter(f, fieldnames=list(row.keys()))
+            if need_header:
+                w.writeheader()
+            w.writerow(row)
+    except Exception as e:
+        logger.error(f'[append_csv] {path} {row} {type(e).__name__}: {e}')
+
+def write_parquet(path: Path, rows: list[dict]) -> None:
+    '''``rows`` を Parquet ファイルとして書き出す。
+
+    pyarrow を直接使う（pandas 非依存）。``rows`` が空ならスキップ（警告のみ）。
+    列スキーマは各列の最初の non-None 値から推論されるので、**同一キーで型が
+    混在するとエラーになる**ことがある点に注意。
+    '''
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    try:
+        if not rows:
+            logger.warning(f'[write_parquet] {path} no rows, skipped')
+            return
+        _ensure_parent(path)
+        pq.write_table(pa.Table.from_pylist(rows), path)
+    except Exception as e:
+        logger.error(f'[write_parquet] {path} {type(e).__name__}: {e}')
+
+def hash_name(key: str) -> str:
+    return hashlib.md5(key.encode()).hexdigest()
+
+def write_text(path: Path, data: str) -> bool:
+    try:
+        _ensure_parent(path)
+        path.write_text(data, encoding='utf-8', errors='replace')
+        return True
+    except Exception as e:
+        logger.error(f'[write_text] {path} {type(e).__name__}: {e}')
+        return False
+
+def write_bytes(path: Path, data: bytes) -> bool:
+    try:
+        _ensure_parent(path)
+        path.write_bytes(data)
+        return True
+    except Exception as e:
+        logger.error(f'[write_bytes] {path} {type(e).__name__}: {e}')
+        return False
+
+def save_log(path: Path, level: str = 'WARNING') -> None:
+    '''コンソール（stderr）に出るログと同じ内容を、指定ファイルにも残す。'''
+    _ensure_parent(path)
+    logger.add(path, level=level, encoding='utf-8')
+
+
+class _SafeWorker:
+    def __init__(self, fn: Callable) -> None:
+        self.fn = fn
+
+    def __call__(self, x):
+        try:
+            return self.fn(x)
+        except Exception as e:
+            logger.error(f'[pool_map] {type(e).__name__}: {e}')
+            return None
+
+
+def _auto_chunksize(n: int, workers: int | None) -> int:
+    '''``chunksize`` を自動で決める（``pool_map`` で未指定のとき）。
+
+    子プロセスへは 1 件ずつより、まとめて送った方が速くなりやすい。そのまとめ数。
+
+    ``w`` は並列数。引数で決まっていなければ ``os.cpu_count()``、それも無ければ 4。
+    この **4** は「CPU が分からないときの仮の並列数」。式 ``n // (w * 4)`` の **4** とは別物。
+
+    ``n // (w * 4)`` の方の **4** は経験則の係数。ざっくり言うとチャンクの個数が
+    ``w * 4`` 前後になりやすく、負荷が均等ならワーカーあたりだいたい **4 回分の塊**
+    を処理するイメージ（厳密ではない）。
+
+    例: ``n=200``, ``w=5`` なら ``200 // 20 = 10`` が chunksize。全体は 20 チャンク、
+    5 人で割ると 1 人あたり平均 4 チャンク（各 10 件）。
+
+    結果は ``min(64, …)`` で上限。塊が大きすぎると **負荷が偏りやすい**。
+    タスクの重さがバラバラなとき、太い塊の中に遅いのが多く入ったワーカーだけが
+    長引き、他は先に終わって手待ちしがち（終盤のムラ）。塊を細かくすると配り直しの
+    機会が増えて和らぎやすい。進捗バーも細かく動きやすい。
+
+    ``max(1, …)`` で下限。割り算で 0 になっても最低 1 件は送る。
+    '''
+    w = workers or os.cpu_count() or 4
+    return max(1, min(64, n // (w * 4)))
+
+
+def pool_map[T, R](
+    worker: Callable[[T], R],
+    items: Iterable[T],
+    workers: int | None = None,
+    *,
+    chunksize: int | None = None,
+) -> list[R | None]:
+    '''``ProcessPoolExecutor`` で ``worker`` を並列実行する。
+
+    子プロセスで例外が出た分は ``None`` で返す。全体は止めない。
+    進捗バーは常に tqdm。
+
+    ``chunksize`` は子へまとめて送る件数。省略なら自動。
+    進捗を細かくしたい・タスクの重さがバラバラで末尾に重いのが残る、なら ``chunksize=1``。
+    '''
+    safe = _SafeWorker(worker)
+    item_list = list(items)
+    cs = chunksize if chunksize is not None else _auto_chunksize(len(item_list), workers)
+    with ProcessPoolExecutor(max_workers=workers) as ex:
+        return list(
+            tqdm(ex.map(safe, item_list, chunksize=cs), total=len(item_list), unit='file')
+        )
+
+def glob_paths(dir_path: Path, pattern: str = '*.html') -> list[str]:
+    '''
+    ``dir_path`` 直下で ``pattern`` に一致するパスを ``str`` のリストで返す。
+
+    ``str`` にしているのは ``pool_map`` 等のプロセスプールへ渡すとき pickle しやすくするため。
+    '''
+    return [str(p) for p in dir_path.glob(pattern)]

domx-0.1.1.dist-info/METADATA

@@ -0,0 +1,239 @@
+Metadata-Version: 2.4
+Name: domx
+Version: 0.1.1
+Summary: 自分用・非汎用
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: patchright>=1.40
+Requires-Dist: playwright>=1.40
+Requires-Dist: selectolax>=0.3
+Requires-Dist: pyarrow>=14.0
+Requires-Dist: camoufox>=0.4
+Requires-Dist: loguru>=0.7
+Requires-Dist: tqdm>=4.66
+
+# domx
+
+自分用・非汎用
+
+## インストール
+
+`uv add domx`  
+`uv run patchright install chromium`  
+`uv run camoufox fetch`
+
+## 使用例
+
+### スクレイピング
+
+```python
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import append_csv, from_here, save_log, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    pref_urls = p.ss('li.item > ul > li > a').urls
+
+    classroom_urls = []
+    for i, url in enumerate(pref_urls, 1):
+        print(f'pref_urls {i}/{len(pref_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        classroom_urls.extend(p.ss('.school-area h4 a').urls)
+
+    for i, url in enumerate(classroom_urls, 1):
+        print(f'classroom_urls {i}/{len(classroom_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        th_grep = p.ss('th').re
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '教室名': p.s('h1 .text01').text,
+            '住所': p.s('.item .mapText').text,
+            '電話番号': p.s('.item .phoneNumber').text,
+            'HP': th_grep.s(r'ホームページ').next('td').s('a').url,
+            '営業時間': th_grep.s(r'営業時間').next('td').text,
+            '定休日': th_grep.s(r'定休日').next('td').text,
+        })
+        p.s('.school-map').screenshot(here(f'media/{i}-screenshot.png'))
+        if (img_url := p.s('.school-area img').src):
+            if (res := p.goto(img_url)) and res.ok:
+                write_bytes(here(f'media/{i}-img.jpg'), res.body())
+```
+
+### スクレイピング(スクショと画像も保存)
+
+```python
+import time
+from urllib.parse import urlencode
+
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import save_log, append_csv, from_here, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+    
+    p.goto('https://example.com/demo/search')
+    prefecture_urls = p.ss('li > a[href^="https://example.com/demo/search/area/"]').urls
+
+    bukken_urls = []
+    for i, prefecture_url in enumerate(prefecture_urls, 1):
+        print(f'{i}/{len(prefecture_urls)} エリア一覧ページ')
+        page_num = 1
+        while True:
+            if not p.goto(f'{prefecture_url}?{urlencode({"page": page_num})}'):
+                break
+            if not (bukken_elems := p.ss('ul li div a[href^="https://example.com"]:has(p)')):
+                break
+            bukken_urls.extend(bukken_elems.urls)
+            page_num += 1
+    
+    for i, url in enumerate(bukken_urls, 1):
+        print(f'{i}/{len(bukken_urls)} 詳細ページ {url}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        
+        dt_grep = p.ss('h4').re.s(r'概要').next('div:has(dl)').ss('dt').re
+        dd_text = lambda pattern: dt_grep.s(pattern).next('dd').text
+
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '価格': dd_text(r'価格'),
+            '所在地': dd_text(r'所在地'),
+            '交通': dd_text(r'交通'),
+            '駐車場': dd_text(r'駐車場'),
+            '備考': dd_text(r'備考'),
+            '情報更新日': dd_text(r'情報更新日'),
+        })
+        
+        page.add_style_tag(content='header, footer.site-footer { visibility: hidden !important; }')
+        
+        p.ss('h4').re.s(r'概要').next('div:has(dl)').screenshot(path=here(f'media/{i}-summary.png'))
+
+        elem_iframe = p.s('iframe[src^="https://example.com"]')
+        elem_iframe.scroll_into_view()
+        time.sleep(3)
+        elem_iframe.screenshot(path=here(f'media/{i}-iframe.png'))
+        
+        main_img_url = p.s('img.w-full.object-contain').src
+        
+        img_desc_grep = p.ss('p.text-left').re.s(r'画像をクリック').next('ul').ss('li p').re
+        img_desc = img_desc_grep.s(r'表紙') or img_desc_grep.s(r'^(?!.*裏面).*')
+        img_url = img_desc.parent('li').s('a').url
+        
+        if main_img_url and (res := p.goto(main_img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-main-img.jpg'), res.body())
+        if img_url and (res := p.goto(img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-img-desc.jpg'), res.body())
+```
+
+### スクレイピング(HTML丸ごと保存)
+
+```python
+from domx import wrap_page
+from domx.browser import camoufox_page
+from domx.utils import append_csv, from_here, hash_name, save_log, write_text
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with camoufox_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    item_urls = p.ss('ul.items > li > a').urls
+
+    for i, url in enumerate(item_urls, 1):
+        print(f'item_urls {i}/{len(item_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        file_name = f'{hash_name(url)}.html'
+        if not write_text(here('html') / file_name, p.html(with_url=True, with_saved_at=True)):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'write_text'})
+            continue
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力
+
+```python
+from domx import wrap_parser
+from domx.utils import from_here, parse_html, save_log, write_parquet
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+results = []
+for i, file_path in enumerate(here('html').glob('*.html'),1):
+    print(f'html {i}')
+    if not (parser := parse_html(file_path)):
+        continue
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    results.append({
+        'URL': p.url,
+        'file_name': file_path.name,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '物件番号': dts.s(r'物件番号').next('dd').text,
+    })
+write_parquet(here('parquet/extract.parquet'), results)
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力(並列処理)
+
+```python
+from pathlib import Path
+
+from domx import wrap_parser
+from domx.utils import from_here, glob_paths, parse_html, pool_map, write_parquet
+
+def main():
+    here = from_here(__file__)
+    html_paths = glob_paths(here('html'), '*.html')
+    results = [r for r in pool_map(extract, html_paths) if r]
+    write_parquet(here('parquet/extract.parquet'), results)
+
+def extract(file_path: str) -> dict | None:
+    if not (parser := parse_html(Path(file_path))):
+        return None
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    return {
+        'URL': p.url,
+        'file_path': file_path,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '価格': dts.s(r'価格').next('dd').text,
+        '設備・条件': dts.s(r'設備').next('dd').text,
+        '備考': dts.s(r'備考').next('dd').text,
+    }
+
+if __name__ == '__main__':
+    main()
+```
+
+## License - ライセンス
+
+[MIT](./LICENSE)
+

domx-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+domx/__init__.py,sha256=F2csRoeqAMIz_PlKzNBZLC75GZ6Bhypj1y8oaglaVkI,692
+domx/browser.py,sha256=nhhabsYOed42va8b5GMt598fG0aOkVIQtwKVPwWph8g,1206
+domx/domx.py,sha256=QjgDTtTDRD0PpIL_2bUfe9abgnFaxGebYcGgyby901c,18219
+domx/utils.py,sha256=gnp15-2Et6rKs7ULgFAGZmQnMvkYZ4VnxN-a1gksPEU,6676
+domx-0.1.1.dist-info/licenses/LICENSE,sha256=q8ED812OTMMCwQSdHvtx6PSnmtRIotcIjKPHMmVseQI,1096
+domx-0.1.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+domx-0.1.1.dist-info/METADATA,sha256=D1bZj6wntjgx5E0YuDv_d7nSox1f0zXJ-iMSbCPSLow,7876
+domx-0.1.1.dist-info/RECORD,,

domx-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any

domx-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nishizawa Takamasa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.