domx 0.1.1__tar.gz

domx-0.1.1/.gitignore

@@ -0,0 +1,218 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

domx-0.1.1/.python-version

@@ -0,0 +1 @@
+3.12

domx-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nishizawa Takamasa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

domx-0.1.1/PKG-INFO

@@ -0,0 +1,239 @@
+Metadata-Version: 2.4
+Name: domx
+Version: 0.1.1
+Summary: 自分用・非汎用
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: patchright>=1.40
+Requires-Dist: playwright>=1.40
+Requires-Dist: selectolax>=0.3
+Requires-Dist: pyarrow>=14.0
+Requires-Dist: camoufox>=0.4
+Requires-Dist: loguru>=0.7
+Requires-Dist: tqdm>=4.66
+
+# domx
+
+自分用・非汎用
+
+## インストール
+
+`uv add domx`  
+`uv run patchright install chromium`  
+`uv run camoufox fetch`
+
+## 使用例
+
+### スクレイピング
+
+```python
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import append_csv, from_here, save_log, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    pref_urls = p.ss('li.item > ul > li > a').urls
+
+    classroom_urls = []
+    for i, url in enumerate(pref_urls, 1):
+        print(f'pref_urls {i}/{len(pref_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        classroom_urls.extend(p.ss('.school-area h4 a').urls)
+
+    for i, url in enumerate(classroom_urls, 1):
+        print(f'classroom_urls {i}/{len(classroom_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        th_grep = p.ss('th').re
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '教室名': p.s('h1 .text01').text,
+            '住所': p.s('.item .mapText').text,
+            '電話番号': p.s('.item .phoneNumber').text,
+            'HP': th_grep.s(r'ホームページ').next('td').s('a').url,
+            '営業時間': th_grep.s(r'営業時間').next('td').text,
+            '定休日': th_grep.s(r'定休日').next('td').text,
+        })
+        p.s('.school-map').screenshot(here(f'media/{i}-screenshot.png'))
+        if (img_url := p.s('.school-area img').src):
+            if (res := p.goto(img_url)) and res.ok:
+                write_bytes(here(f'media/{i}-img.jpg'), res.body())
+```
+
+### スクレイピング(スクショと画像も保存)
+
+```python
+import time
+from urllib.parse import urlencode
+
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import save_log, append_csv, from_here, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+    
+    p.goto('https://example.com/demo/search')
+    prefecture_urls = p.ss('li > a[href^="https://example.com/demo/search/area/"]').urls
+
+    bukken_urls = []
+    for i, prefecture_url in enumerate(prefecture_urls, 1):
+        print(f'{i}/{len(prefecture_urls)} エリア一覧ページ')
+        page_num = 1
+        while True:
+            if not p.goto(f'{prefecture_url}?{urlencode({"page": page_num})}'):
+                break
+            if not (bukken_elems := p.ss('ul li div a[href^="https://example.com"]:has(p)')):
+                break
+            bukken_urls.extend(bukken_elems.urls)
+            page_num += 1
+    
+    for i, url in enumerate(bukken_urls, 1):
+        print(f'{i}/{len(bukken_urls)} 詳細ページ {url}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        
+        dt_grep = p.ss('h4').re.s(r'概要').next('div:has(dl)').ss('dt').re
+        dd_text = lambda pattern: dt_grep.s(pattern).next('dd').text
+
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '価格': dd_text(r'価格'),
+            '所在地': dd_text(r'所在地'),
+            '交通': dd_text(r'交通'),
+            '駐車場': dd_text(r'駐車場'),
+            '備考': dd_text(r'備考'),
+            '情報更新日': dd_text(r'情報更新日'),
+        })
+        
+        page.add_style_tag(content='header, footer.site-footer { visibility: hidden !important; }')
+        
+        p.ss('h4').re.s(r'概要').next('div:has(dl)').screenshot(path=here(f'media/{i}-summary.png'))
+
+        elem_iframe = p.s('iframe[src^="https://example.com"]')
+        elem_iframe.scroll_into_view()
+        time.sleep(3)
+        elem_iframe.screenshot(path=here(f'media/{i}-iframe.png'))
+        
+        main_img_url = p.s('img.w-full.object-contain').src
+        
+        img_desc_grep = p.ss('p.text-left').re.s(r'画像をクリック').next('ul').ss('li p').re
+        img_desc = img_desc_grep.s(r'表紙') or img_desc_grep.s(r'^(?!.*裏面).*')
+        img_url = img_desc.parent('li').s('a').url
+        
+        if main_img_url and (res := p.goto(main_img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-main-img.jpg'), res.body())
+        if img_url and (res := p.goto(img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-img-desc.jpg'), res.body())
+```
+
+### スクレイピング(HTML丸ごと保存)
+
+```python
+from domx import wrap_page
+from domx.browser import camoufox_page
+from domx.utils import append_csv, from_here, hash_name, save_log, write_text
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with camoufox_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    item_urls = p.ss('ul.items > li > a').urls
+
+    for i, url in enumerate(item_urls, 1):
+        print(f'item_urls {i}/{len(item_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        file_name = f'{hash_name(url)}.html'
+        if not write_text(here('html') / file_name, p.html(with_url=True, with_saved_at=True)):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'write_text'})
+            continue
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力
+
+```python
+from domx import wrap_parser
+from domx.utils import from_here, parse_html, save_log, write_parquet
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+results = []
+for i, file_path in enumerate(here('html').glob('*.html'),1):
+    print(f'html {i}')
+    if not (parser := parse_html(file_path)):
+        continue
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    results.append({
+        'URL': p.url,
+        'file_name': file_path.name,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '物件番号': dts.s(r'物件番号').next('dd').text,
+    })
+write_parquet(here('parquet/extract.parquet'), results)
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力(並列処理)
+
+```python
+from pathlib import Path
+
+from domx import wrap_parser
+from domx.utils import from_here, glob_paths, parse_html, pool_map, write_parquet
+
+def main():
+    here = from_here(__file__)
+    html_paths = glob_paths(here('html'), '*.html')
+    results = [r for r in pool_map(extract, html_paths) if r]
+    write_parquet(here('parquet/extract.parquet'), results)
+
+def extract(file_path: str) -> dict | None:
+    if not (parser := parse_html(Path(file_path))):
+        return None
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    return {
+        'URL': p.url,
+        'file_path': file_path,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '価格': dts.s(r'価格').next('dd').text,
+        '設備・条件': dts.s(r'設備').next('dd').text,
+        '備考': dts.s(r'備考').next('dd').text,
+    }
+
+if __name__ == '__main__':
+    main()
+```
+
+## License - ライセンス
+
+[MIT](./LICENSE)
+

domx-0.1.1/README.md

@@ -0,0 +1,223 @@
+# domx
+
+自分用・非汎用
+
+## インストール
+
+`uv add domx`  
+`uv run patchright install chromium`  
+`uv run camoufox fetch`
+
+## 使用例
+
+### スクレイピング
+
+```python
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import append_csv, from_here, save_log, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    pref_urls = p.ss('li.item > ul > li > a').urls
+
+    classroom_urls = []
+    for i, url in enumerate(pref_urls, 1):
+        print(f'pref_urls {i}/{len(pref_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        classroom_urls.extend(p.ss('.school-area h4 a').urls)
+
+    for i, url in enumerate(classroom_urls, 1):
+        print(f'classroom_urls {i}/{len(classroom_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        th_grep = p.ss('th').re
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '教室名': p.s('h1 .text01').text,
+            '住所': p.s('.item .mapText').text,
+            '電話番号': p.s('.item .phoneNumber').text,
+            'HP': th_grep.s(r'ホームページ').next('td').s('a').url,
+            '営業時間': th_grep.s(r'営業時間').next('td').text,
+            '定休日': th_grep.s(r'定休日').next('td').text,
+        })
+        p.s('.school-map').screenshot(here(f'media/{i}-screenshot.png'))
+        if (img_url := p.s('.school-area img').src):
+            if (res := p.goto(img_url)) and res.ok:
+                write_bytes(here(f'media/{i}-img.jpg'), res.body())
+```
+
+### スクレイピング(スクショと画像も保存)
+
+```python
+import time
+from urllib.parse import urlencode
+
+from domx import wrap_page
+from domx.browser import patchright_page
+from domx.utils import save_log, append_csv, from_here, write_bytes
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with patchright_page() as page:
+    p = wrap_page(page)
+    
+    p.goto('https://example.com/demo/search')
+    prefecture_urls = p.ss('li > a[href^="https://example.com/demo/search/area/"]').urls
+
+    bukken_urls = []
+    for i, prefecture_url in enumerate(prefecture_urls, 1):
+        print(f'{i}/{len(prefecture_urls)} エリア一覧ページ')
+        page_num = 1
+        while True:
+            if not p.goto(f'{prefecture_url}?{urlencode({"page": page_num})}'):
+                break
+            if not (bukken_elems := p.ss('ul li div a[href^="https://example.com"]:has(p)')):
+                break
+            bukken_urls.extend(bukken_elems.urls)
+            page_num += 1
+    
+    for i, url in enumerate(bukken_urls, 1):
+        print(f'{i}/{len(bukken_urls)} 詳細ページ {url}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        
+        dt_grep = p.ss('h4').re.s(r'概要').next('div:has(dl)').ss('dt').re
+        dd_text = lambda pattern: dt_grep.s(pattern).next('dd').text
+
+        append_csv(here('csv/scrape.csv'), {
+            'id': i,
+            'URL': page.url,
+            '価格': dd_text(r'価格'),
+            '所在地': dd_text(r'所在地'),
+            '交通': dd_text(r'交通'),
+            '駐車場': dd_text(r'駐車場'),
+            '備考': dd_text(r'備考'),
+            '情報更新日': dd_text(r'情報更新日'),
+        })
+        
+        page.add_style_tag(content='header, footer.site-footer { visibility: hidden !important; }')
+        
+        p.ss('h4').re.s(r'概要').next('div:has(dl)').screenshot(path=here(f'media/{i}-summary.png'))
+
+        elem_iframe = p.s('iframe[src^="https://example.com"]')
+        elem_iframe.scroll_into_view()
+        time.sleep(3)
+        elem_iframe.screenshot(path=here(f'media/{i}-iframe.png'))
+        
+        main_img_url = p.s('img.w-full.object-contain').src
+        
+        img_desc_grep = p.ss('p.text-left').re.s(r'画像をクリック').next('ul').ss('li p').re
+        img_desc = img_desc_grep.s(r'表紙') or img_desc_grep.s(r'^(?!.*裏面).*')
+        img_url = img_desc.parent('li').s('a').url
+        
+        if main_img_url and (res := p.goto(main_img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-main-img.jpg'), res.body())
+        if img_url and (res := p.goto(img_url)) and res.ok:
+            write_bytes(here(f'media/{i}-img-desc.jpg'), res.body())
+```
+
+### スクレイピング(HTML丸ごと保存)
+
+```python
+from domx import wrap_page
+from domx.browser import camoufox_page
+from domx.utils import append_csv, from_here, hash_name, save_log, write_text
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+with camoufox_page() as page:
+    p = wrap_page(page)
+
+    p.goto('https://www.foobarbaz1.jp')
+    item_urls = p.ss('ul.items > li > a').urls
+
+    for i, url in enumerate(item_urls, 1):
+        print(f'item_urls {i}/{len(item_urls)}')
+        if not p.goto(url):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'goto'})
+            continue
+        file_name = f'{hash_name(url)}.html'
+        if not write_text(here('html') / file_name, p.html(with_url=True, with_saved_at=True)):
+            append_csv(here('csv/failed.csv'), {'url': url, 'reason': 'write_text'})
+            continue
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力
+
+```python
+from domx import wrap_parser
+from domx.utils import from_here, parse_html, save_log, write_parquet
+
+here = from_here(__file__)
+save_log(here('log/scraping.log'))
+
+results = []
+for i, file_path in enumerate(here('html').glob('*.html'),1):
+    print(f'html {i}')
+    if not (parser := parse_html(file_path)):
+        continue
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    results.append({
+        'URL': p.url,
+        'file_name': file_path.name,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '物件番号': dts.s(r'物件番号').next('dd').text,
+    })
+write_parquet(here('parquet/extract.parquet'), results)
+```
+
+### ローカルHTMLからデータ抽出&Parquet出力(並列処理)
+
+```python
+from pathlib import Path
+
+from domx import wrap_parser
+from domx.utils import from_here, glob_paths, parse_html, pool_map, write_parquet
+
+def main():
+    here = from_here(__file__)
+    html_paths = glob_paths(here('html'), '*.html')
+    results = [r for r in pool_map(extract, html_paths) if r]
+    write_parquet(here('parquet/extract.parquet'), results)
+
+def extract(file_path: str) -> dict | None:
+    if not (parser := parse_html(Path(file_path))):
+        return None
+    p = wrap_parser(parser)
+    dts = p.ss('dt').re
+    return {
+        'URL': p.url,
+        'file_path': file_path,
+        '教室名': p.s('h1 .text02').text,
+        '住所': p.s('.item .mapText').text,
+        '所在地': dts.s(r'所在地').next('dd').text,
+        '交通': dts.s(r'交通').next('dd').text,
+        '価格': dts.s(r'価格').next('dd').text,
+        '設備・条件': dts.s(r'設備').next('dd').text,
+        '備考': dts.s(r'備考').next('dd').text,
+    }
+
+if __name__ == '__main__':
+    main()
+```
+
+## License - ライセンス
+
+[MIT](./LICENSE)