scrapyrus 0.0.1__tar.gz

scrapyrus-0.0.1/LICENSE.md

@@ -0,0 +1,8 @@
+Copyright 2026, The copyright holders according to COPYING.md
+
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

scrapyrus-0.0.1/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: scrapyrus
+Version: 0.0.1
+Summary: Add short description here
+Maintainer-email: Dominic Kempf <your@email.com>
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: beautifulsoup4
+Requires-Dist: cltk[stanza]>=2.5.1
+Requires-Dist: click
+Requires-Dist: saxonche
+Requires-Dist: pydantic>=2
+Requires-Dist: psycopg[binary]>=3
+Requires-Dist: requests
+Requires-Dist: termgraph>=0.7.6
+Requires-Dist: tqdm
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-cov; extra == "tests"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx_mdinclude; extra == "docs"
+Requires-Dist: sphinx_rtd_theme; extra == "docs"
+Dynamic: license-file
+
+# Welcome to scrapyrus
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/ssciwr/scrapyrus/ci.yml?branch=main)](https://github.com/ssciwr/scrapyrus/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/ssciwr/scrapyrus/branch/main/graph/badge.svg)](https://codecov.io/gh/ssciwr/scrapyrus)
+
+## Installation
+
+The Python package `scrapyrus` can be installed from PyPI:
+
+```
+python -m pip install scrapyrus
+```
+
+## Development installation
+
+If you want to contribute to the development of `scrapyrus`, we recommend
+the following editable installation from this repository:
+
+```
+git clone git@github.com:ssciwr/scrapyrus.git
+cd scrapyrus
+python -m pip install --editable .[tests]
+```
+
+Having done so, the test suite can be run using `pytest`:
+
+```
+python -m pytest
+```
+
+## Docker Compose
+
+The repository includes a Docker Compose setup with PostgreSQL and a profiled
+`scrapyrus` execution container. The PostgreSQL service uses a pgvector-enabled
+image because `scrapyrus embeddings` commands require the `vector` extension:
+
+```
+docker compose up -d postgres
+```
+
+The `scrapyrus` container mounts this repository at `/workspace` and sets
+`SCRAPYRUS_DATABASE_URL` to the PostgreSQL service. It also mounts the local
+`./exchange` directory at `/exchange` for moving data between the host and the
+container. Start it as a one-off shell when you need to run the CLI:
+
+```
+docker compose run --build --rm scrapyrus bash
+scrapyrus metadata ingest
+scrapyrus transcriptions ingest
+scrapyrus embeddings ingest \
+    --inference-server-url <url> --model-name <model> --api-key <key>
+```
+
+Embedding ingestion reads the XML rows created by `transcriptions ingest`, so
+both commands must target the same PostgreSQL database and must run in that
+order.
+
+The `scrapyrus` service is intentionally not started by the default `up`
+command. Some Snap-packaged Docker installations fail to stop idle long-lived
+containers during rebuilds with `cannot stop container: ... permission denied`;
+using one-off `run --rm` containers avoids that host-side recreate path.
+
+If `idp.data` is somewhere else inside the mounted workspace, pass it explicitly:
+
+```
+scrapyrus --idp-data /workspace/idp.data metadata ingest
+```
+
+To export the metadata database tables as CSV files:
+
+```
+scrapyrus metadata dump
+```
+
+If an older checkout already started the Compose stack with the stock PostgreSQL
+image, recreate the database service after updating:
+
+```
+docker compose up -d --force-recreate postgres
+```
+
+## Acknowledgments
+
+This repository was set up using the [SSC Cookiecutter for Python Packages](https://github.com/ssciwr/cookiecutter-python-package).

scrapyrus-0.0.1/README.md

@@ -0,0 +1,85 @@
+# Welcome to scrapyrus
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/ssciwr/scrapyrus/ci.yml?branch=main)](https://github.com/ssciwr/scrapyrus/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/ssciwr/scrapyrus/branch/main/graph/badge.svg)](https://codecov.io/gh/ssciwr/scrapyrus)
+
+## Installation
+
+The Python package `scrapyrus` can be installed from PyPI:
+
+```
+python -m pip install scrapyrus
+```
+
+## Development installation
+
+If you want to contribute to the development of `scrapyrus`, we recommend
+the following editable installation from this repository:
+
+```
+git clone git@github.com:ssciwr/scrapyrus.git
+cd scrapyrus
+python -m pip install --editable .[tests]
+```
+
+Having done so, the test suite can be run using `pytest`:
+
+```
+python -m pytest
+```
+
+## Docker Compose
+
+The repository includes a Docker Compose setup with PostgreSQL and a profiled
+`scrapyrus` execution container. The PostgreSQL service uses a pgvector-enabled
+image because `scrapyrus embeddings` commands require the `vector` extension:
+
+```
+docker compose up -d postgres
+```
+
+The `scrapyrus` container mounts this repository at `/workspace` and sets
+`SCRAPYRUS_DATABASE_URL` to the PostgreSQL service. It also mounts the local
+`./exchange` directory at `/exchange` for moving data between the host and the
+container. Start it as a one-off shell when you need to run the CLI:
+
+```
+docker compose run --build --rm scrapyrus bash
+scrapyrus metadata ingest
+scrapyrus transcriptions ingest
+scrapyrus embeddings ingest \
+    --inference-server-url <url> --model-name <model> --api-key <key>
+```
+
+Embedding ingestion reads the XML rows created by `transcriptions ingest`, so
+both commands must target the same PostgreSQL database and must run in that
+order.
+
+The `scrapyrus` service is intentionally not started by the default `up`
+command. Some Snap-packaged Docker installations fail to stop idle long-lived
+containers during rebuilds with `cannot stop container: ... permission denied`;
+using one-off `run --rm` containers avoids that host-side recreate path.
+
+If `idp.data` is somewhere else inside the mounted workspace, pass it explicitly:
+
+```
+scrapyrus --idp-data /workspace/idp.data metadata ingest
+```
+
+To export the metadata database tables as CSV files:
+
+```
+scrapyrus metadata dump
+```
+
+If an older checkout already started the Compose stack with the stock PostgreSQL
+image, recreate the database service after updating:
+
+```
+docker compose up -d --force-recreate postgres
+```
+
+## Acknowledgments
+
+This repository was set up using the [SSC Cookiecutter for Python Packages](https://github.com/ssciwr/cookiecutter-python-package).

scrapyrus-0.0.1/pyproject.toml

@@ -0,0 +1,68 @@
+# This section describes the requirements of the build/installation
+# process itself. Being able to do this was the original reason to
+# introduce pyproject.toml
+[build-system]
+requires = [
+    "setuptools >=61",
+]
+build-backend = "setuptools.build_meta"
+
+# This section provides general project metadata that is used across
+# a variety of build tools. Notably, the version specified here is the
+# single source of truth for scrapyrus's version
+[project]
+name = "scrapyrus"
+description = "Add short description here"
+readme = "README.md"
+maintainers = [
+    { name = "Dominic Kempf", email = "your@email.com" },
+]
+version = "0.0.1"
+requires-python = ">=3.13"
+license-files = ["LICEN[CS]E*"]
+license = "MIT"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "beautifulsoup4",
+    "cltk[stanza] >=2.5.1",
+    "click",
+    "saxonche",
+    "pydantic >=2",
+    "psycopg[binary] >=3",
+    "requests",
+    "termgraph >=0.7.6",
+    "tqdm",
+]
+
+[project.optional-dependencies]
+tests = [
+    "pytest",
+    "pytest-cov",
+]
+docs = [
+    "sphinx",
+    "sphinx_mdinclude",
+    "sphinx_rtd_theme",
+]
+
+# Command line scripts installed as part of the installation
+[project.scripts]
+scrapyrus = "scrapyrus.__main__:main"
+
+# The following section contains setuptools-specific configuration
+# options. For a full reference of available options, check the overview
+# at https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+scrapyrus = ["images_broken.txt", "transcriptions/xslt/*.xsl"]
+
+# The following is the configuration for the pytest test suite
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]

scrapyrus-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scrapyrus-0.0.1/src/scrapyrus/__init__.py

@@ -0,0 +1,21 @@
+from importlib import metadata as importlib_metadata
+
+from scrapyrus.metadata.base import catalog, table_summary
+from scrapyrus.transcriptions.core import (
+    epidoc_xml_to_text,
+    translation_epidoc_xml_to_text,
+)
+from scrapyrus.transcriptions.search import BM25SearchHit, MetadataFilter, bm25_search
+
+__version__ = importlib_metadata.version(__package__)
+
+__all__ = [
+    "__version__",
+    "BM25SearchHit",
+    "MetadataFilter",
+    "bm25_search",
+    "catalog",
+    "epidoc_xml_to_text",
+    "table_summary",
+    "translation_epidoc_xml_to_text",
+]