thunderdots 0.1.0__tar.gz

thunderdots-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,144 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+    branches:
+      - "**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python-lint:
+    name: Python lint and format
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Setup Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Ruff format check
+        run: uv run ruff format --check
+
+      - name: Ruff lint
+        run: uv run ruff check
+
+  python-tests:
+    name: Python unit tests
+    runs-on: ubuntu-latest
+    needs: python-lint
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Setup Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Run unit tests
+        run: uv run pytest
+
+  package-build:
+    name: Build Python package
+    runs-on: ubuntu-latest
+    needs: python-tests
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Setup Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Build package
+        run: uv build
+
+      - name: Check package metadata
+        run: uvx twine check dist/*
+
+  docs-build:
+    name: Build documentation
+    runs-on: ubuntu-latest
+    needs: python-tests
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Setup Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Build MkDocs documentation
+        run: uv run mkdocs build --strict -f mkdocs/mkdocs.yml
+
+      - name: Upload GitHub Pages artifact
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: mkdocs/site
+
+  docs-deploy:
+    name: Deploy documentation to GitHub Pages
+    runs-on: ubuntu-latest
+    needs: docs-build
+
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

thunderdots-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,96 @@
+name: Release to PyPI
+
+on:
+  release:
+    types:
+      - published
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  pypi-release:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/project/thunderdots/
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Setup Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Check release tag format
+        run: |
+          uv run python - <<'PY'
+          import os
+          import re
+
+          tag = os.environ["GITHUB_REF_NAME"]
+
+          if not re.fullmatch(r"v\d+\.\d+\.\d+([a-zA-Z0-9.\-]+)?", tag):
+              raise SystemExit(
+                  f"Invalid release tag '{tag}'. Expected something like v0.1.0"
+              )
+
+          print(f"Release tag format OK: {tag}")
+          PY
+
+      - name: Build package
+        run: uv build
+
+      - name: Check built package version
+        run: |
+          uv run python - <<'PY'
+          import os
+          import zipfile
+          from pathlib import Path
+          from email.parser import Parser
+
+          tag = os.environ["GITHUB_REF_NAME"]
+          expected = tag[1:] if tag.startswith("v") else tag
+
+          wheels = list(Path("dist").glob("*.whl"))
+          if not wheels:
+              raise SystemExit("No wheel found in dist/")
+
+          wheel = wheels[0]
+          with zipfile.ZipFile(wheel) as zf:
+              metadata_name = next(
+                  name for name in zf.namelist()
+                  if name.endswith(".dist-info/METADATA")
+              )
+              metadata = Parser().parsestr(
+                  zf.read(metadata_name).decode("utf-8")
+              )
+
+          actual = metadata["Version"]
+
+          if actual != expected:
+              raise SystemExit(
+                  f"Built package version '{actual}' does not match release tag '{tag}'"
+              )
+
+          print(f"Built package version OK: {actual}")
+          PY
+
+      - name: Check package metadata
+        run: uvx twine check dist/*
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

thunderdots-0.1.0/.gitignore

@@ -0,0 +1,34 @@
+# Python
+
+.venv/
+venv/
+env/
+ENV/
+env.bak/
+venv.bak/
+__pycache__/
+
+.cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# VS Code
+.vscode/
+
+# PyCharm
+.idea/
+
+# MacOS
+.DS_Store
+
+# scripts
+project_dump.txt
+dump_project.py
+specs
+out_results/
+thunderdots/native/build/*mkdocs/site/
+site/
+
+*.egg-info/
+__version__.py

thunderdots-0.1.0/CITATION.cff

@@ -0,0 +1,47 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it using the metadata from this file."
+title: "ThunderDots"
+abstract: >
+  A Python client for Distributed Text Services (DTS) endpoints, initially built
+  for DoTS. ThunderDots walks DTS collections and subcollections, fetches
+  resources and TEI/XML documents, extracts text fragments from full documents,
+  DTS navigation, or custom TEI XPath rules, filters Dublin Core and extension
+  metadata, validates generated outputs, and exports records for indexing,
+  full-text search, RAG, vector databases, or corpus-analysis pipelines.
+type: software
+authors:
+  - family-names: Terriel
+    given-names: Lucas
+    orcid: "https://orcid.org/0000-0002-9189-258X"
+    affiliation: "École nationale des chartes – PSL"
+repository-code: "https://github.com/chartes/thunderdots"
+url: "https://github.com/chartes/thunderdots"
+license: MIT
+keywords:
+  - DTS
+  - Distributed Text Services
+  - TEI
+  - XML
+  - text extraction
+  - digital humanities
+  - cultural heritage
+  - corpus analysis
+  - full-text search
+  - RAG
+  - Elasticsearch
+  - Qdrant
+  - Python
+version: "0.1.0"
+date-released: "2026-01-01"
+preferred-citation:
+  type: software
+  title: "ThunderDots"
+  authors:
+    - family-names: Terriel
+      given-names: Lucas
+      orcid: "https://orcid.org/0000-0002-9189-258X"
+      affiliation: "École nationale des chartes – PSL"
+  repository-code: "https://github.com/chartes/thunderdots"
+  url: "https://github.com/chartes/thunderdots"
+  license: MIT
+  date-released: "2026-01-01"

thunderdots-0.1.0/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ThunderDots contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thunderdots-0.1.0/Makefile

@@ -0,0 +1,149 @@
+.PHONY: help \
+	dev.sync dev.install \
+	docs.prepare docs.serve docs.build docs.build-dev docs.clean docs.check docs.deploy \
+	tests.unit tests.network tests.all \
+	lint.check lint.format \
+	check clean
+
+# ---------------------------------------------------------------------
+# Tooling
+# ---------------------------------------------------------------------
+
+UV ?= uv
+PYTHON ?= $(UV) run python
+PYTEST ?= $(PYTHON) -m pytest
+RUFF ?= $(PYTHON) -m ruff
+MKDOCS ?= $(PYTHON) -m mkdocs
+
+# ---------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------
+
+DOCS_DIR ?= mkdocs
+DOCS_CONFIG ?= $(DOCS_DIR)/mkdocs.yml
+DOCS_SRC ?= $(DOCS_DIR)/docs
+DOCS_SITE ?= $(DOCS_DIR)/site
+
+ROOT_ASSETS_DIR ?= assets
+DOCS_ASSETS_DIR ?= $(DOCS_SRC)/assets
+
+NOTEBOOKS_DIR ?= notebooks
+DOCS_NOTEBOOKS_DIR ?= $(DOCS_SRC)/notebooks
+USER_NOTEBOOK ?= thunderdots_documentation_utilisateur.ipynb
+
+# ---------------------------------------------------------------------
+# Help
+# ---------------------------------------------------------------------
+
+help:
+	@echo "ThunderDots development commands"
+	@echo ""
+	@echo "Environment:"
+	@echo "  make dev.sync          Sync project dependencies with uv"
+	@echo "  make dev.install       Install project with dev and docs extras"
+	@echo ""
+	@echo "Documentation:"
+	@echo "  make docs.prepare      Copy assets and notebooks into MkDocs docs_dir"
+	@echo "  make docs.serve        Serve MkDocs locally"
+	@echo "  make docs.build        Build documentation in strict mode"
+	@echo "  make docs.build-dev    Build documentation without strict mode"
+	@echo "  make docs.check        Validate documentation build"
+	@echo "  make docs.deploy       Deploy documentation to gh-pages"
+	@echo "  make docs.clean        Remove generated documentation site"
+	@echo ""
+	@echo "Tests:"
+	@echo "  make tests.unit        Run unit tests without network tests"
+	@echo "  make tests.network     Run tests with online DTS tests"
+	@echo "  make tests.all         Run all tests"
+	@echo ""
+	@echo "Linting:"
+	@echo "  make lint.check        Run Ruff checks"
+	@echo "  make lint.format       Format code with Ruff"
+	@echo ""
+	@echo "Global:"
+	@echo "  make check             Run lint + tests + docs check"
+	@echo "  make clean             Remove generated caches and build artefacts"
+
+# ---------------------------------------------------------------------
+# Environment
+# ---------------------------------------------------------------------
+
+dev.sync:
+	$(UV) sync --extra dev --extra docs
+
+dev.install:
+	$(UV) pip install -e ".[dev,docs]"
+
+# ---------------------------------------------------------------------
+# Documentation
+# ---------------------------------------------------------------------
+
+docs.prepare:
+	mkdir -p $(DOCS_ASSETS_DIR)
+	mkdir -p $(DOCS_NOTEBOOKS_DIR)
+	@if [ -d "$(ROOT_ASSETS_DIR)" ]; then \
+		cp -R $(ROOT_ASSETS_DIR)/* $(DOCS_ASSETS_DIR)/ 2>/dev/null || true; \
+	fi
+	@if [ -f "$(NOTEBOOKS_DIR)/$(USER_NOTEBOOK)" ]; then \
+		cp "$(NOTEBOOKS_DIR)/$(USER_NOTEBOOK)" "$(DOCS_NOTEBOOKS_DIR)/$(USER_NOTEBOOK)"; \
+	fi
+
+docs.serve: docs.prepare
+	$(MKDOCS) serve -f $(DOCS_CONFIG)
+
+docs.build: docs.prepare
+	$(MKDOCS) build -f $(DOCS_CONFIG) --strict
+
+docs.build-dev: docs.prepare
+	$(MKDOCS) build -f $(DOCS_CONFIG)
+
+docs.clean:
+	rm -rf $(DOCS_SITE)
+
+docs.check: docs.build
+
+docs.deploy: docs.prepare
+	$(MKDOCS) gh-deploy -f $(DOCS_CONFIG) --force
+
+# ---------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------
+
+tests.unit:
+	$(PYTEST)
+
+tests.network:
+	RUN_NETWORK_TESTS=1 $(PYTEST)
+
+tests.all: tests.network
+
+# ---------------------------------------------------------------------
+# Linting / formatting
+# ---------------------------------------------------------------------
+
+lint.check:
+	$(RUFF) check thunderdots tests
+
+lint.format:
+	$(RUFF) format thunderdots tests
+	$(RUFF) check thunderdots tests --fix
+
+# ---------------------------------------------------------------------
+# Global checks
+# ---------------------------------------------------------------------
+
+check: lint.check tests.unit docs.check
+
+# ---------------------------------------------------------------------
+# Cleaning
+# ---------------------------------------------------------------------
+
+clean: docs.clean
+	rm -rf .pytest_cache
+	rm -rf .ruff_cache
+	rm -rf htmlcov
+	rm -rf dist
+	rm -rf build
+	rm -rf *.egg-info
+	find . -type d -name "__pycache__" -prune -exec rm -rf {} +
+	find . -type f -name "*.pyc" -delete

thunderdots-0.1.0/PKG-INFO

@@ -0,0 +1,194 @@
+Metadata-Version: 2.4
+Name: thunderdots
+Version: 0.1.0
+Summary: ThunderDots is a Python client for DTS (Distributed Text Services) endpoints, initially built for DoTS.
+Author: Lucas Terriel, École nationale des chartes - PSL
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/chartes/thunderdots
+Project-URL: Repository, https://github.com/chartes/thunderdots
+Project-URL: Documentation, https://chartes.github.io/thunderdots/
+Project-URL: Issues, https://github.com/chartes/thunderdots/issues
+Keywords: dts,tei,xml,digital-humanities,corpus,dots
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: edtf>=5.0.1
+Requires-Dist: h2>=4.3.0
+Requires-Dist: httpx[http2]>=0.28.1
+Requires-Dist: jsonschema>=4.26.0
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: notebook>=7.5.6
+Requires-Dist: rich>=14.3.2
+Requires-Dist: tqdm>=4.67.3
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0.2; extra == "dev"
+Requires-Dist: ruff>=0.15.1; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.6.1; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: mkdocs-jupyter>=0.25; extra == "docs"
+Requires-Dist: jupyter; extra == "docs"
+Requires-Dist: ipykernel; extra == "docs"
+Provides-Extra: all
+Requires-Dist: thunderdots[dev,docs]; extra == "all"
+Dynamic: license-file
+
+<h1 align="center">
+  <img src="assets/dots-light.png" width="450"><br>
+  ThunderDots — DTS client for documentary corpora
+</h1>
+
+<p align="center">
+  <strong>Fast DTS crawling, TEI fragmentation, metadata filtering, validation, and export pipelines.</strong>
+</p>
+
+<p align="center">
+  <a href="https://github.com/astral-sh/uv">
+    <img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json" alt="uv">
+  </a>
+  <a href="https://github.com/astral-sh/ruff">
+    <img src="https://img.shields.io/badge/lint-ruff-0A0A0A?logo=ruff&logoColor=white" alt="ruff">
+  </a>
+  <a href="https://github.com/chartes/thunderdots/actions/workflows/ci.yml">
+    <img src="https://github.com/chartes/thunderdots/actions/workflows/ci.yml/badge.svg" alt="CI">
+  </a>
+  <a href="https://github.com/chartes/thunderdots/blob/master/LICENSE.md">
+    <img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT">
+  </a>
+<a href="https://pypi.org/project/thunderdots/">
+  <img src="https://img.shields.io/pypi/v/thunderdots.svg" alt="PyPI version">
+</a>
+</p>
+
+---
+
+## Overview
+
+**ThunderDots** is a Python client for [DTS](https://dtsapi.org/specifications/) (*Distributed Text Services*) endpoints, initially built for [DoTS](https://chartes.github.io/dots_documentation/).
+
+It helps you move from a remote DTS API to structured Python objects and JSON records that can feed indexing pipelines, including full-text search, RAG/vector databases, and corpus-analysis workflows.
+
+ThunderDots focuses on practical documentary workflows: crawling DTS collections, fetching TEI/XML resources, extracting reusable text fragments, selecting metadata, validating outputs, and exporting data to downstream search or indexing systems.
+
+---
+
+## What ThunderDots does
+
+ThunderDots can:
+
+- walk DTS collections and subcollections;
+- fetch resources and TEI/XML documents;
+- extract text fragments from full documents, DTS navigation, or custom TEI XPath rules;
+- preserve or filter Dublin Core and extension metadata;
+- enrich temporal metadata such as dates and coverage ranges;
+- validate generated outputs with JSON Schema;
+- export records to indexing pipelines such as Elasticsearch or Qdrant-compatible formats;
+- cache fetched corpora as JSON and CSV;
+- run synchronous or asynchronous workflows.
+
+---
+
+## Installation
+
+### With `uv`
+
+```bash
+uv add thunderdots
+```
+
+### With pip
+
+```bash
+pip install thunderdots
+```
+
+### For development
+
+```bash
+git clone https://github.com/chartes/thunderdots.git
+cd thunderdots
+
+uv venv
+source .venv/bin/activate
+uv sync --all-extras --dev
+```
+
+or with pip
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+```
+
+## Minimal example
+
+```python
+from thunderdots import ThunderDots
+
+td = ThunderDots(
+    endpoint_dts="https://dots.chartes.psl.eu/api/dts",
+    collection_params={"collection_id": "ENCPOS_1900"},
+    resource_params={"fragment_mode": "document"},
+)
+
+td.fetch()
+results = td.results()
+
+print(td.stats())
+```
+
+## Development
+
+### Run tests
+
+```bash
+pytest
+```
+
+Online DTS tests are opt-in:
+
+```bash
+RUN_NETWORK_TESTS=1 pytest
+```
+
+### Run Ruff (linter, format)
+
+```bash
+ruff format --check
+ruff check
+```
+
+### Build the documentation
+
+```bash
+mkdocs build --strict -f mkdocs/mkdocs.yml
+```
+
+### Create a new PyPI release
+
+Check the [release checklist](./RELEASE.md) for details.
+
+### License
+
+ThunderDots is distributed under the [MIT License](./LICENSE.md).
+
+### Citation
+
+If you use ThunderDots in academic work, please cite it as:
+
+```
+@software{terriel_thunderdots_2026,
+  author       = {Terriel, Lucas},
+  title        = {ThunderDots},
+  year         = {2026},
+  publisher    = {GitHub},
+  institution  = {{École nationale des chartes}},
+  url          = {https://github.com/chartes/thunderdots},
+  note         = {Python client for Distributed Text Services (DTS) via DoTS}
+}
+```
+
+You can also use the repository metadata from [CITATION.cff](./CITATION.cff).