docdistance 1.0.15__tar.gz

docdistance-1.0.15/LICENSE

@@ -0,0 +1,10 @@
+
+The MIT License (MIT)
+Copyright (c) 2026, Stellars Henson <konrad.jelen+github@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

docdistance-1.0.15/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: docdistance
+Version: 1.0.15
+Summary: Project that uses theory of From Word Embeddings To Document Distances / Optimal Transport to give meaningful distance from one document to another, useful if building agentic projects that convert or extract information from one document to another using frontier models but without the ability to calculate KL divergence from logits
+Author: Stellars Henson <konrad.jelen+github@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/stellarshenson/docdistance
+Project-URL: Repository, https://github.com/stellarshenson/docdistance
+Project-URL: Issues, https://github.com/stellarshenson/docdistance/issues
+Keywords: optimal-transport,word-movers-distance,statement-movers-distance,document-similarity,document-distance,embeddings,mmbert,nlp
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Requires-Python: ~=3.13.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: loguru
+Requires-Dist: tqdm
+Requires-Dist: typer
+Requires-Dist: rich
+Requires-Dist: python-dotenv
+Requires-Dist: numpy
+Requires-Dist: pot
+Requires-Dist: transformers
+Requires-Dist: wtpsplit
+Requires-Dist: openvino
+Requires-Dist: torch
+Provides-Extra: dev
+Requires-Dist: build; extra == "dev"
+Requires-Dist: ipykernel; extra == "dev"
+Requires-Dist: ipython; extra == "dev"
+Requires-Dist: nbdime; extra == "dev"
+Requires-Dist: pip; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: matplotlib; extra == "dev"
+Requires-Dist: seaborn; extra == "dev"
+Dynamic: license-file
+
+# docdistance
+
+Semantic distance between two documents via Statement Mover's Distance - optimal transport over mmBERT statement embeddings, after Kusner et al. 2015 (*From Word Embeddings To Document Distances*). A thin frontend to the library; the SOTA docs carry the mechanics, benchmarks, and validation.
+
+- **Input** - two documents, raw text or a file path
+- **Output** - an SMD distance, a 0..1 closeness, a verdict, and the statement alignment
+- **Use** - agentic document conversion and extraction pipelines, where token logits are unavailable and KL divergence cannot be computed
+- **Unit** - statement-level and position-invariant, with an interpretable transport plan
+
+## Theory
+
+A document distance grounded in embeddings and optimal transport, not surface overlap.
+
+- **WMD** - Word Mover's Distance (Kusner et al. 2015) casts document similarity as optimal transport between embedded tokens
+- **SMD** - this project lifts it to statements: segment, embed, transport between the two statement clouds
+- **Beyond cosine** - whole-document cosine collapses when the same claims sit in a different place or order; statement-level transport is position-invariant
+- **Metric** - the ground cost `√(2 − 2cos)` on L2-normalized embeddings is a metric, so the document distance is one too
+- **Logit-free** - an embedding-grounded alternative where token probabilities (KL divergence) are unavailable, as in frontier-model pipelines
+
+## Method
+
+Three stages; the transport plan is the interpretable by-product.
+
+1. **Segment** - split each document into atomic statements with the SAT (Segment Any Text) segmenter
+2. **Embed** - encode each statement with the mmBERT contextual encoder (mean-pooled, L2-normalized)
+3. **Compare** - optimal transport between the two statement clouds (Statement Mover's Distance), optionally unbalanced so added or missing statements are scored, not force-matched
+
+- **Closeness** - `1 − SMD/√2`, on a 0..1 scale
+- **Source-conditioned** - a variant `d(A, B | S)` re-bases the transport onto a shared source `S` and reads off a selection axis and a grounding axis
+
+## Usage
+
+The library is the product; install once, then call it.
+
+```python
+from docdistance import document_distance
+
+result = document_distance("report_v1.md", "report_v2.md")
+print(result.closeness)  # 0..1 similarity, 1 - SMD/sqrt(2)
+print(result.verdict)    # "similar" | "not similar"
+```
+
+```bash
+make install                                   # environment, package, Jupyter kernel
+docdistance install                            # download + cache the models (once)
+docdistance distance a.md b.md                 # rich, coloured verdict
+docdistance distance a.md b.md --json          # machine-readable JSON
+```
+
+- **Offline after install** - distance calls run fully offline once the models are cached
+- **Backend** - `--backend openvino|torch`, default `openvino` (CPU INT8)
+- **Full API and flags** - `docdistance --help` and the SOTA docs
+
+## Documentation
+
+The SOTA documents explain how it works in detail; this README only introduces it.
+
+- `docs/wmd-docdistance-solution-sota.md` - source-free distance: design, mechanism, performance, validation
+- `docs/wmd-wrt-source-docdistance-solution.md` - source-conditioned distance `d(A,B|S)`
+- `docs/mmbert-quantization-solution.md` - the INT8 / FP8 statement encoder
+- `references/papers/from-word-embeddings-to-document-distances.md` - WMD paper digest (Kusner et al. 2015)
+
+> **Note**: Scaffolded with the [copier-data-science](https://github.com/stellarshenson/copier-data-science) template.

docdistance-1.0.15/README.md

@@ -0,0 +1,63 @@
+# docdistance
+
+Semantic distance between two documents via Statement Mover's Distance - optimal transport over mmBERT statement embeddings, after Kusner et al. 2015 (*From Word Embeddings To Document Distances*). A thin frontend to the library; the SOTA docs carry the mechanics, benchmarks, and validation.
+
+- **Input** - two documents, raw text or a file path
+- **Output** - an SMD distance, a 0..1 closeness, a verdict, and the statement alignment
+- **Use** - agentic document conversion and extraction pipelines, where token logits are unavailable and KL divergence cannot be computed
+- **Unit** - statement-level and position-invariant, with an interpretable transport plan
+
+## Theory
+
+A document distance grounded in embeddings and optimal transport, not surface overlap.
+
+- **WMD** - Word Mover's Distance (Kusner et al. 2015) casts document similarity as optimal transport between embedded tokens
+- **SMD** - this project lifts it to statements: segment, embed, transport between the two statement clouds
+- **Beyond cosine** - whole-document cosine collapses when the same claims sit in a different place or order; statement-level transport is position-invariant
+- **Metric** - the ground cost `√(2 − 2cos)` on L2-normalized embeddings is a metric, so the document distance is one too
+- **Logit-free** - an embedding-grounded alternative where token probabilities (KL divergence) are unavailable, as in frontier-model pipelines
+
+## Method
+
+Three stages; the transport plan is the interpretable by-product.
+
+1. **Segment** - split each document into atomic statements with the SAT (Segment Any Text) segmenter
+2. **Embed** - encode each statement with the mmBERT contextual encoder (mean-pooled, L2-normalized)
+3. **Compare** - optimal transport between the two statement clouds (Statement Mover's Distance), optionally unbalanced so added or missing statements are scored, not force-matched
+
+- **Closeness** - `1 − SMD/√2`, on a 0..1 scale
+- **Source-conditioned** - a variant `d(A, B | S)` re-bases the transport onto a shared source `S` and reads off a selection axis and a grounding axis
+
+## Usage
+
+The library is the product; install once, then call it.
+
+```python
+from docdistance import document_distance
+
+result = document_distance("report_v1.md", "report_v2.md")
+print(result.closeness)  # 0..1 similarity, 1 - SMD/sqrt(2)
+print(result.verdict)    # "similar" | "not similar"
+```
+
+```bash
+make install                                   # environment, package, Jupyter kernel
+docdistance install                            # download + cache the models (once)
+docdistance distance a.md b.md                 # rich, coloured verdict
+docdistance distance a.md b.md --json          # machine-readable JSON
+```
+
+- **Offline after install** - distance calls run fully offline once the models are cached
+- **Backend** - `--backend openvino|torch`, default `openvino` (CPU INT8)
+- **Full API and flags** - `docdistance --help` and the SOTA docs
+
+## Documentation
+
+The SOTA documents explain how it works in detail; this README only introduces it.
+
+- `docs/wmd-docdistance-solution-sota.md` - source-free distance: design, mechanism, performance, validation
+- `docs/wmd-wrt-source-docdistance-solution.md` - source-conditioned distance `d(A,B|S)`
+- `docs/mmbert-quantization-solution.md` - the INT8 / FP8 statement encoder
+- `references/papers/from-word-embeddings-to-document-distances.md` - WMD paper digest (Kusner et al. 2015)
+
+> **Note**: Scaffolded with the [copier-data-science](https://github.com/stellarshenson/copier-data-science) template.

docdistance-1.0.15/pyproject.toml

@@ -0,0 +1,90 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "docdistance"
+version = "1.0.15"
+description = "Project that uses theory of From Word Embeddings To Document Distances / Optimal Transport to give meaningful distance from one document to another, useful if building agentic projects that convert or extract information from one document to another using frontier models but without the ability to calculate KL divergence from logits"
+authors = [
+    { name = "Stellars Henson \u003ckonrad.jelen+github@gmail.com\u003e" },
+]
+license = "MIT"
+readme = "README.md"
+requires-python = "~=3.13.0"
+keywords = [
+    "optimal-transport",
+    "word-movers-distance",
+    "statement-movers-distance",
+    "document-similarity",
+    "document-distance",
+    "embeddings",
+    "mmbert",
+    "nlp",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing :: Linguistic",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+]
+
+dependencies = [
+    "loguru",
+    "tqdm",
+    "typer",
+    "rich",
+    "python-dotenv",
+    "numpy",
+    "pot",
+    "transformers",
+    "wtpsplit",
+    "openvino",
+    "torch",
+]
+
+[project.urls]
+Homepage = "https://github.com/stellarshenson/docdistance"
+Repository = "https://github.com/stellarshenson/docdistance"
+Issues = "https://github.com/stellarshenson/docdistance/issues"
+
+[project.scripts]
+docdistance = "docdistance.cli:app"
+
+[project.optional-dependencies]
+dev = [
+    "build",
+    "ipykernel",
+    "ipython",
+    "nbdime",
+    "pip",
+    "pytest",
+    "pytest-cov",
+    "ruff",
+    "twine",
+    "matplotlib",
+    "seaborn",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["docdistance*"]
+exclude = ["tests*"]
+
+[tool.ruff]
+line-length = 99
+src = ["src/docdistance"]
+include = ["pyproject.toml", "src/docdistance/**/*.py"]
+
+[tool.ruff.lint]
+extend-select = ["I"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["docdistance"]
+force-sort-within-sections = true

docdistance-1.0.15/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

docdistance-1.0.15/src/docdistance/__init__.py

@@ -0,0 +1,38 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from docdistance import config  # noqa: F401  (sets up logging + paths on import)
+from docdistance.distance import (
+    DistanceResult,
+    SourceConditionedResult,
+    closeness,
+    compute_distance,
+    compute_source_conditioned,
+    rwmd,
+    smd,
+    wcd,
+)
+from docdistance.pipeline import (
+    DocDistance,
+    document_distance,
+    source_conditioned_distance,
+)
+
+try:
+    __version__ = version("docdistance")
+except PackageNotFoundError:  # running from source, not installed
+    __version__ = "0.0.0"
+
+__all__ = [
+    "DocDistance",
+    "DistanceResult",
+    "SourceConditionedResult",
+    "document_distance",
+    "source_conditioned_distance",
+    "compute_distance",
+    "compute_source_conditioned",
+    "smd",
+    "wcd",
+    "rwmd",
+    "closeness",
+    "__version__",
+]

docdistance-1.0.15/src/docdistance/cli.py

@@ -0,0 +1,229 @@
+"""docdistance command-line interface.
+
+Three subcommands - ``install`` (the only one that downloads models), ``distance`` (symmetric SMD)
+and ``distance-wrt-source`` (source-conditioned). Human output is rich and coloured on a capable
+terminal; ``--json`` emits machine-readable JSON and ``--result-only`` emits the bare result.
+Logs go to stderr (loguru, ``--verbose`` for DEBUG), so stdout carries only the result.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+import json
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+import typer
+
+from docdistance.config import configure_logging
+from docdistance.distance import DEFAULT_THRESHOLD
+
+app = typer.Typer(
+    rich_markup_mode="rich",
+    no_args_is_help=True,
+    add_completion=False,
+    help="[bold]docdistance[/bold] - semantic distance between documents via Statement Mover's Distance "
+    "(optimal transport over mmBERT statement embeddings).",
+)
+
+_out = Console()  # stdout, for the result
+_err = Console(stderr=True)  # stderr, for errors
+
+
+class Backend(str, Enum):
+    openvino = "openvino"
+    torch = "torch"
+
+
+class InstallBackend(str, Enum):
+    openvino = "openvino"
+    torch = "torch"
+    both = "both"
+
+
+def _version_cb(value: bool):
+    if value:
+        from docdistance import __version__
+
+        typer.echo(f"docdistance {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False, "--version", callback=_version_cb, is_eager=True, help="show version and exit"
+    ),
+):
+    """Semantic document distance grounded in optimal-transport theory."""
+
+
+def _run(fn):
+    """Call ``fn`` and turn a missing-model error into a clean message + exit code 1."""
+    from docdistance.encoders import ModelsNotInstalled
+
+    try:
+        return fn()
+    except ModelsNotInstalled as exc:
+        _err.print(f"[bold red]error:[/bold red] {exc}")
+        raise typer.Exit(1)
+
+
+def _emit_distance(r, json_out: bool, result_only: bool) -> None:
+    if result_only:
+        typer.echo(str(r.smd))
+        return
+    if json_out:
+        typer.echo(json.dumps(r.to_dict(), indent=2))
+        return
+    color = "green" if r.verdict == "similar" else "red"
+    grid = Table.grid(padding=(0, 2))
+    grid.add_column(style="bold cyan")
+    grid.add_column()
+    grid.add_row("SMD (distance)", f"{r.smd:.4f}")
+    grid.add_row("closeness", f"{r.closeness * 100:.1f}%")
+    grid.add_row(
+        "verdict", f"[{color}]{r.verdict}[/{color}]  (threshold {r.threshold:.2f} closeness)"
+    )
+    grid.add_row("bounds", f"WCD {r.wcd:.4f}  ≤  RWMD {r.rwmd:.4f}  ≤  SMD {r.smd:.4f}")
+    grid.add_row("statements", f"{r.n_statements_a} vs {r.n_statements_b}")
+    grid.add_row("anisotropy", "on" if r.anisotropy else "off")
+    _out.print(
+        Panel(grid, title="[bold]Document distance[/bold]", border_style=color, expand=False)
+    )
+
+
+def _emit_wrt_source(r, json_out: bool, result_only: bool) -> None:
+    if result_only:
+        typer.echo(f"{r.d_sel},{r.residual_a},{r.residual_b}")
+        return
+    if json_out:
+        typer.echo(json.dumps(r.to_dict(), indent=2))
+        return
+    grid = Table.grid(padding=(0, 2))
+    grid.add_column(style="bold cyan")
+    grid.add_column()
+    grid.add_row("D_sel (selection divergence)", f"{r.d_sel:.4f}")
+    grid.add_row("A → source", f"{r.residual_a:.4f}  (closeness {r.closeness_a * 100:.1f}%)")
+    grid.add_row("B → source", f"{r.residual_b:.4f}  (closeness {r.closeness_b * 100:.1f}%)")
+    grid.add_row(
+        "statements", f"A {r.n_statements_a} / B {r.n_statements_b} / S {r.n_statements_source}"
+    )
+    _out.print(
+        Panel(
+            grid,
+            title="[bold]Source-conditioned distance d(A,B|S)[/bold]",
+            border_style="cyan",
+            expand=False,
+        )
+    )
+    _out.print(
+        "[dim]residual = geometric distance to the source; the reranker + NLI grounding grade and "
+        "numeric verifier are deferred to E02[/dim]"
+    )
+
+
+@app.command(
+    epilog="[bold]Examples[/bold]\n\n"
+    "  docdistance distance report_v1.md report_v2.md\n"
+    '  docdistance distance "first text" "second text" --backend torch\n'
+    "  docdistance distance a.md b.md --json\n"
+    "  docdistance distance a.md b.md --result-only"
+)
+def distance(
+    a: str = typer.Argument(..., help="first document - a file path or raw text"),
+    b: str = typer.Argument(..., help="second document - a file path or raw text"),
+    backend: Backend = typer.Option(
+        Backend.openvino, "--backend", help="statement encoder backend"
+    ),
+    anisotropy: bool = typer.Option(
+        False,
+        "--anisotropy/--no-anisotropy",
+        help="all-but-the-top anisotropy removal - needs a corpus, off by default for a pair",
+    ),
+    threshold: float = typer.Option(
+        DEFAULT_THRESHOLD,
+        "--threshold",
+        help="closeness cutoff for the similar / not-similar verdict",
+    ),
+    json_out: bool = typer.Option(False, "--json", help="machine-readable JSON to stdout"),
+    result_only: bool = typer.Option(
+        False, "--result-only", help="bare SMD scalar to stdout, no clutter"
+    ),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="DEBUG logging to stderr"),
+):
+    """Symmetric distance between two documents - the exact Statement Mover's Distance."""
+    configure_logging(verbose)
+    from docdistance.pipeline import document_distance
+
+    result = _run(
+        lambda: document_distance(
+            a, b, backend=backend.value, anisotropy=anisotropy, threshold=threshold
+        )
+    )
+    _emit_distance(result, json_out, result_only)
+
+
+@app.command(
+    name="distance-wrt-source",
+    epilog="[bold]Examples[/bold]\n\n"
+    "  docdistance distance-wrt-source summary_a.md summary_b.md --source article.md\n"
+    "  docdistance distance-wrt-source a.md b.md -s s.md --json\n"
+    "  docdistance distance-wrt-source a.md b.md -s s.md --result-only   [dim]# D_sel,res_a,res_b[/dim]",
+)
+def distance_wrt_source(
+    a: str = typer.Argument(..., help="first document - a file path or raw text"),
+    b: str = typer.Argument(..., help="second document - a file path or raw text"),
+    source: str = typer.Option(..., "--source", "-s", help="the common source document"),
+    backend: Backend = typer.Option(
+        Backend.openvino, "--backend", help="statement encoder backend"
+    ),
+    anisotropy: bool = typer.Option(
+        False,
+        "--anisotropy/--no-anisotropy",
+        help="anisotropy removal - needs a corpus, off by default",
+    ),
+    json_out: bool = typer.Option(False, "--json", help="machine-readable JSON to stdout"),
+    result_only: bool = typer.Option(
+        False, "--result-only", help="bare comma-separated D_sel,residual_a,residual_b to stdout"
+    ),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="DEBUG logging to stderr"),
+):
+    """Source-conditioned distance d(A, B | S) - selection divergence plus each document's distance to S."""
+    configure_logging(verbose)
+    from docdistance.pipeline import source_conditioned_distance
+
+    result = _run(
+        lambda: source_conditioned_distance(
+            a, b, source, backend=backend.value, anisotropy=anisotropy
+        )
+    )
+    _emit_wrt_source(result, json_out, result_only)
+
+
+@app.command(
+    epilog="[bold]Examples[/bold]\n\n"
+    "  docdistance install               [dim]# both backends[/dim]\n"
+    "  docdistance install --backend openvino",
+)
+def install(
+    backend: InstallBackend = typer.Option(
+        InstallBackend.both, "--backend", help="which encoder weights to fetch"
+    ),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="DEBUG logging to stderr"),
+):
+    """Download and cache the models - the only command that fetches from the Hub (TQDM progress bars)."""
+    configure_logging(verbose)
+    from docdistance.encoders import ModelsNotInstalled, download_models
+
+    try:
+        backends = download_models(backend.value)
+    except ModelsNotInstalled as exc:
+        _err.print(f"[bold red]error:[/bold red] {exc}")
+        raise typer.Exit(1)
+    _out.print(f"[green]models ready:[/green] {', '.join(backends)}")
+
+
+if __name__ == "__main__":
+    app()

docdistance-1.0.15/src/docdistance/config.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+import sys
+
+from dotenv import load_dotenv
+from loguru import logger
+
+########### SETUP ###############
+
+# set up logger - INFO by default (DEBUG only via the CLI --verbose flag), sink to stderr so
+# stdout stays clean for --json / --result-only output
+logger.remove()
+logger.add(sys.stderr, colorize=True, level="INFO")
+
+# If tqdm is installed, configure loguru with tqdm.write
+# https://github.com/Delgan/loguru/issues/135
+try:
+    from tqdm import tqdm
+
+    logger.remove()
+    logger.add(lambda msg: tqdm.write(msg, end="", file=sys.stderr), colorize=True, level="INFO")
+except ModuleNotFoundError:
+    pass
+
+########## VARIABLES ############
+
+# Load environment variables from .env file if it exists
+load_dotenv()
+
+# paths
+PROJ_ROOT = Path(__file__).resolve().parents[2]
+DATA_DIR = PROJ_ROOT / "data"
+RAW_DATA_DIR = DATA_DIR / "raw"
+INTERIM_DATA_DIR = DATA_DIR / "interim"
+PROCESSED_DATA_DIR = DATA_DIR / "processed"
+EXTERNAL_DATA_DIR = DATA_DIR / "external"
+MODELS_DIR = PROJ_ROOT / "models"
+REPORTS_DIR = PROJ_ROOT / "reports"
+FIGURES_DIR = REPORTS_DIR / "figures"
+
+# log current root dir (debug so it never pollutes machine-readable stdout)
+logger.debug(f"PROJ_ROOT path is: {PROJ_ROOT}")
+
+########## MODELS ###############
+
+# segmenter (wtpsplit SaT) and the mmBERT statement encoders, by backend
+SAT_MODEL = "sat-3l-sm"
+MMBERT_TORCH_MODEL = "jhu-clsp/mmBERT-base"
+MMBERT_OPENVINO_LOCAL = MODELS_DIR / "02-mmbert-openvino-int8"
+MMBERT_OPENVINO_HF = "stellars/mmBERT-base-openvino-int8"
+
+
+def configure_logging(verbose: bool = False) -> None:
+    """Re-point loguru at stderr at INFO, or DEBUG when ``verbose`` - the CLI calls this first.
+
+    stderr keeps stdout reserved for the result so ``--json`` and ``--result-only`` stay machine-parseable.
+    """
+    level = "DEBUG" if verbose else "INFO"
+    logger.remove()
+    try:
+        from tqdm import tqdm
+
+        logger.add(
+            lambda msg: tqdm.write(msg, end="", file=sys.stderr), colorize=True, level=level
+        )
+    except ModuleNotFoundError:
+        logger.add(sys.stderr, colorize=True, level=level)

docdistance-1.0.15/src/docdistance/dataset.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from loguru import logger
+from tqdm import tqdm
+import typer
+
+from docdistance.config import PROCESSED_DATA_DIR, RAW_DATA_DIR
+
+app = typer.Typer()
+
+
+@app.command()
+def main(
+    # ---- REPLACE DEFAULT PATHS AS APPROPRIATE ----
+    input_path: Path = RAW_DATA_DIR / "dataset.csv",
+    output_path: Path = PROCESSED_DATA_DIR / "dataset.csv",
+    # ----------------------------------------------
+):
+    # ---- REPLACE THIS WITH YOUR OWN CODE ----
+    logger.info("Processing dataset...")
+    for i in tqdm(range(10), total=10):
+        if i == 5:
+            logger.info("Something happened for iteration 5.")
+    logger.success("Processing dataset complete.")
+    # -----------------------------------------
+
+
+if __name__ == "__main__":
+    app()