echr-py 0.2.2__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {echr_py-0.2.2 → echr_py-0.2.5}/CITATION.cff +2 -2
- {echr_py-0.2.2/echr_py.egg-info → echr_py-0.2.5}/PKG-INFO +31 -9
- {echr_py-0.2.2 → echr_py-0.2.5}/README.md +30 -8
- {echr_py-0.2.2 → echr_py-0.2.5/echr_py.egg-info}/PKG-INFO +31 -9
- {echr_py-0.2.2 → echr_py-0.2.5}/echr_py.egg-info/SOURCES.txt +3 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/__init__.py +1 -1
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/_aio.py +47 -7
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/_sync.py +4 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/__init__.py +2 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/corpus.py +88 -7
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/rescue.py +62 -2
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/cli.py +55 -8
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/config.py +5 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/citation_authority.json +1 -1
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/aio.py +3 -3
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/client.py +6 -3
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/downloader.py +18 -10
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/raw_download.py +3 -3
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/client.py +11 -3
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/downloader.py +12 -6
- echr_py-0.2.5/hudoc_py/mcp/calllog.py +173 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/mcp/server.py +17 -4
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/case.py +31 -5
- echr_py-0.2.5/hudoc_py/models/docx_structure.py +129 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/conversion.py +20 -2
- echr_py-0.2.5/hudoc_py/text/docx_structure.py +488 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/opinions.py +62 -7
- {echr_py-0.2.2 → echr_py-0.2.5}/pyproject.toml +1 -1
- {echr_py-0.2.2 → echr_py-0.2.5}/scripts/public_package_files.txt +3 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/LICENSE +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/MANIFEST.in +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/RIGHTS.md +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/SECURITY.md +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/echr_py.egg-info/dependency_links.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/echr_py.egg-info/entry_points.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/echr_py.egg-info/requires.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/echr_py.egg-info/top_level.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/bundle.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/ecli.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/bilingual/reconcile.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/cache/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/cache/store.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/artifacts.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/authority.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/benchmarks.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/catalog.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/extractor.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/graph.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/metrics.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/models.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/occurrences.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/paragraphs.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/reporter.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/citations/resolver.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/citation_authority.csv +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/citation_authority_eng.csv +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/citation_authority_eng.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/citation_authority_supplements.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/historical_citation_catalog.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/data/kpthesaurus_eng.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/_sync.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/bridge.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/collections.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/imports.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/queries.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/related_scraper.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/execution/textract.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/extractors/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/extractors/base.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/adapters.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/assets/LICENSE-D3.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/assets/d3.v7.min.js +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/export.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/models.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/graphs/template.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/gui/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/gui/app.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/llm/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/llm/base.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/llm/batch.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/llm/client.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/llm/providers.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/browse.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/codebook.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/export.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/paragraphs.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/registry.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/local/search.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/dsl.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/queries.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/main/url.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/mcp/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/mcp/__main__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/mcp/install.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/citation.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/common.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/document.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/exec_case.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/exec_document.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/models/version.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/pipeline/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/pipeline/runner.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/pipeline/validation.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/retrieval/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/retrieval/benchmark.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/retrieval/embeddings.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/retrieval/hybrid.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/retrieval/providers.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/batch.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/hooks.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/jobs.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/models.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/runner.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/schemas/citation-use-minimal.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/schemas/citation-use-multiaxial.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/spec.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/studies/taxonomies.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/composition.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/dispositive.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/judges.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/segmentation.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/text/spine.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/thesaurus.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/utils/__init__.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/utils/downloads.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/utils/jsonl.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/utils/retry.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/utils/runlog.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/hudoc_py/versions.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/release-requirements.txt +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/scripts/check_sdist.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/scripts/check_wheel.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/scripts/public_payload_hashes.json +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/scripts/verify_release.py +0 -0
- {echr_py-0.2.2 → echr_py-0.2.5}/setup.cfg +0 -0
|
@@ -5,8 +5,8 @@ type: software
|
|
|
5
5
|
authors:
|
|
6
6
|
- family-names: Ridi
|
|
7
7
|
given-names: Niccolo
|
|
8
|
-
version: 0.2.
|
|
9
|
-
date-released: 2026-08
|
|
8
|
+
version: 0.2.5
|
|
9
|
+
date-released: 2026-10-08
|
|
10
10
|
license: MIT
|
|
11
11
|
repository-code: "https://github.com/niccoloridi/echr-py"
|
|
12
12
|
url: "https://github.com/niccoloridi/echr-py"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: echr-py
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Pythonic acquisition of HUDOC and HUDOC-EXEC metadata and source documents for ECtHR research.
|
|
5
5
|
Author-email: Niccolo Ridi <niccoloridi@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -102,7 +102,7 @@ Dynamic: license-file
|
|
|
102
102
|
<a href="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml"><img src="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
103
103
|
<img src="https://img.shields.io/badge/python-3.11%E2%80%933.14-3776AB.svg" alt="Python 3.11–3.14">
|
|
104
104
|
<a href="https://github.com/niccoloridi/echr-py/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT license"></a>
|
|
105
|
-
<img src="https://img.shields.io/badge/version-0.2.
|
|
105
|
+
<img src="https://img.shields.io/badge/version-0.2.5-blue.svg" alt="Version 0.2.5">
|
|
106
106
|
</p>
|
|
107
107
|
|
|
108
108
|
`echr-py` is a research engineering toolkit for the case law of the European
|
|
@@ -112,10 +112,6 @@ Court of Human Rights. It connects public
|
|
|
112
112
|
research artifacts through Python, a command-line interface, and the Model
|
|
113
113
|
Context Protocol (MCP).
|
|
114
114
|
|
|
115
|
-
The distribution and primary command are now named `echr-py`. The established
|
|
116
|
-
`hudoc_py` Python import and the legacy `hudoc-py` command remain available for
|
|
117
|
-
backward compatibility.
|
|
118
|
-
|
|
119
115
|
It is designed for work where provenance matters. A result can retain the exact
|
|
120
116
|
HUDOC document, language version, section, paragraph, individual opinion,
|
|
121
117
|
footnote, citation span, cited procedural document, cited-paragraph pinpoint,
|
|
@@ -344,6 +340,17 @@ echr-py versions download \
|
|
|
344
340
|
The acquisition manifest records each language-specific item ID, rendition
|
|
345
341
|
type, requested-format outcome, HTTP status, path, byte length, and SHA-256.
|
|
346
342
|
|
|
343
|
+
### DOCX structure, opt-in
|
|
344
|
+
|
|
345
|
+
HUDOC's HTML is generated from the Registry's Word files and keeps only hashed
|
|
346
|
+
formatting classes. The DOCX keeps the template's named styles (`Ju_H_Head`,
|
|
347
|
+
`Ju_Para`, `Ju_Quot`, `Ju_Judges`, `Opi_H_Head`, `Opi_Para`), present from the
|
|
348
|
+
first judgment of 1961 onward. `--docx-structure` (or `docx_structure=True`)
|
|
349
|
+
reads them into an advisory `DocxStructure` with sections, numbered paragraphs,
|
|
350
|
+
quotations, the bench, separate opinions and footnotes, and reports where it
|
|
351
|
+
agrees with the HTML segmentation. HTML stays canonical for offsets, citation
|
|
352
|
+
loci and benchmarks. See [docs/text-segmentation.md](https://github.com/niccoloridi/echr-py/blob/main/docs/text-segmentation.md#docx-structure-opt-in-advisory).
|
|
353
|
+
|
|
347
354
|
## Local lexical, dense, and hybrid retrieval
|
|
348
355
|
|
|
349
356
|
Build a portable paragraph index from any acquired corpus:
|
|
@@ -495,7 +502,7 @@ without cloning the repository:
|
|
|
495
502
|
|
|
496
503
|
```bash
|
|
497
504
|
claude mcp add --scope user --transport stdio echr-py -- \
|
|
498
|
-
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.
|
|
505
|
+
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.5' echr-py mcp
|
|
499
506
|
claude mcp get echr-py
|
|
500
507
|
```
|
|
501
508
|
|
|
@@ -555,7 +562,7 @@ study jobs are an explicit opt-in.
|
|
|
555
562
|
|
|
556
563
|
| Command group | Purpose |
|
|
557
564
|
| --- | --- |
|
|
558
|
-
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition |
|
|
565
|
+
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition; `--docx-structure` adds the advisory Registry-style structure |
|
|
559
566
|
| `versions …`, `corpus …` | Language versions and reproducible corpora |
|
|
560
567
|
| `local …`, `embeddings …` | Offline browsing and paragraph retrieval |
|
|
561
568
|
| `gui` | Optional local Streamlit corpus browser |
|
|
@@ -563,7 +570,7 @@ study jobs are an explicit opt-in.
|
|
|
563
570
|
| `graph metrics|export|html|gexf` | Network metrics and portable graph output |
|
|
564
571
|
| `study validate|plan|run|status|resume|export` | Bounded optional dataset studies |
|
|
565
572
|
| `exec …` | HUDOC-EXEC record discovery, download, and source conversion |
|
|
566
|
-
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in |
|
|
573
|
+
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in; `--call-log PATH` records every tool call (tool, arguments, returned item IDs and ECLIs, version, time) as a replayable JSONL log |
|
|
567
574
|
|
|
568
575
|
```bash
|
|
569
576
|
echr-py --help
|
|
@@ -635,6 +642,21 @@ cd echr-py
|
|
|
635
642
|
python -m pip install -e ".[all,dev]"
|
|
636
643
|
```
|
|
637
644
|
|
|
645
|
+
### Names
|
|
646
|
+
|
|
647
|
+
The distribution and primary command are `echr-py`. The Python import is
|
|
648
|
+
`hudoc_py`, and the earlier `hudoc-py` command still works; both remain
|
|
649
|
+
available so existing code and scripts keep running.
|
|
650
|
+
|
|
651
|
+
```python
|
|
652
|
+
import hudoc_py # import name
|
|
653
|
+
```
|
|
654
|
+
|
|
655
|
+
```bash
|
|
656
|
+
echr-py --version # primary command
|
|
657
|
+
hudoc-py --version # legacy alias, same program
|
|
658
|
+
```
|
|
659
|
+
|
|
638
660
|
## Documentation
|
|
639
661
|
|
|
640
662
|
- [Worked examples](https://github.com/niccoloridi/echr-py/blob/main/docs/examples.md)
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<a href="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml"><img src="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
15
15
|
<img src="https://img.shields.io/badge/python-3.11%E2%80%933.14-3776AB.svg" alt="Python 3.11–3.14">
|
|
16
16
|
<a href="https://github.com/niccoloridi/echr-py/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT license"></a>
|
|
17
|
-
<img src="https://img.shields.io/badge/version-0.2.
|
|
17
|
+
<img src="https://img.shields.io/badge/version-0.2.5-blue.svg" alt="Version 0.2.5">
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
20
|
`echr-py` is a research engineering toolkit for the case law of the European
|
|
@@ -24,10 +24,6 @@ Court of Human Rights. It connects public
|
|
|
24
24
|
research artifacts through Python, a command-line interface, and the Model
|
|
25
25
|
Context Protocol (MCP).
|
|
26
26
|
|
|
27
|
-
The distribution and primary command are now named `echr-py`. The established
|
|
28
|
-
`hudoc_py` Python import and the legacy `hudoc-py` command remain available for
|
|
29
|
-
backward compatibility.
|
|
30
|
-
|
|
31
27
|
It is designed for work where provenance matters. A result can retain the exact
|
|
32
28
|
HUDOC document, language version, section, paragraph, individual opinion,
|
|
33
29
|
footnote, citation span, cited procedural document, cited-paragraph pinpoint,
|
|
@@ -256,6 +252,17 @@ echr-py versions download \
|
|
|
256
252
|
The acquisition manifest records each language-specific item ID, rendition
|
|
257
253
|
type, requested-format outcome, HTTP status, path, byte length, and SHA-256.
|
|
258
254
|
|
|
255
|
+
### DOCX structure, opt-in
|
|
256
|
+
|
|
257
|
+
HUDOC's HTML is generated from the Registry's Word files and keeps only hashed
|
|
258
|
+
formatting classes. The DOCX keeps the template's named styles (`Ju_H_Head`,
|
|
259
|
+
`Ju_Para`, `Ju_Quot`, `Ju_Judges`, `Opi_H_Head`, `Opi_Para`), present from the
|
|
260
|
+
first judgment of 1961 onward. `--docx-structure` (or `docx_structure=True`)
|
|
261
|
+
reads them into an advisory `DocxStructure` with sections, numbered paragraphs,
|
|
262
|
+
quotations, the bench, separate opinions and footnotes, and reports where it
|
|
263
|
+
agrees with the HTML segmentation. HTML stays canonical for offsets, citation
|
|
264
|
+
loci and benchmarks. See [docs/text-segmentation.md](https://github.com/niccoloridi/echr-py/blob/main/docs/text-segmentation.md#docx-structure-opt-in-advisory).
|
|
265
|
+
|
|
259
266
|
## Local lexical, dense, and hybrid retrieval
|
|
260
267
|
|
|
261
268
|
Build a portable paragraph index from any acquired corpus:
|
|
@@ -407,7 +414,7 @@ without cloning the repository:
|
|
|
407
414
|
|
|
408
415
|
```bash
|
|
409
416
|
claude mcp add --scope user --transport stdio echr-py -- \
|
|
410
|
-
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.
|
|
417
|
+
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.5' echr-py mcp
|
|
411
418
|
claude mcp get echr-py
|
|
412
419
|
```
|
|
413
420
|
|
|
@@ -467,7 +474,7 @@ study jobs are an explicit opt-in.
|
|
|
467
474
|
|
|
468
475
|
| Command group | Purpose |
|
|
469
476
|
| --- | --- |
|
|
470
|
-
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition |
|
|
477
|
+
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition; `--docx-structure` adds the advisory Registry-style structure |
|
|
471
478
|
| `versions …`, `corpus …` | Language versions and reproducible corpora |
|
|
472
479
|
| `local …`, `embeddings …` | Offline browsing and paragraph retrieval |
|
|
473
480
|
| `gui` | Optional local Streamlit corpus browser |
|
|
@@ -475,7 +482,7 @@ study jobs are an explicit opt-in.
|
|
|
475
482
|
| `graph metrics|export|html|gexf` | Network metrics and portable graph output |
|
|
476
483
|
| `study validate|plan|run|status|resume|export` | Bounded optional dataset studies |
|
|
477
484
|
| `exec …` | HUDOC-EXEC record discovery, download, and source conversion |
|
|
478
|
-
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in |
|
|
485
|
+
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in; `--call-log PATH` records every tool call (tool, arguments, returned item IDs and ECLIs, version, time) as a replayable JSONL log |
|
|
479
486
|
|
|
480
487
|
```bash
|
|
481
488
|
echr-py --help
|
|
@@ -547,6 +554,21 @@ cd echr-py
|
|
|
547
554
|
python -m pip install -e ".[all,dev]"
|
|
548
555
|
```
|
|
549
556
|
|
|
557
|
+
### Names
|
|
558
|
+
|
|
559
|
+
The distribution and primary command are `echr-py`. The Python import is
|
|
560
|
+
`hudoc_py`, and the earlier `hudoc-py` command still works; both remain
|
|
561
|
+
available so existing code and scripts keep running.
|
|
562
|
+
|
|
563
|
+
```python
|
|
564
|
+
import hudoc_py # import name
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
```bash
|
|
568
|
+
echr-py --version # primary command
|
|
569
|
+
hudoc-py --version # legacy alias, same program
|
|
570
|
+
```
|
|
571
|
+
|
|
550
572
|
## Documentation
|
|
551
573
|
|
|
552
574
|
- [Worked examples](https://github.com/niccoloridi/echr-py/blob/main/docs/examples.md)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: echr-py
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Pythonic acquisition of HUDOC and HUDOC-EXEC metadata and source documents for ECtHR research.
|
|
5
5
|
Author-email: Niccolo Ridi <niccoloridi@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -102,7 +102,7 @@ Dynamic: license-file
|
|
|
102
102
|
<a href="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml"><img src="https://github.com/niccoloridi/echr-py/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
103
103
|
<img src="https://img.shields.io/badge/python-3.11%E2%80%933.14-3776AB.svg" alt="Python 3.11–3.14">
|
|
104
104
|
<a href="https://github.com/niccoloridi/echr-py/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT license"></a>
|
|
105
|
-
<img src="https://img.shields.io/badge/version-0.2.
|
|
105
|
+
<img src="https://img.shields.io/badge/version-0.2.5-blue.svg" alt="Version 0.2.5">
|
|
106
106
|
</p>
|
|
107
107
|
|
|
108
108
|
`echr-py` is a research engineering toolkit for the case law of the European
|
|
@@ -112,10 +112,6 @@ Court of Human Rights. It connects public
|
|
|
112
112
|
research artifacts through Python, a command-line interface, and the Model
|
|
113
113
|
Context Protocol (MCP).
|
|
114
114
|
|
|
115
|
-
The distribution and primary command are now named `echr-py`. The established
|
|
116
|
-
`hudoc_py` Python import and the legacy `hudoc-py` command remain available for
|
|
117
|
-
backward compatibility.
|
|
118
|
-
|
|
119
115
|
It is designed for work where provenance matters. A result can retain the exact
|
|
120
116
|
HUDOC document, language version, section, paragraph, individual opinion,
|
|
121
117
|
footnote, citation span, cited procedural document, cited-paragraph pinpoint,
|
|
@@ -344,6 +340,17 @@ echr-py versions download \
|
|
|
344
340
|
The acquisition manifest records each language-specific item ID, rendition
|
|
345
341
|
type, requested-format outcome, HTTP status, path, byte length, and SHA-256.
|
|
346
342
|
|
|
343
|
+
### DOCX structure, opt-in
|
|
344
|
+
|
|
345
|
+
HUDOC's HTML is generated from the Registry's Word files and keeps only hashed
|
|
346
|
+
formatting classes. The DOCX keeps the template's named styles (`Ju_H_Head`,
|
|
347
|
+
`Ju_Para`, `Ju_Quot`, `Ju_Judges`, `Opi_H_Head`, `Opi_Para`), present from the
|
|
348
|
+
first judgment of 1961 onward. `--docx-structure` (or `docx_structure=True`)
|
|
349
|
+
reads them into an advisory `DocxStructure` with sections, numbered paragraphs,
|
|
350
|
+
quotations, the bench, separate opinions and footnotes, and reports where it
|
|
351
|
+
agrees with the HTML segmentation. HTML stays canonical for offsets, citation
|
|
352
|
+
loci and benchmarks. See [docs/text-segmentation.md](https://github.com/niccoloridi/echr-py/blob/main/docs/text-segmentation.md#docx-structure-opt-in-advisory).
|
|
353
|
+
|
|
347
354
|
## Local lexical, dense, and hybrid retrieval
|
|
348
355
|
|
|
349
356
|
Build a portable paragraph index from any acquired corpus:
|
|
@@ -495,7 +502,7 @@ without cloning the repository:
|
|
|
495
502
|
|
|
496
503
|
```bash
|
|
497
504
|
claude mcp add --scope user --transport stdio echr-py -- \
|
|
498
|
-
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.
|
|
505
|
+
"$(command -v uvx)" --from 'echr-py[mcp]==0.2.5' echr-py mcp
|
|
499
506
|
claude mcp get echr-py
|
|
500
507
|
```
|
|
501
508
|
|
|
@@ -555,7 +562,7 @@ study jobs are an explicit opt-in.
|
|
|
555
562
|
|
|
556
563
|
| Command group | Purpose |
|
|
557
564
|
| --- | --- |
|
|
558
|
-
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition |
|
|
565
|
+
| `search`, `count`, `fetch-case`, `smart-fetch` | HUDOC metadata and document acquisition; `--docx-structure` adds the advisory Registry-style structure |
|
|
559
566
|
| `versions …`, `corpus …` | Language versions and reproducible corpora |
|
|
560
567
|
| `local …`, `embeddings …` | Offline browsing and paragraph retrieval |
|
|
561
568
|
| `gui` | Optional local Streamlit corpus browser |
|
|
@@ -563,7 +570,7 @@ study jobs are an explicit opt-in.
|
|
|
563
570
|
| `graph metrics|export|html|gexf` | Network metrics and portable graph output |
|
|
564
571
|
| `study validate|plan|run|status|resume|export` | Bounded optional dataset studies |
|
|
565
572
|
| `exec …` | HUDOC-EXEC record discovery, download, and source conversion |
|
|
566
|
-
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in |
|
|
573
|
+
| `mcp` | Read-only MCP server by default; bounded jobs are opt-in; `--call-log PATH` records every tool call (tool, arguments, returned item IDs and ECLIs, version, time) as a replayable JSONL log |
|
|
567
574
|
|
|
568
575
|
```bash
|
|
569
576
|
echr-py --help
|
|
@@ -635,6 +642,21 @@ cd echr-py
|
|
|
635
642
|
python -m pip install -e ".[all,dev]"
|
|
636
643
|
```
|
|
637
644
|
|
|
645
|
+
### Names
|
|
646
|
+
|
|
647
|
+
The distribution and primary command are `echr-py`. The Python import is
|
|
648
|
+
`hudoc_py`, and the earlier `hudoc-py` command still works; both remain
|
|
649
|
+
available so existing code and scripts keep running.
|
|
650
|
+
|
|
651
|
+
```python
|
|
652
|
+
import hudoc_py # import name
|
|
653
|
+
```
|
|
654
|
+
|
|
655
|
+
```bash
|
|
656
|
+
echr-py --version # primary command
|
|
657
|
+
hudoc-py --version # legacy alias, same program
|
|
658
|
+
```
|
|
659
|
+
|
|
638
660
|
## Documentation
|
|
639
661
|
|
|
640
662
|
- [Worked examples](https://github.com/niccoloridi/echr-py/blob/main/docs/examples.md)
|
|
@@ -91,6 +91,7 @@ hudoc_py/main/queries.py
|
|
|
91
91
|
hudoc_py/main/url.py
|
|
92
92
|
hudoc_py/mcp/__init__.py
|
|
93
93
|
hudoc_py/mcp/__main__.py
|
|
94
|
+
hudoc_py/mcp/calllog.py
|
|
94
95
|
hudoc_py/mcp/install.py
|
|
95
96
|
hudoc_py/mcp/server.py
|
|
96
97
|
hudoc_py/models/__init__.py
|
|
@@ -98,6 +99,7 @@ hudoc_py/models/case.py
|
|
|
98
99
|
hudoc_py/models/citation.py
|
|
99
100
|
hudoc_py/models/common.py
|
|
100
101
|
hudoc_py/models/document.py
|
|
102
|
+
hudoc_py/models/docx_structure.py
|
|
101
103
|
hudoc_py/models/exec_case.py
|
|
102
104
|
hudoc_py/models/exec_document.py
|
|
103
105
|
hudoc_py/models/version.py
|
|
@@ -123,6 +125,7 @@ hudoc_py/text/__init__.py
|
|
|
123
125
|
hudoc_py/text/composition.py
|
|
124
126
|
hudoc_py/text/conversion.py
|
|
125
127
|
hudoc_py/text/dispositive.py
|
|
128
|
+
hudoc_py/text/docx_structure.py
|
|
126
129
|
hudoc_py/text/judges.py
|
|
127
130
|
hudoc_py/text/opinions.py
|
|
128
131
|
hudoc_py/text/segmentation.py
|
|
@@ -7,6 +7,7 @@ These coroutines are the canonical implementation; the sync facade in
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import asyncio
|
|
10
|
+
from collections.abc import Iterable
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
@@ -14,7 +15,11 @@ import aiohttp
|
|
|
14
15
|
|
|
15
16
|
from . import config
|
|
16
17
|
from .main.client import AsyncHudocClient
|
|
17
|
-
from .main.downloader import
|
|
18
|
+
from .main.downloader import (
|
|
19
|
+
download_headers,
|
|
20
|
+
fetch_document_docx,
|
|
21
|
+
fetch_document_html,
|
|
22
|
+
)
|
|
18
23
|
from .main.dsl import Q
|
|
19
24
|
from .main.queries import resolve_sort
|
|
20
25
|
from .models import Case, CaseCollection
|
|
@@ -134,7 +139,7 @@ async def hydrate_texts(
|
|
|
134
139
|
if case.text_source_itemid != case.itemid:
|
|
135
140
|
counts["fallback_used"] += 1
|
|
136
141
|
|
|
137
|
-
async with aiohttp.ClientSession(headers=
|
|
142
|
+
async with aiohttp.ClientSession(headers=download_headers()) as session:
|
|
138
143
|
await asyncio.gather(*(_one(session, c) for c in collection))
|
|
139
144
|
return counts
|
|
140
145
|
|
|
@@ -216,6 +221,7 @@ async def smart_fetch(
|
|
|
216
221
|
rich_sections: bool = False,
|
|
217
222
|
french_fallback: bool = True,
|
|
218
223
|
concurrency: int = config.HUDOC_CONCURRENCY,
|
|
224
|
+
docx_structure: bool = False,
|
|
219
225
|
**filters: Any,
|
|
220
226
|
) -> CaseCollection:
|
|
221
227
|
"""Search, keep the ``top`` matches, and fetch their texts concurrently.
|
|
@@ -245,6 +251,8 @@ async def smart_fetch(
|
|
|
245
251
|
french_fallback=french_fallback,
|
|
246
252
|
concurrency=concurrency,
|
|
247
253
|
)
|
|
254
|
+
if docx_structure and collection:
|
|
255
|
+
await hydrate_docx_structures(collection, concurrency=concurrency)
|
|
248
256
|
|
|
249
257
|
return collection
|
|
250
258
|
|
|
@@ -261,6 +269,7 @@ async def fetch_case(
|
|
|
261
269
|
french_fallback: bool = True,
|
|
262
270
|
rescue: bool = False,
|
|
263
271
|
docx_out: str | Path | None = None,
|
|
272
|
+
docx_structure: bool = False,
|
|
264
273
|
) -> Case | None:
|
|
265
274
|
"""Fetch metadata (and optionally full text) for a single case.
|
|
266
275
|
|
|
@@ -295,7 +304,7 @@ async def fetch_case(
|
|
|
295
304
|
case = Case.model_validate(rows[0])
|
|
296
305
|
|
|
297
306
|
if with_text and case.itemid:
|
|
298
|
-
async with aiohttp.ClientSession(headers=
|
|
307
|
+
async with aiohttp.ClientSession(headers=download_headers()) as session:
|
|
299
308
|
loaded = await _fetch_case_text(
|
|
300
309
|
session,
|
|
301
310
|
case,
|
|
@@ -320,12 +329,43 @@ async def fetch_case(
|
|
|
320
329
|
french_fallback=True,
|
|
321
330
|
)
|
|
322
331
|
|
|
323
|
-
if docx_out is not None and case.itemid:
|
|
324
|
-
await fetch_docx(case.itemid, out=docx_out)
|
|
332
|
+
if (docx_out is not None or docx_structure) and case.itemid:
|
|
333
|
+
data = await fetch_docx(case.itemid, out=docx_out)
|
|
334
|
+
if docx_structure:
|
|
335
|
+
attach_docx_structure(case, data)
|
|
325
336
|
|
|
326
337
|
return case
|
|
327
338
|
|
|
328
339
|
|
|
340
|
+
def attach_docx_structure(case: Case, data: bytes | None) -> None:
|
|
341
|
+
"""Set ``case.docx_structure`` from DOCX bytes (an empty, diagnosed structure when absent)."""
|
|
342
|
+
from .models.docx_structure import DocxStructure
|
|
343
|
+
from .text.docx_structure import build_docx_structure
|
|
344
|
+
|
|
345
|
+
if data is None:
|
|
346
|
+
case.docx_structure = DocxStructure(
|
|
347
|
+
itemid=case.itemid, sha256="", byte_count=0, diagnostics=["docx_unavailable"]
|
|
348
|
+
)
|
|
349
|
+
return
|
|
350
|
+
case.docx_structure = build_docx_structure(data, itemid=case.itemid)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
async def hydrate_docx_structures(
|
|
354
|
+
cases: Iterable[Case], *, concurrency: int = config.HUDOC_CONCURRENCY
|
|
355
|
+
) -> int:
|
|
356
|
+
"""Fetch each case's DOCX and attach its structure; returns the number attached."""
|
|
357
|
+
semaphore = asyncio.Semaphore(max(1, concurrency))
|
|
358
|
+
targets = [c for c in cases if c.itemid]
|
|
359
|
+
|
|
360
|
+
async def one(case: Case) -> None:
|
|
361
|
+
async with semaphore:
|
|
362
|
+
data = await fetch_docx(case.itemid) # type: ignore[arg-type]
|
|
363
|
+
attach_docx_structure(case, data)
|
|
364
|
+
|
|
365
|
+
await asyncio.gather(*(one(c) for c in targets))
|
|
366
|
+
return sum(1 for c in targets if c.docx_structure is not None and c.docx_structure.byte_count)
|
|
367
|
+
|
|
368
|
+
|
|
329
369
|
async def fetch_docx(itemid: str, *, out: str | Path | None = None) -> bytes | None:
|
|
330
370
|
"""Fetch a document's raw DOCX bytes; optionally write them to ``out``.
|
|
331
371
|
|
|
@@ -333,7 +373,7 @@ async def fetch_docx(itemid: str, *, out: str | Path | None = None) -> bytes | N
|
|
|
333
373
|
DOCX is a separate binary payload from the HTML body and is deliberately
|
|
334
374
|
not stored on ``Case.text``.
|
|
335
375
|
"""
|
|
336
|
-
async with aiohttp.ClientSession(headers=
|
|
376
|
+
async with aiohttp.ClientSession(headers=download_headers()) as session:
|
|
337
377
|
data = await fetch_document_docx(session, itemid)
|
|
338
378
|
if data is not None and out is not None:
|
|
339
379
|
Path(out).write_bytes(data)
|
|
@@ -342,7 +382,7 @@ async def fetch_docx(itemid: str, *, out: str | Path | None = None) -> bytes | N
|
|
|
342
382
|
|
|
343
383
|
async def fetch_text(itemid: str, *, format: str = "text") -> str | None:
|
|
344
384
|
"""Fetch and convert a single document's body."""
|
|
345
|
-
async with aiohttp.ClientSession(headers=
|
|
385
|
+
async with aiohttp.ClientSession(headers=download_headers()) as session:
|
|
346
386
|
html = await fetch_document_html(session, itemid)
|
|
347
387
|
if html is None:
|
|
348
388
|
return None
|
|
@@ -65,6 +65,7 @@ def smart_fetch(
|
|
|
65
65
|
rich_sections: bool = False,
|
|
66
66
|
french_fallback: bool = True,
|
|
67
67
|
concurrency: int = config.HUDOC_CONCURRENCY,
|
|
68
|
+
docx_structure: bool = False,
|
|
68
69
|
**filters: Any,
|
|
69
70
|
) -> CaseCollection:
|
|
70
71
|
"""Search, keep the ``top`` matches, and fetch their texts concurrently."""
|
|
@@ -80,6 +81,7 @@ def smart_fetch(
|
|
|
80
81
|
rich_sections=rich_sections,
|
|
81
82
|
french_fallback=french_fallback,
|
|
82
83
|
concurrency=concurrency,
|
|
84
|
+
docx_structure=docx_structure,
|
|
83
85
|
**filters,
|
|
84
86
|
)
|
|
85
87
|
)
|
|
@@ -97,6 +99,7 @@ def fetch_case(
|
|
|
97
99
|
french_fallback: bool = True,
|
|
98
100
|
rescue: bool = False,
|
|
99
101
|
docx_out: str | Path | None = None,
|
|
102
|
+
docx_structure: bool = False,
|
|
100
103
|
) -> Case | None:
|
|
101
104
|
return _run(
|
|
102
105
|
_aio.fetch_case(
|
|
@@ -110,6 +113,7 @@ def fetch_case(
|
|
|
110
113
|
french_fallback=french_fallback,
|
|
111
114
|
rescue=rescue,
|
|
112
115
|
docx_out=docx_out,
|
|
116
|
+
docx_structure=docx_structure,
|
|
113
117
|
)
|
|
114
118
|
)
|
|
115
119
|
|
|
@@ -33,6 +33,7 @@ from .rescue import (
|
|
|
33
33
|
find_french_sibling,
|
|
34
34
|
rescue_candidates,
|
|
35
35
|
rescue_french,
|
|
36
|
+
sibling_conflicts,
|
|
36
37
|
)
|
|
37
38
|
|
|
38
39
|
|
|
@@ -70,6 +71,7 @@ __all__ = [
|
|
|
70
71
|
"ReconcileStats",
|
|
71
72
|
"ReconcileInvariantError",
|
|
72
73
|
"rescue_french",
|
|
74
|
+
"sibling_conflicts",
|
|
73
75
|
"rescue_candidates",
|
|
74
76
|
"find_french_sibling",
|
|
75
77
|
"apply_rescue_mapping",
|
|
@@ -31,9 +31,10 @@ from pydantic import BaseModel, Field, field_validator
|
|
|
31
31
|
|
|
32
32
|
from .. import config
|
|
33
33
|
from .._aio import _fetch_case_text, search
|
|
34
|
-
from ..main.downloader import
|
|
34
|
+
from ..main.downloader import download_headers
|
|
35
35
|
from ..main.dsl import Q
|
|
36
36
|
from ..models.case import Case, CaseCollection
|
|
37
|
+
from ..models.common import Sections
|
|
37
38
|
from ..utils.jsonl import append_jsonl, append_jsonl_many, iter_jsonl, load_processed_ids
|
|
38
39
|
from .reconcile import ExtraSiblingPolicy, ReconcileStats, reconcile
|
|
39
40
|
from .rescue import RescueStats, rescue_french
|
|
@@ -51,6 +52,7 @@ class CorpusReport(BaseModel):
|
|
|
51
52
|
language_texts: dict[str, int] | None = None
|
|
52
53
|
citations: dict[str, Any] | None = None
|
|
53
54
|
rich_tables: dict[str, int] | None = None
|
|
55
|
+
docx_structures: dict[str, int] | None = None
|
|
54
56
|
selection: dict[str, int] | None = None
|
|
55
57
|
out_dir: str = ""
|
|
56
58
|
|
|
@@ -184,12 +186,15 @@ def _save_language_versions(cases: list[Case], entries: list[SelectionEntry], pa
|
|
|
184
186
|
|
|
185
187
|
ownership = _language_version_ownership(entries)
|
|
186
188
|
rows = [
|
|
187
|
-
{
|
|
189
|
+
{
|
|
190
|
+
**case.model_dump(mode="json", exclude={"text", "sections", "docx_structure"}),
|
|
191
|
+
**ownership[case.itemid],
|
|
192
|
+
}
|
|
188
193
|
for case in cases
|
|
189
194
|
if case.itemid in ownership
|
|
190
195
|
]
|
|
191
196
|
columns = [
|
|
192
|
-
*[name for name in Case.model_fields if name not in {"text", "sections"}],
|
|
197
|
+
*[name for name in Case.model_fields if name not in {"text", "sections", "docx_structure"}],
|
|
193
198
|
"primary_itemid",
|
|
194
199
|
"selection_ecli",
|
|
195
200
|
"selected_language",
|
|
@@ -369,6 +374,7 @@ async def build_corpus(
|
|
|
369
374
|
text_format: str = "text",
|
|
370
375
|
rescue: bool = True,
|
|
371
376
|
save_docx: bool = False,
|
|
377
|
+
docx_structure: bool = False,
|
|
372
378
|
resolve_case_citations: bool = False,
|
|
373
379
|
citation_authority: str | Path | None = None,
|
|
374
380
|
citation_overrides: str | Path | None = None,
|
|
@@ -378,6 +384,9 @@ async def build_corpus(
|
|
|
378
384
|
concurrency: int = config.HUDOC_CONCURRENCY,
|
|
379
385
|
**filters: Any,
|
|
380
386
|
) -> CorpusReport:
|
|
387
|
+
# The DOCX agreement report needs the HTML segmentation, which only the rich
|
|
388
|
+
# split persists; requesting the DOCX layer therefore implies rich sections.
|
|
389
|
+
rich_sections = rich_sections or docx_structure
|
|
381
390
|
"""Build a reconciled, text-hydrated corpus under ``out_dir``.
|
|
382
391
|
|
|
383
392
|
The default search covers both languages and all six doctypes, so ENG/FRE
|
|
@@ -464,8 +473,8 @@ async def build_corpus(
|
|
|
464
473
|
failures_path=failures_path,
|
|
465
474
|
)
|
|
466
475
|
|
|
467
|
-
# 5. Optional raw DOCX download.
|
|
468
|
-
if save_docx:
|
|
476
|
+
# 5. Optional raw DOCX download (forced on when the DOCX structure layer is requested).
|
|
477
|
+
if save_docx or docx_structure:
|
|
469
478
|
from ..main.downloader import AsyncDocumentDownloader
|
|
470
479
|
|
|
471
480
|
downloader = AsyncDocumentDownloader(
|
|
@@ -516,6 +525,16 @@ async def build_corpus(
|
|
|
516
525
|
write_resolution_artifacts(citation_result, citation_dir)
|
|
517
526
|
report.citations = citation_result.report.model_dump(mode="json")
|
|
518
527
|
|
|
528
|
+
# 8. Optional advisory DOCX structure layer: one record per case with the
|
|
529
|
+
# Registry-style structure and its agreement with the HTML segmentation.
|
|
530
|
+
if docx_structure:
|
|
531
|
+
persisted_sections = (
|
|
532
|
+
out / "language-sections.jsonl" if language_version_cases else out / "sections.jsonl"
|
|
533
|
+
)
|
|
534
|
+
report.docx_structures = _write_docx_structures(
|
|
535
|
+
result.cases, out, sections_path=persisted_sections
|
|
536
|
+
)
|
|
537
|
+
|
|
519
538
|
(out / "report.json").write_text(report.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
|
520
539
|
from .bundle import generate_corpus_manifest
|
|
521
540
|
|
|
@@ -560,7 +579,7 @@ async def _hydrate_to_jsonl(
|
|
|
560
579
|
return case, False
|
|
561
580
|
return case, True
|
|
562
581
|
|
|
563
|
-
async with aiohttp.ClientSession(headers=
|
|
582
|
+
async with aiohttp.ClientSession(headers=download_headers()) as session:
|
|
564
583
|
results = await asyncio.gather(*(_one(session, c) for c in todo))
|
|
565
584
|
for case, ok in results:
|
|
566
585
|
if not ok:
|
|
@@ -609,7 +628,7 @@ def _save_parquet(cases: list[Case], path: Path, *, drop_text: bool) -> None:
|
|
|
609
628
|
rows = [c.model_dump(mode="json") for c in cases]
|
|
610
629
|
df = pd.DataFrame(rows) if rows else pd.DataFrame(columns=list(Case.model_fields))
|
|
611
630
|
if drop_text:
|
|
612
|
-
df = df.drop(columns=[c for c in ("text", "sections") if c in df.columns])
|
|
631
|
+
df = df.drop(columns=[c for c in ("text", "sections", "docx_structure") if c in df.columns])
|
|
613
632
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
614
633
|
df.to_parquet(path, index=False)
|
|
615
634
|
|
|
@@ -627,3 +646,65 @@ def load_cases(path: str | Path) -> CaseCollection:
|
|
|
627
646
|
|
|
628
647
|
|
|
629
648
|
__all__ = ["build_corpus", "CorpusReport", "load_cases", "load_selection", "SelectionEntry"]
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _load_persisted_sections(path: Path) -> dict[str, Sections]:
|
|
652
|
+
"""Read ``sections.jsonl`` (or ``language-sections.jsonl``) back into ``Sections`` by item ID."""
|
|
653
|
+
from ..utils.jsonl import iter_jsonl
|
|
654
|
+
|
|
655
|
+
if not path.exists():
|
|
656
|
+
return {}
|
|
657
|
+
restored: dict[str, Sections] = {}
|
|
658
|
+
for record in iter_jsonl(path):
|
|
659
|
+
itemid = str(record.get("itemid") or "")
|
|
660
|
+
if itemid and isinstance(record.get("sections"), dict) and itemid not in restored:
|
|
661
|
+
restored[itemid] = Sections.model_validate(record["sections"])
|
|
662
|
+
return restored
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _write_docx_structures(
|
|
666
|
+
cases: list[Case], out: Path, *, sections_path: Path | None = None
|
|
667
|
+
) -> dict[str, int]:
|
|
668
|
+
"""Parse the DOCX files saved under ``docs/docx`` and write ``docx_structure.jsonl``.
|
|
669
|
+
|
|
670
|
+
Sections for the agreement report come from the in-memory case when it was
|
|
671
|
+
hydrated in this run, and otherwise from the persisted ``sections.jsonl`` so
|
|
672
|
+
resumed builds report agreement for every case.
|
|
673
|
+
"""
|
|
674
|
+
import json
|
|
675
|
+
|
|
676
|
+
from ..text.docx_structure import build_docx_structure, compare_docx_structure
|
|
677
|
+
|
|
678
|
+
persisted = _load_persisted_sections(sections_path) if sections_path is not None else {}
|
|
679
|
+
docx_dir = out / "docs" / "docx"
|
|
680
|
+
counts = {"cases": 0, "with_docx": 0, "sections_agree": 0, "opinions_agree": 0}
|
|
681
|
+
with (out / "docx_structure.jsonl").open("w", encoding="utf-8") as handle:
|
|
682
|
+
for case in cases:
|
|
683
|
+
if not case.itemid:
|
|
684
|
+
continue
|
|
685
|
+
counts["cases"] += 1
|
|
686
|
+
path = docx_dir / f"{case.itemid}.docx"
|
|
687
|
+
record: dict[str, Any] = {
|
|
688
|
+
"itemid": case.itemid,
|
|
689
|
+
"docx_path": str(path.relative_to(out)) if path.exists() else None,
|
|
690
|
+
}
|
|
691
|
+
if path.exists():
|
|
692
|
+
structure = build_docx_structure(path.read_bytes(), itemid=case.itemid)
|
|
693
|
+
case.docx_structure = structure
|
|
694
|
+
counts["with_docx"] += 1
|
|
695
|
+
record["structure"] = structure.model_dump(mode="json")
|
|
696
|
+
sections = case.sections or persisted.get(case.itemid)
|
|
697
|
+
if sections is not None:
|
|
698
|
+
agreement = compare_docx_structure(
|
|
699
|
+
case
|
|
700
|
+
if case.sections is not None
|
|
701
|
+
else case.model_copy(update={"sections": sections}),
|
|
702
|
+
structure,
|
|
703
|
+
)
|
|
704
|
+
record["agreement"] = agreement.model_dump(mode="json")
|
|
705
|
+
counts["sections_agree"] += int(agreement.sections_agree)
|
|
706
|
+
counts["opinions_agree"] += int(agreement.opinion_counts_agree)
|
|
707
|
+
else:
|
|
708
|
+
record["structure"] = None
|
|
709
|
+
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
710
|
+
return counts
|