folio-resolve 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_resolve-0.1.0/.github/workflows/publish.yml +51 -0
- folio_resolve-0.1.0/.gitignore +10 -0
- folio_resolve-0.1.0/LICENSE +21 -0
- folio_resolve-0.1.0/PKG-INFO +169 -0
- folio_resolve-0.1.0/README.md +130 -0
- folio_resolve-0.1.0/THIRD-PARTY.md +58 -0
- folio_resolve-0.1.0/docs/BYOK.md +293 -0
- folio_resolve-0.1.0/docs/migration/SCHEDULE.md +36 -0
- folio_resolve-0.1.0/docs/plans/2026-07-15-001-feat-folio-matching-v0-plan.md +118 -0
- folio_resolve-0.1.0/pyproject.toml +83 -0
- folio_resolve-0.1.0/src/folio_resolve/__init__.py +107 -0
- folio_resolve-0.1.0/src/folio_resolve/annotate/__init__.py +46 -0
- folio_resolve-0.1.0/src/folio_resolve/annotate/feedback_store.py +95 -0
- folio_resolve-0.1.0/src/folio_resolve/annotate/lifecycle.py +71 -0
- folio_resolve-0.1.0/src/folio_resolve/annotate/models.py +135 -0
- folio_resolve-0.1.0/src/folio_resolve/annotate/render.py +69 -0
- folio_resolve-0.1.0/src/folio_resolve/blocklist.py +145 -0
- folio_resolve-0.1.0/src/folio_resolve/calibration.py +89 -0
- folio_resolve-0.1.0/src/folio_resolve/data/__init__.py +1 -0
- folio_resolve-0.1.0/src/folio_resolve/data/alias_blocklist.json +36 -0
- folio_resolve-0.1.0/src/folio_resolve/decompose.py +105 -0
- folio_resolve-0.1.0/src/folio_resolve/domain_prior.py +198 -0
- folio_resolve-0.1.0/src/folio_resolve/embedding.py +139 -0
- folio_resolve-0.1.0/src/folio_resolve/entity_ruler.py +114 -0
- folio_resolve-0.1.0/src/folio_resolve/gates.py +124 -0
- folio_resolve-0.1.0/src/folio_resolve/judge.py +161 -0
- folio_resolve-0.1.0/src/folio_resolve/matching/__init__.py +5 -0
- folio_resolve-0.1.0/src/folio_resolve/matching/aho_corasick.py +139 -0
- folio_resolve-0.1.0/src/folio_resolve/ontology.py +162 -0
- folio_resolve-0.1.0/src/folio_resolve/pipeline.py +210 -0
- folio_resolve-0.1.0/src/folio_resolve/reconciler.py +230 -0
- folio_resolve-0.1.0/src/folio_resolve/resolve.py +133 -0
- folio_resolve-0.1.0/src/folio_resolve/scoring.py +298 -0
- folio_resolve-0.1.0/src/folio_resolve/sources.py +72 -0
- folio_resolve-0.1.0/src/folio_resolve/spec.py +109 -0
- folio_resolve-0.1.0/tests/conftest.py +38 -0
- folio_resolve-0.1.0/tests/test_annotate.py +103 -0
- folio_resolve-0.1.0/tests/test_judge.py +62 -0
- folio_resolve-0.1.0/tests/test_matching_and_ruler.py +66 -0
- folio_resolve-0.1.0/tests/test_new_capabilities.py +201 -0
- folio_resolve-0.1.0/tests/test_pipeline.py +89 -0
- folio_resolve-0.1.0/tests/test_reconciler.py +51 -0
- folio_resolve-0.1.0/tests/test_resolve.py +108 -0
- folio_resolve-0.1.0/tests/test_scoring.py +48 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishes folio-resolve to PyPI via PyPI Trusted Publishing (OIDC) — no API
|
|
4
|
+
# tokens or secrets are stored in this repo. Fires when a GitHub Release is
|
|
5
|
+
# published, or manually via workflow_dispatch.
|
|
6
|
+
#
|
|
7
|
+
# One-time PyPI setup (owner, before the first release):
|
|
8
|
+
# PyPI -> project "folio-resolve" -> Publishing -> add a Trusted Publisher:
|
|
9
|
+
# Owner: damienriehl / Repository: folio-resolve
|
|
10
|
+
# Workflow: publish.yml / Environment: pypi
|
|
11
|
+
|
|
12
|
+
on:
|
|
13
|
+
release:
|
|
14
|
+
types: [published]
|
|
15
|
+
workflow_dispatch:
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
build:
|
|
19
|
+
name: Build sdist + wheel
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Install uv
|
|
25
|
+
uses: astral-sh/setup-uv@v5
|
|
26
|
+
|
|
27
|
+
- name: Build distributions
|
|
28
|
+
run: uv build
|
|
29
|
+
|
|
30
|
+
- name: Upload build artifacts
|
|
31
|
+
uses: actions/upload-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
name: Publish to PyPI
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
environment: pypi
|
|
41
|
+
permissions:
|
|
42
|
+
id-token: write # required for Trusted Publishing (OIDC)
|
|
43
|
+
steps:
|
|
44
|
+
- name: Download build artifacts
|
|
45
|
+
uses: actions/download-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: dist
|
|
48
|
+
path: dist/
|
|
49
|
+
|
|
50
|
+
- name: Publish to PyPI
|
|
51
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Damien Riehl
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: folio-resolve
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Shared FOLIO source-text-to-concept resolution engine: scorer, span decomposition, gates, alias blocklist, domain-prior judge, and annotate primitives.
|
|
5
|
+
Project-URL: Homepage, https://github.com/damienriehl/folio-resolve
|
|
6
|
+
Project-URL: Repository, https://github.com/damienriehl/folio-resolve
|
|
7
|
+
Project-URL: Issues, https://github.com/damienriehl/folio-resolve/issues
|
|
8
|
+
Author: Damien Riehl
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: entity-matching,entity-resolution,folio,legal,nlp,ontology,taxonomy
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Legal Industry
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Requires-Dist: pydantic<3.0,>=2.7
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
30
|
+
Provides-Extra: embedding
|
|
31
|
+
Requires-Dist: faiss-cpu<2.0,>=1.8; extra == 'embedding'
|
|
32
|
+
Requires-Dist: numpy<3.0,>=1.26; extra == 'embedding'
|
|
33
|
+
Requires-Dist: sentence-transformers<4.0,>=3.0; extra == 'embedding'
|
|
34
|
+
Provides-Extra: folio
|
|
35
|
+
Requires-Dist: folio-python[search]>=0.1.5; extra == 'folio'
|
|
36
|
+
Provides-Extra: spacy
|
|
37
|
+
Requires-Dist: spacy<4.0,>=3.7; extra == 'spacy'
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
40
|
+
# folio-resolve
|
|
41
|
+
|
|
42
|
+
**The shared FOLIO source-text→concept matching engine.** One pinned, MIT-licensed Python library
|
|
43
|
+
that maps arbitrary source text (book headings, deposition transcripts, intake narratives, task
|
|
44
|
+
titles) to concepts in [FOLIO](https://folio.openlegalstandard.org) — the open legal ontology of
|
|
45
|
+
18,000+ concepts.
|
|
46
|
+
|
|
47
|
+
> This is a **lift-and-improve** extraction. The matching intelligence already existed across
|
|
48
|
+
> `folio-mapper`, `folio-enrich`, and `folio-insights` — but it had **diverged** (enrich literally
|
|
49
|
+
> forked mapper's scorer) and was informally shared through a fragile `sys.path` hack. `folio-resolve`
|
|
50
|
+
> is the single source of truth those repos now pin, plus the capabilities the recorded failures demanded.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Why this exists
|
|
55
|
+
|
|
56
|
+
Three repos independently built the same engine and drifted apart:
|
|
57
|
+
|
|
58
|
+
- **folio-mapper** wrote the canonical, word-order-invariant scorer + a 4-stage pipeline + a FAISS embedding index.
|
|
59
|
+
- **folio-enrich** *copied* mapper's scorer (its `search.py` says "ported from folio-mapper" in the code) and added an Aho-Corasick entity ruler, a reconciler, a domain-prior judge, and a feature-rich annotate/feedback UI.
|
|
60
|
+
- **folio-insights** `sys.path`-imports **both** siblings at runtime — documented as fragile.
|
|
61
|
+
|
|
62
|
+
A book-annotation review (Ch02) then surfaced failures that no amount of local patching would fix:
|
|
63
|
+
place-names over-scoring (Slovenia → 99 units), homonyms (Action ≠ Auction), conjoined compound
|
|
64
|
+
headings that match nothing, metadata tagged as substance, and pure-semantic maps with no shared
|
|
65
|
+
label token (Presumptions → Litigation Burdens of Proof). Those became this library's new capabilities.
|
|
66
|
+
|
|
67
|
+
## What it does
|
|
68
|
+
|
|
69
|
+
| Capability | Module | Solves |
|
|
70
|
+
|---|---|---|
|
|
71
|
+
| Word-order-invariant relevance scoring | `scoring` | "arbitration rules" = "rules of arbitration" |
|
|
72
|
+
| Multi-strategy label search + legal expansions | `scoring`, `pipeline` | recall on paraphrases |
|
|
73
|
+
| Pure-Python Aho-Corasick entity ruler | `entity_ruler`, `matching` | trusted exact-label spans, no spaCy/C deps |
|
|
74
|
+
| Candidate reconciliation (ruler + LLM + semantic) | `reconciler` | one clean candidate set with provenance |
|
|
75
|
+
| **Span decomposition** (conjunction split + shared head) | `decompose` | "Findings of Fact **and** Conclusions of Law" → both siblings |
|
|
76
|
+
| **Place-name / short-label gates** | `gates` | kills Slovenia→99 and Presumptions→Northern Mariana Islands@90 |
|
|
77
|
+
| **Alias/homonym blocklist** | `blocklist` | deterministic Action ≠ Auction guard |
|
|
78
|
+
| **Metadata/front-matter exclusion** | `sources` | never tag the copyright page |
|
|
79
|
+
| **Multi-tag domain prior** (auto-suggest + validate/add) | `domain_prior` | Defenses → *Litigation* Defenses; corpus carries many subjects |
|
|
80
|
+
| **Score calibration** (weak-band recalibration) | `calibration` | verdict-labeled score→P(correct) fit |
|
|
81
|
+
| LLM judge interface + domain-prior prompts | `judge` | context-aware disambiguation, verdict enforcement |
|
|
82
|
+
| **Annotate primitives** (confidence, per-tag verdicts, notes, reject/restore, insights) | `annotate` | the self-improving feedback loop, as a library |
|
|
83
|
+
| 4-stage pipeline: filter → expand → rank → judge | `pipeline` | the build-once-use-many entry point |
|
|
84
|
+
|
|
85
|
+
## Personas
|
|
86
|
+
|
|
87
|
+
- **Book-extraction pipelines** (`folio-insights`, `books`) — offline, CLI-first tagging of treatise text.
|
|
88
|
+
- **Intake / matter classifiers** (`alea-intake`, `mootloop`) — narrative → FOLIO concepts for routing.
|
|
89
|
+
- **Interactive mappers** (`folio-mapper`, `clio-skills`) — a UI over the library's ranked candidates.
|
|
90
|
+
- **The FOLIO ontology team** — a reference implementation of "good" source-text→concept matching.
|
|
91
|
+
|
|
92
|
+
## Use cases
|
|
93
|
+
|
|
94
|
+
- Tag a chapter of a litigation treatise with FOLIO concepts, with a litigation domain prior so
|
|
95
|
+
"Defenses" resolves to *Litigation Defenses*, not the generic sense.
|
|
96
|
+
- Resolve a compound heading like *"Proposed Findings of Fact and Conclusions of Law"* to the two
|
|
97
|
+
sibling concepts it actually names.
|
|
98
|
+
- Suggest the subject tags for a corpus ("Personal Injury Depositions" → *Personal Injury* +
|
|
99
|
+
*Deposition*) and let a human validate/add via a FOLIO taxonomy-tree picker.
|
|
100
|
+
- Feed a reviewer's `wrong` verdict on a homonym straight into the alias blocklist so the mistake
|
|
101
|
+
never recurs — the self-improving loop.
|
|
102
|
+
|
|
103
|
+
## Install
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
uv add folio-resolve # core (pure-Python, only pydantic)
|
|
107
|
+
uv add "folio-resolve[folio]" # + folio-python live ontology adapter
|
|
108
|
+
uv add "folio-resolve[embedding]" # + sentence-transformers / faiss for the semantic path
|
|
109
|
+
uv add "folio-resolve[spacy]" # + optional spaCy ruler adapter
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The **core is pure-Python** — the scorer, decomposition, gates, blocklist, domain-prior, reconciler,
|
|
113
|
+
calibration, and annotate primitives depend on nothing heavier than `pydantic`. FAISS,
|
|
114
|
+
sentence-transformers, spaCy, and folio-python live behind `Protocol` seams with working pure-Python
|
|
115
|
+
defaults, so the whole test suite runs with no model downloads and no network.
|
|
116
|
+
|
|
117
|
+
## Bring Your Own Key (BYOK)
|
|
118
|
+
|
|
119
|
+
`folio-resolve` is **key-agnostic** — it never reads an env var, instantiates a provider SDK, or makes
|
|
120
|
+
a network call on its own. The **zero-key deterministic core** (ruler, scoring, decomposition, gates,
|
|
121
|
+
blocklist, metadata exclusion, calibration, annotate) runs fully offline and free. Three optional stages
|
|
122
|
+
accept a provider through a `typing.Protocol` seam you fill with an object you construct:
|
|
123
|
+
|
|
124
|
+
| Stage | Protocol | Buys you | Absent → |
|
|
125
|
+
|---|---|---|---|
|
|
126
|
+
| **Judge** | `Judge.complete(system, user) -> str` | context-aware disambiguation + verdict enforcement | items pass through **unjudged** |
|
|
127
|
+
| **Embeddings** | `EmbeddingProvider` (`embed`/`embed_batch`/`dimension`) | semantic recall for no-shared-token maps | local `all-MiniLM-L6-v2` default (no key), or skipped |
|
|
128
|
+
| **Domain-prior suggestions** | `DomainPriorSuggester(ontology)` | auto-suggest corpus subject tags | supply tags manually |
|
|
129
|
+
|
|
130
|
+
You own the key, the vendor (OpenAI / Gemini / Anthropic / local), and the spend. The library ships the
|
|
131
|
+
judge **prompt builders** and **deterministic verdict enforcement**; you supply only the raw model call.
|
|
132
|
+
Graceful degradation is the default — no key means deterministic-only output with items marked
|
|
133
|
+
`unjudged`, never a crash. Reference cost: **≈ $0.12 / chapter on `gemini-2.5-flash-lite`** (~1,875
|
|
134
|
+
calls, ~652K tokens, 464 units). Full guide, env-var conventions, and a minimal wiring example per
|
|
135
|
+
vendor: **[docs/BYOK.md](docs/BYOK.md)**.
|
|
136
|
+
|
|
137
|
+
## Quick start
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from folio_resolve import InMemoryOntology, Concept, MatchPipeline, DomainPrior
|
|
141
|
+
|
|
142
|
+
ontology = InMemoryOntology([
|
|
143
|
+
Concept(iri="R-defenses", label="Litigation Defenses", branch="Objectives"),
|
|
144
|
+
Concept(iri="R-arb", label="Arbitration Rules", branch="Service"),
|
|
145
|
+
])
|
|
146
|
+
pipe = MatchPipeline(ontology=ontology)
|
|
147
|
+
|
|
148
|
+
# word-order-invariant
|
|
149
|
+
pipe.match("rules of arbitration") # -> Arbitration Rules
|
|
150
|
+
|
|
151
|
+
# domain prior threads a subject into the (optional) judge
|
|
152
|
+
prior = DomainPrior.from_manifest_subjects("treatise", [("R-lit", "Litigation")])
|
|
153
|
+
pipe.match("Defenses", domain_prior=prior)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## Development
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
uv sync --extra dev
|
|
160
|
+
uv run pytest # full suite, pure-Python, no network
|
|
161
|
+
uv run mypy src # strict
|
|
162
|
+
uv run ruff check
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## License & attribution
|
|
166
|
+
|
|
167
|
+
MIT — see [LICENSE](LICENSE). Every extracted component and dependency is logged in
|
|
168
|
+
[THIRD-PARTY.md](THIRD-PARTY.md). The migration schedule for consumer repos is in
|
|
169
|
+
[docs/migration/SCHEDULE.md](docs/migration/SCHEDULE.md).
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# folio-resolve
|
|
2
|
+
|
|
3
|
+
**The shared FOLIO source-text→concept matching engine.** One pinned, MIT-licensed Python library
|
|
4
|
+
that maps arbitrary source text (book headings, deposition transcripts, intake narratives, task
|
|
5
|
+
titles) to concepts in [FOLIO](https://folio.openlegalstandard.org) — the open legal ontology of
|
|
6
|
+
18,000+ concepts.
|
|
7
|
+
|
|
8
|
+
> This is a **lift-and-improve** extraction. The matching intelligence already existed across
|
|
9
|
+
> `folio-mapper`, `folio-enrich`, and `folio-insights` — but it had **diverged** (enrich literally
|
|
10
|
+
> forked mapper's scorer) and was informally shared through a fragile `sys.path` hack. `folio-resolve`
|
|
11
|
+
> is the single source of truth those repos now pin, plus the capabilities the recorded failures demanded.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Why this exists
|
|
16
|
+
|
|
17
|
+
Three repos independently built the same engine and drifted apart:
|
|
18
|
+
|
|
19
|
+
- **folio-mapper** wrote the canonical, word-order-invariant scorer + a 4-stage pipeline + a FAISS embedding index.
|
|
20
|
+
- **folio-enrich** *copied* mapper's scorer (its `search.py` says "ported from folio-mapper" in the code) and added an Aho-Corasick entity ruler, a reconciler, a domain-prior judge, and a feature-rich annotate/feedback UI.
|
|
21
|
+
- **folio-insights** `sys.path`-imports **both** siblings at runtime — documented as fragile.
|
|
22
|
+
|
|
23
|
+
A book-annotation review (Ch02) then surfaced failures that no amount of local patching would fix:
|
|
24
|
+
place-names over-scoring (Slovenia → 99 units), homonyms (Action ≠ Auction), conjoined compound
|
|
25
|
+
headings that match nothing, metadata tagged as substance, and pure-semantic maps with no shared
|
|
26
|
+
label token (Presumptions → Litigation Burdens of Proof). Those became this library's new capabilities.
|
|
27
|
+
|
|
28
|
+
## What it does
|
|
29
|
+
|
|
30
|
+
| Capability | Module | Solves |
|
|
31
|
+
|---|---|---|
|
|
32
|
+
| Word-order-invariant relevance scoring | `scoring` | "arbitration rules" = "rules of arbitration" |
|
|
33
|
+
| Multi-strategy label search + legal expansions | `scoring`, `pipeline` | recall on paraphrases |
|
|
34
|
+
| Pure-Python Aho-Corasick entity ruler | `entity_ruler`, `matching` | trusted exact-label spans, no spaCy/C deps |
|
|
35
|
+
| Candidate reconciliation (ruler + LLM + semantic) | `reconciler` | one clean candidate set with provenance |
|
|
36
|
+
| **Span decomposition** (conjunction split + shared head) | `decompose` | "Findings of Fact **and** Conclusions of Law" → both siblings |
|
|
37
|
+
| **Place-name / short-label gates** | `gates` | kills Slovenia→99 and Presumptions→Northern Mariana Islands@90 |
|
|
38
|
+
| **Alias/homonym blocklist** | `blocklist` | deterministic Action ≠ Auction guard |
|
|
39
|
+
| **Metadata/front-matter exclusion** | `sources` | never tag the copyright page |
|
|
40
|
+
| **Multi-tag domain prior** (auto-suggest + validate/add) | `domain_prior` | Defenses → *Litigation* Defenses; corpus carries many subjects |
|
|
41
|
+
| **Score calibration** (weak-band recalibration) | `calibration` | verdict-labeled score→P(correct) fit |
|
|
42
|
+
| LLM judge interface + domain-prior prompts | `judge` | context-aware disambiguation, verdict enforcement |
|
|
43
|
+
| **Annotate primitives** (confidence, per-tag verdicts, notes, reject/restore, insights) | `annotate` | the self-improving feedback loop, as a library |
|
|
44
|
+
| 4-stage pipeline: filter → expand → rank → judge | `pipeline` | the build-once-use-many entry point |
|
|
45
|
+
|
|
46
|
+
## Personas
|
|
47
|
+
|
|
48
|
+
- **Book-extraction pipelines** (`folio-insights`, `books`) — offline, CLI-first tagging of treatise text.
|
|
49
|
+
- **Intake / matter classifiers** (`alea-intake`, `mootloop`) — narrative → FOLIO concepts for routing.
|
|
50
|
+
- **Interactive mappers** (`folio-mapper`, `clio-skills`) — a UI over the library's ranked candidates.
|
|
51
|
+
- **The FOLIO ontology team** — a reference implementation of "good" source-text→concept matching.
|
|
52
|
+
|
|
53
|
+
## Use cases
|
|
54
|
+
|
|
55
|
+
- Tag a chapter of a litigation treatise with FOLIO concepts, with a litigation domain prior so
|
|
56
|
+
"Defenses" resolves to *Litigation Defenses*, not the generic sense.
|
|
57
|
+
- Resolve a compound heading like *"Proposed Findings of Fact and Conclusions of Law"* to the two
|
|
58
|
+
sibling concepts it actually names.
|
|
59
|
+
- Suggest the subject tags for a corpus ("Personal Injury Depositions" → *Personal Injury* +
|
|
60
|
+
*Deposition*) and let a human validate/add via a FOLIO taxonomy-tree picker.
|
|
61
|
+
- Feed a reviewer's `wrong` verdict on a homonym straight into the alias blocklist so the mistake
|
|
62
|
+
never recurs — the self-improving loop.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv add folio-resolve # core (pure-Python, only pydantic)
|
|
68
|
+
uv add "folio-resolve[folio]" # + folio-python live ontology adapter
|
|
69
|
+
uv add "folio-resolve[embedding]" # + sentence-transformers / faiss for the semantic path
|
|
70
|
+
uv add "folio-resolve[spacy]" # + optional spaCy ruler adapter
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The **core is pure-Python** — the scorer, decomposition, gates, blocklist, domain-prior, reconciler,
|
|
74
|
+
calibration, and annotate primitives depend on nothing heavier than `pydantic`. FAISS,
|
|
75
|
+
sentence-transformers, spaCy, and folio-python live behind `Protocol` seams with working pure-Python
|
|
76
|
+
defaults, so the whole test suite runs with no model downloads and no network.
|
|
77
|
+
|
|
78
|
+
## Bring Your Own Key (BYOK)
|
|
79
|
+
|
|
80
|
+
`folio-resolve` is **key-agnostic** — it never reads an env var, instantiates a provider SDK, or makes
|
|
81
|
+
a network call on its own. The **zero-key deterministic core** (ruler, scoring, decomposition, gates,
|
|
82
|
+
blocklist, metadata exclusion, calibration, annotate) runs fully offline and free. Three optional stages
|
|
83
|
+
accept a provider through a `typing.Protocol` seam you fill with an object you construct:
|
|
84
|
+
|
|
85
|
+
| Stage | Protocol | Buys you | Absent → |
|
|
86
|
+
|---|---|---|---|
|
|
87
|
+
| **Judge** | `Judge.complete(system, user) -> str` | context-aware disambiguation + verdict enforcement | items pass through **unjudged** |
|
|
88
|
+
| **Embeddings** | `EmbeddingProvider` (`embed`/`embed_batch`/`dimension`) | semantic recall for no-shared-token maps | local `all-MiniLM-L6-v2` default (no key), or skipped |
|
|
89
|
+
| **Domain-prior suggestions** | `DomainPriorSuggester(ontology)` | auto-suggest corpus subject tags | supply tags manually |
|
|
90
|
+
|
|
91
|
+
You own the key, the vendor (OpenAI / Gemini / Anthropic / local), and the spend. The library ships the
|
|
92
|
+
judge **prompt builders** and **deterministic verdict enforcement**; you supply only the raw model call.
|
|
93
|
+
Graceful degradation is the default — no key means deterministic-only output with items marked
|
|
94
|
+
`unjudged`, never a crash. Reference cost: **≈ $0.12 / chapter on `gemini-2.5-flash-lite`** (~1,875
|
|
95
|
+
calls, ~652K tokens, 464 units). Full guide, env-var conventions, and a minimal wiring example per
|
|
96
|
+
vendor: **[docs/BYOK.md](docs/BYOK.md)**.
|
|
97
|
+
|
|
98
|
+
## Quick start
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from folio_resolve import InMemoryOntology, Concept, MatchPipeline, DomainPrior
|
|
102
|
+
|
|
103
|
+
ontology = InMemoryOntology([
|
|
104
|
+
Concept(iri="R-defenses", label="Litigation Defenses", branch="Objectives"),
|
|
105
|
+
Concept(iri="R-arb", label="Arbitration Rules", branch="Service"),
|
|
106
|
+
])
|
|
107
|
+
pipe = MatchPipeline(ontology=ontology)
|
|
108
|
+
|
|
109
|
+
# word-order-invariant
|
|
110
|
+
pipe.match("rules of arbitration") # -> Arbitration Rules
|
|
111
|
+
|
|
112
|
+
# domain prior threads a subject into the (optional) judge
|
|
113
|
+
prior = DomainPrior.from_manifest_subjects("treatise", [("R-lit", "Litigation")])
|
|
114
|
+
pipe.match("Defenses", domain_prior=prior)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Development
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
uv sync --extra dev
|
|
121
|
+
uv run pytest # full suite, pure-Python, no network
|
|
122
|
+
uv run mypy src # strict
|
|
123
|
+
uv run ruff check
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## License & attribution
|
|
127
|
+
|
|
128
|
+
MIT — see [LICENSE](LICENSE). Every extracted component and dependency is logged in
|
|
129
|
+
[THIRD-PARTY.md](THIRD-PARTY.md). The migration schedule for consumer repos is in
|
|
130
|
+
[docs/migration/SCHEDULE.md](docs/migration/SCHEDULE.md).
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Third-Party Components & Attribution
|
|
2
|
+
|
|
3
|
+
`folio-resolve` is MIT-licensed. This file logs (1) the internal components lifted into this
|
|
4
|
+
package and (2) every third-party dependency with its license, per house policy.
|
|
5
|
+
|
|
6
|
+
## Components extracted from sibling repos (lift-and-improve)
|
|
7
|
+
|
|
8
|
+
All three source repos are Damien Riehl's own MIT-licensed projects; this is a consolidation, not
|
|
9
|
+
a third-party incorporation. Logged here for provenance.
|
|
10
|
+
|
|
11
|
+
| Component | Source repo · file | Reused as |
|
|
12
|
+
|---|---|---|
|
|
13
|
+
| Word-order-invariant scorer (`_compute_relevance_score`, `_word_overlap`, `_tokenize`, `_content_words`, `LEGAL_TERM_EXPANSIONS`, `SEARCH_STOPWORDS`, `BRANCH_SIGNAL_WORDS`) | folio-mapper · `backend/app/services/folio_service.py` | `scoring.py` |
|
|
14
|
+
| Multi-strategy search term generation | folio-mapper · `_generate_search_terms`; folio-enrich · `folio/search.py` (itself "ported from folio-mapper") | `scoring.generate_search_terms`, `pipeline.py` |
|
|
15
|
+
| 4-stage pipeline (filter → expand → rank → judge) | folio-mapper · `backend/app/services/pipeline/` | `pipeline.py` |
|
|
16
|
+
| LLM judge verdict enforcement + 90+/70-89/50-69 calibration prompt | folio-mapper · `pipeline/stage3_judge.py`, `pipeline/prompts.py` | `judge.py` |
|
|
17
|
+
| FAISS embedding index (IndexFlatIP, `all-MiniLM-L6-v2`, disk cache) | folio-mapper · `embedding/folio_index.py` | `embedding.py` (Protocol + optional adapter) |
|
|
18
|
+
| Aho-Corasick matcher (containment-aware overlap, word boundaries) | folio-enrich · `matching/aho_corasick.py` | `matching/aho_corasick.py` (reimplemented pure-Python; C dep dropped) |
|
|
19
|
+
| Entity ruler + pattern builder (label_type-encoded IDs) | folio-enrich · `entity_ruler/{ruler,pattern_builder}.py` | `entity_ruler.py` |
|
|
20
|
+
| Reconciler (diminishing boost, ruler-only gate, embedding triage) | folio-enrich · `reconciliation/reconciler.py` | `reconciler.py` |
|
|
21
|
+
| Domain-prior judge (contextual rerank + branch judge threading document_type) | folio-enrich · `llm/prompts/contextual_rerank.py`, `concept/branch_judge.py` | `judge.py`, `domain_prior.py` |
|
|
22
|
+
| Ontology-neutral spec layer (FOLIO_SPEC + CANON_SPEC, lemma denylist) | folio-enrich · `ontology/spec.py` | `spec.py` |
|
|
23
|
+
| Annotate models (Span, StageEvent, FeedbackItem, Annotation, FeedbackEntry, InsightsSummary) | folio-enrich · `models/annotation.py`, `models/feedback.py` | `annotate/models.py` |
|
|
24
|
+
| Feedback store + insights aggregation | folio-enrich · `storage/feedback_store.py` | `annotate/feedback_store.py` |
|
|
25
|
+
| Reject/restore/promote/cascade/bulk-reject lifecycle | folio-enrich · `api/routes/enrich.py` | `annotate/lifecycle.py` |
|
|
26
|
+
| Boundary-sweep span layout | folio-enrich · `frontend/index.html:renderAnnotatedText` | `annotate/render.py` (ported to Python) |
|
|
27
|
+
|
|
28
|
+
New in this package (no prior implementation): `decompose.py`, `gates.py`, `blocklist.py`,
|
|
29
|
+
`sources.py`, the multi-tag `domain_prior.py`, `calibration.py`, and the per-tag `TagVerdict`.
|
|
30
|
+
|
|
31
|
+
## Runtime dependencies
|
|
32
|
+
|
|
33
|
+
| Package | License | Used for | Tier |
|
|
34
|
+
|---|---|---|---|
|
|
35
|
+
| [pydantic](https://github.com/pydantic/pydantic) | MIT | annotate data models | core (required) |
|
|
36
|
+
| [folio-python](https://pypi.org/project/folio-python/) | MIT | live FOLIO ontology + `search_by_label` | optional (`folio` extra) |
|
|
37
|
+
| [faiss-cpu](https://github.com/facebookresearch/faiss) | MIT | ANN index for the semantic path | optional (`embedding` extra) |
|
|
38
|
+
| [sentence-transformers](https://github.com/UKPLab/sentence-transformers) | Apache-2.0 | `all-MiniLM-L6-v2` local embeddings | optional (`embedding` extra) |
|
|
39
|
+
| [numpy](https://numpy.org) | BSD-3-Clause | vector math for embeddings | optional (`embedding` extra) |
|
|
40
|
+
| [spaCy](https://github.com/explosion/spaCy) | MIT | optional EntityRuler adapter | optional (`spacy` extra) |
|
|
41
|
+
|
|
42
|
+
The **pure-Python Aho-Corasick** implementation in `matching/aho_corasick.py` deliberately removes
|
|
43
|
+
the `pyahocorasick` (BSD-3) C-extension dependency that folio-enrich required.
|
|
44
|
+
|
|
45
|
+
## Dev dependencies
|
|
46
|
+
|
|
47
|
+
| Package | License |
|
|
48
|
+
|---|---|
|
|
49
|
+
| [pytest](https://github.com/pytest-dev/pytest) | MIT |
|
|
50
|
+
| [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) | Apache-2.0 |
|
|
51
|
+
| [mypy](https://github.com/python/mypy) | MIT |
|
|
52
|
+
| [ruff](https://github.com/astral-sh/ruff) | MIT |
|
|
53
|
+
|
|
54
|
+
## Ontology data
|
|
55
|
+
|
|
56
|
+
FOLIO (Federated Open Legal Information Ontology) — CC-BY-4.0, © SALI Alliance. This package does
|
|
57
|
+
not vendor FOLIO; it is fetched at runtime via `folio-python`. Consumers displaying FOLIO concepts
|
|
58
|
+
should carry the CC-BY attribution.
|