fair-ly-accurate 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. fair_ly_accurate-0.3.0/.github/workflows/ci.yml +47 -0
  2. fair_ly_accurate-0.3.0/.github/workflows/publish.yml +58 -0
  3. fair_ly_accurate-0.3.0/.gitignore +15 -0
  4. fair_ly_accurate-0.3.0/CHANGELOG.md +90 -0
  5. fair_ly_accurate-0.3.0/LICENSE +21 -0
  6. fair_ly_accurate-0.3.0/PKG-INFO +168 -0
  7. fair_ly_accurate-0.3.0/README.md +118 -0
  8. fair_ly_accurate-0.3.0/RELEASING.md +57 -0
  9. fair_ly_accurate-0.3.0/docs/methods.md +352 -0
  10. fair_ly_accurate-0.3.0/examples/README.md +30 -0
  11. fair_ly_accurate-0.3.0/examples/sample_records.jsonl +3 -0
  12. fair_ly_accurate-0.3.0/gold/README.md +24 -0
  13. fair_ly_accurate-0.3.0/gold/affiliation.tsv +34 -0
  14. fair_ly_accurate-0.3.0/gold/funderName.tsv +27 -0
  15. fair_ly_accurate-0.3.0/gold/publisher.tsv +28 -0
  16. fair_ly_accurate-0.3.0/gold/rights.tsv +24 -0
  17. fair_ly_accurate-0.3.0/gold/subject.tsv +30 -0
  18. fair_ly_accurate-0.3.0/pyproject.toml +56 -0
  19. fair_ly_accurate-0.3.0/src/fairly/__init__.py +31 -0
  20. fair_ly_accurate-0.3.0/src/fairly/authorities/__init__.py +13 -0
  21. fair_ly_accurate-0.3.0/src/fairly/authorities/base.py +22 -0
  22. fair_ly_accurate-0.3.0/src/fairly/authorities/geocode.py +96 -0
  23. fair_ly_accurate-0.3.0/src/fairly/authorities/ror.py +111 -0
  24. fair_ly_accurate-0.3.0/src/fairly/cleaners.py +149 -0
  25. fair_ly_accurate-0.3.0/src/fairly/cli.py +154 -0
  26. fair_ly_accurate-0.3.0/src/fairly/cluster.py +105 -0
  27. fair_ly_accurate-0.3.0/src/fairly/config.py +45 -0
  28. fair_ly_accurate-0.3.0/src/fairly/controlled.py +88 -0
  29. fair_ly_accurate-0.3.0/src/fairly/embeddings.py +69 -0
  30. fair_ly_accurate-0.3.0/src/fairly/holdout.py +36 -0
  31. fair_ly_accurate-0.3.0/src/fairly/io.py +141 -0
  32. fair_ly_accurate-0.3.0/src/fairly/licenses.py +143 -0
  33. fair_ly_accurate-0.3.0/src/fairly/pipeline.py +88 -0
  34. fair_ly_accurate-0.3.0/src/fairly/presets.py +82 -0
  35. fair_ly_accurate-0.3.0/src/fairly/synonyms.py +69 -0
  36. fair_ly_accurate-0.3.0/src/fairly/validate.py +122 -0
  37. fair_ly_accurate-0.3.0/tests/test_cleaners.py +43 -0
  38. fair_ly_accurate-0.3.0/tests/test_cluster.py +41 -0
  39. fair_ly_accurate-0.3.0/tests/test_controlled.py +57 -0
  40. fair_ly_accurate-0.3.0/tests/test_holdout.py +20 -0
  41. fair_ly_accurate-0.3.0/tests/test_io.py +47 -0
  42. fair_ly_accurate-0.3.0/tests/test_ror.py +55 -0
  43. fair_ly_accurate-0.3.0/tests/test_synonyms_presets_pipeline.py +67 -0
  44. fair_ly_accurate-0.3.0/tests/test_validate.py +37 -0
@@ -0,0 +1,47 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ test:
14
+ name: Test (py${{ matrix.python-version }})
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ fail-fast: false
18
+ matrix:
19
+ # Match requires-python = ">=3.9"; test the ends of the range.
20
+ python-version: ["3.9", "3.12"]
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+
24
+ - uses: actions/setup-python@v5
25
+ with:
26
+ python-version: ${{ matrix.python-version }}
27
+
28
+ - name: Install (with dev + cluster/geo extras)
29
+ run: |
30
+ python -m pip install --upgrade pip
31
+ python -m pip install -e ".[all,dev]"
32
+
33
+ - name: Run tests
34
+ run: python -m pytest -q
35
+
36
+ build:
37
+ name: Build & check distributions
38
+ runs-on: ubuntu-latest
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+ - uses: actions/setup-python@v5
42
+ with:
43
+ python-version: "3.12"
44
+ - run: |
45
+ python -m pip install --upgrade build twine
46
+ python -m build
47
+ python -m twine check dist/*
@@ -0,0 +1,58 @@
1
+ name: Publish
2
+
3
+ # Build, then publish to PyPI via Trusted Publishing (OIDC) — no API tokens stored.
4
+ # Triggered when you publish a GitHub Release (which carries a vX.Y.Z tag).
5
+ on:
6
+ release:
7
+ types: [published]
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ build:
14
+ name: Build distributions
15
+ runs-on: ubuntu-latest
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+
19
+ - uses: actions/setup-python@v5
20
+ with:
21
+ python-version: "3.12"
22
+
23
+ - name: Build sdist + wheel
24
+ run: |
25
+ python -m pip install --upgrade build twine
26
+ python -m build
27
+ python -m twine check dist/*
28
+
29
+ - name: Verify the release tag matches the package version
30
+ run: |
31
+ pkg_version="$(ls dist/*.tar.gz | sed -E 's#.*/fair_ly_accurate-(.*)\.tar\.gz#\1#')"
32
+ tag_version="${GITHUB_REF_NAME#v}"
33
+ echo "package=$pkg_version tag=$tag_version"
34
+ if [ "$pkg_version" != "$tag_version" ]; then
35
+ echo "::error::Release tag $GITHUB_REF_NAME does not match package version $pkg_version. Bump the version in pyproject.toml and src/fairly/__init__.py before tagging."
36
+ exit 1
37
+ fi
38
+
39
+ - uses: actions/upload-artifact@v4
40
+ with:
41
+ name: dist
42
+ path: dist/
43
+
44
+ publish-pypi:
45
+ name: Publish to PyPI
46
+ needs: build
47
+ runs-on: ubuntu-latest
48
+ environment:
49
+ name: pypi
50
+ url: https://pypi.org/p/fair-ly-accurate
51
+ permissions:
52
+ id-token: write # OIDC token for Trusted Publishing
53
+ steps:
54
+ - uses: actions/download-artifact@v4
55
+ with:
56
+ name: dist
57
+ path: dist/
58
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,15 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .eggs/
5
+ build/
6
+ dist/
7
+ .pytest_cache/
8
+ .venv/
9
+ venv/
10
+ *.npy
11
+ *.pickle
12
+ *.emb.*
13
+ .ror_cache/
14
+ .geo_cache/
15
+ .DS_Store
@@ -0,0 +1,90 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [0.3.0] - 2026-08-27
9
+
10
+ ### Changed
11
+ - **First public PyPI release.** Distribution name is `fair-ly-accurate`
12
+ (`pip install fair-ly-accurate`); the import name remains `fairly`.
13
+
14
+ ### Added
15
+ - Automated publishing via **PyPI Trusted Publishing (OIDC)** on GitHub Release
16
+ (`.github/workflows/publish.yml`), a CI workflow running the test suite on
17
+ Python 3.9/3.12 (`.github/workflows/ci.yml`), and a `RELEASING.md` runbook.
18
+
19
+ ## [0.2.1] - 2026-07-16
20
+
21
+ ### Added
22
+ - Licenses: parse Creative Commons `rightsUri` values and spelled-out CC names.
23
+
24
+ ## [0.2.0] - 2026-07-14
25
+
26
+ ### Added
27
+ - **Controlled-vocabulary normalization** (`fairly.controlled.ControlledVocabulary`): a
28
+ deterministic normalizer for fixed-vocabulary fields (license ids, country codes, units,
29
+ resource types). Folds surface strings (`fold_key`: strip diacritics/case/punctuation) and
30
+ resolves them against the canonical set + an alias map to a single canonical id — never
31
+ merging two distinct ids (`CC-BY-SA-4.0` vs `CC-BY-NC-4.0`), which embedding clustering
32
+ cannot guarantee. New `strategy="controlled"` for `FieldConfig`/`Normalizer`.
33
+ - **`fairly.licenses`**: a ready SPDX/CC license `ControlledVocabulary` + `classify()`
34
+ (allowed/blocked/unknown) policy. The `rights` preset now uses it instead of clustering.
35
+ - Documented in `docs/methods.md` §6.5. On the posters.science corpus this approach
36
+ reclassified ~6,100 license values a plain whitelist match had missed.
37
+
38
+ ## [0.1.1] - 2026-07-10
39
+
40
+ ### Added
41
+ - **Reference gold sets** (`gold/`) for `affiliation`, `funderName`, `publisher`,
42
+ `subject`, `rights` — true synonym groups plus adversarial near-misses.
43
+ - **V-measure validation of the default presets.** A sweep (eps 0.05–1.0) against
44
+ the gold sets confirms every preset `eps` sits on its field's optimal plateau
45
+ (funder/subject V=1.00, publisher 0.99, affiliation 0.96, rights 0.82) and
46
+ documents the per-field over-merge cliff. Results and interpretation added to
47
+ `docs/methods.md` §8.
48
+
49
+ ### Changed
50
+ - Docs and preset docstrings now describe the `eps` values as V-measure validated
51
+ (not "proposed"). No `eps` values changed — validation confirmed the existing
52
+ settings. `rights` flagged as better served by a controlled vocabulary.
53
+
54
+ ## [0.1.0] - 2026-07-10
55
+
56
+ Initial release. A generalizable, authority-aware synonym-normalization tool for
57
+ FAIR / DataCite metadata fields, distilled from the methods proven on the
58
+ posters.science corpus.
59
+
60
+ ### Added
61
+ - **Synonym-lustre engine** (`fairly.cluster`): gte-large embeddings + HDBSCAN
62
+ (`min_cluster_size=2, min_samples=1, cluster_selection_epsilon, leaf`) with a
63
+ most-frequent-canonical (centroid tie-break) rule; disk-cached embeddings keyed
64
+ on model id + cache version.
65
+ - **Abbreviation / short-token holdout** (`fairly.holdout`): `< 4`-char and
66
+ all-caps single-token terms are held out of clustering (map to themselves) to
67
+ prevent short-string over-merges (`ZHAW → Z`).
68
+ - **ROR authority split** (`fairly.authorities.ror`): build a `name → ROR id`
69
+ index from a ROR v2 dump (ambiguous names dropped) and split semantic clusters
70
+ that span two or more institutions.
71
+ - **Geocoding normalization** (`fairly.authorities.geocode`): geocode place
72
+ fields (Nominatim, cached, rate-limited) and group by `(country, city)` →
73
+ `"City, Country"`; pluggable resolver.
74
+ - **Deterministic cleaners** (`fairly.cleaners`): composable steps + `ORG`,
75
+ `SUBJECT`, `MINIMAL` presets (balanced quote-unwrap, preamble strip, placeholder
76
+ / generic / hedge / citation drop, word-boundary-safe).
77
+ - **V-measure validation** (`fairly.validate`): gold-set homogeneity /
78
+ completeness / V-measure across an `eps` sweep, with explicit noise handling, to
79
+ tune `cluster_selection_epsilon` per field.
80
+ - **DataCite field presets** (`fairly.presets`): `publisher`, `funderName`,
81
+ `affiliation`, `subject`, `rights`, `geoLocationPlace`.
82
+ - **Field-path I/O** (`fairly.io`): CSV/TSV columns and dotted JSON/JSONL paths
83
+ through nested dicts and lists (`creators[].affiliation[].name`).
84
+ - **CLI** (`fairly`): `presets`, `build-ror-index`, `validate`, `normalize`.
85
+ - Packaging (`fairly-synonyms`), optional `cluster` / `geo` / `all` extras, MIT
86
+ license, and a test suite (30 tests) covering the pure-logic and clustering paths.
87
+
88
+ [0.2.0]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.2.0
89
+ [0.1.1]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.1.1
90
+ [0.1.0]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.1.0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 FAIR Data Innovations Hub
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,168 @@
1
+ Metadata-Version: 2.5
2
+ Name: fair-ly-accurate
3
+ Version: 0.3.0
4
+ Summary: Fair-ly accurate text synonyms for data cleaning: authority-aware synonym collapsing and normalization of FAIR / DataCite metadata fields.
5
+ Project-URL: Homepage, https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning
6
+ Project-URL: Repository, https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning
7
+ Author: FAIR Data Innovations Hub
8
+ License: MIT License
9
+
10
+ Copyright (c) 2026 FAIR Data Innovations Hub
11
+
12
+ Permission is hereby granted, free of charge, to any person obtaining a copy
13
+ of this software and associated documentation files (the "Software"), to deal
14
+ in the Software without restriction, including without limitation the rights
15
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+ copies of the Software, and to permit persons to whom the Software is
17
+ furnished to do so, subject to the following conditions:
18
+
19
+ The above copyright notice and this permission notice shall be included in all
20
+ copies or substantial portions of the Software.
21
+
22
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+ SOFTWARE.
29
+ License-File: LICENSE
30
+ Keywords: DataCite,FAIR,ROR,clustering,entity-resolution,metadata,normalization,synonyms
31
+ Classifier: Development Status :: 3 - Alpha
32
+ Classifier: Intended Audience :: Science/Research
33
+ Classifier: License :: OSI Approved :: MIT License
34
+ Classifier: Programming Language :: Python :: 3
35
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
36
+ Requires-Python: >=3.9
37
+ Requires-Dist: numpy>=1.21
38
+ Provides-Extra: all
39
+ Requires-Dist: geopy>=2.3; extra == 'all'
40
+ Requires-Dist: scikit-learn>=1.3; extra == 'all'
41
+ Requires-Dist: sentence-transformers>=2.2; extra == 'all'
42
+ Provides-Extra: cluster
43
+ Requires-Dist: scikit-learn>=1.3; extra == 'cluster'
44
+ Requires-Dist: sentence-transformers>=2.2; extra == 'cluster'
45
+ Provides-Extra: dev
46
+ Requires-Dist: pytest>=7; extra == 'dev'
47
+ Provides-Extra: geo
48
+ Requires-Dist: geopy>=2.3; extra == 'geo'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # fair-ly accurate text synonyms for data cleaning
52
+
53
+ **`fairly`** collapses spelling and format variants of FAIR / DataCite metadata
54
+ field values onto a single canonical form — *fairly accurately* — so that
55
+ `Univ. of California, Berkeley`, `University of California-Berkeley` and
56
+ `UC Berkeley` all become one publisher/affiliation, while genuinely different
57
+ entities (`University of Washington` vs `Washington University`) stay apart.
58
+
59
+ It packages four cooperating strategies, each earning its place against a real
60
+ failure mode:
61
+
62
+ | Strategy | What it does | Guards against |
63
+ |---|---|---|
64
+ | **Synonym-lustre** | embed distinct values (`gte-large`) → cluster (HDBSCAN) → map each cluster to its most-frequent member | spelling/format/punctuation variants |
65
+ | **Abbreviation holdout** | keep acronyms and `< 4`-char tokens *out* of clustering | `ZHAW → Z`, `LUH → HU` over-merges |
66
+ | **ROR authority split** | split a cluster that spans two real institutions using a local ROR dump | `University of Washington` vs `Washington University` |
67
+ | **Geocoding** | for place fields, geocode and group by `(country, city)` → `"City, Country"` | `Graz` merging into `Vienna` ("city in Austria") |
68
+ | **Controlled vocab** | for fixed-vocabulary fields (licenses, codes), deterministically fold + alias-resolve to a canonical id | `CC-BY-SA` collapsing into `CC-BY-NC` — clustering can't tell them apart |
69
+
70
+ Quality is measured, not assumed: a **V-measure** harness tunes the clustering
71
+ threshold per field against a small gold set.
72
+
73
+ > Method background and per-field recommendations: **[docs/methods.md](docs/methods.md)**.
74
+
75
+ ## Install
76
+
77
+ ```bash
78
+ pip install fair-ly-accurate # core (cleaners, ROR index, I/O, V-measure math)
79
+ pip install "fair-ly-accurate[cluster]" # + embeddings & HDBSCAN (semantic clustering)
80
+ pip install "fair-ly-accurate[geo]" # + geopy (place geocoding)
81
+ pip install "fair-ly-accurate[all]" # everything
82
+ ```
83
+
84
+ (The distribution installs as `fair-ly-accurate`; the import name is `fairly`: `from fairly import Normalizer`.)
85
+
86
+ The heavy ML/geo dependencies are optional extras, so the deterministic cleaners,
87
+ ROR index and field-path I/O install light.
88
+
89
+ ## Quickstart (CLI)
90
+
91
+ ```bash
92
+ # 0. (once, for institution fields) build a ROR index from a ROR v2 dump
93
+ fairly build-ror-index --dump v2-ror-data.json --out ror_index.pickle
94
+
95
+ # 1. tune the threshold for a field against a gold set (variant<TAB>true_canonical)
96
+ fairly validate --gold affiliation_gold.tsv --preset affiliation --field affiliation
97
+
98
+ # 2. build + apply a synonym map to a DataCite-shaped field, with review + ROR split
99
+ fairly normalize \
100
+ --input records.jsonl \
101
+ --field-path 'creators[].affiliation[].name' \
102
+ --preset affiliation \
103
+ --ror-index ror_index.pickle \
104
+ --review affiliation_review.tsv \
105
+ --output records.normalized.jsonl
106
+ ```
107
+
108
+ The `--field-path` understands dotted paths through nested dicts and lists, e.g.
109
+ `publisher.name`, `fundingReferences[].funderName`, `subjects[].subject`,
110
+ `creators[].affiliation[].name`. Tabular inputs use the column name as the path.
111
+
112
+ **Always eyeball the `--review` TSV before trusting an apply** — especially for
113
+ homogeneous fields like funders, where over-merge is easy. Add `--dry-run` to
114
+ build the map and review without writing.
115
+
116
+ ## Quickstart (library)
117
+
118
+ ```python
119
+ from fairly import Normalizer, get_preset
120
+ from fairly.authorities.ror import RorIndex
121
+ from fairly.io import collect_counts, read_records, map_values
122
+
123
+ records = read_records("records.jsonl")
124
+ counts = collect_counts(records, "fundingReferences[].funderName")
125
+
126
+ norm = Normalizer(get_preset("funderName"), cache_dir=".cache")
127
+ smap = norm.build(counts, authority=RorIndex.load("ror_index.pickle"))
128
+ smap.write_review("funder_review.tsv", counts) # review gate
129
+
130
+ for rec in records:
131
+ map_values(rec, "fundingReferences[].funderName", lambda v: norm.normalize(v, smap))
132
+ ```
133
+
134
+ ## Field presets
135
+
136
+ Built-in `FieldConfig` presets for common DataCite/FAIR fields (`fairly presets`):
137
+
138
+ | Field | Strategy | eps | Holdout | Authority |
139
+ |---|---|---|---|---|
140
+ | `publisher` | semantic | 0.30 | yes | ROR |
141
+ | `funderName` | semantic | 0.07 | yes | ROR |
142
+ | `affiliation` | semantic | 0.07 | yes | ROR |
143
+ | `subject` | semantic | 0.10 | yes | — |
144
+ | `rights` | **controlled** | — | — | SPDX/CC vocab |
145
+ | `geoLocationPlace` | geocode | — | — | — |
146
+
147
+ These `eps` values are **V-measure validated** against the curated gold sets in
148
+ [`gold/`](gold/) — each sits on its field's optimal plateau (funder/subject 1.00,
149
+ publisher 0.99, affiliation 0.96; `rights` 0.82, so prefer a controlled vocabulary
150
+ for licenses). **Re-tune for your data** with `fairly validate` — see
151
+ [docs/methods.md](docs/methods.md#8-v-measure-validation) for the full sweep and why
152
+ each field wants a different threshold.
153
+
154
+ ## Why these choices
155
+
156
+ - **Most-frequent canonical** gives clean, recognizable labels (the common form
157
+ wins), with centroid-closeness only breaking ties.
158
+ - **Holdout** exists because short/all-caps tokens carry too little signal for an
159
+ embedding to place; left in, they become spurious join keys.
160
+ - **Authority split** is what lets us cluster *aggressively* for coverage without
161
+ paying for it in precision: the embedding proposes merges, ROR/geocoding vetoes
162
+ the wrong ones.
163
+ - **V-measure** turns "did we over-merge?" into a number: homogeneity penalizes
164
+ over-merge (the costly direction), completeness penalizes under-merge.
165
+
166
+ ## License
167
+
168
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,118 @@
1
+ # fair-ly accurate text synonyms for data cleaning
2
+
3
+ **`fairly`** collapses spelling and format variants of FAIR / DataCite metadata
4
+ field values onto a single canonical form — *fairly accurately* — so that
5
+ `Univ. of California, Berkeley`, `University of California-Berkeley` and
6
+ `UC Berkeley` all become one publisher/affiliation, while genuinely different
7
+ entities (`University of Washington` vs `Washington University`) stay apart.
8
+
9
+ It packages four cooperating strategies, each earning its place against a real
10
+ failure mode:
11
+
12
+ | Strategy | What it does | Guards against |
13
+ |---|---|---|
14
+ | **Synonym-lustre** | embed distinct values (`gte-large`) → cluster (HDBSCAN) → map each cluster to its most-frequent member | spelling/format/punctuation variants |
15
+ | **Abbreviation holdout** | keep acronyms and `< 4`-char tokens *out* of clustering | `ZHAW → Z`, `LUH → HU` over-merges |
16
+ | **ROR authority split** | split a cluster that spans two real institutions using a local ROR dump | `University of Washington` vs `Washington University` |
17
+ | **Geocoding** | for place fields, geocode and group by `(country, city)` → `"City, Country"` | `Graz` merging into `Vienna` ("city in Austria") |
18
+ | **Controlled vocab** | for fixed-vocabulary fields (licenses, codes), deterministically fold + alias-resolve to a canonical id | `CC-BY-SA` collapsing into `CC-BY-NC` — clustering can't tell them apart |
19
+
20
+ Quality is measured, not assumed: a **V-measure** harness tunes the clustering
21
+ threshold per field against a small gold set.
22
+
23
+ > Method background and per-field recommendations: **[docs/methods.md](docs/methods.md)**.
24
+
25
+ ## Install
26
+
27
+ ```bash
28
+ pip install fair-ly-accurate # core (cleaners, ROR index, I/O, V-measure math)
29
+ pip install "fair-ly-accurate[cluster]" # + embeddings & HDBSCAN (semantic clustering)
30
+ pip install "fair-ly-accurate[geo]" # + geopy (place geocoding)
31
+ pip install "fair-ly-accurate[all]" # everything
32
+ ```
33
+
34
+ (The distribution installs as `fair-ly-accurate`; the import name is `fairly`: `from fairly import Normalizer`.)
35
+
36
+ The heavy ML/geo dependencies are optional extras, so the deterministic cleaners,
37
+ ROR index and field-path I/O install light.
38
+
39
+ ## Quickstart (CLI)
40
+
41
+ ```bash
42
+ # 0. (once, for institution fields) build a ROR index from a ROR v2 dump
43
+ fairly build-ror-index --dump v2-ror-data.json --out ror_index.pickle
44
+
45
+ # 1. tune the threshold for a field against a gold set (variant<TAB>true_canonical)
46
+ fairly validate --gold affiliation_gold.tsv --preset affiliation --field affiliation
47
+
48
+ # 2. build + apply a synonym map to a DataCite-shaped field, with review + ROR split
49
+ fairly normalize \
50
+ --input records.jsonl \
51
+ --field-path 'creators[].affiliation[].name' \
52
+ --preset affiliation \
53
+ --ror-index ror_index.pickle \
54
+ --review affiliation_review.tsv \
55
+ --output records.normalized.jsonl
56
+ ```
57
+
58
+ The `--field-path` understands dotted paths through nested dicts and lists, e.g.
59
+ `publisher.name`, `fundingReferences[].funderName`, `subjects[].subject`,
60
+ `creators[].affiliation[].name`. Tabular inputs use the column name as the path.
61
+
62
+ **Always eyeball the `--review` TSV before trusting an apply** — especially for
63
+ homogeneous fields like funders, where over-merge is easy. Add `--dry-run` to
64
+ build the map and review without writing.
65
+
66
+ ## Quickstart (library)
67
+
68
+ ```python
69
+ from fairly import Normalizer, get_preset
70
+ from fairly.authorities.ror import RorIndex
71
+ from fairly.io import collect_counts, read_records, map_values
72
+
73
+ records = read_records("records.jsonl")
74
+ counts = collect_counts(records, "fundingReferences[].funderName")
75
+
76
+ norm = Normalizer(get_preset("funderName"), cache_dir=".cache")
77
+ smap = norm.build(counts, authority=RorIndex.load("ror_index.pickle"))
78
+ smap.write_review("funder_review.tsv", counts) # review gate
79
+
80
+ for rec in records:
81
+ map_values(rec, "fundingReferences[].funderName", lambda v: norm.normalize(v, smap))
82
+ ```
83
+
84
+ ## Field presets
85
+
86
+ Built-in `FieldConfig` presets for common DataCite/FAIR fields (`fairly presets`):
87
+
88
+ | Field | Strategy | eps | Holdout | Authority |
89
+ |---|---|---|---|---|
90
+ | `publisher` | semantic | 0.30 | yes | ROR |
91
+ | `funderName` | semantic | 0.07 | yes | ROR |
92
+ | `affiliation` | semantic | 0.07 | yes | ROR |
93
+ | `subject` | semantic | 0.10 | yes | — |
94
+ | `rights` | **controlled** | — | — | SPDX/CC vocab |
95
+ | `geoLocationPlace` | geocode | — | — | — |
96
+
97
+ These `eps` values are **V-measure validated** against the curated gold sets in
98
+ [`gold/`](gold/) — each sits on its field's optimal plateau (funder/subject 1.00,
99
+ publisher 0.99, affiliation 0.96; `rights` 0.82, so prefer a controlled vocabulary
100
+ for licenses). **Re-tune for your data** with `fairly validate` — see
101
+ [docs/methods.md](docs/methods.md#8-v-measure-validation) for the full sweep and why
102
+ each field wants a different threshold.
103
+
104
+ ## Why these choices
105
+
106
+ - **Most-frequent canonical** gives clean, recognizable labels (the common form
107
+ wins), with centroid-closeness only breaking ties.
108
+ - **Holdout** exists because short/all-caps tokens carry too little signal for an
109
+ embedding to place; left in, they become spurious join keys.
110
+ - **Authority split** is what lets us cluster *aggressively* for coverage without
111
+ paying for it in precision: the embedding proposes merges, ROR/geocoding vetoes
112
+ the wrong ones.
113
+ - **V-measure** turns "did we over-merge?" into a number: homogeneity penalizes
114
+ over-merge (the costly direction), completeness penalizes under-merge.
115
+
116
+ ## License
117
+
118
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,57 @@
1
+ # Releasing `fair-ly-accurate` to PyPI
2
+
3
+ Publishing is automated with **PyPI Trusted Publishing (OIDC)** — GitHub Actions
4
+ mints a short-lived token per run, so there are no API tokens to store or rotate.
5
+ The workflow lives in [`.github/workflows/publish.yml`](.github/workflows/publish.yml):
6
+ publishing a **GitHub Release** builds the distributions and uploads them to PyPI.
7
+
8
+ ## One-time setup
9
+
10
+ Register this repo as a *trusted publisher* on PyPI. Because the project does not
11
+ exist yet, use the "pending publisher" form:
12
+
13
+ - **PyPI** — https://pypi.org/manage/account/publishing/ → *Add a pending publisher*
14
+ - PyPI Project Name: `fair-ly-accurate`
15
+ - Owner: `fairdataihub`
16
+ - Repository name: `fair-ly-accurate-text-synonyms-for-data-cleaning`
17
+ - Workflow name: `publish.yml`
18
+ - Environment name: `pypi`
19
+
20
+ The first successful publish creates the `fair-ly-accurate` project owned by the account
21
+ that registered the pending publisher.
22
+
23
+ (Optional, recommended) In GitHub → *Settings → Environments*, create an
24
+ environment named `pypi` and add yourself as a **required reviewer**, so a publish
25
+ needs a human click even though the repo may have other maintainers with write
26
+ access.
27
+
28
+ ## Cutting a release
29
+
30
+ 1. Bump the version in **both** places (they must match, or the workflow fails the build):
31
+ - `pyproject.toml` → `version = "X.Y.Z"`
32
+ - `src/fairly/__init__.py` → `__version__ = "X.Y.Z"`
33
+ - Add a `CHANGELOG.md` entry.
34
+ 2. Commit, then tag and push:
35
+ ```bash
36
+ git commit -am "Release X.Y.Z"
37
+ git tag vX.Y.Z
38
+ git push && git push --tags
39
+ ```
40
+ 3. Create a **GitHub Release** for that tag
41
+ (Releases → Draft a new release → choose `vX.Y.Z` → Publish). That triggers the
42
+ PyPI publish job.
43
+ 4. Confirm:
44
+ ```bash
45
+ pip install fair-ly-accurate==X.Y.Z
46
+ ```
47
+
48
+ PyPI refuses to overwrite an existing version, so every release needs a fresh
49
+ version number.
50
+
51
+ ## Local build check
52
+
53
+ ```bash
54
+ python -m pip install build twine
55
+ python -m build
56
+ python -m twine check dist/*
57
+ ```