fair-ly-accurate 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fair_ly_accurate-0.3.0/.github/workflows/ci.yml +47 -0
- fair_ly_accurate-0.3.0/.github/workflows/publish.yml +58 -0
- fair_ly_accurate-0.3.0/.gitignore +15 -0
- fair_ly_accurate-0.3.0/CHANGELOG.md +90 -0
- fair_ly_accurate-0.3.0/LICENSE +21 -0
- fair_ly_accurate-0.3.0/PKG-INFO +168 -0
- fair_ly_accurate-0.3.0/README.md +118 -0
- fair_ly_accurate-0.3.0/RELEASING.md +57 -0
- fair_ly_accurate-0.3.0/docs/methods.md +352 -0
- fair_ly_accurate-0.3.0/examples/README.md +30 -0
- fair_ly_accurate-0.3.0/examples/sample_records.jsonl +3 -0
- fair_ly_accurate-0.3.0/gold/README.md +24 -0
- fair_ly_accurate-0.3.0/gold/affiliation.tsv +34 -0
- fair_ly_accurate-0.3.0/gold/funderName.tsv +27 -0
- fair_ly_accurate-0.3.0/gold/publisher.tsv +28 -0
- fair_ly_accurate-0.3.0/gold/rights.tsv +24 -0
- fair_ly_accurate-0.3.0/gold/subject.tsv +30 -0
- fair_ly_accurate-0.3.0/pyproject.toml +56 -0
- fair_ly_accurate-0.3.0/src/fairly/__init__.py +31 -0
- fair_ly_accurate-0.3.0/src/fairly/authorities/__init__.py +13 -0
- fair_ly_accurate-0.3.0/src/fairly/authorities/base.py +22 -0
- fair_ly_accurate-0.3.0/src/fairly/authorities/geocode.py +96 -0
- fair_ly_accurate-0.3.0/src/fairly/authorities/ror.py +111 -0
- fair_ly_accurate-0.3.0/src/fairly/cleaners.py +149 -0
- fair_ly_accurate-0.3.0/src/fairly/cli.py +154 -0
- fair_ly_accurate-0.3.0/src/fairly/cluster.py +105 -0
- fair_ly_accurate-0.3.0/src/fairly/config.py +45 -0
- fair_ly_accurate-0.3.0/src/fairly/controlled.py +88 -0
- fair_ly_accurate-0.3.0/src/fairly/embeddings.py +69 -0
- fair_ly_accurate-0.3.0/src/fairly/holdout.py +36 -0
- fair_ly_accurate-0.3.0/src/fairly/io.py +141 -0
- fair_ly_accurate-0.3.0/src/fairly/licenses.py +143 -0
- fair_ly_accurate-0.3.0/src/fairly/pipeline.py +88 -0
- fair_ly_accurate-0.3.0/src/fairly/presets.py +82 -0
- fair_ly_accurate-0.3.0/src/fairly/synonyms.py +69 -0
- fair_ly_accurate-0.3.0/src/fairly/validate.py +122 -0
- fair_ly_accurate-0.3.0/tests/test_cleaners.py +43 -0
- fair_ly_accurate-0.3.0/tests/test_cluster.py +41 -0
- fair_ly_accurate-0.3.0/tests/test_controlled.py +57 -0
- fair_ly_accurate-0.3.0/tests/test_holdout.py +20 -0
- fair_ly_accurate-0.3.0/tests/test_io.py +47 -0
- fair_ly_accurate-0.3.0/tests/test_ror.py +55 -0
- fair_ly_accurate-0.3.0/tests/test_synonyms_presets_pipeline.py +67 -0
- fair_ly_accurate-0.3.0/tests/test_validate.py +37 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
test:
|
|
14
|
+
name: Test (py${{ matrix.python-version }})
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
strategy:
|
|
17
|
+
fail-fast: false
|
|
18
|
+
matrix:
|
|
19
|
+
# Match requires-python = ">=3.9"; test the ends of the range.
|
|
20
|
+
python-version: ["3.9", "3.12"]
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: ${{ matrix.python-version }}
|
|
27
|
+
|
|
28
|
+
- name: Install (with dev + cluster/geo extras)
|
|
29
|
+
run: |
|
|
30
|
+
python -m pip install --upgrade pip
|
|
31
|
+
python -m pip install -e ".[all,dev]"
|
|
32
|
+
|
|
33
|
+
- name: Run tests
|
|
34
|
+
run: python -m pytest -q
|
|
35
|
+
|
|
36
|
+
build:
|
|
37
|
+
name: Build & check distributions
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v4
|
|
41
|
+
- uses: actions/setup-python@v5
|
|
42
|
+
with:
|
|
43
|
+
python-version: "3.12"
|
|
44
|
+
- run: |
|
|
45
|
+
python -m pip install --upgrade build twine
|
|
46
|
+
python -m build
|
|
47
|
+
python -m twine check dist/*
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
# Build, then publish to PyPI via Trusted Publishing (OIDC) — no API tokens stored.
|
|
4
|
+
# Triggered when you publish a GitHub Release (which carries a vX.Y.Z tag).
|
|
5
|
+
on:
|
|
6
|
+
release:
|
|
7
|
+
types: [published]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build:
|
|
14
|
+
name: Build distributions
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12"
|
|
22
|
+
|
|
23
|
+
- name: Build sdist + wheel
|
|
24
|
+
run: |
|
|
25
|
+
python -m pip install --upgrade build twine
|
|
26
|
+
python -m build
|
|
27
|
+
python -m twine check dist/*
|
|
28
|
+
|
|
29
|
+
- name: Verify the release tag matches the package version
|
|
30
|
+
run: |
|
|
31
|
+
pkg_version="$(ls dist/*.tar.gz | sed -E 's#.*/fair_ly_accurate-(.*)\.tar\.gz#\1#')"
|
|
32
|
+
tag_version="${GITHUB_REF_NAME#v}"
|
|
33
|
+
echo "package=$pkg_version tag=$tag_version"
|
|
34
|
+
if [ "$pkg_version" != "$tag_version" ]; then
|
|
35
|
+
echo "::error::Release tag $GITHUB_REF_NAME does not match package version $pkg_version. Bump the version in pyproject.toml and src/fairly/__init__.py before tagging."
|
|
36
|
+
exit 1
|
|
37
|
+
fi
|
|
38
|
+
|
|
39
|
+
- uses: actions/upload-artifact@v4
|
|
40
|
+
with:
|
|
41
|
+
name: dist
|
|
42
|
+
path: dist/
|
|
43
|
+
|
|
44
|
+
publish-pypi:
|
|
45
|
+
name: Publish to PyPI
|
|
46
|
+
needs: build
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
environment:
|
|
49
|
+
name: pypi
|
|
50
|
+
url: https://pypi.org/p/fair-ly-accurate
|
|
51
|
+
permissions:
|
|
52
|
+
id-token: write # OIDC token for Trusted Publishing
|
|
53
|
+
steps:
|
|
54
|
+
- uses: actions/download-artifact@v4
|
|
55
|
+
with:
|
|
56
|
+
name: dist
|
|
57
|
+
path: dist/
|
|
58
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.3.0] - 2026-08-27
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- **First public PyPI release.** Distribution name is `fair-ly-accurate`
|
|
12
|
+
(`pip install fair-ly-accurate`); the import name remains `fairly`.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
- Automated publishing via **PyPI Trusted Publishing (OIDC)** on GitHub Release
|
|
16
|
+
(`.github/workflows/publish.yml`), a CI workflow running the test suite on
|
|
17
|
+
Python 3.9/3.12 (`.github/workflows/ci.yml`), and a `RELEASING.md` runbook.
|
|
18
|
+
|
|
19
|
+
## [0.2.1] - 2026-07-16
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
- Licenses: parse Creative Commons `rightsUri` values and spelled-out CC names.
|
|
23
|
+
|
|
24
|
+
## [0.2.0] - 2026-07-14
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
- **Controlled-vocabulary normalization** (`fairly.controlled.ControlledVocabulary`): a
|
|
28
|
+
deterministic normalizer for fixed-vocabulary fields (license ids, country codes, units,
|
|
29
|
+
resource types). Folds surface strings (`fold_key`: strip diacritics/case/punctuation) and
|
|
30
|
+
resolves them against the canonical set + an alias map to a single canonical id — never
|
|
31
|
+
merging two distinct ids (`CC-BY-SA-4.0` vs `CC-BY-NC-4.0`), which embedding clustering
|
|
32
|
+
cannot guarantee. New `strategy="controlled"` for `FieldConfig`/`Normalizer`.
|
|
33
|
+
- **`fairly.licenses`**: a ready SPDX/CC license `ControlledVocabulary` + `classify()`
|
|
34
|
+
(allowed/blocked/unknown) policy. The `rights` preset now uses it instead of clustering.
|
|
35
|
+
- Documented in `docs/methods.md` §6.5. On the posters.science corpus this approach
|
|
36
|
+
reclassified ~6,100 license values a plain whitelist match had missed.
|
|
37
|
+
|
|
38
|
+
## [0.1.1] - 2026-07-10
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
- **Reference gold sets** (`gold/`) for `affiliation`, `funderName`, `publisher`,
|
|
42
|
+
`subject`, `rights` — true synonym groups plus adversarial near-misses.
|
|
43
|
+
- **V-measure validation of the default presets.** A sweep (eps 0.05–1.0) against
|
|
44
|
+
the gold sets confirms every preset `eps` sits on its field's optimal plateau
|
|
45
|
+
(funder/subject V=1.00, publisher 0.99, affiliation 0.96, rights 0.82) and
|
|
46
|
+
documents the per-field over-merge cliff. Results and interpretation added to
|
|
47
|
+
`docs/methods.md` §8.
|
|
48
|
+
|
|
49
|
+
### Changed
|
|
50
|
+
- Docs and preset docstrings now describe the `eps` values as V-measure validated
|
|
51
|
+
(not "proposed"). No `eps` values changed — validation confirmed the existing
|
|
52
|
+
settings. `rights` flagged as better served by a controlled vocabulary.
|
|
53
|
+
|
|
54
|
+
## [0.1.0] - 2026-07-10
|
|
55
|
+
|
|
56
|
+
Initial release. A generalizable, authority-aware synonym-normalization tool for
|
|
57
|
+
FAIR / DataCite metadata fields, distilled from the methods proven on the
|
|
58
|
+
posters.science corpus.
|
|
59
|
+
|
|
60
|
+
### Added
|
|
61
|
+
- **Synonym-lustre engine** (`fairly.cluster`): gte-large embeddings + HDBSCAN
|
|
62
|
+
(`min_cluster_size=2, min_samples=1, cluster_selection_epsilon, leaf`) with a
|
|
63
|
+
most-frequent-canonical (centroid tie-break) rule; disk-cached embeddings keyed
|
|
64
|
+
on model id + cache version.
|
|
65
|
+
- **Abbreviation / short-token holdout** (`fairly.holdout`): `< 4`-char and
|
|
66
|
+
all-caps single-token terms are held out of clustering (map to themselves) to
|
|
67
|
+
prevent short-string over-merges (`ZHAW → Z`).
|
|
68
|
+
- **ROR authority split** (`fairly.authorities.ror`): build a `name → ROR id`
|
|
69
|
+
index from a ROR v2 dump (ambiguous names dropped) and split semantic clusters
|
|
70
|
+
that span two or more institutions.
|
|
71
|
+
- **Geocoding normalization** (`fairly.authorities.geocode`): geocode place
|
|
72
|
+
fields (Nominatim, cached, rate-limited) and group by `(country, city)` →
|
|
73
|
+
`"City, Country"`; pluggable resolver.
|
|
74
|
+
- **Deterministic cleaners** (`fairly.cleaners`): composable steps + `ORG`,
|
|
75
|
+
`SUBJECT`, `MINIMAL` presets (balanced quote-unwrap, preamble strip, placeholder
|
|
76
|
+
/ generic / hedge / citation drop, word-boundary-safe).
|
|
77
|
+
- **V-measure validation** (`fairly.validate`): gold-set homogeneity /
|
|
78
|
+
completeness / V-measure across an `eps` sweep, with explicit noise handling, to
|
|
79
|
+
tune `cluster_selection_epsilon` per field.
|
|
80
|
+
- **DataCite field presets** (`fairly.presets`): `publisher`, `funderName`,
|
|
81
|
+
`affiliation`, `subject`, `rights`, `geoLocationPlace`.
|
|
82
|
+
- **Field-path I/O** (`fairly.io`): CSV/TSV columns and dotted JSON/JSONL paths
|
|
83
|
+
through nested dicts and lists (`creators[].affiliation[].name`).
|
|
84
|
+
- **CLI** (`fairly`): `presets`, `build-ror-index`, `validate`, `normalize`.
|
|
85
|
+
- Packaging (`fairly-synonyms`), optional `cluster` / `geo` / `all` extras, MIT
|
|
86
|
+
license, and a test suite (30 tests) covering the pure-logic and clustering paths.
|
|
87
|
+
|
|
88
|
+
[0.2.0]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.2.0
|
|
89
|
+
[0.1.1]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.1.1
|
|
90
|
+
[0.1.0]: https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 FAIR Data Innovations Hub
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fair-ly-accurate
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Fair-ly accurate text synonyms for data cleaning: authority-aware synonym collapsing and normalization of FAIR / DataCite metadata fields.
|
|
5
|
+
Project-URL: Homepage, https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning
|
|
6
|
+
Project-URL: Repository, https://github.com/fairdataihub/fair-ly-accurate-text-synonyms-for-data-cleaning
|
|
7
|
+
Author: FAIR Data Innovations Hub
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 FAIR Data Innovations Hub
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: DataCite,FAIR,ROR,clustering,entity-resolution,metadata,normalization,synonyms
|
|
31
|
+
Classifier: Development Status :: 3 - Alpha
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
36
|
+
Requires-Python: >=3.9
|
|
37
|
+
Requires-Dist: numpy>=1.21
|
|
38
|
+
Provides-Extra: all
|
|
39
|
+
Requires-Dist: geopy>=2.3; extra == 'all'
|
|
40
|
+
Requires-Dist: scikit-learn>=1.3; extra == 'all'
|
|
41
|
+
Requires-Dist: sentence-transformers>=2.2; extra == 'all'
|
|
42
|
+
Provides-Extra: cluster
|
|
43
|
+
Requires-Dist: scikit-learn>=1.3; extra == 'cluster'
|
|
44
|
+
Requires-Dist: sentence-transformers>=2.2; extra == 'cluster'
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
47
|
+
Provides-Extra: geo
|
|
48
|
+
Requires-Dist: geopy>=2.3; extra == 'geo'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
# fair-ly accurate text synonyms for data cleaning
|
|
52
|
+
|
|
53
|
+
**`fairly`** collapses spelling and format variants of FAIR / DataCite metadata
|
|
54
|
+
field values onto a single canonical form — *fairly accurately* — so that
|
|
55
|
+
`Univ. of California, Berkeley`, `University of California-Berkeley` and
|
|
56
|
+
`UC Berkeley` all become one publisher/affiliation, while genuinely different
|
|
57
|
+
entities (`University of Washington` vs `Washington University`) stay apart.
|
|
58
|
+
|
|
59
|
+
It packages four cooperating strategies, each earning its place against a real
|
|
60
|
+
failure mode:
|
|
61
|
+
|
|
62
|
+
| Strategy | What it does | Guards against |
|
|
63
|
+
|---|---|---|
|
|
64
|
+
| **Synonym-lustre** | embed distinct values (`gte-large`) → cluster (HDBSCAN) → map each cluster to its most-frequent member | spelling/format/punctuation variants |
|
|
65
|
+
| **Abbreviation holdout** | keep acronyms and `< 4`-char tokens *out* of clustering | `ZHAW → Z`, `LUH → HU` over-merges |
|
|
66
|
+
| **ROR authority split** | split a cluster that spans two real institutions using a local ROR dump | `University of Washington` vs `Washington University` |
|
|
67
|
+
| **Geocoding** | for place fields, geocode and group by `(country, city)` → `"City, Country"` | `Graz` merging into `Vienna` ("city in Austria") |
|
|
68
|
+
| **Controlled vocab** | for fixed-vocabulary fields (licenses, codes), deterministically fold + alias-resolve to a canonical id | `CC-BY-SA` collapsing into `CC-BY-NC` — clustering can't tell them apart |
|
|
69
|
+
|
|
70
|
+
Quality is measured, not assumed: a **V-measure** harness tunes the clustering
|
|
71
|
+
threshold per field against a small gold set.
|
|
72
|
+
|
|
73
|
+
> Method background and per-field recommendations: **[docs/methods.md](docs/methods.md)**.
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install fair-ly-accurate # core (cleaners, ROR index, I/O, V-measure math)
|
|
79
|
+
pip install "fair-ly-accurate[cluster]" # + embeddings & HDBSCAN (semantic clustering)
|
|
80
|
+
pip install "fair-ly-accurate[geo]" # + geopy (place geocoding)
|
|
81
|
+
pip install "fair-ly-accurate[all]" # everything
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
(The distribution installs as `fair-ly-accurate`; the import name is `fairly`: `from fairly import Normalizer`.)
|
|
85
|
+
|
|
86
|
+
The heavy ML/geo dependencies are optional extras, so the deterministic cleaners,
|
|
87
|
+
ROR index and field-path I/O install light.
|
|
88
|
+
|
|
89
|
+
## Quickstart (CLI)
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
# 0. (once, for institution fields) build a ROR index from a ROR v2 dump
|
|
93
|
+
fairly build-ror-index --dump v2-ror-data.json --out ror_index.pickle
|
|
94
|
+
|
|
95
|
+
# 1. tune the threshold for a field against a gold set (variant<TAB>true_canonical)
|
|
96
|
+
fairly validate --gold affiliation_gold.tsv --preset affiliation --field affiliation
|
|
97
|
+
|
|
98
|
+
# 2. build + apply a synonym map to a DataCite-shaped field, with review + ROR split
|
|
99
|
+
fairly normalize \
|
|
100
|
+
--input records.jsonl \
|
|
101
|
+
--field-path 'creators[].affiliation[].name' \
|
|
102
|
+
--preset affiliation \
|
|
103
|
+
--ror-index ror_index.pickle \
|
|
104
|
+
--review affiliation_review.tsv \
|
|
105
|
+
--output records.normalized.jsonl
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The `--field-path` understands dotted paths through nested dicts and lists, e.g.
|
|
109
|
+
`publisher.name`, `fundingReferences[].funderName`, `subjects[].subject`,
|
|
110
|
+
`creators[].affiliation[].name`. Tabular inputs use the column name as the path.
|
|
111
|
+
|
|
112
|
+
**Always eyeball the `--review` TSV before trusting an apply** — especially for
|
|
113
|
+
homogeneous fields like funders, where over-merge is easy. Add `--dry-run` to
|
|
114
|
+
build the map and review without writing.
|
|
115
|
+
|
|
116
|
+
## Quickstart (library)
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from fairly import Normalizer, get_preset
|
|
120
|
+
from fairly.authorities.ror import RorIndex
|
|
121
|
+
from fairly.io import collect_counts, read_records, map_values
|
|
122
|
+
|
|
123
|
+
records = read_records("records.jsonl")
|
|
124
|
+
counts = collect_counts(records, "fundingReferences[].funderName")
|
|
125
|
+
|
|
126
|
+
norm = Normalizer(get_preset("funderName"), cache_dir=".cache")
|
|
127
|
+
smap = norm.build(counts, authority=RorIndex.load("ror_index.pickle"))
|
|
128
|
+
smap.write_review("funder_review.tsv", counts) # review gate
|
|
129
|
+
|
|
130
|
+
for rec in records:
|
|
131
|
+
map_values(rec, "fundingReferences[].funderName", lambda v: norm.normalize(v, smap))
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Field presets
|
|
135
|
+
|
|
136
|
+
Built-in `FieldConfig` presets for common DataCite/FAIR fields (`fairly presets`):
|
|
137
|
+
|
|
138
|
+
| Field | Strategy | eps | Holdout | Authority |
|
|
139
|
+
|---|---|---|---|---|
|
|
140
|
+
| `publisher` | semantic | 0.30 | yes | ROR |
|
|
141
|
+
| `funderName` | semantic | 0.07 | yes | ROR |
|
|
142
|
+
| `affiliation` | semantic | 0.07 | yes | ROR |
|
|
143
|
+
| `subject` | semantic | 0.10 | yes | — |
|
|
144
|
+
| `rights` | **controlled** | — | — | SPDX/CC vocab |
|
|
145
|
+
| `geoLocationPlace` | geocode | — | — | — |
|
|
146
|
+
|
|
147
|
+
These `eps` values are **V-measure validated** against the curated gold sets in
|
|
148
|
+
[`gold/`](gold/) — each sits on its field's optimal plateau (funder/subject 1.00,
|
|
149
|
+
publisher 0.99, affiliation 0.96; `rights` 0.82, so prefer a controlled vocabulary
|
|
150
|
+
for licenses). **Re-tune for your data** with `fairly validate` — see
|
|
151
|
+
[docs/methods.md](docs/methods.md#8-v-measure-validation) for the full sweep and why
|
|
152
|
+
each field wants a different threshold.
|
|
153
|
+
|
|
154
|
+
## Why these choices
|
|
155
|
+
|
|
156
|
+
- **Most-frequent canonical** gives clean, recognizable labels (the common form
|
|
157
|
+
wins), with centroid-closeness only breaking ties.
|
|
158
|
+
- **Holdout** exists because short/all-caps tokens carry too little signal for an
|
|
159
|
+
embedding to place; left in, they become spurious join keys.
|
|
160
|
+
- **Authority split** is what lets us cluster *aggressively* for coverage without
|
|
161
|
+
paying for it in precision: the embedding proposes merges, ROR/geocoding vetoes
|
|
162
|
+
the wrong ones.
|
|
163
|
+
- **V-measure** turns "did we over-merge?" into a number: homogeneity penalizes
|
|
164
|
+
over-merge (the costly direction), completeness penalizes under-merge.
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# fair-ly accurate text synonyms for data cleaning
|
|
2
|
+
|
|
3
|
+
**`fairly`** collapses spelling and format variants of FAIR / DataCite metadata
|
|
4
|
+
field values onto a single canonical form — *fairly accurately* — so that
|
|
5
|
+
`Univ. of California, Berkeley`, `University of California-Berkeley` and
|
|
6
|
+
`UC Berkeley` all become one publisher/affiliation, while genuinely different
|
|
7
|
+
entities (`University of Washington` vs `Washington University`) stay apart.
|
|
8
|
+
|
|
9
|
+
It packages four cooperating strategies, each earning its place against a real
|
|
10
|
+
failure mode:
|
|
11
|
+
|
|
12
|
+
| Strategy | What it does | Guards against |
|
|
13
|
+
|---|---|---|
|
|
14
|
+
| **Synonym-lustre** | embed distinct values (`gte-large`) → cluster (HDBSCAN) → map each cluster to its most-frequent member | spelling/format/punctuation variants |
|
|
15
|
+
| **Abbreviation holdout** | keep acronyms and `< 4`-char tokens *out* of clustering | `ZHAW → Z`, `LUH → HU` over-merges |
|
|
16
|
+
| **ROR authority split** | split a cluster that spans two real institutions using a local ROR dump | `University of Washington` vs `Washington University` |
|
|
17
|
+
| **Geocoding** | for place fields, geocode and group by `(country, city)` → `"City, Country"` | `Graz` merging into `Vienna` ("city in Austria") |
|
|
18
|
+
| **Controlled vocab** | for fixed-vocabulary fields (licenses, codes), deterministically fold + alias-resolve to a canonical id | `CC-BY-SA` collapsing into `CC-BY-NC` — clustering can't tell them apart |
|
|
19
|
+
|
|
20
|
+
Quality is measured, not assumed: a **V-measure** harness tunes the clustering
|
|
21
|
+
threshold per field against a small gold set.
|
|
22
|
+
|
|
23
|
+
> Method background and per-field recommendations: **[docs/methods.md](docs/methods.md)**.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install fair-ly-accurate # core (cleaners, ROR index, I/O, V-measure math)
|
|
29
|
+
pip install "fair-ly-accurate[cluster]" # + embeddings & HDBSCAN (semantic clustering)
|
|
30
|
+
pip install "fair-ly-accurate[geo]" # + geopy (place geocoding)
|
|
31
|
+
pip install "fair-ly-accurate[all]" # everything
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
(The distribution installs as `fair-ly-accurate`; the import name is `fairly`: `from fairly import Normalizer`.)
|
|
35
|
+
|
|
36
|
+
The heavy ML/geo dependencies are optional extras, so the deterministic cleaners,
|
|
37
|
+
ROR index and field-path I/O install light.
|
|
38
|
+
|
|
39
|
+
## Quickstart (CLI)
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
# 0. (once, for institution fields) build a ROR index from a ROR v2 dump
|
|
43
|
+
fairly build-ror-index --dump v2-ror-data.json --out ror_index.pickle
|
|
44
|
+
|
|
45
|
+
# 1. tune the threshold for a field against a gold set (variant<TAB>true_canonical)
|
|
46
|
+
fairly validate --gold affiliation_gold.tsv --preset affiliation --field affiliation
|
|
47
|
+
|
|
48
|
+
# 2. build + apply a synonym map to a DataCite-shaped field, with review + ROR split
|
|
49
|
+
fairly normalize \
|
|
50
|
+
--input records.jsonl \
|
|
51
|
+
--field-path 'creators[].affiliation[].name' \
|
|
52
|
+
--preset affiliation \
|
|
53
|
+
--ror-index ror_index.pickle \
|
|
54
|
+
--review affiliation_review.tsv \
|
|
55
|
+
--output records.normalized.jsonl
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The `--field-path` understands dotted paths through nested dicts and lists, e.g.
|
|
59
|
+
`publisher.name`, `fundingReferences[].funderName`, `subjects[].subject`,
|
|
60
|
+
`creators[].affiliation[].name`. Tabular inputs use the column name as the path.
|
|
61
|
+
|
|
62
|
+
**Always eyeball the `--review` TSV before trusting an apply** — especially for
|
|
63
|
+
homogeneous fields like funders, where over-merge is easy. Add `--dry-run` to
|
|
64
|
+
build the map and review without writing.
|
|
65
|
+
|
|
66
|
+
## Quickstart (library)
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from fairly import Normalizer, get_preset
|
|
70
|
+
from fairly.authorities.ror import RorIndex
|
|
71
|
+
from fairly.io import collect_counts, read_records, map_values
|
|
72
|
+
|
|
73
|
+
records = read_records("records.jsonl")
|
|
74
|
+
counts = collect_counts(records, "fundingReferences[].funderName")
|
|
75
|
+
|
|
76
|
+
norm = Normalizer(get_preset("funderName"), cache_dir=".cache")
|
|
77
|
+
smap = norm.build(counts, authority=RorIndex.load("ror_index.pickle"))
|
|
78
|
+
smap.write_review("funder_review.tsv", counts) # review gate
|
|
79
|
+
|
|
80
|
+
for rec in records:
|
|
81
|
+
map_values(rec, "fundingReferences[].funderName", lambda v: norm.normalize(v, smap))
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Field presets
|
|
85
|
+
|
|
86
|
+
Built-in `FieldConfig` presets for common DataCite/FAIR fields (`fairly presets`):
|
|
87
|
+
|
|
88
|
+
| Field | Strategy | eps | Holdout | Authority |
|
|
89
|
+
|---|---|---|---|---|
|
|
90
|
+
| `publisher` | semantic | 0.30 | yes | ROR |
|
|
91
|
+
| `funderName` | semantic | 0.07 | yes | ROR |
|
|
92
|
+
| `affiliation` | semantic | 0.07 | yes | ROR |
|
|
93
|
+
| `subject` | semantic | 0.10 | yes | — |
|
|
94
|
+
| `rights` | **controlled** | — | — | SPDX/CC vocab |
|
|
95
|
+
| `geoLocationPlace` | geocode | — | — | — |
|
|
96
|
+
|
|
97
|
+
These `eps` values are **V-measure validated** against the curated gold sets in
|
|
98
|
+
[`gold/`](gold/) — each sits on its field's optimal plateau (funder/subject 1.00,
|
|
99
|
+
publisher 0.99, affiliation 0.96; `rights` 0.82, so prefer a controlled vocabulary
|
|
100
|
+
for licenses). **Re-tune for your data** with `fairly validate` — see
|
|
101
|
+
[docs/methods.md](docs/methods.md#8-v-measure-validation) for the full sweep and why
|
|
102
|
+
each field wants a different threshold.
|
|
103
|
+
|
|
104
|
+
## Why these choices
|
|
105
|
+
|
|
106
|
+
- **Most-frequent canonical** gives clean, recognizable labels (the common form
|
|
107
|
+
wins), with centroid-closeness only breaking ties.
|
|
108
|
+
- **Holdout** exists because short/all-caps tokens carry too little signal for an
|
|
109
|
+
embedding to place; left in, they become spurious join keys.
|
|
110
|
+
- **Authority split** is what lets us cluster *aggressively* for coverage without
|
|
111
|
+
paying for it in precision: the embedding proposes merges, ROR/geocoding vetoes
|
|
112
|
+
the wrong ones.
|
|
113
|
+
- **V-measure** turns "did we over-merge?" into a number: homogeneity penalizes
|
|
114
|
+
over-merge (the costly direction), completeness penalizes under-merge.
|
|
115
|
+
|
|
116
|
+
## License
|
|
117
|
+
|
|
118
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Releasing `fair-ly-accurate` to PyPI
|
|
2
|
+
|
|
3
|
+
Publishing is automated with **PyPI Trusted Publishing (OIDC)** — GitHub Actions
|
|
4
|
+
mints a short-lived token per run, so there are no API tokens to store or rotate.
|
|
5
|
+
The workflow lives in [`.github/workflows/publish.yml`](.github/workflows/publish.yml):
|
|
6
|
+
publishing a **GitHub Release** builds the distributions and uploads them to PyPI.
|
|
7
|
+
|
|
8
|
+
## One-time setup
|
|
9
|
+
|
|
10
|
+
Register this repo as a *trusted publisher* on PyPI. Because the project does not
|
|
11
|
+
exist yet, use the "pending publisher" form:
|
|
12
|
+
|
|
13
|
+
- **PyPI** — https://pypi.org/manage/account/publishing/ → *Add a pending publisher*
|
|
14
|
+
- PyPI Project Name: `fair-ly-accurate`
|
|
15
|
+
- Owner: `fairdataihub`
|
|
16
|
+
- Repository name: `fair-ly-accurate-text-synonyms-for-data-cleaning`
|
|
17
|
+
- Workflow name: `publish.yml`
|
|
18
|
+
- Environment name: `pypi`
|
|
19
|
+
|
|
20
|
+
The first successful publish creates the `fair-ly-accurate` project owned by the account
|
|
21
|
+
that registered the pending publisher.
|
|
22
|
+
|
|
23
|
+
(Optional, recommended) In GitHub → *Settings → Environments*, create an
|
|
24
|
+
environment named `pypi` and add yourself as a **required reviewer**, so a publish
|
|
25
|
+
needs a human click even though the repo may have other maintainers with write
|
|
26
|
+
access.
|
|
27
|
+
|
|
28
|
+
## Cutting a release
|
|
29
|
+
|
|
30
|
+
1. Bump the version in **both** places (they must match, or the workflow fails the build):
|
|
31
|
+
- `pyproject.toml` → `version = "X.Y.Z"`
|
|
32
|
+
- `src/fairly/__init__.py` → `__version__ = "X.Y.Z"`
|
|
33
|
+
- Add a `CHANGELOG.md` entry.
|
|
34
|
+
2. Commit, then tag and push:
|
|
35
|
+
```bash
|
|
36
|
+
git commit -am "Release X.Y.Z"
|
|
37
|
+
git tag vX.Y.Z
|
|
38
|
+
git push && git push --tags
|
|
39
|
+
```
|
|
40
|
+
3. Create a **GitHub Release** for that tag
|
|
41
|
+
(Releases → Draft a new release → choose `vX.Y.Z` → Publish). That triggers the
|
|
42
|
+
PyPI publish job.
|
|
43
|
+
4. Confirm:
|
|
44
|
+
```bash
|
|
45
|
+
pip install fair-ly-accurate==X.Y.Z
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
PyPI refuses to overwrite an existing version, so every release needs a fresh
|
|
49
|
+
version number.
|
|
50
|
+
|
|
51
|
+
## Local build check
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
python -m pip install build twine
|
|
55
|
+
python -m build
|
|
56
|
+
python -m twine check dist/*
|
|
57
|
+
```
|