matchlab 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/workflows/cd.yml +5 -5
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/workflows/ci.yml +3 -3
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/workflows/codeql.yml +3 -3
- {matchlab-0.1.2 → matchlab-0.2.0}/.pre-commit-config.yaml +1 -1
- {matchlab-0.1.2/src/matchlab.egg-info → matchlab-0.2.0}/PKG-INFO +3 -3
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/linkers/splink.md +2 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/pyproject.toml +3 -2
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/hash.py +2 -4
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/samples.py +5 -2
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/linkers/deterministic.py +1 -1
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/linkers/splinklinker.py +8 -6
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/resolvers/components.py +8 -5
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/stores/duckdb.py +39 -38
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/compare.py +5 -2
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/base.py +1 -1
- {matchlab-0.1.2 → matchlab-0.2.0/src/matchlab.egg-info}/PKG-INFO +3 -3
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/requires.txt +2 -2
- matchlab-0.2.0/src/matchlab.egg-info/scm_version.json +8 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/core/test_hash.py +15 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/core/test_resolver_output.py +5 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_linkers_deterministic.py +4 -10
- {matchlab-0.1.2 → matchlab-0.2.0}/test/resolvers/test_components.py +26 -4
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/test_scores.py +13 -4
- matchlab-0.2.0/uv.lock +1807 -0
- matchlab-0.1.2/src/matchlab.egg-info/scm_version.json +0 -8
- matchlab-0.1.2/uv.lock +0 -2063
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/CODEOWNERS +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/SECURITY_CHECKLIST.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/actions/setup-toolchain/action.yml +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/dependabot.yml +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/pull_request_template.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/skills/lucid-docs/README.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/skills/lucid-docs/SKILL.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.github/skills/lucid-docs/readability.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.gitignore +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.vscode/launch.json +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/.vscode/settings.json +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/LICENSE +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/README.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/README.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/__main__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/cases.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/data.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/measure.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/measurements.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/report.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/results/.gitkeep +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/runner.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/benchmarks/topologies.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/core/exceptions.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/core/hash.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/core/resolver_output.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/core/schemas.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/core/testkit.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/evaluation/judgements.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/evaluation/metrics.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/evaluation/sampling.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/evaluation/tui.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/document.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/lineage.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/progress.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/resources.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/specs.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/dedupers/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/dedupers/naive.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/linkers/deterministic.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/linkers/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/models/linkers/weighted-deterministic.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/recordstep.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/resolvers/components.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/resolvers/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/sources/dataframe.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/sources/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/sources/relational.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/transformers/clean.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/transformers/explode.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/transformers/group.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/transformers/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/plan/steps/transformers/select.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/stores/duckdb.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/api/stores/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/assets/matchlab-icon-dark.png +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/assets/matchlab-icon-light.svg +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/assets/matchlab-logo-dark.svg +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/assets/matchlab-logo-light.svg +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/contributing.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/glossary.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/build-a-plan.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/custom-methodologies.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/evaluate.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/install.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/matchbox-to-matchlab.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/query.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/guide/serialise.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/index.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/use-cases.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/docs/why-matchlab.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/justfile +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/scripts/test_eval.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/setup.cfg +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/cli.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/dataframes.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/exceptions.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/kinds.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/logging.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/resolver_output.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/schemas.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/sql.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/core/versioning.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/document.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/judgements.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/metrics.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/app.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/modals.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/styles.tcss +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/widgets/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/widgets/assignment.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/widgets/styling.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/eval/tui/widgets/table.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/lineage.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/comparison.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/dedupers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/dedupers/base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/dedupers/naive.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/linkers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/linkers/base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/linkers/weighteddeterministic.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/models/models.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/progress.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/recordstep.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/resolvers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/resolvers/base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/resolvers/resolvers.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/resources.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/sources/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/sources/base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/sources/dataframe.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/sources/relational.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/sources/sources.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/specs.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/steps.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/stores/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/stores/base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/stores/default.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/_generate.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/entities.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/features.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/linked.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/matchers.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/models.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/testkit/sources.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/clean.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/explode.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/group.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/select.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab/transformers/transform.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/SOURCES.txt +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/dependency_links.txt +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/entry_points.txt +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/scm_file_list.json +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/src/matchlab.egg-info/top_level.txt +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/README.md +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/adapters/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/adapters/conftest.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/adapters/test_base.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/adapters/test_contract.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/adapters/test_duckdb.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/conftest.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/core/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/core/test_eval.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/core/test_versioning.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/eval/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/eval/test_samples.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/eval/tui/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/eval/tui/test_app.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/eval/tui/test_widgets.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_comparison.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_dedupers_deterministic.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_linkers_probabilistic.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_linkers_splink.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/models/test_models.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/plan/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/plan/test_ground_truth.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/plan/test_plan.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/plan/test_steps.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/resolvers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/resolvers/test_resolvers.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/sources/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/sources/conftest.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/sources/test_locations.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_benchmarks.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_cli.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_document.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_lineage.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_progress.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/test_resources.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/test_entities.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/test_linked.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/test_models.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/testkit/test_sources.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/__init__.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/test_clean.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/test_explode.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/test_group.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/test_select.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/test/transformers/test_transform.py +0 -0
- {matchlab-0.1.2 → matchlab-0.2.0}/zensical.toml +0 -0
|
@@ -12,7 +12,7 @@ jobs:
|
|
|
12
12
|
|
|
13
13
|
steps:
|
|
14
14
|
- name: Checkout repository
|
|
15
|
-
uses: actions/checkout@
|
|
15
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
16
16
|
with:
|
|
17
17
|
fetch-depth: 0
|
|
18
18
|
|
|
@@ -25,7 +25,7 @@ jobs:
|
|
|
25
25
|
run: uv build
|
|
26
26
|
|
|
27
27
|
- name: Upload package artifacts
|
|
28
|
-
uses: actions/upload-artifact@
|
|
28
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
29
29
|
with:
|
|
30
30
|
name: package-dist
|
|
31
31
|
path: ./dist
|
|
@@ -48,13 +48,13 @@ jobs:
|
|
|
48
48
|
|
|
49
49
|
steps:
|
|
50
50
|
- name: Download package artifacts
|
|
51
|
-
uses: actions/download-artifact@
|
|
51
|
+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
52
52
|
with:
|
|
53
53
|
name: package-dist
|
|
54
54
|
path: ./dist
|
|
55
55
|
|
|
56
56
|
- name: Publish package to PyPI
|
|
57
|
-
uses: pypa/gh-action-pypi-publish@
|
|
57
|
+
uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
|
|
58
58
|
|
|
59
59
|
deploy-docs:
|
|
60
60
|
needs: build
|
|
@@ -71,4 +71,4 @@ jobs:
|
|
|
71
71
|
steps:
|
|
72
72
|
- name: Deploy documentation to GitHub Pages
|
|
73
73
|
id: deployment
|
|
74
|
-
uses: actions/deploy-pages@
|
|
74
|
+
uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1
|
|
@@ -18,7 +18,7 @@ jobs:
|
|
|
18
18
|
|
|
19
19
|
steps:
|
|
20
20
|
- name: Checkout repository
|
|
21
|
-
uses: actions/checkout@
|
|
21
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
22
22
|
|
|
23
23
|
- name: Setup toolchain
|
|
24
24
|
uses: ./.github/actions/setup-toolchain
|
|
@@ -33,7 +33,7 @@ jobs:
|
|
|
33
33
|
|
|
34
34
|
steps:
|
|
35
35
|
- name: Checkout repository
|
|
36
|
-
uses: actions/checkout@
|
|
36
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
37
37
|
|
|
38
38
|
- name: Setup toolchain
|
|
39
39
|
uses: ./.github/actions/setup-toolchain
|
|
@@ -51,7 +51,7 @@ jobs:
|
|
|
51
51
|
|
|
52
52
|
steps:
|
|
53
53
|
- name: Checkout repository
|
|
54
|
-
uses: actions/checkout@
|
|
54
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
55
55
|
with:
|
|
56
56
|
fetch-depth: 0
|
|
57
57
|
|
|
@@ -34,10 +34,10 @@ jobs:
|
|
|
34
34
|
|
|
35
35
|
steps:
|
|
36
36
|
- name: Checkout repository
|
|
37
|
-
uses: actions/checkout@
|
|
37
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
38
38
|
|
|
39
39
|
- name: Initialise CodeQL
|
|
40
|
-
uses: github/codeql-action/init@
|
|
40
|
+
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
|
41
41
|
with:
|
|
42
42
|
languages: ${{ matrix.language }}
|
|
43
43
|
|
|
@@ -45,6 +45,6 @@ jobs:
|
|
|
45
45
|
queries: security-extended
|
|
46
46
|
|
|
47
47
|
- name: Perform CodeQL Analysis
|
|
48
|
-
uses: github/codeql-action/analyze@
|
|
48
|
+
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
|
49
49
|
with:
|
|
50
50
|
category: "/language:${{matrix.language}}"
|
|
@@ -22,7 +22,7 @@ repos:
|
|
|
22
22
|
|
|
23
23
|
# Mandatory internal hooks
|
|
24
24
|
- repo: https://github.com/uktrade/github-standards
|
|
25
|
-
rev: v1.4.
|
|
25
|
+
rev: v1.4.5 # update periodically with pre-commit autoupdate
|
|
26
26
|
hooks:
|
|
27
27
|
- id: run-security-scan
|
|
28
28
|
verbose: false
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: matchlab
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A local-first library for building, running and evaluating entity resolution pipelines.
|
|
5
5
|
Author: Department for Business and Trade
|
|
6
6
|
Project-URL: Documentation, https://uktrade.github.io/matchlab/
|
|
@@ -23,12 +23,12 @@ Requires-Dist: fastdsu>=0.0.0
|
|
|
23
23
|
Requires-Dist: frozendict>=2.4.6
|
|
24
24
|
Requires-Dist: pandas>=2.3.3
|
|
25
25
|
Requires-Dist: platformdirs>=4.5.0
|
|
26
|
-
Requires-Dist: polars-hash>=0.
|
|
26
|
+
Requires-Dist: polars-hash>=0.9.3
|
|
27
27
|
Requires-Dist: polars>=1.32.3
|
|
28
28
|
Requires-Dist: pyarrow>=17.0.0
|
|
29
29
|
Requires-Dist: pydantic>=2.9.2
|
|
30
30
|
Requires-Dist: rich>=13.9.4
|
|
31
|
-
Requires-Dist: splink<
|
|
31
|
+
Requires-Dist: splink<5.1.0,>=5.0.0
|
|
32
32
|
Requires-Dist: sqlalchemy>=2.0.35
|
|
33
33
|
Requires-Dist: sqlglot[c]>=30.0.3
|
|
34
34
|
Requires-Dist: textual>=0.80
|
|
@@ -24,12 +24,12 @@ dependencies = [
|
|
|
24
24
|
"frozendict>=2.4.6",
|
|
25
25
|
"pandas>=2.3.3",
|
|
26
26
|
"platformdirs>=4.5.0",
|
|
27
|
-
"polars-hash>=0.
|
|
27
|
+
"polars-hash>=0.9.3",
|
|
28
28
|
"polars>=1.32.3",
|
|
29
29
|
"pyarrow>=17.0.0",
|
|
30
30
|
"pydantic>=2.9.2",
|
|
31
31
|
"rich>=13.9.4",
|
|
32
|
-
"splink>=
|
|
32
|
+
"splink>=5.0.0,<5.1.0",
|
|
33
33
|
"sqlalchemy>=2.0.35",
|
|
34
34
|
"sqlglot[c]>=30.0.3",
|
|
35
35
|
"textual>=0.80",
|
|
@@ -132,6 +132,7 @@ docstring-code-format = true
|
|
|
132
132
|
[tool.pytest.ini_options]
|
|
133
133
|
testpaths = ["test"]
|
|
134
134
|
asyncio_mode = "auto"
|
|
135
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
135
136
|
pythonpath = ["."]
|
|
136
137
|
addopts = "-n auto --dist=loadgroup"
|
|
137
138
|
norecursedirs = "src"
|
|
@@ -71,11 +71,9 @@ def hash_rows(
|
|
|
71
71
|
|
|
72
72
|
if method == HashMethod.XXH3_128:
|
|
73
73
|
row_hashes = df_processed.select(
|
|
74
|
-
#
|
|
75
|
-
# and leaves require binary
|
|
74
|
+
# Preserve the 16-byte format used by stored leaves and fingerprints.
|
|
76
75
|
plh.concat_str(str_concatenation)
|
|
77
|
-
.nchash.xxh3_128()
|
|
78
|
-
.cast(pl.Binary)
|
|
76
|
+
.nchash.xxh3_128(return_binary=True, byte_order="little")
|
|
79
77
|
.alias("row_hash")
|
|
80
78
|
)
|
|
81
79
|
return row_hashes["row_hash"]
|
|
@@ -10,6 +10,7 @@ from collections.abc import Sequence
|
|
|
10
10
|
from typing import TYPE_CHECKING, Any, TypeAlias
|
|
11
11
|
|
|
12
12
|
import polars as pl
|
|
13
|
+
import pyarrow as pa
|
|
13
14
|
from fastdsu import connected_components
|
|
14
15
|
from pydantic import BaseModel
|
|
15
16
|
|
|
@@ -254,8 +255,10 @@ def _merged_resolver_output(readings: list[Reading]) -> pl.DataFrame:
|
|
|
254
255
|
)
|
|
255
256
|
|
|
256
257
|
# Find components
|
|
257
|
-
components: pl.DataFrame = pl.
|
|
258
|
-
|
|
258
|
+
components: pl.DataFrame = pl.DataFrame(
|
|
259
|
+
pa.record_batch(
|
|
260
|
+
connected_components(edges["src"].to_arrow(), edges["dst"].to_arrow())
|
|
261
|
+
)
|
|
259
262
|
).rename({"key": "leaf", "label": "_component"})
|
|
260
263
|
|
|
261
264
|
# Content address components by leaves
|
|
@@ -163,7 +163,7 @@ class DeterministicLinker(Linker):
|
|
|
163
163
|
max_est: int = self._get_max_cardinality(con, query)
|
|
164
164
|
logger.info(f"Round {round_num}: Estimated max cardinality: {max_est:,}")
|
|
165
165
|
|
|
166
|
-
return con.
|
|
166
|
+
return con.sql(query).pl()
|
|
167
167
|
|
|
168
168
|
def _get_max_cardinality(
|
|
169
169
|
self, con: duckdb.DuckDBPyConnection, query: SQLQuery
|
|
@@ -109,9 +109,9 @@ class SplinkLinker(Linker):
|
|
|
109
109
|
|
|
110
110
|
>>> from splink import SettingsCreator, block_on
|
|
111
111
|
... import splink.comparison_library as cl
|
|
112
|
-
... import splink.comparison_template_library as ctl
|
|
113
112
|
...
|
|
114
113
|
... splink_settings = SettingsCreator(
|
|
114
|
+
... link_type="link_only",
|
|
115
115
|
... retain_matching_columns=False,
|
|
116
116
|
... retain_intermediate_calculation_columns=False,
|
|
117
117
|
... blocking_rules_to_generate_predictions=[
|
|
@@ -119,12 +119,13 @@ class SplinkLinker(Linker):
|
|
|
119
119
|
... block_on("postcode"),
|
|
120
120
|
... ],
|
|
121
121
|
... comparisons=[
|
|
122
|
-
... cl.
|
|
122
|
+
... cl.JaroWinklerAtThresholds(
|
|
123
123
|
... "company_name",
|
|
124
124
|
... [0.9, 0.6],
|
|
125
|
+
... ).configure(
|
|
125
126
|
... term_frequency_adjustments=True
|
|
126
127
|
... ),
|
|
127
|
-
...
|
|
128
|
+
... cl.PostcodeComparison("postcode"),
|
|
128
129
|
... ]
|
|
129
130
|
... )
|
|
130
131
|
"""
|
|
@@ -224,6 +225,9 @@ class SplinkLinker(Linker):
|
|
|
224
225
|
# Convert to pandas for Splink compatibility
|
|
225
226
|
left_pd = left.with_columns(pl.col(self.left_id).cast(pl.String)).to_pandas()
|
|
226
227
|
right_pd = right.with_columns(pl.col(self.right_id).cast(pl.String)).to_pandas()
|
|
228
|
+
db_api = DuckDBAPI()
|
|
229
|
+
left_table = db_api.register(left_pd, dataset_display_name="l")
|
|
230
|
+
right_table = db_api.register(right_pd, dataset_display_name="r")
|
|
227
231
|
|
|
228
232
|
# A copy, because building a Splink linker writes into the settings it is
|
|
229
233
|
# given: it stamps a random `linker_uid` onto them. Handed our own object,
|
|
@@ -232,10 +236,8 @@ class SplinkLinker(Linker):
|
|
|
232
236
|
# before. A step has to address the same artifact whether or not it has been
|
|
233
237
|
# run, and a random value could never be part of that address anyway.
|
|
234
238
|
self._linker = SplinkLibLinkerClass(
|
|
235
|
-
|
|
236
|
-
input_table_aliases=["l", "r"],
|
|
239
|
+
splink_dataframe_or_dataframes=[left_table, right_table],
|
|
237
240
|
settings=deepcopy(self.linker_settings),
|
|
238
|
-
db_api=DuckDBAPI(),
|
|
239
241
|
)
|
|
240
242
|
|
|
241
243
|
for func in self.linker_training_functions:
|
|
@@ -4,6 +4,7 @@ from collections.abc import Mapping
|
|
|
4
4
|
from typing import Annotated, ClassVar
|
|
5
5
|
|
|
6
6
|
import polars as pl
|
|
7
|
+
import pyarrow as pa
|
|
7
8
|
from fastdsu import connected_components
|
|
8
9
|
from pydantic import Field
|
|
9
10
|
|
|
@@ -43,7 +44,7 @@ class Components(ResolverMethod):
|
|
|
43
44
|
]
|
|
44
45
|
|
|
45
46
|
if not filtered:
|
|
46
|
-
return pl.
|
|
47
|
+
return pl.DataFrame(SCHEMA_CLUSTERS.empty_table())
|
|
47
48
|
|
|
48
49
|
edges = pl.concat(filtered).select(
|
|
49
50
|
pl.col("left_id").alias("src"),
|
|
@@ -51,10 +52,12 @@ class Components(ResolverMethod):
|
|
|
51
52
|
)
|
|
52
53
|
|
|
53
54
|
return (
|
|
54
|
-
pl.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
pl.DataFrame(
|
|
56
|
+
pa.record_batch(
|
|
57
|
+
connected_components(
|
|
58
|
+
edges["src"].to_arrow(),
|
|
59
|
+
edges["dst"].to_arrow(),
|
|
60
|
+
)
|
|
58
61
|
)
|
|
59
62
|
)
|
|
60
63
|
.rename({"label": "parent_id", "key": "child_id"})
|
|
@@ -162,10 +162,10 @@ class DuckDBStore(Store):
|
|
|
162
162
|
"""Create the schema, recreating the store if it predates this version."""
|
|
163
163
|
existing = self.conn.execute(
|
|
164
164
|
"SELECT table_name FROM information_schema.tables WHERE table_name = 'meta'"
|
|
165
|
-
).
|
|
165
|
+
).fetchall()
|
|
166
166
|
if existing:
|
|
167
|
-
|
|
168
|
-
if
|
|
167
|
+
rows = self.conn.execute("SELECT schema_version FROM meta").fetchall()
|
|
168
|
+
if rows and rows[0][0] == _SCHEMA_VERSION:
|
|
169
169
|
return
|
|
170
170
|
|
|
171
171
|
# Either a pre-versioned store or an older version, so start clean. Artifacts
|
|
@@ -196,10 +196,10 @@ class DuckDBStore(Store):
|
|
|
196
196
|
self.conn.register(name, df.to_arrow())
|
|
197
197
|
|
|
198
198
|
def _kind(self, fp: Fingerprint) -> StepKind | None:
|
|
199
|
-
|
|
199
|
+
rows = self.conn.execute(
|
|
200
200
|
"SELECT kind FROM artifacts WHERE fp = ?", [fp]
|
|
201
|
-
).
|
|
202
|
-
return StepKind(
|
|
201
|
+
).fetchall()
|
|
202
|
+
return StepKind(rows[0][0]) if rows else None
|
|
203
203
|
|
|
204
204
|
def _register_artifact(self, fp: Fingerprint, kind: StepKind) -> None:
|
|
205
205
|
self.conn.execute(
|
|
@@ -266,15 +266,15 @@ class DuckDBStore(Store):
|
|
|
266
266
|
# as much as a missing row does.
|
|
267
267
|
resident = self.conn.execute(
|
|
268
268
|
"SELECT sum(memory_usage_bytes) FROM duckdb_memory()"
|
|
269
|
-
).
|
|
270
|
-
size = (resident[0] if resident else 0) or 0
|
|
269
|
+
).fetchall()
|
|
270
|
+
size = (resident[0][0] if resident else 0) or 0
|
|
271
271
|
free, path = 0, None
|
|
272
272
|
else:
|
|
273
273
|
self.conn.execute("CHECKPOINT")
|
|
274
274
|
blocks = self.conn.execute(
|
|
275
275
|
"SELECT block_size, free_blocks FROM pragma_database_size()"
|
|
276
|
-
).
|
|
277
|
-
free = blocks[0] * blocks[1] if blocks else 0
|
|
276
|
+
).fetchall()
|
|
277
|
+
free = blocks[0][0] * blocks[0][1] if blocks else 0
|
|
278
278
|
# `self.path` is the constructor's argument verbatim, so it's resolved
|
|
279
279
|
# here. Otherwise a store opened as "./run.duckdb" would report itself
|
|
280
280
|
# against a working directory that has since moved on.
|
|
@@ -285,7 +285,7 @@ class DuckDBStore(Store):
|
|
|
285
285
|
if candidate.exists()
|
|
286
286
|
)
|
|
287
287
|
|
|
288
|
-
labelled = self.conn.execute("SELECT count(*) FROM labels").
|
|
288
|
+
labelled = self.conn.execute("SELECT count(*) FROM labels").fetchall()
|
|
289
289
|
return DuckDBStoreStats(
|
|
290
290
|
location=self.path,
|
|
291
291
|
path=path,
|
|
@@ -295,7 +295,7 @@ class DuckDBStore(Store):
|
|
|
295
295
|
"SELECT kind, count(*) FROM artifacts GROUP BY kind"
|
|
296
296
|
).fetchall()
|
|
297
297
|
),
|
|
298
|
-
labels=labelled[0] if labelled else 0,
|
|
298
|
+
labels=labelled[0][0] if labelled else 0,
|
|
299
299
|
free_bytes=free,
|
|
300
300
|
)
|
|
301
301
|
|
|
@@ -336,13 +336,13 @@ class DuckDBStore(Store):
|
|
|
336
336
|
def read_source_extract(self, fp: Fingerprint) -> pl.DataFrame:
|
|
337
337
|
if self._kind(fp) is not StepKind.SOURCE:
|
|
338
338
|
raise KeyError(f"No stored source for fingerprint {fp.hex()}")
|
|
339
|
-
return self.conn.
|
|
339
|
+
return self.conn.sql(f'SELECT * FROM "{self._extract_table(fp)}"').pl()
|
|
340
340
|
|
|
341
341
|
def read_source_leaves(self, fp: Fingerprint) -> pl.DataFrame:
|
|
342
342
|
if self._kind(fp) is not StepKind.SOURCE:
|
|
343
343
|
raise KeyError(f"No stored source for fingerprint {fp.hex()}")
|
|
344
|
-
return self.conn.
|
|
345
|
-
"SELECT key, leaf FROM source_leaves WHERE fp = ?", [fp]
|
|
344
|
+
return self.conn.sql(
|
|
345
|
+
"SELECT key, leaf FROM source_leaves WHERE fp = ?", params=[fp]
|
|
346
346
|
).pl()
|
|
347
347
|
|
|
348
348
|
# -- identifiers ------------------------------------------------------------------
|
|
@@ -357,20 +357,20 @@ class DuckDBStore(Store):
|
|
|
357
357
|
raise KeyError(f"No stored source for fingerprint {source_fp.hex()}")
|
|
358
358
|
|
|
359
359
|
if resolver_fp is None:
|
|
360
|
-
return self.conn.
|
|
360
|
+
return self.conn.sql(
|
|
361
361
|
"SELECT leaf AS id, CAST(? AS VARCHAR) AS source, key, leaf "
|
|
362
362
|
"FROM source_leaves WHERE fp = ?",
|
|
363
|
-
[source_name, source_fp],
|
|
363
|
+
params=[source_name, source_fp],
|
|
364
364
|
).pl()
|
|
365
365
|
|
|
366
366
|
if self._kind(resolver_fp) is not StepKind.RESOLVER:
|
|
367
367
|
raise KeyError(f"No stored resolver for fingerprint {resolver_fp.hex()}")
|
|
368
368
|
# Both predicates matter. `source` keeps this from scanning every source's
|
|
369
369
|
# rows, and `resolver_output` holds one generation per collect of the plan.
|
|
370
|
-
return self.conn.
|
|
370
|
+
return self.conn.sql(
|
|
371
371
|
"SELECT root AS id, source, key, leaf "
|
|
372
372
|
"FROM resolver_output WHERE fp = ? AND source = ?",
|
|
373
|
-
[resolver_fp, source_name],
|
|
373
|
+
params=[resolver_fp, source_name],
|
|
374
374
|
).pl()
|
|
375
375
|
|
|
376
376
|
# -- maintenance ------------------------------------------------------------------
|
|
@@ -464,13 +464,13 @@ class DuckDBStore(Store):
|
|
|
464
464
|
|
|
465
465
|
database = self.conn.execute(
|
|
466
466
|
"SELECT database_name FROM duckdb_databases() WHERE NOT internal LIMIT 1"
|
|
467
|
-
).
|
|
468
|
-
if database
|
|
467
|
+
).fetchall()
|
|
468
|
+
if not database: # a connected store always has one
|
|
469
469
|
return
|
|
470
470
|
|
|
471
471
|
self.conn.execute("CHECKPOINT")
|
|
472
472
|
self.conn.execute(f"ATTACH '{temporary}' AS pruned")
|
|
473
|
-
self.conn.execute(f'COPY FROM DATABASE "{database[0]}" TO pruned')
|
|
473
|
+
self.conn.execute(f'COPY FROM DATABASE "{database[0][0]}" TO pruned')
|
|
474
474
|
self.conn.execute("DETACH pruned")
|
|
475
475
|
self.conn.close()
|
|
476
476
|
|
|
@@ -500,8 +500,8 @@ class DuckDBStore(Store):
|
|
|
500
500
|
def read_model(self, fp: Fingerprint) -> pl.DataFrame:
|
|
501
501
|
if self._kind(fp) is not StepKind.MODEL:
|
|
502
502
|
raise KeyError(f"No stored model for fingerprint {fp.hex()}")
|
|
503
|
-
return self.conn.
|
|
504
|
-
"SELECT left_id, right_id, score FROM model_edges WHERE fp = ?", [fp]
|
|
503
|
+
return self.conn.sql(
|
|
504
|
+
"SELECT left_id, right_id, score FROM model_edges WHERE fp = ?", params=[fp]
|
|
505
505
|
).pl()
|
|
506
506
|
|
|
507
507
|
# -- transforms -------------------------------------------------------------------
|
|
@@ -519,7 +519,7 @@ class DuckDBStore(Store):
|
|
|
519
519
|
def read_transform(self, fp: Fingerprint) -> pl.DataFrame:
|
|
520
520
|
if self._kind(fp) is not StepKind.TRANSFORM:
|
|
521
521
|
raise KeyError(f"No stored transform for fingerprint {fp.hex()}")
|
|
522
|
-
return self.conn.
|
|
522
|
+
return self.conn.sql(f'SELECT * FROM "{self._transform_table(fp)}"').pl()
|
|
523
523
|
|
|
524
524
|
# -- resolvers --------------------------------------------------------------------
|
|
525
525
|
|
|
@@ -554,8 +554,9 @@ class DuckDBStore(Store):
|
|
|
554
554
|
def read_resolver(self, fp: Fingerprint) -> pl.DataFrame:
|
|
555
555
|
if self._kind(fp) is not StepKind.RESOLVER:
|
|
556
556
|
raise KeyError(f"No stored resolver for fingerprint {fp.hex()}")
|
|
557
|
-
return self.conn.
|
|
558
|
-
"SELECT root, leaf, key, source FROM resolver_output WHERE fp = ?",
|
|
557
|
+
return self.conn.sql(
|
|
558
|
+
"SELECT root, leaf, key, source FROM resolver_output WHERE fp = ?",
|
|
559
|
+
params=[fp],
|
|
559
560
|
).pl()
|
|
560
561
|
|
|
561
562
|
# -- evaluation -------------------------------------------------------------------
|
|
@@ -583,15 +584,15 @@ class DuckDBStore(Store):
|
|
|
583
584
|
self, tag: str | None = None
|
|
584
585
|
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
585
586
|
if tag is None:
|
|
586
|
-
judgements = self.conn.
|
|
587
|
+
judgements = self.conn.sql(
|
|
587
588
|
"SELECT user_name, endorsed, shown FROM judgements"
|
|
588
589
|
).pl()
|
|
589
590
|
else:
|
|
590
|
-
judgements = self.conn.
|
|
591
|
+
judgements = self.conn.sql(
|
|
591
592
|
"SELECT user_name, endorsed, shown FROM judgements WHERE tag = ?",
|
|
592
|
-
[tag],
|
|
593
|
+
params=[tag],
|
|
593
594
|
).pl()
|
|
594
|
-
expansion = self.conn.
|
|
595
|
+
expansion = self.conn.sql("SELECT root, leaves FROM expansion").pl()
|
|
595
596
|
|
|
596
597
|
# Present empty results with the right columns/dtypes. We deliberately do not
|
|
597
598
|
# re-validate against the arrow transport schemas here. Those pin `leaves` to a
|
|
@@ -624,10 +625,10 @@ class DuckDBStore(Store):
|
|
|
624
625
|
)
|
|
625
626
|
|
|
626
627
|
def find(self, label: str) -> Fingerprint | None:
|
|
627
|
-
|
|
628
|
+
rows = self.conn.execute(
|
|
628
629
|
"SELECT fp FROM labels WHERE label = ?", [label]
|
|
629
|
-
).
|
|
630
|
-
return
|
|
630
|
+
).fetchall()
|
|
631
|
+
return rows[0][0] if rows else None
|
|
631
632
|
|
|
632
633
|
def labels(self) -> list[str]:
|
|
633
634
|
return [
|
|
@@ -638,12 +639,12 @@ class DuckDBStore(Store):
|
|
|
638
639
|
]
|
|
639
640
|
|
|
640
641
|
def source_key_field(self, fp: Fingerprint) -> str:
|
|
641
|
-
|
|
642
|
+
rows = self.conn.execute(
|
|
642
643
|
"SELECT key_field FROM source_meta WHERE fp = ?", [fp]
|
|
643
|
-
).
|
|
644
|
-
if
|
|
644
|
+
).fetchall()
|
|
645
|
+
if not rows:
|
|
645
646
|
raise KeyError(f"No stored source for fingerprint {fp.hex()}")
|
|
646
|
-
return
|
|
647
|
+
return rows[0][0]
|
|
647
648
|
|
|
648
649
|
def resolver_output_sources(self, fp: Fingerprint) -> dict[str, Fingerprint]:
|
|
649
650
|
return {
|
|
@@ -14,6 +14,7 @@ clusters were subsets" tells you the matcher is too strict, in a way a number ca
|
|
|
14
14
|
from collections import Counter
|
|
15
15
|
|
|
16
16
|
import polars as pl
|
|
17
|
+
import pyarrow as pa
|
|
17
18
|
from fastdsu import connected_components
|
|
18
19
|
|
|
19
20
|
from matchlab.testkit.entities import Cluster, EntityReference
|
|
@@ -144,8 +145,10 @@ def scores_to_clusters(
|
|
|
144
145
|
edges = pl.concat([matched, self_edges])
|
|
145
146
|
|
|
146
147
|
components = (
|
|
147
|
-
pl.
|
|
148
|
-
|
|
148
|
+
pl.DataFrame(
|
|
149
|
+
pa.record_batch(
|
|
150
|
+
connected_components(edges["src"].to_arrow(), edges["dst"].to_arrow())
|
|
151
|
+
)
|
|
149
152
|
)
|
|
150
153
|
.rename({"key": "id", "label": "component"})
|
|
151
154
|
.group_by("component")
|
|
@@ -71,4 +71,4 @@ def run_sql(query: SQLQuery, data: pl.DataFrame) -> pl.DataFrame:
|
|
|
71
71
|
"""Execute one DuckDB query against `data`, registered as `data`."""
|
|
72
72
|
with duckdb.connect(":memory:") as connection:
|
|
73
73
|
connection.register("data", data)
|
|
74
|
-
return connection.
|
|
74
|
+
return connection.sql(query).pl()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: matchlab
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A local-first library for building, running and evaluating entity resolution pipelines.
|
|
5
5
|
Author: Department for Business and Trade
|
|
6
6
|
Project-URL: Documentation, https://uktrade.github.io/matchlab/
|
|
@@ -23,12 +23,12 @@ Requires-Dist: fastdsu>=0.0.0
|
|
|
23
23
|
Requires-Dist: frozendict>=2.4.6
|
|
24
24
|
Requires-Dist: pandas>=2.3.3
|
|
25
25
|
Requires-Dist: platformdirs>=4.5.0
|
|
26
|
-
Requires-Dist: polars-hash>=0.
|
|
26
|
+
Requires-Dist: polars-hash>=0.9.3
|
|
27
27
|
Requires-Dist: polars>=1.32.3
|
|
28
28
|
Requires-Dist: pyarrow>=17.0.0
|
|
29
29
|
Requires-Dist: pydantic>=2.9.2
|
|
30
30
|
Requires-Dist: rich>=13.9.4
|
|
31
|
-
Requires-Dist: splink<
|
|
31
|
+
Requires-Dist: splink<5.1.0,>=5.0.0
|
|
32
32
|
Requires-Dist: sqlalchemy>=2.0.35
|
|
33
33
|
Requires-Dist: sqlglot[c]>=30.0.3
|
|
34
34
|
Requires-Dist: textual>=0.80
|
|
@@ -5,12 +5,12 @@ fastdsu>=0.0.0
|
|
|
5
5
|
frozendict>=2.4.6
|
|
6
6
|
pandas>=2.3.3
|
|
7
7
|
platformdirs>=4.5.0
|
|
8
|
-
polars-hash>=0.
|
|
8
|
+
polars-hash>=0.9.3
|
|
9
9
|
polars>=1.32.3
|
|
10
10
|
pyarrow>=17.0.0
|
|
11
11
|
pydantic>=2.9.2
|
|
12
12
|
rich>=13.9.4
|
|
13
|
-
splink<
|
|
13
|
+
splink<5.1.0,>=5.0.0
|
|
14
14
|
sqlalchemy>=2.0.35
|
|
15
15
|
sqlglot[c]>=30.0.3
|
|
16
16
|
textual>=0.80
|
|
@@ -24,6 +24,21 @@ methods = pytest.mark.parametrize(
|
|
|
24
24
|
# -- row hashing ----------------------------------------------------------------------
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
def test_hash_rows_compatibility() -> None:
|
|
28
|
+
"""Dependency upgrades must preserve the hashes that identify stored leaves."""
|
|
29
|
+
data = pl.DataFrame(
|
|
30
|
+
{"name": ["acme", None, "beta"], "town": ["london", "leeds", ""]}
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
hashes = hash_rows(data, columns=["name", "town"])
|
|
34
|
+
|
|
35
|
+
assert hashes.to_list() == [
|
|
36
|
+
bytes.fromhex("bf468096d54b94f7092bd76ef2a60bf4"),
|
|
37
|
+
bytes.fromhex("05a8dd348c59da251f00af558abfb4f3"),
|
|
38
|
+
bytes.fromhex("2f757c1cc37b78a49bdb6c2f8a6ee432"),
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
|
|
27
42
|
@methods
|
|
28
43
|
def test_hash_rows_all_dtypes(method: HashMethod) -> None:
|
|
29
44
|
"""One hash per row over the full spread of dtypes a source can present.
|
|
@@ -57,6 +57,11 @@ def test_root_id_order_invariant() -> None:
|
|
|
57
57
|
assert root_id_of([1, 2]) != root_id_of([1, 3])
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
def test_root_id_compatibility() -> None:
|
|
61
|
+
"""Dependency upgrades must preserve root IDs used by stored judgements."""
|
|
62
|
+
assert root_id_of([1, 2, 18446744073709551615]) == 10545872694498418499
|
|
63
|
+
|
|
64
|
+
|
|
60
65
|
# -- the layered fall-through scenario -------------------------------------------------
|
|
61
66
|
|
|
62
67
|
|
|
@@ -151,23 +151,14 @@ def configure_splink_linker(
|
|
|
151
151
|
if not shared_fields:
|
|
152
152
|
raise ValueError("Must have at least one shared field")
|
|
153
153
|
|
|
154
|
-
deterministic_matching_rules: list[str] = []
|
|
155
154
|
blocking_rules_to_generate_predictions: list[BlockingRuleCreator] = []
|
|
156
155
|
comparisons: list[ComparisonCreator] = []
|
|
157
156
|
|
|
158
157
|
for field in shared_fields:
|
|
159
|
-
deterministic_matching_rules.append(f"l.{field} = r.{field}")
|
|
160
158
|
blocking_rules_to_generate_predictions.append(brl.block_on(field))
|
|
161
|
-
comparisons.append(cl.ExactMatch(field).configure(m_probabilities=[1,
|
|
159
|
+
comparisons.append(cl.ExactMatch(field).configure(m_probabilities=[1, 1e-300]))
|
|
162
160
|
|
|
163
161
|
linker_training_functions = [
|
|
164
|
-
{
|
|
165
|
-
"function": "estimate_probability_two_random_records_match",
|
|
166
|
-
"arguments": {
|
|
167
|
-
"deterministic_matching_rules": deterministic_matching_rules,
|
|
168
|
-
"recall": 1,
|
|
169
|
-
},
|
|
170
|
-
},
|
|
171
162
|
{
|
|
172
163
|
"function": "estimate_u_using_random_sampling",
|
|
173
164
|
"arguments": {"max_pairs": 1e4},
|
|
@@ -178,8 +169,11 @@ def configure_splink_linker(
|
|
|
178
169
|
# maximisation, which many of these tests can't run, since several have only one
|
|
179
170
|
# shared field. A pinned m is fine here, since these tests only check raw
|
|
180
171
|
# match/no-match behaviour, not a calibrated probability.
|
|
172
|
+
# Splink 5 rejects zero m/u values, so use its probability floor for mismatches.
|
|
181
173
|
linker_settings = SettingsCreator(
|
|
182
174
|
link_type="link_only",
|
|
175
|
+
# Splink 5 cannot predict when the no-match fixture estimates a zero prior.
|
|
176
|
+
probability_two_random_records_match=0.01,
|
|
183
177
|
retain_matching_columns=False,
|
|
184
178
|
retain_intermediate_calculation_columns=False,
|
|
185
179
|
blocking_rules_to_generate_predictions=blocking_rules_to_generate_predictions,
|