matchbox-db 0.2.2__tar.gz

matchbox_db-0.2.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Department for Business and Trade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

matchbox_db-0.2.2/PKG-INFO

@@ -0,0 +1,160 @@
+Metadata-Version: 2.2
+Name: matchbox-db
+Version: 0.2.2
+Summary: A framework for orchestrating and comparing data linking and deduplication methodologies.
+Author: Department for Business and Trade
+License: MIT License
+        
+        Copyright (c) 2024 Department for Business and Trade
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Documentation, https://uktrade.github.io/matchbox/
+Project-URL: Repository, https://github.com/uktrade/matchbox.git
+Requires-Python: <3.14,>=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1.7
+Requires-Dist: connectorx>=0.3.3
+Requires-Dist: duckdb>=1.1.1
+Requires-Dist: faker>=36.1.1
+Requires-Dist: frozendict>=2.4.6
+Requires-Dist: httpx>=0.28.0
+Requires-Dist: matplotlib>=3.9.2
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: psycopg2>=2.9.10
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: pydantic-settings>=2.5.2
+Requires-Dist: pydantic>=2.9.2
+Requires-Dist: rustworkx>=0.15.1
+Requires-Dist: splink<4.1.0,>=4.0.5
+Requires-Dist: sqlalchemy>=2.0.35
+Requires-Dist: rich>=13.9.4
+Provides-Extra: server
+Requires-Dist: boto3>=1.35.99; extra == "server"
+Requires-Dist: ddtrace>=3.2.1; extra == "server"
+Requires-Dist: fastapi[standard]<0.116.0,>=0.115.0; extra == "server"
+Requires-Dist: pg-bulk-ingest>=0.0.54; extra == "server"
+Requires-Dist: python-multipart>=0.0.18; extra == "server"
+Requires-Dist: tomli>=2.0.1; extra == "server"
+
+# 🔥 Matchbox DB
+
+Record matching is a chore. Matchbox is a match pipeline orchestration tool that aims to:
+
+* Make matching an iterative, collaborative, measurable problem
+* Compose sources, dedupers and linkers and make the results very easy to query
+* Allow organisations to know they have matching records without having to share the data
+* Allow matching pipelines to run iteratively
+* Support batch and real-time matching 
+
+Matchbox doesn't store raw data, instead indexing the data in your warehouse and leaving permissioning at the level of the user, service or pipeline. 
+
+## Installation
+To install the matchbox client:
+```
+pip install "matchbox-db"
+```
+
+To install the full package, including the server features:
+
+```
+pip install "matchbox-db[server]"
+```
+
+## Running the server locally
+
+To run the server locally, run:
+
+```bash
+docker compose up --build
+```
+
+## Running the server locally with Datadog (monitoring) integration
+
+1. Run:
+
+   ```
+   cp ./environments/datadog-agent-private-sample.env ./environments/.datadog-agent-private.env
+   ```
+
+2. Populate the newly-created ` ./environments/.datadog-agent-private.env` with a Datadog API key.
+
+
+3. Run the server using:
+
+   ```bash
+   docker compose --profile monitoring up --build
+   ```
+
+
+## Use cases
+
+### Data architects and engineers
+
+* Reconcile entities across disparate datasets
+* Rationalise about the quality of different entity matching pipelines and serve up the best
+* Run matching pipelines without recomputing them every time
+* Lay the foundation for the nouns of a semantic layer
+
+### Data analysts and scientists
+
+* Use your team's best matching methods when retrieving entities, always
+* Measurably improve methodologies when they don't work for you
+* When you link new datasets, allow others to use your work easily and securely
+
+### Service owners
+
+* Understand the broader business entities in your service, not just what you have
+* Enrich other services with data generated in yours without giving away any permissioning powers
+* Empower your users to label matched entities and let other services use that information
+
+## Structure
+
+> [!CAUTION]
+> Some of the below is aspirational. Matchbox is in alpha and under heavy construction.
+
+The project is loosely formed into a client/server structure.
+
+### Server
+
+The parts of matchbox intended for deployment. Allows different backends as long as they can meet the standards of the adapter and tests.
+
+### Client
+
+The parts of matchbox intended for users and services to call a matchbox server, and to insert matched data in the right structure.
+
+If the dataset isn't already in matchbox, it'll need to be indexed.
+
+Pipelines using this part of matchbox will:
+
+1. Use `matchbox.query()` to retrieve source data from the perspective of a particular resolution point
+2. Use `matchbox.process()` to clean the data with standardised processes
+3. Use `matchbox.make_model()` with `matchbox.dedupers` and `matchbox.linkers` to create a new model
+4. Generate probabilistic model outputs using `model.run()`
+5. Upload the probabilites to matchbox with `results.to_matchbox()`
+6. Label data, or use existing data, to decide the probability threshold that you're willing to consider "truth" for your new model
+7. Use `model.roc_curve()` and other tools to make your decision
+8. Update `model.truth` to codify it
+
+With the truth threshold set to `1.0` by default, deterministic methodologies are ready for others to use from step five!
+
+## Development
+
+See our full development guide and coding standards in [CONTRIBUTING.md](./docs/contributing.md)

matchbox_db-0.2.2/README.md

@@ -0,0 +1,104 @@
+# 🔥 Matchbox DB
+
+Record matching is a chore. Matchbox is a match pipeline orchestration tool that aims to:
+
+* Make matching an iterative, collaborative, measurable problem
+* Compose sources, dedupers and linkers and make the results very easy to query
+* Allow organisations to know they have matching records without having to share the data
+* Allow matching pipelines to run iteratively
+* Support batch and real-time matching 
+
+Matchbox doesn't store raw data, instead indexing the data in your warehouse and leaving permissioning at the level of the user, service or pipeline. 
+
+## Installation
+To install the matchbox client:
+```
+pip install "matchbox-db"
+```
+
+To install the full package, including the server features:
+
+```
+pip install "matchbox-db[server]"
+```
+
+## Running the server locally
+
+To run the server locally, run:
+
+```bash
+docker compose up --build
+```
+
+## Running the server locally with Datadog (monitoring) integration
+
+1. Run:
+
+   ```
+   cp ./environments/datadog-agent-private-sample.env ./environments/.datadog-agent-private.env
+   ```
+
+2. Populate the newly-created ` ./environments/.datadog-agent-private.env` with a Datadog API key.
+
+
+3. Run the server using:
+
+   ```bash
+   docker compose --profile monitoring up --build
+   ```
+
+
+## Use cases
+
+### Data architects and engineers
+
+* Reconcile entities across disparate datasets
+* Rationalise about the quality of different entity matching pipelines and serve up the best
+* Run matching pipelines without recomputing them every time
+* Lay the foundation for the nouns of a semantic layer
+
+### Data analysts and scientists
+
+* Use your team's best matching methods when retrieving entities, always
+* Measurably improve methodologies when they don't work for you
+* When you link new datasets, allow others to use your work easily and securely
+
+### Service owners
+
+* Understand the broader business entities in your service, not just what you have
+* Enrich other services with data generated in yours without giving away any permissioning powers
+* Empower your users to label matched entities and let other services use that information
+
+## Structure
+
+> [!CAUTION]
+> Some of the below is aspirational. Matchbox is in alpha and under heavy construction.
+
+The project is loosely formed into a client/server structure.
+
+### Server
+
+The parts of matchbox intended for deployment. Allows different backends as long as they can meet the standards of the adapter and tests.
+
+### Client
+
+The parts of matchbox intended for users and services to call a matchbox server, and to insert matched data in the right structure.
+
+If the dataset isn't already in matchbox, it'll need to be indexed.
+
+Pipelines using this part of matchbox will:
+
+1. Use `matchbox.query()` to retrieve source data from the perspective of a particular resolution point
+2. Use `matchbox.process()` to clean the data with standardised processes
+3. Use `matchbox.make_model()` with `matchbox.dedupers` and `matchbox.linkers` to create a new model
+4. Generate probabilistic model outputs using `model.run()`
+5. Upload the probabilites to matchbox with `results.to_matchbox()`
+6. Label data, or use existing data, to decide the probability threshold that you're willing to consider "truth" for your new model
+7. Use `model.roc_curve()` and other tools to make your decision
+8. Update `model.truth` to codify it
+
+With the truth threshold set to `1.0` by default, deterministic methodologies are ready for others to use from step five!
+
+## Development
+
+See our full development guide and coding standards in [CONTRIBUTING.md](./docs/contributing.md)

matchbox_db-0.2.2/pyproject.toml

@@ -0,0 +1,130 @@
+[project]
+name = "matchbox-db"
+version = "0.2.2"
+description = "A framework for orchestrating and comparing data linking and deduplication methodologies."
+authors = [{ name="Department for Business and Trade" }]
+license = {file = "LICENSE"}
+readme = "README.md"
+requires-python = ">=3.11,<3.14"
+dependencies = [
+    "click>=8.1.7",
+    "connectorx>=0.3.3",
+    "duckdb>=1.1.1",
+    "faker>=36.1.1",
+    "frozendict>=2.4.6",
+    "httpx>=0.28.0",
+    "matplotlib>=3.9.2",
+    "pandas>=2.2.3",
+    "psycopg2>=2.9.10",
+    "pyarrow>=17.0.0",
+    "pydantic-settings>=2.5.2",
+    "pydantic>=2.9.2",
+    "rustworkx>=0.15.1",
+    "splink>=4.0.5,<4.1.0",
+    "sqlalchemy>=2.0.35",
+    "rich>=13.9.4",
+]
+
+[project.optional-dependencies]
+server = [
+    "boto3>=1.35.99",
+    "ddtrace>=3.2.1",
+    "fastapi[standard]>=0.115.0,<0.116.0",
+    "pg-bulk-ingest>=0.0.54",
+    "python-multipart>=0.0.18",
+    "tomli>=2.0.1",
+]
+
+[dependency-groups]
+dev = [
+    "docker>=7.1.0",
+    "griffe-inherited-docstrings>=1.1.1",
+    "ipykernel>=6.29.5",
+    "ipywidgets>=8.1.5",
+    "mkdocs>=1.6.1",
+    "mkdocs-autorefs>=1.3.0",
+    "mkdocs-material>=9.5.50",
+    "mkdocstrings[python]>=0.27.0",
+    "moto[s3]>=5.0.26",
+    "pre-commit>=3.8.0",
+    "pytest>=8.3.3",
+    "pytest-asyncio>=0.25.2",
+    "pytest-cov>=5.0.0",
+    "pytest-env>=1.1.5",
+    "respx>=0.22.0",
+    "ruff>=0.9.0",
+    "snakeviz>=2.2.2",
+    "tomli-w>=1.1.0",
+]
+typing = [
+    "boto3-stubs[s3]>=1.35.99",
+    "polars>=1.11.0",
+    "pyarrow-stubs>=17.16",
+]
+
+[project.urls]
+"Documentation" = "https://uktrade.github.io/matchbox/"
+"Repository" = "https://github.com/uktrade/matchbox.git"
+
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
+
+[tool.uv]
+default-groups = ["dev", "typing"]
+package = true
+upgrade-package = ["ruff"]
+
+[tool.ruff]
+exclude = [
+    "*.ipynb"
+]
+line-length = 88
+indent-width = 4
+target-version = "py311"
+src = ["src"]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "B",
+    "D"
+]
+ignore = []
+fixable = ["ALL"]
+unfixable = []
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"**/{test,docs}/*" = ["D"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-format = true
+
+[tool.pytest.ini_options]
+testpaths = ["test"]
+pythonpath = ["."]
+addopts = "--cov=matchbox test/ --log-disable=pg_bulk_ingest"
+norecursedirs = "src"
+log_cli = false  # Set to true to enable logs and individual tests
+log_cli_level = "ERROR"  # Set to "INFO" to see more detailed logging
+log_cli_format = "%(asctime)s [%(levelname)s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_date_format = "%X"
+asyncio_mode = "strict"
+asyncio_default_fixture_loop_scope = "function"
+filterwarnings = [
+    'ignore:.*__fields.*:pydantic.PydanticDeprecatedSince20:unittest.mock',
+    'ignore:.*__fields_set__.*:pydantic.PydanticDeprecatedSince20:unittest.mock'
+]
+markers = [
+    "docker: marks tests that require resources in Docker to be available",
+]

matchbox_db-0.2.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

matchbox_db-0.2.2/src/matchbox/__init__.py

@@ -0,0 +1,13 @@
+"""Matchbox."""
+
+from matchbox.common.exceptions import MatchboxClientSettingsException
+from matchbox.common.logging import logger
+
+try:
+    # Environment variables must be loaded first for other imports to work
+    from matchbox.client import *  # noqa: E402, F403
+except MatchboxClientSettingsException:
+    logger.warning(
+        "Impossible to initialise client. "
+        "Please ignore if running in server mode. Otherwise, check your .env file.",
+    )

matchbox_db-0.2.2/src/matchbox/client/__init__.py

@@ -0,0 +1,9 @@
+"""All client-side functionalities of Matchbox."""
+
+from matchbox.client.helpers.cleaner import process
+from matchbox.client.helpers.index import index
+from matchbox.client.helpers.selector import match, query
+from matchbox.client.models.models import make_model
+from matchbox.client.visualisation import draw_resolution_graph
+
+__all__ = ("process", "index", "match", "query", "make_model", "draw_resolution_graph")