matchbox-db 0.6.2__tar.gz → 0.6.3.dev96__tar.gz

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/.github/workflows/prerelease.yml

@@ -7,8 +7,10 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-development-image:
+  build:
     runs-on: ubuntu-latest
+    outputs:
+      mb_version: ${{ steps.get_version.outputs.mb_version }}
     permissions:
       contents: write
       packages: write
@@ -34,11 +36,21 @@ jobs:
           python-version: "3.11"
 
       - name: Extract development version
+        id: get_version
         run: |
-          echo "MB_VERSION=$( \
-            SETUPTOOLS_SCM_PRETEND_METADATA='{distance=1}' \
-            uv run --frozen python -m setuptools_scm \
-          )" >> $GITHUB_ENV
+          echo "mb_version=$(uv run --frozen python -m setuptools_scm | sed 's/+.*//')" \
+          >> "$GITHUB_OUTPUT"
+
+      - name: Build package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.get_version.outputs.mb_version }}
+        run: uv build
+
+      - name: Upload package artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-dist
+          path: ./dist
 
       - name: Extract tag metadata for Docker
         id: meta
@@ -46,7 +58,7 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
-            type=raw,value=${{ env.MB_VERSION }}
+            type=raw,value=${{ steps.get_version.outputs.mb_version }}
             type=raw,value=development
 
       - name: Build and push Docker image
@@ -56,7 +68,7 @@ jobs:
           file: src/matchbox/server/Dockerfile
           push: true
           build-args: |
-            MB_VERSION=${{ env.MB_VERSION }}
+            MB_VERSION=${{ steps.get_version.outputs.mb_version }}
           tags: ${{ steps.meta.outputs.tags }}
 
       - name: Delete existing development release if it exists
@@ -76,5 +88,23 @@ jobs:
             
             May be unstable.
             
-            **Version:** ${{ env.MB_VERSION }}
+            **Version:** ${{ steps.get_version.outputs.mb_version }}
             **Commit:** ${{ github.sha }}
+
+  deploy-package:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    
+    environment: pypi
+    
+    steps:
+      - name: Download package artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: package-dist
+          path: ./dist
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

{matchbox_db-0.6.2/src/matchbox_db.egg-info → matchbox_db-0.6.3.dev96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matchbox-db
-Version: 0.6.2
+Version: 0.6.3.dev96
 Summary: A framework for orchestrating and comparing data linking and deduplication methodologies.
 Author: Department for Business and Trade
 Project-URL: Documentation, https://uktrade.github.io/matchbox/
@@ -14,7 +14,6 @@ Requires-Dist: duckdb>=1.1.1
 Requires-Dist: faker>=36.1.1
 Requires-Dist: frozendict>=2.4.6
 Requires-Dist: httpx>=0.28.0
-Requires-Dist: matplotlib>=3.9.2
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: polars-hash>=0.5.3
 Requires-Dist: polars>=1.32.3

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/docs/api/common/factories/index.md

@@ -51,7 +51,7 @@ source_testkit = source_factory()
 
 # Setup store
 tracker = InMemoryUploadTracker()
-upload_id = tracker.add_source(source_testkit.source_config)
+upload_id = tracker.add_source(source_testkit.source.resolution_path)
 ```
 
 
@@ -83,7 +83,7 @@ source_factory(
 By default, each `SourceTestkit` or `ModelTestkit` creates a new [`DAG`][matchbox.client.dags.DAG]. If membership to the right DAG is important, you can either set it manually:
 
 ```python
-dag = DAG("companies", new=True)
+dag = DAG("companies")
 source_testkit = source_factory(dag=dag)
 ```
 
@@ -91,7 +91,7 @@ Or, you can unpack your objects into `DAG` methods:
 
 ```python
 source_testkit = source_factory()
-dag = DAG("companies", new=True)
+dag = DAG("companies")
 dag.source(**source_testkit.into_dag())
 ```
 

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/docs/client/evaluation.md

@@ -63,7 +63,7 @@ from sqlalchemy import create_engine
 
 engine = create_engine('postgresql://')
 
-dag = DAG(name="companies", new=True)
+dag = DAG(name="companies").new_run()
 
 source = dag.source(...) # source parameters must be completed
 

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/docs/client/link-data.md

@@ -78,10 +78,10 @@ You're now ready to create your first [`DAG`][matchbox.client.dags.DAG].
 === "Example"
     ```python
     from matchbox.client.dags import DAG
-    dag = DAG(name="companies", new=True)
+    dag = DAG(name="companies").new_run()
     ```
 
-A DAG needs a name, which will be used to identify this DAG once you publish it to the Matchbox server. You also need to specify that this is a new, unpublished DAG. If you don't do that, Matchbox will look for a DAG with this name that was already published.
+A DAG needs a name, which will be used to identify this DAG once you publish it to the Matchbox server. You also need to use the `.new_run()` method to prepare the DAG to send results to the server.
 
 This DAG will own all the sources and models you define later.
 
@@ -418,7 +418,7 @@ Once you're happy with your results, you need to publish your DAG so that other
 
 === "Example"
     ```python    
-    dag.publish()
+    dag.set_default()
     ```
 
 
@@ -527,16 +527,18 @@ This example demonstrates how you can:
 
 ### Re-run a previous DAG
 
-You might want to publish a new version of your DAG based on newer data. You can retrieve the old DAG and inspect it. You can't sync or publish it, as it will be read-only. However, you can generate a new version from it explicitly
+You might want to publish a new run of your DAG based on newer data. You can retrieve the old DAG and inspect it. You can't sync or publish it, as it will be read-only. However, you can generate a new run from it explicitly
 
 === "Example"
     ```python    
-    # Retrieve published DAG
-    old_dag = DAG(name="companies")
-    # Generate a new DAG from it
-    new_dag = old_dag.clone()
-    new_dag.run_and_sync()
-    new_dag.publish()
+    # Create a new DAG identical to the previous default
+    dag = DAG(name="companies").load_default(
+        location=RelationalDBLocation(name="dbname", client=engine)
+    ).new_run()
+    # Run new DAG
+    dag.run_and_sync()
+    # Make the DAG the new default
+    dag.set_default()
     ```
 
 ## Best practices

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/docs/client/look-up.md

@@ -8,7 +8,7 @@ Given a key and a source, you can retrieve all keys resolving to the same entity
     ```python
     from matchbox.client.dags import DAG
 
-    matches = DAG("companies").lookup_key(
+    matches = DAG("companies").load_default().lookup_key(
         from_source="datahub_companies",
         to_sources=["companies_house"],
         key="8534735",
@@ -32,5 +32,5 @@ You can download an entire lookup as a PyArrow table.
     ```python
     from matchbox.client.dags import DAG
 
-    lookup = DAG("companies").extract_lookup()
+    lookup = DAG("companies").load_default().extract_lookup()
     ```

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/mkdocs.yml

@@ -47,7 +47,6 @@ nav:
       - Sources: api/common/factories/sources.md
       - Models: api/common/factories/models.md
       - Scenarios: api/common/factories/scenarios.md
-    - Graph: api/common/graph.md
     - Hashing: api/common/hash.md
     - Logging: api/common/logging.md
     - Transform: api/common/transform.md

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/pyproject.toml

@@ -13,7 +13,6 @@ dependencies = [
     "faker>=36.1.1",
     "frozendict>=2.4.6",
     "httpx>=0.28.0",
-    "matplotlib>=3.9.2",
     "pandas>=2.2.3",
     "polars-hash>=0.5.3",
     "polars>=1.32.3",

{matchbox_db-0.6.2 → matchbox_db-0.6.3.dev96}/src/matchbox/client/_handler.py

@@ -8,6 +8,7 @@ from importlib.metadata import version
 from io import BytesIO
 
 import httpx
+import polars as pl
 from pyarrow import Table
 from pyarrow.parquet import read_table
 from tenacity import (
@@ -32,33 +33,37 @@ from matchbox.common.dtos import (
     BackendCountableType,
     BackendParameterType,
     BackendResourceType,
+    Collection,
+    CollectionName,
     LoginAttempt,
     LoginResult,
     Match,
+    ModelResolutionPath,
     NotFoundError,
     Resolution,
-    ResolutionOperationStatus,
+    ResolutionPath,
     ResolutionType,
+    ResourceOperationStatus,
+    Run,
+    RunID,
+    SourceResolutionPath,
     UploadStage,
     UploadStatus,
 )
 from matchbox.common.eval import Judgement, ModelComparison
 from matchbox.common.exceptions import (
+    MatchboxCollectionNotFoundError,
     MatchboxDataNotFound,
     MatchboxDeletionNotConfirmed,
     MatchboxEmptyServerResponse,
     MatchboxResolutionNotFoundError,
+    MatchboxRunNotFoundError,
     MatchboxServerFileError,
     MatchboxTooManySamplesRequested,
     MatchboxUnhandledServerResponse,
     MatchboxUnparsedClientRequest,
     MatchboxUserNotFoundError,
 )
-from matchbox.common.graph import (
-    ModelResolutionName,
-    ResolutionName,
-    SourceResolutionName,
-)
 from matchbox.common.hash import hash_to_base64
 from matchbox.common.logging import logger
 
@@ -117,6 +122,10 @@ def handle_http_code(res: httpx.Response) -> httpx.Response:
     if res.status_code == 404:
         error = NotFoundError.model_validate(res.json())
         match error.entity:
+            case BackendResourceType.COLLECTION:
+                raise MatchboxCollectionNotFoundError(error.details)
+            case BackendResourceType.RUN:
+                raise MatchboxRunNotFoundError(error.details)
             case BackendResourceType.RESOLUTION:
                 raise MatchboxResolutionNotFoundError(error.details)
             case BackendResourceType.CLUSTER:
@@ -127,7 +136,7 @@ def handle_http_code(res: httpx.Response) -> httpx.Response:
                 raise RuntimeError(f"Unexpected 404 error: {error.details}")
 
     if res.status_code == 409:
-        error = ResolutionOperationStatus.model_validate(res.json())
+        error = ResourceOperationStatus.model_validate(res.json())
         raise MatchboxDeletionNotConfirmed(message=error.details)
 
     if res.status_code == 422:
@@ -180,9 +189,9 @@ def login(user_name: str) -> int:
 
 @http_retry
 def query(
-    source: SourceResolutionName,
+    source: SourceResolutionPath,
     return_leaf_id: bool,
-    resolution: ResolutionName | None = None,
+    resolution: ResolutionPath | None = None,
     threshold: int | None = None,
     limit: int | None = None,
 ) -> Table:
@@ -194,8 +203,10 @@ def query(
         "/query",
         params=url_params(
             {
-                "source": source,
-                "resolution": resolution,
+                "collection": source.collection,
+                "run_id": source.run,
+                "source": source.name,
+                "resolution": resolution.name if resolution else None,
                 "return_leaf_id": return_leaf_id,
                 "threshold": threshold,
                 "limit": limit,
@@ -222,16 +233,16 @@ def query(
 
 @http_retry
 def match(
-    targets: list[SourceResolutionName],
-    source: SourceResolutionName,
+    targets: list[SourceResolutionPath],
+    source: SourceResolutionPath,
     key: str,
-    resolution: ResolutionName,
+    resolution: ResolutionPath,
     threshold: int | None = None,
 ) -> list[Match]:
     """Match a source against a list of targets."""
     log_prefix = f"Query {source}"
     logger.debug(
-        f"{key} to {', '.join(targets)} using {resolution}",
+        f"{key} to {', '.join(str(targets))} using {resolution}",
         prefix=log_prefix,
     )
 
@@ -239,10 +250,12 @@ def match(
         "/match",
         params=url_params(
             {
-                "targets": targets,
-                "source": source,
+                "collection": resolution.collection,
+                "run_id": resolution.run,
+                "targets": [t.name for t in targets],
+                "source": source.name,
                 "key": key,
-                "resolution": resolution,
+                "resolution": resolution.name,
                 "threshold": threshold,
             }
         ),
@@ -258,42 +271,125 @@ def match(
     return matches
 
 
-# Resolution management
+# Collection management
+
+
+@http_retry
+def get_collection(name: CollectionName) -> Collection:
+    """Get all runs and resolutions in a collection."""
+    log_prefix = f"Collection {name}"
+    logger.debug("Retrieving", prefix=log_prefix)
+
+    res = CLIENT.get(f"/collections/{name}")
+    return Collection.model_validate(res.json())
+
+
+@http_retry
+def create_collection(name: CollectionName) -> ResourceOperationStatus:
+    """Create a new collection."""
+    log_prefix = f"Collection {name}"
+    logger.debug("Creating", prefix=log_prefix)
+
+    res = CLIENT.post(
+        f"/collections/{name}",
+    )
+
+    return ResourceOperationStatus.model_validate(res.json())
+
+
+# Run management
 
 
 @http_retry
-def get_leaf_source_resolutions(name: ModelResolutionName) -> list[Resolution]:
-    log_prefix = f"Resolution {name}"
+def get_run(collection: CollectionName, run_id: RunID) -> Run:
+    """Get all resolutions in a run."""
+    log_prefix = f"Collection {collection}, run {run_id}"
     logger.debug("Retrieving", prefix=log_prefix)
 
-    res = CLIENT.get(f"/resolutions/{name}/sources")
+    res = CLIENT.get(f"/collections/{collection}/runs/{run_id}")
+    return Run.model_validate(res.json())
+
+
+@http_retry
+def create_run(collection: CollectionName) -> ResourceOperationStatus:
+    """Create a new run."""
+    log_prefix = f"Collection {collection}, new run"
+    logger.debug("Creating", prefix=log_prefix)
+
+    res = CLIENT.post(f"/collections/{collection}/runs")
+
+    return Run.model_validate(res.json())
+
+
+@http_retry
+def delete_run(
+    collection: CollectionName, run_id: RunID, certain: bool = False
+) -> ResourceOperationStatus:
+    """Delete a run in Matchbox."""
+    log_prefix = f"Collection {collection}, run {run_id}"
+    logger.debug("Deleting", prefix=log_prefix)
+
+    res = CLIENT.delete(
+        f"/collections/{collection}/runs/{run_id}",
+        params={"certain": certain},
+    )
+    return ResourceOperationStatus.model_validate(res.json())
+
 
-    return [Resolution.model_validate(s) for s in res.json()]
+@http_retry
+def set_run_mutable(
+    collection: CollectionName, run_id: RunID, mutable: bool
+) -> ResourceOperationStatus:
+    """Set a run as mutable for a collection."""
+    log_prefix = f"Collection {collection}, run {run_id}"
+    logger.debug("Setting mutability", prefix=log_prefix)
+
+    res = CLIENT.patch(f"/collections/{collection}/runs/{run_id}/mutable", json=mutable)
+    return ResourceOperationStatus.model_validate(res.json())
+
+
+@http_retry
+def set_run_default(
+    collection: CollectionName, run_id: RunID, default: bool
+) -> ResourceOperationStatus:
+    """Set a run as the default run for a collection."""
+    log_prefix = f"Collection {collection}, run {run_id}"
+    logger.debug("Setting as default", prefix=log_prefix)
+
+    res = CLIENT.patch(f"/collections/{collection}/runs/{run_id}/default", json=default)
+    return ResourceOperationStatus.model_validate(res.json())
+
+
+# Resolution management
 
 
 @http_retry
 def create_resolution(
     resolution: Resolution,
-) -> ResolutionOperationStatus | UploadStatus:
+    path: ResolutionPath,
+) -> ResourceOperationStatus:
     """Create a resolution (model or source)."""
-    log_prefix = f"Resolution {resolution.name}"
+    log_prefix = f"Resolution {path}"
     logger.debug("Creating", prefix=log_prefix)
 
-    res = CLIENT.post("/resolutions", json=resolution.model_dump())
+    res = CLIENT.post(
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}",
+        json=resolution.model_dump(),
+    )
 
-    return ResolutionOperationStatus.model_validate(res.json())
+    return ResourceOperationStatus.model_validate(res.json())
 
 
 @http_retry
 def get_resolution(
-    name: ResolutionName, validate_type: ResolutionType | None = None
+    path: ResolutionPath, validate_type: ResolutionType | None = None
 ) -> Resolution | None:
     """Get a resolution from Matchbox."""
-    log_prefix = f"Resolution {name}"
+    log_prefix = f"Resolution {path}"
     logger.debug("Retrieving metadata", prefix=log_prefix)
 
     res = CLIENT.get(
-        f"/resolutions/{name}",
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}",
         params=url_params({"validate_type": validate_type}),
     )
     return Resolution.model_validate(res.json())
@@ -301,17 +397,19 @@ def get_resolution(
 
 @http_retry
 def set_data(
-    name: ResolutionName, data: Table, validate_type: ResolutionType
+    path: ResolutionPath, data: pl.DataFrame | Table, validate_type: ResolutionType
 ) -> UploadStatus:
     """Upload source hashes or model results to server."""
-    log_prefix = f"Resolution {name}"
+    log_prefix = f"Resolution {path}"
     logger.debug("Uploading results", prefix=log_prefix)
 
-    buffer = table_to_buffer(table=data)
+    data_arrow = data.to_arrow() if isinstance(data, pl.DataFrame) else data
+    buffer = table_to_buffer(table=data_arrow)
 
     # Initialise upload
     metadata_res = CLIENT.post(
-        f"/resolutions/{name}/data", params=url_params({"validate_type": validate_type})
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}/data",
+        params=url_params({"validate_type": validate_type}),
     )
 
     upload = UploadStatus.model_validate(metadata_res.json())
@@ -343,70 +441,103 @@ def set_data(
 
 
 @http_retry
-def get_results(name: ModelResolutionName) -> Table:
+def get_results(path: ModelResolutionPath) -> Table:
     """Get model results from Matchbox."""
-    log_prefix = f"Model {name}"
+    log_prefix = f"Model {path}"
     logger.debug("Retrieving results", prefix=log_prefix)
 
-    res = CLIENT.get(f"/resolutions/{name}/data")
+    res = CLIENT.get(
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}/data"
+    )
     buffer = BytesIO(res.content)
     return read_table(buffer)
 
 
 @http_retry
-def set_truth(name: ModelResolutionName, truth: int) -> ResolutionOperationStatus:
+def set_truth(path: ModelResolutionPath, truth: int) -> ResourceOperationStatus:
     """Set the truth threshold for a model in Matchbox."""
-    log_prefix = f"Model {name}"
+    log_prefix = f"Model {path}"
     logger.debug("Setting truth value", prefix=log_prefix)
 
-    res = CLIENT.patch(f"/resolutions/{name}/truth", json=truth)
-    return ResolutionOperationStatus.model_validate(res.json())
+    res = CLIENT.patch(
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}/truth",
+        json=truth,
+    )
+    return ResourceOperationStatus.model_validate(res.json())
 
 
 @http_retry
-def get_truth(name: ModelResolutionName) -> int:
+def get_truth(path: ModelResolutionPath) -> int:
     """Get the truth threshold for a model in Matchbox."""
-    log_prefix = f"Model {name}"
+    log_prefix = f"Model {path}"
     logger.debug("Retrieving truth value", prefix=log_prefix)
 
-    res = CLIENT.get(f"/resolutions/{name}/truth")
+    res = CLIENT.get(
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}/truth"
+    )
     return res.json()
 
 
 @http_retry
 def delete_resolution(
-    name: ModelResolutionName, certain: bool = False
-) -> ResolutionOperationStatus:
+    path: ModelResolutionPath, certain: bool = False
+) -> ResourceOperationStatus:
     """Delete a resolution in Matchbox."""
-    log_prefix = f"Model {name}"
+    log_prefix = f"Model {path}"
     logger.debug("Deleting", prefix=log_prefix)
 
-    res = CLIENT.delete(f"/resolutions/{name}", params={"certain": certain})
-    return ResolutionOperationStatus.model_validate(res.json())
+    res = CLIENT.delete(
+        f"/collections/{path.collection}/runs/{path.run}/resolutions/{path.name}",
+        params={"certain": certain},
+    )
+    return ResourceOperationStatus.model_validate(res.json())
 
 
 # Evaluation
 
 
 @http_retry
-def sample_for_eval(n: int, resolution: ModelResolutionName, user_id: int) -> Table:
+def sample_for_eval(n: int, resolution: ModelResolutionPath, user_id: int) -> Table:
+    """Sample model results for evaluation."""
     res = CLIENT.get(
         "/eval/samples",
-        params=url_params({"n": n, "resolution": resolution, "user_id": user_id}),
+        params=url_params(
+            {
+                "n": n,
+                "collection": resolution.collection,
+                "run_id": resolution.run,
+                "resolution": resolution.name,
+                "user_id": user_id,
+            }
+        ),
     )
 
     return read_table(BytesIO(res.content))
 
 
 @http_retry
-def compare_models(resolutions: list[ModelResolutionName]) -> ModelComparison:
-    res = CLIENT.get("/eval/compare", params=url_params({"resolutions": resolutions}))
+def compare_models(
+    resolutions: list[ModelResolutionPath],
+) -> ModelComparison:
+    """Get a model comparison for a set of model resolutions."""
+    qualified_resolution = [
+        ModelResolutionPath(
+            collection=resolution.collection,
+            run=resolution.run,
+            name=resolution,
+        )
+        for resolution in resolutions
+    ]
+    res = CLIENT.post(
+        "/eval/compare", json=[r.model_dump() for r in qualified_resolution]
+    )
     scores = {resolution: tuple(pr) for resolution, pr in res.json().items()}
     return scores
 
 
 @http_retry
 def send_eval_judgement(judgement: Judgement) -> None:
+    """Send judgements to the server."""
     logger.debug(
         f"Submitting judgement {judgement.shown}:{judgement.endorsed} "
         f"for {judgement.user_id}"
@@ -416,6 +547,7 @@ def send_eval_judgement(judgement: Judgement) -> None:
 
 @http_retry
 def download_eval_data() -> tuple[Table, Table]:
+    """Download all judgements from the server."""
     logger.debug("Retrieving all judgements.")
     res = CLIENT.get("/eval/judgements")
 
@@ -432,7 +564,7 @@ def download_eval_data() -> tuple[Table, Table]:
     check_schema(SCHEMA_JUDGEMENTS, judgements.schema)
     check_schema(SCHEMA_CLUSTER_EXPANSION, expansion.schema)
 
-    return judgements, expansion
+    return pl.from_arrow(judgements), pl.from_arrow(expansion)
 
 
 # Admin