matchbox-db 0.6.3.dev96__tar.gz → 0.7.1__tar.gz

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
         run: cp environments/development.env .env
 
       - name: Run databases and API
-        run: docker compose up -d --wait
+        run: docker compose --env-file .env --env-file environments/containers.env up -d --wait
 
       - name: Run Python tests
         run: uv run pytest

{matchbox_db-0.6.3.dev96/src/matchbox_db.egg-info → matchbox_db-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matchbox-db
-Version: 0.6.3.dev96
+Version: 0.7.1
 Summary: A framework for orchestrating and comparing data linking and deduplication methodologies.
 Author: Department for Business and Trade
 Project-URL: Documentation, https://uktrade.github.io/matchbox/

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/docker-compose.yml

@@ -3,7 +3,7 @@ services:
     image: postgres:17
     restart: always
     ports:
-      - "${MB__DEV__WAREHOUSE_PORT:-7654}:5432"
+      - "${MB__DEV__WAREHOUSE_PORT}:5432"
     volumes:
       - warehouse_data:/var/lib/postgresql/data
     environment:
@@ -17,8 +17,8 @@ services:
       - matchbox_store:/data
     command: server --console-address ":9001" /data
     ports:
-      - "${MB__DEV__DATASTORE_PORT:-9000}:9000"
-      - "${MB__DEV__DATASTORE_CONSOLE_PORT-9001}:9001"
+      - "${MB__DEV__DATASTORE_PORT}:9000"
+      - "${MB__DEV__DATASTORE_CONSOLE_PORT}:9001"
     healthcheck:
       test: ["CMD", "mc", "ready", "local"]
       interval: 5s
@@ -31,12 +31,11 @@ services:
     build:
       context: .
       args:
-        ENV_FILE: containers.env
         MB_VERSION: ${MB_VERSION-0.0.0dev0}
       dockerfile: src/matchbox/server/Dockerfile
       target: dev
     ports:
-      - "${MB__DEV__API_PORT-8000}:8000"
+      - "${MB__DEV__API_PORT}:8000"
     command: fastapi dev --host 0.0.0.0 src/matchbox/server/api
     depends_on:
       - postgres
@@ -59,7 +58,7 @@ services:
     image: postgres:17
     restart: always
     ports:
-      - "${MB__DEV__POSTGRES_BACKEND_PORT:-5432}:5432"
+      - "${MB__DEV__POSTGRES_BACKEND_PORT}:5432"
     volumes:
       - matchbox_data:/var/lib/postgresql/data
     environment:
@@ -70,30 +69,25 @@ services:
     image: redis:8
     restart: always
     ports:
-      - "${MB__DEV__REDIS_PORT:-6379}:6379"
+      - "${MB__DEV__REDIS_PORT}:6379"
   worker:
     build:
       context: .
       dockerfile: src/matchbox/server/Dockerfile
       target: dev
       args:
-        ENV_FILE: containers.env
         MB_VERSION: ${MB_VERSION-0.0.0dev0}
     command: celery -A matchbox.server.uploads.celery worker --loglevel=info
     depends_on:
       - postgres
       - filestorage
       - redis
-    environment:
-      - CELERY_BROKER_URL=redis://redis:6379/0
   flower:
     image: mher/flower
     container_name: flower
     ports:
-      - "${MB__DEV__FLOWER_PORT:-5555}:5555"
-    environment:
-      - CELERY_BROKER_URL=redis://redis:6379/0 
-    command: celery flower --broker=${CELERY_BROKER_URL}
+      - "${MB__DEV__FLOWER_PORT}:5555"
+    command: celery --broker=${MB__SERVER__REDIS_URI} flower
     depends_on:
       - redis
 

matchbox_db-0.7.1/docs/api/client/locations.md

@@ -0,0 +1,17 @@
+::: matchbox.client.locations
+    options:
+        show_root_heading: true
+        show_root_full_path: true
+        members_order: source
+        show_if_no_docstring: true
+        docstring_style: google
+        show_signature_annotations: true
+        separate_signature: true
+        extra:
+            show_root_docstring: true
+        filters:
+            - "!^[A-Z]$"  # Excludes single-letter uppercase variables (like T, P, R)
+            - "!^_"       # Excludes private attributes
+            - "!_logger$"  # Excludes logger variables
+            - "!_path$"    # Excludes path variables
+            - "!model_config" # Excludes Pydantic configuration

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/docs/client/evaluation.md

@@ -55,9 +55,6 @@ Choosing the right threshold for your model involves balancing precision and rec
 To evaluate this in code, first you need to build and run a model outside of a DAG:
 
 ```python
-from matchbox.client.models import Model
-from matchbox.client.queries import Query
-from matchbox.client.sources import Source
 from matchbox.client.models.dedupers import NaiveDeduper
 from sqlalchemy import create_engine
 

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/docs/client/link-data.md

@@ -97,10 +97,12 @@ The `key_field` is the field in your source that contains some unique code that
     ```python
     from matchbox.client import RelationalDBLocation
 
+    warehouse = RelationalDBLocation(name="dbname").set_client(engine)
+
     # Companies House data
     companies_house = dag.source(
         name="companies_house",
-        location=RelationalDBLocation(name="dbname", client=engine),
+        location=warehouse,
         extract_transform="""
             select
                 number::str as company_number,
@@ -117,7 +119,7 @@ The `key_field` is the field in your source that contains some unique code that
     # Exporters data
     exporters = dag.source(
         name="hmrc_exporters",
-        location=RelationalDBLocation(name="dbname", client=engine),
+        location=warehouse,
         extract_transform="""
             select
                 id,
@@ -134,7 +136,7 @@ The `key_field` is the field in your source that contains some unique code that
 
 Each [`Source`][matchbox.client.sources.Source] object requires:
 
-- A `location`, such as [`RelationalDBLocation`][matchbox.client.sources.RelationalDBLocation]. This will need a `name`, and `client`
+- A `location`, such as [`RelationalDBLocation`][matchbox.client.locations.RelationalDBLocation]. This will need a `name`, and `client`
     - The name of a location is a way of tagging it, such that later on you can filter sources you want to retrieve from the server
     - For a relational database, a SQLAlchemy engine is your client
 - An `extract_transform` string, which will take data from the location and transform it into your key and index fields. Its syntax will depend on the type of location
@@ -165,6 +167,7 @@ Dedupe steps identify and resolve duplicates within a single source.
         },
         truth=1.0,
     )
+    ```
 
 A query can optionally take instructions on how to clean the data. These are defined using a dictionary where:
 
@@ -532,9 +535,7 @@ You might want to publish a new run of your DAG based on newer data. You can ret
 === "Example"
     ```python    
     # Create a new DAG identical to the previous default
-    dag = DAG(name="companies").load_default(
-        location=RelationalDBLocation(name="dbname", client=engine)
-    ).new_run()
+    dag = DAG(name="companies").load_default().set_client(engine).new_run()
     # Run new DAG
     dag.run_and_sync()
     # Make the DAG the new default

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/docs/contributing.md

@@ -9,7 +9,7 @@ This document describes how you can get started with developing Matchbox.
 
 ## Setup
 
-Set up environment variables by creating a `.env` file under project directory. See [`/environments/development.env`](https://github.com/uktrade/matchbox/blob/main/environments/development.env) for sensible defaults.
+Set up environment variables by creating a `.env` file under project directory. See [`/environments/development.env`](https://github.com/uktrade/matchbox/blob/main/environments/development.env) for sensible defaults, which should work as is with the Docker Compose set-up and the tests.
 
 This project is managed by [uv](https://docs.astral.sh/uv/), linted and formated with [ruff](https://docs.astral.sh/ruff/), and tested with [pytest](https://docs.pytest.org/en/stable/). [Docker](https://www.docker.com) is used for local development. Documentation is build using [mkdocs](https://www.mkdocs.org).
 
@@ -33,24 +33,36 @@ Task running is done with [just](https://just.systems/man/en/). To see all avail
 just -l
 ```
 
+## Starting containers
+
+All containers required to run a development version of Matchbox can be built and launched as follows:
+
+```shell
+just build -d --wait
+```
+
+!!! tip "Docker configuration"
+    If you want to change the ports on which the various Docker containers listen (typically to avoid clashes with other projects) you need to change the variables starting with `MB__DEV_` in `.env`.
+
 ## Run tests
 
 !!! note
 
     Your `.env` file needs to be correctly configured for tests to be loaded.
 
-A just task is provided to run all tests.
+If the Docker containers are already up, you can run all tests:
 
 ```shell
-just test
+pytest
 ```
 
-If you're running tests with some other method, such as your IDE or pytest directly, you'll need to start the local backends and mock warehouse in Docker.
+Otherwise, you can build and launch the containers, as well as run the tests with a single command:
 
 ```shell
-just build -d --wait
+just test
 ```
 
+
 ## Database Migrations for PostgreSQL backend
 
 Migrations for the PostgreSQL backend are managed by [Alembic](https://alembic.sqlalchemy.org/en/latest/).

matchbox_db-0.7.1/environments/containers.env

@@ -0,0 +1,5 @@
+# .env overrides for Docker containers
+
+MB__SERVER__REDIS_URI=redis://redis:6379/0
+MB__SERVER__DATASTORE__HOST=filestorage
+MB__SERVER__POSTGRES__HOST=postgres

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/environments/development.env

@@ -1,4 +1,10 @@
-# Set Docker Compose ports
+## COPY THIS FILE to .env in the project root ##
+
+# Port configuration for Docker Compose
+# If you change these values, you might have to update a corresponding
+# variable in the server section. For example, if you change the Postgres
+# port, you won't be able to run local migrations without also changing
+# MB__SERVER__POSTGRES__PORT in this file
 MB__DEV__API_PORT=8000
 MB__DEV__DATASTORE_CONSOLE_PORT=9003
 MB__DEV__DATASTORE_PORT=9002
@@ -7,25 +13,26 @@ MB__DEV__POSTGRES_BACKEND_PORT=5432
 MB__DEV__REDIS_PORT=6379
 MB__DEV__FLOWER_PORT=5555
 
-# Configure client
+# Client configuration
 MB__CLIENT__API_ROOT=http://localhost:8000
 MB__CLIENT__TIMEOUT=10
 MB__CLIENT__RETRY_DELAY=2
 MB__CLIENT__USER=scott.mcgregor
 MB__CLIENT__BATCH_SIZE=250_000
 
-# Configure server (when run outside of Docker)
-MB__SERVER__BACKEND_TYPE=postgres
-MB__SERVER__BATCH_SIZE=250_000
+# Server configuration for when the API is run outside of Docker
+# e.g. when running in a debugger, or when checking local DB migrations
 MB__SERVER__LOG_LEVEL=INFO
+MB__SERVER__BATCH_SIZE=250_000
+MB__SERVER__BACKEND_TYPE=postgres
 MB__SERVER__AUTHORISATION=False
-MB__SERVER__PUBLIC_KEY=
+
 MB__SERVER__TASK_RUNNER=celery
 MB__SERVER__REDIS_URI=redis://localhost:6379/0
 MB__SERVER__UPLOADS_EXPIRY_MINUTES=10_080
 
 MB__SERVER__DATASTORE__HOST=localhost
-MB__SERVER__DATASTORE__PORT=9002
+MB__SERVER__DATASTORE__PORT=9000
 MB__SERVER__DATASTORE__ACCESS_KEY_ID=access_key_id
 MB__SERVER__DATASTORE__SECRET_ACCESS_KEY=secret_access_key
 MB__SERVER__DATASTORE__DEFAULT_REGION=eu-west-2

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/justfile

@@ -9,7 +9,7 @@ mod eval 'src/matchbox/client/eval/justfile'
 build *DOCKER_ARGS:
     uv sync --extra server
     MB_VERSION=$(uv run --frozen python -m setuptools_scm) \
-    docker compose --env-file=environments/containers.env up --build {{DOCKER_ARGS}}
+    docker compose --env-file .env --env-file environments/containers.env up --build {{DOCKER_ARGS}}
 
 # Delete all compiled Python files
 clean:

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/mkdocs.yml

@@ -25,6 +25,7 @@ nav:
   - Client:
     - Overview: api/client/index.md
     - DAGs: api/client/dags.md
+    - Locations: api/client/locations.md
     - Sources: api/client/sources.md
     - Queries: api/client/queries.md
     - Models: api/client/models.md

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/src/matchbox/client/__init__.py

@@ -1,7 +1,7 @@
 """All client-side functionalities of Matchbox."""
 
 from matchbox.client.dags import DAG
-from matchbox.client.sources import RelationalDBLocation
+from matchbox.client.locations import RelationalDBLocation
 
 __all__ = (
     "DAG",

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/src/matchbox/client/_handler.py

@@ -11,6 +11,7 @@ import httpx
 import polars as pl
 from pyarrow import Table
 from pyarrow.parquet import read_table
+from pydantic import ValidationError
 from tenacity import (
     retry,
     retry_if_exception_type,
@@ -120,7 +121,12 @@ def handle_http_code(res: httpx.Response) -> httpx.Response:
             raise RuntimeError(f"Unexpected 400 error: {res.content}")
 
     if res.status_code == 404:
-        error = NotFoundError.model_validate(res.json())
+        try:
+            error = NotFoundError.model_validate(res.json())
+        # Validation will fail if endpoint does not exist
+        except ValidationError as e:
+            raise RuntimeError(f"Error with request {res._request}: {res}") from e
+
         match error.entity:
             case BackendResourceType.COLLECTION:
                 raise MatchboxCollectionNotFoundError(error.details)

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/src/matchbox/client/dags.py

@@ -3,14 +3,15 @@
 import datetime
 import json
 from collections import defaultdict
-from typing import Self
+from typing import Any, Self
 
 from pyarrow import Table as ArrowTable
 
 from matchbox.client import _handler
+from matchbox.client.locations import Location
 from matchbox.client.models import Model
 from matchbox.client.queries import Query
-from matchbox.client.sources import Location, Source
+from matchbox.client.sources import Source
 from matchbox.common.dtos import (
     CollectionName,
     ModelResolutionName,
@@ -123,12 +124,11 @@ class DAG:
         self,
         name: ResolutionName,
         resolution: Resolution,
-        location: Location,
     ) -> None:
         """Convert a resolution to a Source or Model and add to DAG."""
         if resolution.resolution_type == ResolutionType.SOURCE:
             self.source(
-                location=location,
+                location=Location.from_config(resolution.config.location_config),
                 name=SourceResolutionName(name),
                 extract_transform=resolution.config.extract_transform,
                 key_field=resolution.config.key_field,
@@ -301,14 +301,16 @@ class DAG:
 
         return self
 
-    def load_default(self, location: Location) -> Self:
-        """Attach to default run in this collection, loading all DAG nodes.
+    def set_client(self, client: Any) -> Self:
+        """Assign a client to all sources at once."""
+        for node in self.nodes.values():
+            if isinstance(node, Source):
+                node.location.set_client(client)
 
-        Args:
-            location: The Location object that will be attached to nodes coming
-                from default Run. Can be updated per-source after instantiation if
-                necessary.
-        """
+        return self
+
+    def load_default(self) -> Self:
+        """Attach to default run in this collection, loading all DAG nodes."""
         collection = _handler.get_collection(self.name)
 
         run = _handler.get_run(collection=self.name, run_id=collection.default_run)
@@ -322,7 +324,7 @@ class DAG:
         )
 
         for name, resolution in sorted_resolutions:
-            self.add_resolution(name=name, resolution=resolution, location=location)
+            self.add_resolution(name=name, resolution=resolution)
 
         return self
 

{matchbox_db-0.6.3.dev96 → matchbox_db-0.7.1}/src/matchbox/client/eval/utils.py

@@ -8,8 +8,9 @@ from sqlalchemy.exc import OperationalError
 
 from matchbox.client import _handler
 from matchbox.client.dags import DAG
+from matchbox.client.locations import Location
 from matchbox.client.results import Results
-from matchbox.client.sources import Location, Source
+from matchbox.client.sources import Source
 from matchbox.common.dtos import ModelResolutionPath, ResolutionPath, ResolutionType
 from matchbox.common.eval import (
     ModelComparison,
@@ -78,8 +79,8 @@ def get_samples(
             )
             continue
 
-        location = Location.from_config(
-            resolution.config.location_config, client=client
+        location = Location.from_config(resolution.config.location_config).set_client(
+            client
         )
         source = Source.from_resolution(
             resolution=resolution,