ukam-os-builder 0.1.0.dev6__tar.gz → 0.1.0.dev8__tar.gz

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/.github/workflows/ci.yml

@@ -3,9 +3,9 @@ name: Build & package
 on:
   workflow_dispatch:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
-      - "src/**"
+      - "ukam_os_builder/**"
       - "tests/**"
       - "pyproject.toml"
       - "uv.lock"
@@ -65,4 +65,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: dist
-          path: dist/*
+          path: dist/*

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ukam-os-builder
-Version: 0.1.0.dev6
+Version: 0.1.0.dev8
 Summary: Download, process and transform OS address data (NGD or ABP) for UK address matching
 Project-URL: Homepage, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
 Project-URL: Repository, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
@@ -32,11 +32,14 @@ Build OS address data for `uk_address_matcher` from either NGD (National Geograp
 
 - Python `3.10+`
 - OS Data Hub package and version IDs
-- Network access to OS Downloads API
+- Network access to OS Downloads API for downloads or remote listing
+- Existing downloaded archives if you want to run offline without re-downloading
 - Credentials in `.env`:
   - `OS_PROJECT_API_KEY`
   - `OS_PROJECT_API_SECRET`
 
+If the required zip files already exist in your downloads directory, the build can now continue offline without contacting OS Data Hub. `--list-only` still requires network access because it queries remote package metadata.
+
 ## Install from PyPI
 
 ```bash

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/README.md

@@ -6,11 +6,14 @@ Build OS address data for `uk_address_matcher` from either NGD (National Geograp
 
 - Python `3.10+`
 - OS Data Hub package and version IDs
-- Network access to OS Downloads API
+- Network access to OS Downloads API for downloads or remote listing
+- Existing downloaded archives if you want to run offline without re-downloading
 - Credentials in `.env`:
   - `OS_PROJECT_API_KEY`
   - `OS_PROJECT_API_SECRET`
 
+If the required zip files already exist in your downloads directory, the build can now continue offline without contacting OS Data Hub. `--list-only` still requires network access because it queries remote package metadata.
+
 ## Install from PyPI
 
 ```bash

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ukam-os-builder"
-version = "0.1.0.dev6"
+version = "0.1.0.dev8"
 description = "Download, process and transform OS address data (NGD or ABP) for UK address matching"
 readme = "README.md"
 requires-python = ">=3.10"

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/tests/test_api.py

@@ -6,6 +6,7 @@ from textwrap import dedent
 from typing import Literal
 
 import pytest
+import requests
 
 from ukam_os_builder.api.api import create_config_and_env, run_from_config
 
@@ -295,3 +296,74 @@ def test_run_from_config_applies_schema_path_override(
 
     assert calls["step"] == "split"
     assert calls["schema_path"] == custom_schema.resolve()
+
+
+def test_run_from_config_continues_when_api_preflight_is_offline(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    monkeypatch.setenv("OS_PROJECT_API_KEY", "key")
+    monkeypatch.setenv("OS_PROJECT_API_SECRET", "secret")
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        source:
+          type: ngd
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    calls: dict[str, object] = {}
+
+    def fake_check_api(_settings: object) -> None:
+        raise requests.exceptions.ConnectionError("offline")
+
+    def fake_run_pipeline(step: str, settings: object, force: bool, list_only: bool) -> None:
+        calls["step"] = step
+        calls["list_only"] = list_only
+
+    monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", fake_check_api)
+    monkeypatch.setattr("ukam_os_builder.api.api.run_pipeline", fake_run_pipeline)
+
+    with caplog.at_level("WARNING"):
+        run_from_config(config_path=config_path, step="all")
+
+    assert calls["step"] == "all"
+    assert calls["list_only"] is False
+    assert "Could not reach OS Data Hub during API preflight" in caplog.text
+
+
+def test_run_from_config_raises_when_list_only_api_preflight_is_offline(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    monkeypatch.setenv("OS_PROJECT_API_KEY", "key")
+    monkeypatch.setenv("OS_PROJECT_API_SECRET", "secret")
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        source:
+          type: ngd
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    def fake_check_api(_settings: object) -> None:
+        raise requests.exceptions.ConnectionError("offline")
+
+    monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", fake_check_api)
+    monkeypatch.setattr("ukam_os_builder.api.api.run_pipeline", lambda **_kwargs: None)
+
+    with pytest.raises(requests.exceptions.ConnectionError, match="offline"):
+        run_from_config(config_path=config_path, step="download", list_only=True)

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/ukam_os_builder/__init__.py

@@ -8,7 +8,7 @@ from ukam_os_builder.os_builder.inspect_results import (
     inspect_flatfile_variants,
 )
 
-__version__ = "0.1.0.dev6"
+__version__ = "0.1.0.dev8"
 
 __all__ = [
     "create_config_and_env",

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/ukam_os_builder/api/api.py

@@ -5,6 +5,7 @@ import os
 from pathlib import Path
 from typing import Any, Literal
 
+import requests
 import yaml
 
 from ukam_os_builder.api.settings import Settings, SettingsError, load_settings
@@ -344,7 +345,16 @@ def run_from_config(
 
     has_api_key = bool(os.environ.get("OS_PROJECT_API_KEY"))
     if check_api and has_api_key:
-        get_package_version(settings)
+        try:
+            get_package_version(settings)
+        except requests.exceptions.RequestException as exc:
+            if list_only:
+                raise
+            logger.warning(
+                "Could not reach OS Data Hub during API preflight (%s). "
+                "Continuing so local downloads can be used if available.",
+                exc.__class__.__name__,
+            )
 
     overwrite_effective = overwrite if overwrite is not None else bool(force)
     run_pipeline(step=step, settings=settings, force=overwrite_effective, list_only=list_only)

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/ukam_os_builder/os_builder/os_hub.py

@@ -253,6 +253,29 @@ def download_file(
     return True
 
 
+def _use_existing_archives_or_raise(
+    downloads_dir: Path,
+    reason: str,
+    original_exc: Exception,
+) -> list[Path]:
+    """Fall back to existing local archives, or re-raise with a helpful message."""
+    existing_archives = _find_existing_download_archives(downloads_dir)
+    if existing_archives:
+        logger.warning(
+            "%s; using %d existing archive(s) in %s and skipping download "
+            "(MD5 verification against the OS Data Hub will be skipped).",
+            reason,
+            len(existing_archives),
+            downloads_dir,
+        )
+        return existing_archives
+
+    raise ValueError(
+        f"{reason}. No local zip files were found in {downloads_dir}, "
+        "so download cannot be skipped."
+    ) from original_exc
+
+
 def run_download_step(
     settings: Any,
     force: bool = False,
@@ -266,22 +289,23 @@ def run_download_step(
     except ValueError as exc:
         if list_only:
             raise
-
-        existing_archives = _find_existing_download_archives(downloads_dir)
-        if existing_archives:
-            logger.warning(
-                "No API key found; using %d existing archive(s) in %s and skipping download.",
-                len(existing_archives),
-                downloads_dir,
-            )
-            return existing_archives
-
-        raise ValueError(
-            f"{exc} No local zip files were found in {downloads_dir}, so download cannot be skipped."
-        ) from exc
+        return _use_existing_archives_or_raise(
+            downloads_dir,
+            reason="No API key found",
+            original_exc=exc,
+        )
 
     logger.info("Fetching package metadata...")
-    metadata = get_package_version(settings)
+    try:
+        metadata = get_package_version(settings)
+    except (requests.exceptions.RequestException, OSError) as exc:
+        if list_only:
+            raise
+        return _use_existing_archives_or_raise(
+            downloads_dir,
+            reason=f"Could not reach OS Data Hub ({exc.__class__.__name__})",
+            original_exc=exc,
+        )
     items = list_downloads(metadata)
 
     if list_only:

{ukam_os_builder-0.1.0.dev6 → ukam_os_builder-0.1.0.dev8}/uv.lock

@@ -1421,7 +1421,7 @@ wheels = [
 
 [[package]]
 name = "ukam-os-builder"
-version = "0.1.0.dev6"
+version = "0.1.0.dev8"
 source = { editable = "." }
 dependencies = [
     { name = "duckdb" },