ukam-os-builder 0.1.0.dev3__tar.gz → 0.1.0.dev5__tar.gz

ukam_os_builder-0.1.0.dev5/.github/workflows/e2e.yml

@@ -0,0 +1,147 @@
+name: End-to-end tests
+
+on:
+    pull_request:
+        branches: [main]
+
+permissions:
+    contents: read
+
+concurrency:
+    group: e2e-${{ github.head_ref || github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    e2e:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                    - source: ngd
+                      package_id: "18296"
+                      version_id: "118120"
+                    - source: abp
+                      package_id: "0040206240"
+                      version_id: "6777574"
+
+        name: E2E – ${{ matrix.source }}
+
+        steps:
+            - uses: actions/checkout@v5
+
+            - name: Mask API credentials
+              run: |
+                  echo "::add-mask::${{ secrets.OS_PROJECT_API_KEY }}"
+                  echo "::add-mask::${{ secrets.OS_PROJECT_API_SECRET }}"
+
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.10"
+
+            - name: Set up uv
+              uses: astral-sh/setup-uv@v7
+              with:
+                  enable-cache: true
+                  cache-dependency-glob: "uv.lock"
+
+            - name: Install dependencies
+              run: uv sync --all-extras --all-groups
+
+            - name: Write config.yaml
+              run: |
+                  printf '%s\n' \
+                    'paths:'                                          \
+                    '  work_dir: ./data'                              \
+                    ''                                                \
+                    'source:'                                         \
+                    '  type: ${{ matrix.source }}'                    \
+                    ''                                                \
+                    'os_downloads:'                                   \
+                    '  package_id: "${{ matrix.package_id }}"'        \
+                    '  version_id: "${{ matrix.version_id }}"'        \
+                    ''                                                \
+                    'processing:'                                     \
+                    '  parquet_compression: zstd'                     \
+                    '  parquet_compression_level: 9'                  \
+                    '  num_chunks: 1'                                 \
+                    > config.yaml
+
+            - name: Run full pipeline
+              env:
+                  OS_PROJECT_API_KEY: ${{ secrets.OS_PROJECT_API_KEY }}
+                  OS_PROJECT_API_SECRET: ${{ secrets.OS_PROJECT_API_SECRET }}
+              run: uv run ukam-os-build --verbose
+
+            - name: Verify output files exist
+              run: |
+                  echo "=== Output directory ==="
+                  ls -lhR data/output/
+                  echo ""
+                  echo "=== Checking for parquet files ==="
+                  count=$(find data/output -name '*.parquet' | wc -l)
+                  echo "Found $count parquet file(s) in data/output/"
+                  if [ "$count" -eq 0 ]; then
+                    echo "::error::No parquet output files found!"
+                    exit 1
+                  fi
+
+            - name: Preview first output row
+              run: |
+                  uv run python -c "
+                  import duckdb
+                  con = duckdb.connect()
+                  con.sql(\"SELECT * FROM read_parquet('data/output/*.parquet')\").show(max_rows=1, max_width=10000)
+                  "
+
+            # ── Second run: offline (no API credentials) ──────────────
+            - name: Record download file timestamps
+              run: |
+                  stat -c '%n %Y' data/downloads/* | sort > /tmp/downloads_before.txt
+                  echo "=== Download file timestamps ==="
+                  cat /tmp/downloads_before.txt
+
+            - name: Remove everything except downloads and block API access
+              run: |
+                  find data -mindepth 1 -maxdepth 1 ! -name downloads -exec rm -rf {} +
+                  echo "=== Remaining data tree ==="
+                  find data -type f | sort
+
+            - name: Re-run pipeline without API credentials
+              run: |
+                  unset OS_PROJECT_API_KEY OS_PROJECT_API_SECRET
+                  uv run ukam-os-build --verbose --overwrite
+
+            - name: Verify output files exist (offline run)
+              run: |
+                  echo "=== Output directory ==="
+                  ls -lhR data/output/
+                  echo ""
+                  echo "=== Checking for parquet files ==="
+                  count=$(find data/output -name '*.parquet' | wc -l)
+                  echo "Found $count parquet file(s) in data/output/"
+                  if [ "$count" -eq 0 ]; then
+                    echo "::error::No parquet output files found on offline run!"
+                    exit 1
+                  fi
+
+            - name: Preview first output row (offline run)
+              run: |
+                  uv run python -c "
+                  import duckdb
+                  con = duckdb.connect()
+                  con.sql(\"SELECT * FROM read_parquet('data/output/*.parquet')\").show(max_rows=1, max_width=10000)
+                  "
+
+            - name: Verify downloads were not modified
+              run: |
+                  stat -c '%n %Y' data/downloads/* | sort > /tmp/downloads_after.txt
+                  echo "=== Download file timestamps after offline run ==="
+                  cat /tmp/downloads_after.txt
+                  if ! diff -q /tmp/downloads_before.txt /tmp/downloads_after.txt; then
+                    echo "::error::Download file timestamps changed – files were unexpectedly modified!"
+                    diff /tmp/downloads_before.txt /tmp/downloads_after.txt
+                    exit 1
+                  fi
+                  echo "Download timestamps unchanged – existing archives were reused as expected."

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/.github/workflows/release-pypi.yml

@@ -12,6 +12,7 @@ permissions:
 jobs:
   publish:
     runs-on: ubuntu-latest
+    environment: pypi
 
     # Set up such that PyPI Trusted Publishing (OIDC) can work.
     permissions:
@@ -51,36 +52,53 @@ jobs:
 
             core.setOutput('release_sha', tagSha);
 
-      - name: Find successful build artifact run
+      - name: Wait for successful CI build artifact
         id: find_build
         uses: actions/github-script@v7
         with:
           script: |
             const { owner, repo } = context.repo;
             const sha = '${{ steps.main_guard.outputs.release_sha }}';
-
-            const runs = await github.rest.actions.listWorkflowRuns({
-              owner,
-              repo,
-              workflow_id: 'ci.yml',
-              head_sha: sha,
-              event: 'push',
-              status: 'completed',
-              per_page: 50,
-            });
-
-            const run = runs.data.workflow_runs.find((r) => r.conclusion === 'success');
-
-            if (!run) {
-              core.setFailed(
-                `No successful Build & package run found for commit ${sha}. ` +
-                'Wait for the main build to pass, then re-run this release workflow.'
-              );
-              return;
+            const maxAttempts = 30;   // 30 × 20 s = 10 minutes
+            const delayMs = 20_000;   // 20 seconds between polls
+
+            for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+              const runs = await github.rest.actions.listWorkflowRuns({
+                owner,
+                repo,
+                workflow_id: 'ci.yml',
+                head_sha: sha,
+                event: 'push',
+                status: 'completed',
+                per_page: 50,
+              });
+
+              const success = runs.data.workflow_runs.find(r => r.conclusion === 'success');
+              if (success) {
+                core.info(`Found successful CI run ${success.id} (${success.html_url})`);
+                core.setOutput('run_id', String(success.id));
+                return;
+              }
+
+              const failed = runs.data.workflow_runs.find(r => r.conclusion === 'failure');
+              if (failed) {
+                core.setFailed(
+                  `CI run ${failed.id} failed for commit ${sha}. ` +
+                  'Fix CI before releasing.'
+                );
+                return;
+              }
+
+              if (attempt < maxAttempts) {
+                core.info(`Attempt ${attempt}/${maxAttempts}: CI not finished yet — waiting ${delayMs / 1000}s …`);
+                await new Promise(r => setTimeout(r, delayMs));
+              }
             }
 
-            core.info(`Using build run id ${run.id} from ${run.html_url}`);
-            core.setOutput('run_id', String(run.id));
+            core.setFailed(
+              `No successful CI run found for commit ${sha} after ${maxAttempts} attempts (≈10 min). ` +
+              'Check whether the CI workflow was triggered for this commit.'
+            );
 
       - name: Download built dist artifact
         uses: actions/download-artifact@v4

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ukam-os-builder
-Version: 0.1.0.dev3
+Version: 0.1.0.dev5
 Summary: Download, process and transform OS address data (NGD or ABP) for UK address matching
 Project-URL: Homepage, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
 Project-URL: Repository, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
@@ -183,7 +183,7 @@ ukam-os-build --config config.yaml
 
 1. `download` - fetch package metadata and zip files from OS Data Hub.
 2. `extract` - extract CSVs from downloaded zip files and convert to parquet.
-3. `split` - ABP only: split raw records into parquet staging files.
+3. `split` - ABP only: split raw records and write only parquet staging files used by flatfile generation (`street_descriptor`, `blpu`, `lpi`, `delivery_point`, `organisation`, `classification`).
 4. `flatfile` - transform and deduplicate into final output parquet file(s).
 
 All stages are idempotent. Use `--overwrite` to regenerate outputs (`--force` is accepted as a backward-compatible alias).

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/README.md

@@ -157,7 +157,7 @@ ukam-os-build --config config.yaml
 
 1. `download` - fetch package metadata and zip files from OS Data Hub.
 2. `extract` - extract CSVs from downloaded zip files and convert to parquet.
-3. `split` - ABP only: split raw records into parquet staging files.
+3. `split` - ABP only: split raw records and write only parquet staging files used by flatfile generation (`street_descriptor`, `blpu`, `lpi`, `delivery_point`, `organisation`, `classification`).
 4. `flatfile` - transform and deduplicate into final output parquet file(s).
 
 All stages are idempotent. Use `--overwrite` to regenerate outputs (`--force` is accepted as a backward-compatible alias).

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ukam-os-builder"
-version = "0.1.0.dev3"
+version = "0.1.0.dev5"
 description = "Download, process and transform OS address data (NGD or ABP) for UK address matching"
 readme = "README.md"
 requires-python = ">=3.10"

ukam_os_builder-0.1.0.dev5/tests/test_extract_source_filtering.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ukam_os_builder.os_builder.extract import (
+    _filter_zips_for_source,
+    _should_convert_csv_to_parquet,
+)
+
+
+def test_filter_zips_for_source_prefers_ngd_named_zips() -> None:
+    zip_files = [
+        Path("add_gb_builtaddress.zip"),
+        Path("AddressBasePremium_FULL_2025-12-15_002.zip"),
+    ]
+
+    filtered = _filter_zips_for_source(zip_files, "ngd")
+
+    assert filtered == [Path("add_gb_builtaddress.zip")]
+
+
+def test_should_convert_csv_to_parquet_skips_non_ngd_for_ngd_source() -> None:
+    ngd_csv = Path("add_gb_builtaddress.csv")
+    abp_csv = Path("AddressBasePremium_FULL_2025-12-15_002.csv")
+
+    assert _should_convert_csv_to_parquet(ngd_csv, "ngd") is True
+    assert _should_convert_csv_to_parquet(abp_csv, "ngd") is False
+
+
+def test_filter_zips_for_source_excludes_ngd_historicaddress() -> None:
+    zip_files = [
+        Path("add_gb_builtaddress.zip"),
+        Path("add_gb_historicaddress.zip"),
+        Path("add_gb_historicaddress_altadd.zip"),
+        Path("add_gb_prebuildaddress.zip"),
+    ]
+
+    filtered = _filter_zips_for_source(zip_files, "ngd")
+
+    assert Path("add_gb_builtaddress.zip") in filtered
+    assert Path("add_gb_prebuildaddress.zip") in filtered
+    assert Path("add_gb_historicaddress.zip") not in filtered
+    assert Path("add_gb_historicaddress_altadd.zip") not in filtered
+
+
+def test_should_convert_csv_to_parquet_skips_ngd_historicaddress() -> None:
+    assert _should_convert_csv_to_parquet(Path("add_gb_builtaddress.csv"), "ngd") is True
+    assert _should_convert_csv_to_parquet(Path("add_gb_historicaddress.csv"), "ngd") is False
+    assert _should_convert_csv_to_parquet(Path("add_gb_historicaddress_altadd.csv"), "ngd") is False

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/tests/test_settings.py

@@ -122,7 +122,9 @@ def test_load_settings_uses_work_dir_for_default_subpaths(
     assert settings.paths.output_dir == (tmp_path / "custom_data/output").resolve()
 
 
-def test_load_settings_requires_env_vars(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+def test_load_settings_allows_missing_env_vars(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
     monkeypatch.delenv("OS_PROJECT_API_KEY", raising=False)
     monkeypatch.delenv("OS_PROJECT_API_SECRET", raising=False)
 
@@ -139,8 +141,10 @@ def test_load_settings_requires_env_vars(tmp_path: Path, monkeypatch: pytest.Mon
         """,
     )
 
-    with pytest.raises(SettingsError, match="OS_PROJECT_API_KEY"):
-        load_settings(config_path, load_env=False)
+    settings = load_settings(config_path, load_env=False)
+
+    assert settings.os_downloads.api_key is None
+    assert settings.os_downloads.api_secret is None
 
 
 def test_load_settings_validates_positive_read_timeout(

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/tests/test_smoke.py

@@ -121,7 +121,6 @@ def _prepare_test_parquet(settings: Settings) -> None:
         "add_gb_builtaddress_altadd.csv",
         "add_gb_royalmailaddress.csv",
         "add_gb_prebuildaddress.csv",
-        "add_gb_historicaddress.csv",
     ]
 
     for csv_name in sample_files:

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/__init__.py

@@ -8,7 +8,7 @@ from ukam_os_builder.os_builder.inspect_results import (
     inspect_flatfile_variants,
 )
 
-__version__ = "0.1.0.dev3"
+__version__ = "0.1.0.dev5"
 
 __all__ = [
     "create_config_and_env",

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/api/api.py

@@ -8,7 +8,7 @@ from typing import Any, Literal
 import yaml
 
 from ukam_os_builder.api.settings import Settings, SettingsError, load_settings
-from ukam_os_builder.os_builder.os_hub import get_package_version
+from ukam_os_builder.os_builder.os_hub import _get_manifest_path, get_package_version
 from ukam_os_builder.pipeline import run as run_pipeline
 from ukam_os_builder.pipeline import supported_steps_for_source
 
@@ -333,11 +333,6 @@ def run_from_config(
         parquet_compression_level=parquet_compression_level,
     )
     logger.info("Resolved work_dir: %s", settings.paths.work_dir)
-    logger.info("Resolved downloads_dir: %s", settings.paths.downloads_dir)
-    logger.info("Resolved extracted_dir: %s", settings.paths.extracted_dir)
-    logger.info("Resolved parquet_dir: %s", settings.paths.parquet_dir)
-    logger.info("Resolved output_dir: %s", settings.paths.output_dir)
-
     source_type = settings.source.type
     if step != "all":
         supported_steps = supported_steps_for_source(source_type)
@@ -347,9 +342,24 @@ def run_from_config(
                 f"--step {step} is not valid for source {source_type}. Valid steps: {valid_steps}"
             )
 
-    if check_api:
+    has_api_key = bool(os.environ.get("OS_PROJECT_API_KEY"))
+    if check_api and has_api_key:
         get_package_version(settings)
 
     overwrite_effective = overwrite if overwrite is not None else bool(force)
     run_pipeline(step=step, settings=settings, force=overwrite_effective, list_only=list_only)
+
+    logger.info(
+        "✅ Pipeline run completed\n\n"
+        "Where you need to look:\n"
+        "  • downloads_dir (raw OS Hub extracts): %s%s\n"
+        "  • output_dir (final files for address matcher): %s%s\n",
+        str(settings.paths.downloads_dir),
+        "",
+        str(settings.paths.output_dir),
+        "",
+    )
+
+    _get_manifest_path(settings)
+
     return settings

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/api/settings.py

@@ -41,8 +41,8 @@ class OSDownloadSettings(StrictBaseModel):
 
     package_id: str
     version_id: str
-    api_key: SecretStr
-    api_secret: SecretStr
+    api_key: SecretStr | None = None
+    api_secret: SecretStr | None = None
     connect_timeout_seconds: int = 30
     read_timeout_seconds: int = 300
 
@@ -57,6 +57,8 @@ class OSDownloadSettings(StrictBaseModel):
     @field_validator("api_key", "api_secret", mode="before")
     @classmethod
     def _validate_secret(cls, value: Any) -> Any:
+        if value is None:
+            return value
         if isinstance(value, str) and not value.strip():
             raise ValueError("must be non-empty")
         return value
@@ -182,22 +184,11 @@ def _load_yaml(config_path: Path) -> dict[str, Any]:
     return config
 
 
-def _validate_env_vars() -> tuple[str, str]:
-    """Validate required environment variables exist."""
+def _load_env_vars() -> tuple[str | None, str | None]:
+    """Load API credentials from environment variables if available."""
     api_key = os.environ.get("OS_PROJECT_API_KEY")
     api_secret = os.environ.get("OS_PROJECT_API_SECRET")
 
-    if not api_key:
-        raise SettingsError(
-            "OS_PROJECT_API_KEY not found in environment. "
-            "Create a .env file with OS_PROJECT_API_KEY=<your-key>"
-        )
-    if not api_secret:
-        raise SettingsError(
-            "OS_PROJECT_API_SECRET not found in environment. "
-            "Create a .env file with OS_PROJECT_API_SECRET=<your-secret>"
-        )
-
     return api_key, api_secret
 
 
@@ -216,8 +207,7 @@ def load_settings(
         Complete Settings object with resolved paths.
 
     Raises:
-        SettingsError: If config file is missing or invalid,
-                       or if required environment variables are not set.
+        SettingsError: If config file is missing or invalid.
     """
     config_path = Path(config_path).resolve()
     base_dir = config_path.parent
@@ -232,8 +222,8 @@ def load_settings(
     # Load YAML config
     config = _load_yaml(config_path)
 
-    # Validate environment variables
-    api_key, api_secret = _validate_env_vars()
+    # Load environment variables (optional)
+    api_key, api_secret = _load_env_vars()
 
     resolved_paths = resolve_paths(config=config, config_dir=base_dir)
 

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/cli.py

@@ -122,7 +122,7 @@ def main(argv: list[str] | None = None) -> int:
         config_path = Path(args.config).resolve()
         console.print(f"[green]✓[/green] Loaded config: [bold]{config_path}[/bold]")
         console.print(f"[cyan]Step:[/cyan] {args.step}")
-        console.print("[cyan]Checking OS API credentials and connectivity...[/cyan]")
+        console.print("[cyan]Starting pipeline...[/cyan]")
 
         run_from_config(
             config_path=config_path,
@@ -145,8 +145,6 @@ def main(argv: list[str] | None = None) -> int:
             parquet_compression=args.parquet_compression,
             parquet_compression_level=args.parquet_compression_level,
         )
-        logger.info("Pipeline run completed")
-        console.print("[green]✓[/green] API connectivity check passed")
         console.print("[bold green]Build completed successfully[/bold green]")
         return 0
     except (SettingsError, ValueError) as exc:

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/data_sources/abp/split_raw.py

@@ -1,8 +1,8 @@
 """Split raw ABP data module.
 
 Reads raw ABP CSV files (which contain all record types mixed together),
-splits them by record identifier (10/11/15/21/24/28/31/32/99 etc.),
-and writes one parquet file per record type.
+filters to the record types needed for flatfile creation, and writes
+one parquet file per required record type.
 """
 
 from __future__ import annotations
@@ -19,8 +19,8 @@ from ukam_os_builder.api.settings import Settings, create_duckdb_connection
 
 logger = logging.getLogger(__name__)
 
-# Record identifier to table name mapping
-RECORD_TYPE_MAP = {
+# All known ABP record identifiers
+ALL_RECORD_TYPE_MAP = {
     "10": "header",
     "11": "street",
     "15": "street_descriptor",
@@ -35,6 +35,16 @@ RECORD_TYPE_MAP = {
     "99": "trailer",
 }
 
+# Record identifiers needed for ABP flatfile creation
+RECORD_TYPE_MAP = {
+    "15": "street_descriptor",
+    "21": "blpu",
+    "24": "lpi",
+    "28": "delivery_point",
+    "31": "organisation",
+    "32": "classification",
+}
+
 DEFAULT_SCHEMA_PATH = Path(__file__).resolve().parent / "schemas" / "abp_schema.yaml"
 
 
@@ -169,12 +179,23 @@ def split_raw_to_parquet(
             input_counts[name] = count
             logger.debug("Record type %s (%s): %d lines", rid, name, count)
 
+        unused_rids = sorted(set(ALL_RECORD_TYPE_MAP) - set(RECORD_TYPE_MAP))
+        rid_list_sql = ", ".join([f"'{rid}'" for rid in unused_rids])
+        ignored_input = con.execute(f"""
+            SELECT COUNT(*)
+            FROM lines_with_rid
+            WHERE rid IN ({rid_list_sql})
+        """).fetchone()[0]
+
         total_input = sum(input_counts.values())
-        logger.info("Total input lines (with valid record IDs): %d", total_input)
+        logger.info("Total input lines (processed record IDs): %d", total_input)
+        if ignored_input > 0:
+            logger.info("Ignored input lines (unused record IDs): %d", ignored_input)
         if total_input == 0:
             raise ValueError(
                 "No ABP record identifiers found in extracted CSV input. "
-                "Ensure --source abp is used with ABP raw extracts (record IDs 10/11/15/21/24/28/31/32/99)."
+                "Ensure --source abp is used with ABP raw extracts "
+                "(required record IDs: 15/21/24/28/31/32)."
             )
 
         # 4) Process each record type
@@ -279,7 +300,7 @@ def split_raw_to_parquet(
         total_output = sum(output_counts.values())
         logger.info("")
         logger.info("=== Validation: Line count check ===")
-        logger.info("Input lines (with valid record IDs): %d", total_input)
+        logger.info("Input lines (processed record IDs): %d", total_input)
         logger.info("Output rows (parquet): %d", total_output)
 
         if total_input == total_output:

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/data_sources/abp/transform/stages/combine.py

@@ -33,7 +33,7 @@ def combine_and_dedupe(con: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyRelatio
         ),
         ranked AS (
             SELECT *,
-                CASE logical_status WHEN 1 THEN 0 WHEN 3 THEN 1 WHEN 6 THEN 2 WHEN 8 THEN 3 ELSE 9 END AS status_rank,
+                CASE logical_status WHEN 1 THEN 0 WHEN 3 THEN 1 WHEN 6 THEN 2 ELSE 9 END AS status_rank,
                 CASE source WHEN 'LPI' THEN 0 WHEN 'ORGANISATION' THEN 1 WHEN 'DELIVERY_POINT' THEN 2 WHEN 'CUSTOM_LEVEL' THEN 3 ELSE 4 END AS source_rank
             FROM normalized
         ),

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/data_sources/abp/transform/stages/lpi.py

@@ -70,15 +70,15 @@ matching messy user input. We output variants based on **Logical Status**:
     locally known as "Rose Cottage").
 3.  **Provisional (6):** The address assigned during planning/construction, which
     might change before the house is built.
-4.  **Historic (8):** An old address. If "10 High St" is renumbered to "12 High St",
-    the old address is kept as Historic. This helps match old datasets.
+
+Historic addresses (logical_status=8) are excluded from output.
 
 ------------------------------------------------------------------------------
 Key Columns Explained
 ------------------------------------------------------------------------------
 *   `uprn`: The "Golden Key". Use this to link this address to other data.
 *   `base_address`: The constructed full address string.
-*   `logical_status`: 1=Current, 6=Provisional, 8=Historic.
+*   `logical_status`: 1=Current, 6=Provisional.
 *   `official_flag`: 'Y' indicates this is the "official" version, 'N' suggests
     it might be an unofficial alias.
 *   `language`: 'ENG' (English) or 'CYM' (Welsh). Streets in Wales often have
@@ -183,7 +183,6 @@ def prepare_lpi_base(con: duckdb.DuckDBPyConnection) -> None:
                 WHEN 1 THEN 0
                 WHEN 3 THEN 1
                 WHEN 6 THEN 2
-                WHEN 8 THEN 3
                 ELSE 9
             END AS status_rank
         FROM lpi l
@@ -192,7 +191,7 @@ def prepare_lpi_base(con: duckdb.DuckDBPyConnection) -> None:
         LEFT JOIN _sd_best_by_lang sd_lang ON sd_lang.usrn = l.usrn AND sd_lang.language = l.language
         LEFT JOIN _sd_best_any sd_any ON sd_any.usrn = l.usrn
         WHERE (b.addressbase_postal != 'N' OR b.addressbase_postal IS NULL)
-          AND l.logical_status IN (1, 3, 6, 8)
+          AND l.logical_status IN (1, 3, 6)
     """)
 
     # Deduplicated distinct addresses
@@ -266,7 +265,6 @@ def render_variants(con: duckdb.DuckDBPyConnection) -> None:
                 WHEN 1 THEN 'APPROVED'
                 WHEN 3 THEN 'ALTERNATIVE'
                 WHEN 6 THEN 'PROVISIONAL'
-                WHEN 8 THEN 'HISTORICAL'
             END AS variant_label,
             (logical_status = 1) AS is_primary
         FROM lpi_base_distinct

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/data_sources/ngd/to_flatfile.py

@@ -2,7 +2,7 @@
 
 Transforms the extracted parquet files into a single flatfile suitable for
 UK address matching. This includes:
-- Processing core feature types (Built Address, Historic Address, etc.)
+- Processing core feature types (Built Address, Pre-Build Address, etc.)
 - Processing alternate address records
 - Processing Royal Mail addresses
 - Handling Welsh language variants
@@ -27,8 +27,6 @@ logger = logging.getLogger(__name__)
 FEATURE_TYPE_BY_STEM = {
     "add_gb_builtaddress": "Built Address",
     "add_gb_builtaddress_altadd": "Built Address",
-    "add_gb_historicaddress": "Historic Address",
-    "add_gb_historicaddress_altadd": "Historic Address",
     "add_gb_nonaddressableobject": "Non-Addressable Object",
     "add_gb_nonaddressableobject_altadd": "Non-Addressable Object",
     "add_gb_prebuildaddress": "Pre-Build Address",
@@ -39,7 +37,6 @@ FEATURE_TYPE_BY_STEM = {
 # Core feature stems (contain fulladdress and classification fields)
 CORE_FEATURE_STEMS = {
     "add_gb_builtaddress",
-    "add_gb_historicaddress",
     "add_gb_nonaddressableobject",
     "add_gb_prebuildaddress",
 }
@@ -47,7 +44,6 @@ CORE_FEATURE_STEMS = {
 # Alternate address stems (no classification fields)
 ALTADD_STEMS = {
     "add_gb_builtaddress_altadd",
-    "add_gb_historicaddress_altadd",
     "add_gb_nonaddressableobject_altadd",
     "add_gb_prebuildaddress_altadd",
 }
@@ -57,7 +53,6 @@ CORE_FEATURE_PRIORITY = {
     "add_gb_builtaddress": 1,
     "add_gb_prebuildaddress": 2,
     "add_gb_nonaddressableobject": 3,
-    "add_gb_historicaddress": 4,
 }
 
 
@@ -71,7 +66,7 @@ def _create_metadata_lookup_view(
     This view is used to enrich Royal Mail and alternate address records
     with metadata (classificationcode, parentuprn, etc.) by UPRN lookup.
 
-    Uses priority ranking (Built > Pre-Build > Non-Addressable > Historic)
+    Uses priority ranking (Built > Pre-Build > Non-Addressable)
     to dedupe when a UPRN exists in multiple core files.
 
     Args:
@@ -156,7 +151,7 @@ def _create_core_feature_view(
     parquet_path: Path,
     uprn_predicate: str | None = None,
 ) -> None:
-    """Create view for core feature types (Built, Historic, Pre-Build, Non-Addressable).
+    """Create view for core feature types (Built, Pre-Build, Non-Addressable).
 
     These tables have fulladdress, classification fields, and Welsh language columns.
     Produces both English and Welsh (where available) address records.
@@ -413,11 +408,76 @@ def _enrich_with_metadata(con: duckdb.DuckDBPyConnection) -> None:
     con.execute(sql)
 
 
+def _create_custom_level_rows(con: duckdb.DuckDBPyConnection) -> None:
+    """Generate custom level-based address variants and insert into enriched table.
+
+    Parses the ``floorlevel`` column (VARCHAR) from the enriched address table,
+    maps integer floor levels to words (-1=BASEMENT … 6=SIXTH), and prepends the
+    word to the existing ``address_concat`` to create additional address variants.
+
+    These rows use ``feature_type='Custom Level'`` so they receive the lowest
+    dedup priority and never override official address data.
+    """
+    sql = """
+        INSERT INTO all_full_addresses_enriched
+        WITH level_parsed AS (
+            SELECT
+                uprn, address_concat, postcode, filename,
+                classificationcode, parentuprn, rootuprn,
+                hierarchylevel, floorlevel, lowestfloorlevel, highestfloorlevel,
+                address_status, build_status,
+                CASE
+                    WHEN split_part(floorlevel, ',', 1) ~ '^-?[0-9]+$'
+                        THEN CAST(split_part(floorlevel, ',', 1) AS INTEGER)
+                    ELSE NULL
+                END AS level_int
+            FROM all_full_addresses_enriched
+            WHERE floorlevel IS NOT NULL
+              AND address_concat IS NOT NULL
+              AND address_concat <> ''
+        ),
+        level_words AS (
+            SELECT
+                *,
+                CASE level_int
+                    WHEN -1 THEN 'BASEMENT'
+                    WHEN 0 THEN 'GROUND'
+                    WHEN 1 THEN 'FIRST'
+                    WHEN 2 THEN 'SECOND'
+                    WHEN 3 THEN 'THIRD'
+                    WHEN 4 THEN 'FOURTH'
+                    WHEN 5 THEN 'FIFTH'
+                    WHEN 6 THEN 'SIXTH'
+                END AS level_word
+            FROM level_parsed
+            WHERE level_int BETWEEN -1 AND 6
+        )
+        SELECT
+            uprn,
+            TRIM(concat(level_word, ' ', address_concat)) AS address_concat,
+            postcode,
+            'CUSTOM_LEVEL' AS filename,
+            classificationcode,
+            parentuprn,
+            rootuprn,
+            hierarchylevel,
+            floorlevel,
+            lowestfloorlevel,
+            highestfloorlevel,
+            'Custom Level' AS feature_type,
+            address_status,
+            build_status
+        FROM level_words
+        WHERE level_word IS NOT NULL;
+    """
+    con.execute(sql)
+
+
 def _create_dedup_view(con: duckdb.DuckDBPyConnection) -> None:
     """Create deduplicated view of all addresses.
 
     Priority rules for deduplication:
-    - Feature type: Built Address -> Pre-Build -> Royal Mail -> Historic -> Non-Addressable
+    - Feature type: Built Address -> Pre-Build -> Royal Mail -> Non-Addressable
     - Address status: Approved -> Provisional -> Alternative -> Historical
     - Build status: Built Complete -> Under Construction -> Prebuild -> Historic -> Demolished
 
@@ -433,8 +493,8 @@ def _create_dedup_view(con: duckdb.DuckDBPyConnection) -> None:
               WHEN 'Built Address' THEN 1
               WHEN 'Pre-Build Address' THEN 2
               WHEN 'Royal Mail Address' THEN 3
-              WHEN 'Historic Address' THEN 4
               WHEN 'Non-Addressable Object' THEN 5
+              WHEN 'Custom Level' THEN 6
               ELSE 9
             END AS feature_type_rank,
             CASE
@@ -460,7 +520,7 @@ def _create_dedup_view(con: duckdb.DuckDBPyConnection) -> None:
                 build_status_rank
             ) AS rn
           FROM all_full_addresses_enriched
-          WHERE feature_type != 'Non-Addressable Object'
+          WHERE feature_type NOT IN ('Non-Addressable Object')
         )
         SELECT
           uprn,
@@ -641,6 +701,10 @@ def run_flatfile_step(settings: Settings, force: bool = False) -> list[Path]:
         logger.info("Enriching addresses with metadata from core files...")
         _enrich_with_metadata(con)
 
+        # Generate custom level variants
+        logger.info("Generating custom level address variants...")
+        _create_custom_level_rows(con)
+
         # Create deduplicated view
         logger.info("Creating deduplicated view...")
         _create_dedup_view(con)

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/os_builder/extract.py

@@ -11,6 +11,9 @@ from ukam_os_builder.api.settings import Settings
 
 logger = logging.getLogger(__name__)
 
+# NGD file stems to exclude (historic addresses are not used in output)
+_NGD_EXCLUDED_STEMS = {"historicaddress"}
+
 
 def find_downloaded_zips(downloads_dir: Path) -> list[Path]:
     """Find all downloaded zip files in a directory."""
@@ -22,11 +25,20 @@ def find_downloaded_zips(downloads_dir: Path) -> list[Path]:
     return zip_files
 
 
+def _is_excluded_ngd_file(name: str) -> bool:
+    """Return True if *name* matches an excluded NGD stem (e.g. historicaddress)."""
+    name_lower = name.lower()
+    return any(stem in name_lower for stem in _NGD_EXCLUDED_STEMS)
+
+
 def _filter_zips_for_source(zip_files: list[Path], source: str) -> list[Path]:
     source_lower = source.lower()
     if source_lower == "ngd":
         ngd_zips = [
-            zip_path for zip_path in zip_files if zip_path.name.lower().startswith("add_gb_")
+            zip_path
+            for zip_path in zip_files
+            if zip_path.name.lower().startswith("add_gb_")
+            and not _is_excluded_ngd_file(zip_path.name)
         ]
         return ngd_zips or zip_files
     if source_lower == "abp":
@@ -39,7 +51,8 @@ def _filter_zips_for_source(zip_files: list[Path], source: str) -> list[Path]:
 
 def _should_convert_csv_to_parquet(csv_path: Path, source: str) -> bool:
     if source.lower() == "ngd":
-        return csv_path.name.lower().startswith("add_gb_")
+        name_lower = csv_path.name.lower()
+        return name_lower.startswith("add_gb_") and not _is_excluded_ngd_file(name_lower)
     return True
 
 

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/ukam_os_builder/os_builder/os_hub.py

@@ -9,9 +9,25 @@ from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
 import requests
 
+from ukam_os_builder.api.settings import Settings
+
 logger = logging.getLogger(__name__)
 
 API_BASE_URL = "https://api.os.uk/downloads/v1"
+
+# NGD file stems to exclude (historic addresses are not used in output)
+_NGD_EXCLUDED_STEMS = {"historicaddress"}
+
+
+def _should_skip_ngd_download(filename: str, settings: object) -> bool:
+    """Return True if *filename* is an NGD historic-address archive."""
+    source_type = getattr(getattr(settings, "source", None), "type", "")
+    if source_type != "ngd":
+        return False
+    name_lower = filename.lower()
+    return any(stem in name_lower for stem in _NGD_EXCLUDED_STEMS)
+
+
 DEFAULT_CHUNK_SIZE = 1024 * 1024 * 20  # 20 MiB
 DEFAULT_CONNECT_TIMEOUT_SECONDS = 30
 DEFAULT_READ_TIMEOUT_SECONDS = 300
@@ -65,6 +81,13 @@ def _require_api_key(settings: Any) -> str:
     return api_key
 
 
+def _find_existing_download_archives(downloads_dir: Path) -> list[Path]:
+    """Find existing local archives that can be used for extract step."""
+    if not downloads_dir.exists():
+        return []
+    return sorted(downloads_dir.glob("*.zip"))
+
+
 def get_package_version(settings: Any) -> dict:
     """Fetch package version metadata from the OS Data Hub API."""
     package_id = settings.os_downloads.package_id
@@ -236,9 +259,27 @@ def run_download_step(
     list_only: bool = False,
 ) -> list[Path]:
     """Run the OS Data Hub download step for any compatible settings object."""
-    api_key = _require_api_key(settings)
     downloads_dir = settings.paths.downloads_dir
 
+    try:
+        api_key = _require_api_key(settings)
+    except ValueError as exc:
+        if list_only:
+            raise
+
+        existing_archives = _find_existing_download_archives(downloads_dir)
+        if existing_archives:
+            logger.warning(
+                "No API key found; using %d existing archive(s) in %s and skipping download.",
+                len(existing_archives),
+                downloads_dir,
+            )
+            return existing_archives
+
+        raise ValueError(
+            f"{exc} No local zip files were found in {downloads_dir}, so download cannot be skipped."
+        ) from exc
+
     logger.info("Fetching package metadata...")
     metadata = get_package_version(settings)
     items = list_downloads(metadata)
@@ -268,6 +309,11 @@ def run_download_step(
                 logger.warning("No URL for %s, skipping", item.filename)
                 continue
 
+            # Skip NGD historic address files — they are excluded from output
+            if _should_skip_ngd_download(item.filename, settings):
+                logger.info("Skipping historic address file: %s", item.filename)
+                continue
+
             dest_path = downloads_dir / item.filename
             was_downloaded = download_file(
                 url=item.url,
@@ -287,3 +333,54 @@ def run_download_step(
 
     logger.info("Download complete: %d file(s)", len(downloaded))
     return downloaded
+
+
+def _get_manifest_path(settings: Settings) -> Path | None:
+    downloads_dir = settings.paths.downloads_dir.resolve()
+    source_type = settings.source.type  # "abp" | "ngd"
+
+    if source_type == "abp":
+        candidates = list(downloads_dir.glob("*-Order_Details.txt"))
+        if not candidates:
+            logger.info("➡️ Manifest (ABP order details) not found. Check: %s", downloads_dir)
+            return None
+
+        manifest = max(candidates, key=lambda p: p.stat().st_mtime).resolve()
+
+        if len(candidates) > 1:
+            logger.warning(
+                "Multiple ABP manifests found in %s. Using newest: %s",
+                downloads_dir,
+                manifest,
+            )
+
+        logger.info("➡️ Manifest (ABP order details): %s", manifest)
+        return manifest
+
+    elif source_type == "ngd":
+        candidates = list(
+            downloads_dir.glob("*_orderSummary.json")
+        )  # adjust if it's "*.orderSummary.json"
+        if not candidates:
+            logger.info("➡️ Manifests (NGD order summaries) not found. Check: %s", downloads_dir)
+            return None
+
+        built_candidates = list(downloads_dir.glob("*builtaddress*_orderSummary.json"))
+        built_manifest = (
+            max(built_candidates, key=lambda p: p.stat().st_mtime).resolve()
+            if built_candidates
+            else None
+        )
+
+        logger.info(
+            "➡️ Manifests (NGD order summaries): %s (%d files)\n"
+            "    ↳ Built address order summary: %s",
+            downloads_dir,
+            len(candidates),
+            built_manifest if built_manifest else "(not found)",
+        )
+
+        return downloads_dir
+
+    logger.warning("Unknown source type %r. No manifest lookup performed.", source_type)
+    return None

{ukam_os_builder-0.1.0.dev3 → ukam_os_builder-0.1.0.dev5}/uv.lock

@@ -1421,7 +1421,7 @@ wheels = [
 
 [[package]]
 name = "ukam-os-builder"
-version = "0.1.0.dev3"
+version = "0.1.0.dev5"
 source = { editable = "." }
 dependencies = [
     { name = "duckdb" },

ukam_os_builder-0.1.0.dev3/tests/test_extract_source_filtering.py

@@ -1,27 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from ukam_os_builder.os_builder.extract import (
-    _filter_zips_for_source,
-    _should_convert_csv_to_parquet,
-)
-
-
-def test_filter_zips_for_source_prefers_ngd_named_zips() -> None:
-    zip_files = [
-        Path("add_gb_builtaddress.zip"),
-        Path("AddressBasePremium_FULL_2025-12-15_002.zip"),
-    ]
-
-    filtered = _filter_zips_for_source(zip_files, "ngd")
-
-    assert filtered == [Path("add_gb_builtaddress.zip")]
-
-
-def test_should_convert_csv_to_parquet_skips_non_ngd_for_ngd_source() -> None:
-    ngd_csv = Path("add_gb_builtaddress.csv")
-    abp_csv = Path("AddressBasePremium_FULL_2025-12-15_002.csv")
-
-    assert _should_convert_csv_to_parquet(ngd_csv, "ngd") is True
-    assert _should_convert_csv_to_parquet(abp_csv, "ngd") is False