ukam-os-builder 0.1.0.dev2__tar.gz → 0.1.0.dev4__tar.gz

ukam_os_builder-0.1.0.dev4/.github/workflows/e2e.yml

@@ -0,0 +1,147 @@
+name: End-to-end tests
+
+on:
+    pull_request:
+        branches: [main]
+
+permissions:
+    contents: read
+
+concurrency:
+    group: e2e-${{ github.head_ref || github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    e2e:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                    - source: ngd
+                      package_id: "18296"
+                      version_id: "118120"
+                    - source: abp
+                      package_id: "0040206240"
+                      version_id: "6777574"
+
+        name: E2E – ${{ matrix.source }}
+
+        steps:
+            - uses: actions/checkout@v5
+
+            - name: Mask API credentials
+              run: |
+                  echo "::add-mask::${{ secrets.OS_PROJECT_API_KEY }}"
+                  echo "::add-mask::${{ secrets.OS_PROJECT_API_SECRET }}"
+
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.10"
+
+            - name: Set up uv
+              uses: astral-sh/setup-uv@v7
+              with:
+                  enable-cache: true
+                  cache-dependency-glob: "uv.lock"
+
+            - name: Install dependencies
+              run: uv sync --all-extras --all-groups
+
+            - name: Write config.yaml
+              run: |
+                  printf '%s\n' \
+                    'paths:'                                          \
+                    '  work_dir: ./data'                              \
+                    ''                                                \
+                    'source:'                                         \
+                    '  type: ${{ matrix.source }}'                    \
+                    ''                                                \
+                    'os_downloads:'                                   \
+                    '  package_id: "${{ matrix.package_id }}"'        \
+                    '  version_id: "${{ matrix.version_id }}"'        \
+                    ''                                                \
+                    'processing:'                                     \
+                    '  parquet_compression: zstd'                     \
+                    '  parquet_compression_level: 9'                  \
+                    '  num_chunks: 1'                                 \
+                    > config.yaml
+
+            - name: Run full pipeline
+              env:
+                  OS_PROJECT_API_KEY: ${{ secrets.OS_PROJECT_API_KEY }}
+                  OS_PROJECT_API_SECRET: ${{ secrets.OS_PROJECT_API_SECRET }}
+              run: uv run ukam-os-build --verbose
+
+            - name: Verify output files exist
+              run: |
+                  echo "=== Output directory ==="
+                  ls -lhR data/output/
+                  echo ""
+                  echo "=== Checking for parquet files ==="
+                  count=$(find data/output -name '*.parquet' | wc -l)
+                  echo "Found $count parquet file(s) in data/output/"
+                  if [ "$count" -eq 0 ]; then
+                    echo "::error::No parquet output files found!"
+                    exit 1
+                  fi
+
+            - name: Preview first output row
+              run: |
+                  uv run python -c "
+                  import duckdb
+                  con = duckdb.connect()
+                  con.sql(\"SELECT * FROM read_parquet('data/output/*.parquet')\").show(max_rows=1, max_width=10000)
+                  "
+
+            # ── Second run: offline (no API credentials) ──────────────
+            - name: Record download file timestamps
+              run: |
+                  stat -c '%n %Y' data/downloads/* | sort > /tmp/downloads_before.txt
+                  echo "=== Download file timestamps ==="
+                  cat /tmp/downloads_before.txt
+
+            - name: Remove everything except downloads and block API access
+              run: |
+                  find data -mindepth 1 -maxdepth 1 ! -name downloads -exec rm -rf {} +
+                  echo "=== Remaining data tree ==="
+                  find data -type f | sort
+
+            - name: Re-run pipeline without API credentials
+              run: |
+                  unset OS_PROJECT_API_KEY OS_PROJECT_API_SECRET
+                  uv run ukam-os-build --verbose --overwrite
+
+            - name: Verify output files exist (offline run)
+              run: |
+                  echo "=== Output directory ==="
+                  ls -lhR data/output/
+                  echo ""
+                  echo "=== Checking for parquet files ==="
+                  count=$(find data/output -name '*.parquet' | wc -l)
+                  echo "Found $count parquet file(s) in data/output/"
+                  if [ "$count" -eq 0 ]; then
+                    echo "::error::No parquet output files found on offline run!"
+                    exit 1
+                  fi
+
+            - name: Preview first output row (offline run)
+              run: |
+                  uv run python -c "
+                  import duckdb
+                  con = duckdb.connect()
+                  con.sql(\"SELECT * FROM read_parquet('data/output/*.parquet')\").show(max_rows=1, max_width=10000)
+                  "
+
+            - name: Verify downloads were not modified
+              run: |
+                  stat -c '%n %Y' data/downloads/* | sort > /tmp/downloads_after.txt
+                  echo "=== Download file timestamps after offline run ==="
+                  cat /tmp/downloads_after.txt
+                  if ! diff -q /tmp/downloads_before.txt /tmp/downloads_after.txt; then
+                    echo "::error::Download file timestamps changed – files were unexpectedly modified!"
+                    diff /tmp/downloads_before.txt /tmp/downloads_after.txt
+                    exit 1
+                  fi
+                  echo "Download timestamps unchanged – existing archives were reused as expected."

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/.github/workflows/release-pypi.yml

@@ -12,6 +12,7 @@ permissions:
 jobs:
   publish:
     runs-on: ubuntu-latest
+    environment: pypi
 
     # Set up such that PyPI Trusted Publishing (OIDC) can work.
     permissions:
@@ -51,36 +52,53 @@ jobs:
 
             core.setOutput('release_sha', tagSha);
 
-      - name: Find successful build artifact run
+      - name: Wait for successful CI build artifact
         id: find_build
         uses: actions/github-script@v7
         with:
           script: |
             const { owner, repo } = context.repo;
             const sha = '${{ steps.main_guard.outputs.release_sha }}';
-
-            const runs = await github.rest.actions.listWorkflowRuns({
-              owner,
-              repo,
-              workflow_id: 'ci.yml',
-              head_sha: sha,
-              event: 'push',
-              status: 'completed',
-              per_page: 50,
-            });
-
-            const run = runs.data.workflow_runs.find((r) => r.conclusion === 'success');
-
-            if (!run) {
-              core.setFailed(
-                `No successful Build & package run found for commit ${sha}. ` +
-                'Wait for the main build to pass, then re-run this release workflow.'
-              );
-              return;
+            const maxAttempts = 30;   // 30 × 20 s = 10 minutes
+            const delayMs = 20_000;   // 20 seconds between polls
+
+            for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+              const runs = await github.rest.actions.listWorkflowRuns({
+                owner,
+                repo,
+                workflow_id: 'ci.yml',
+                head_sha: sha,
+                event: 'push',
+                status: 'completed',
+                per_page: 50,
+              });
+
+              const success = runs.data.workflow_runs.find(r => r.conclusion === 'success');
+              if (success) {
+                core.info(`Found successful CI run ${success.id} (${success.html_url})`);
+                core.setOutput('run_id', String(success.id));
+                return;
+              }
+
+              const failed = runs.data.workflow_runs.find(r => r.conclusion === 'failure');
+              if (failed) {
+                core.setFailed(
+                  `CI run ${failed.id} failed for commit ${sha}. ` +
+                  'Fix CI before releasing.'
+                );
+                return;
+              }
+
+              if (attempt < maxAttempts) {
+                core.info(`Attempt ${attempt}/${maxAttempts}: CI not finished yet — waiting ${delayMs / 1000}s …`);
+                await new Promise(r => setTimeout(r, delayMs));
+              }
             }
 
-            core.info(`Using build run id ${run.id} from ${run.html_url}`);
-            core.setOutput('run_id', String(run.id));
+            core.setFailed(
+              `No successful CI run found for commit ${sha} after ${maxAttempts} attempts (≈10 min). ` +
+              'Check whether the CI workflow was triggered for this commit.'
+            );
 
       - name: Download built dist artifact
         uses: actions/download-artifact@v4

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/.gitignore

@@ -5,6 +5,7 @@ data/
 !tests/data/**
 scripts/os_docs.md
 .env
+config.yaml
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[codz]

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/AGENTS.md

@@ -7,7 +7,7 @@ This project transforms NGD (National Geographic Database) data into a clean fla
 ## Repository Structure
 
 ```
-├── config.yaml         # Pipeline configuration
+├── config.example.yaml # Pipeline configuration template (copy to config.yaml)
 ├── script.py           # Main entry point
 ├── pyproject.toml      # Project metadata and dependencies
 ├── README.md           # User documentation

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ukam-os-builder
-Version: 0.1.0.dev2
+Version: 0.1.0.dev4
 Summary: Download, process and transform OS address data (NGD or ABP) for UK address matching
 Project-URL: Homepage, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
 Project-URL: Repository, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
@@ -140,7 +140,13 @@ result = inspect_flatfile_variants(config_path="config.yaml", top_offset=0, show
 <summary>Configure manually</summary>
 
 If you prefer not to use the setup wizard, edit `config.yaml` directly.
-Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, then adjust `paths` and `processing` as needed.
+Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`.
+
+Most users only need one path setting:
+
+- `paths.work_dir` (default `./data`, relative to the config file directory)
+
+The tool derives all other directories automatically under `work_dir`.
 
 </details>
 
@@ -153,7 +159,7 @@ Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, the
 
 ### Command notes
 
-- `--list-only` is only valid with `--step download` or `--step all`.
+- `step` only supports `download` and `all` to simplify usage. Use `--overwrite` to re-run a step with the same parameters.
 - CLI overrides take precedence over values in `config.yaml`.
 - By default, `ukam-os-build` loads `.env` from the same directory as your config, unless `--env-file` is supplied.
 
@@ -177,7 +183,7 @@ ukam-os-build --config config.yaml
 
 1. `download` - fetch package metadata and zip files from OS Data Hub.
 2. `extract` - extract CSVs from downloaded zip files and convert to parquet.
-3. `split` - ABP only: split raw records into parquet staging files.
+3. `split` - ABP only: split raw records and write only parquet staging files used by flatfile generation (`street_descriptor`, `blpu`, `lpi`, `delivery_point`, `organisation`, `classification`).
 4. `flatfile` - transform and deduplicate into final output parquet file(s).
 
 All stages are idempotent. Use `--overwrite` to regenerate outputs (`--force` is accepted as a backward-compatible alias).
@@ -294,25 +300,6 @@ When the same UPRN and address combination appears in multiple sources, records
 4. Historic
 5. Demolished
 
-## Manual Download
-
-If you prefer to download manually:
-- Sign in to https://osdatahub.os.uk/
-- Create a datapackage with NGD address features
-- Download the zip file
-
-To run the pipeline from a manual download:
-
-1. Place the zip in the downloads directory configured in `config.yaml`
-   - By default this is `data/downloads/`
-   - The extract step looks for `*.zip` files in this folder
-
-2. Run the pipeline starting from extract:
-
-```bash
-ukam-os-build --config config.yaml --step extract
-ukam-os-build --config config.yaml --step flatfile
-```
 
 ## OS Downloads API
 
@@ -348,10 +335,6 @@ source:
 
 paths:
   work_dir: ./data
-  downloads_dir: ./data/downloads
-  extracted_dir: ./data/extracted
-  parquet_dir: ./data/parquet
-  output_dir: ./data/output
 
 os_downloads:
   package_id: "<your_package_id>"
@@ -366,6 +349,34 @@ processing:
   # duckdb_memory_limit: "8GB"
 ```
 
+By default, the tool creates these directories under `paths.work_dir`:
+
+- downloads: `<work_dir>/downloads`
+- extracted: `<work_dir>/extracted`
+- parquet: `<work_dir>/parquet`
+- output: `<work_dir>/output`
+
+<details>
+<summary>Advanced: override default directories</summary>
+
+Most users won’t need this.
+
+If you need to customize locations, use `paths.overrides`:
+
+```yaml
+paths:
+  work_dir: ./data
+  overrides:
+    downloads_dir: ./somewhere/downloads
+    extracted_dir: /mnt/fast/extracted
+    parquet_dir: ./data/parquet
+    output_dir: ./output
+```
+
+Override keys replace derived defaults. Relative paths are resolved relative to the directory containing `config.yaml`.
+
+</details>
+
 ## Smoke test
 
 ```bash

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/README.md

@@ -114,7 +114,13 @@ result = inspect_flatfile_variants(config_path="config.yaml", top_offset=0, show
 <summary>Configure manually</summary>
 
 If you prefer not to use the setup wizard, edit `config.yaml` directly.
-Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, then adjust `paths` and `processing` as needed.
+Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`.
+
+Most users only need one path setting:
+
+- `paths.work_dir` (default `./data`, relative to the config file directory)
+
+The tool derives all other directories automatically under `work_dir`.
 
 </details>
 
@@ -127,7 +133,7 @@ Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, the
 
 ### Command notes
 
-- `--list-only` is only valid with `--step download` or `--step all`.
+- `step` only supports `download` and `all` to simplify usage. Use `--overwrite` to re-run a step with the same parameters.
 - CLI overrides take precedence over values in `config.yaml`.
 - By default, `ukam-os-build` loads `.env` from the same directory as your config, unless `--env-file` is supplied.
 
@@ -151,7 +157,7 @@ ukam-os-build --config config.yaml
 
 1. `download` - fetch package metadata and zip files from OS Data Hub.
 2. `extract` - extract CSVs from downloaded zip files and convert to parquet.
-3. `split` - ABP only: split raw records into parquet staging files.
+3. `split` - ABP only: split raw records and write only parquet staging files used by flatfile generation (`street_descriptor`, `blpu`, `lpi`, `delivery_point`, `organisation`, `classification`).
 4. `flatfile` - transform and deduplicate into final output parquet file(s).
 
 All stages are idempotent. Use `--overwrite` to regenerate outputs (`--force` is accepted as a backward-compatible alias).
@@ -268,25 +274,6 @@ When the same UPRN and address combination appears in multiple sources, records
 4. Historic
 5. Demolished
 
-## Manual Download
-
-If you prefer to download manually:
-- Sign in to https://osdatahub.os.uk/
-- Create a datapackage with NGD address features
-- Download the zip file
-
-To run the pipeline from a manual download:
-
-1. Place the zip in the downloads directory configured in `config.yaml`
-   - By default this is `data/downloads/`
-   - The extract step looks for `*.zip` files in this folder
-
-2. Run the pipeline starting from extract:
-
-```bash
-ukam-os-build --config config.yaml --step extract
-ukam-os-build --config config.yaml --step flatfile
-```
 
 ## OS Downloads API
 
@@ -322,10 +309,6 @@ source:
 
 paths:
   work_dir: ./data
-  downloads_dir: ./data/downloads
-  extracted_dir: ./data/extracted
-  parquet_dir: ./data/parquet
-  output_dir: ./data/output
 
 os_downloads:
   package_id: "<your_package_id>"
@@ -340,6 +323,34 @@ processing:
   # duckdb_memory_limit: "8GB"
 ```
 
+By default, the tool creates these directories under `paths.work_dir`:
+
+- downloads: `<work_dir>/downloads`
+- extracted: `<work_dir>/extracted`
+- parquet: `<work_dir>/parquet`
+- output: `<work_dir>/output`
+
+<details>
+<summary>Advanced: override default directories</summary>
+
+Most users won’t need this.
+
+If you need to customize locations, use `paths.overrides`:
+
+```yaml
+paths:
+  work_dir: ./data
+  overrides:
+    downloads_dir: ./somewhere/downloads
+    extracted_dir: /mnt/fast/extracted
+    parquet_dir: ./data/parquet
+    output_dir: ./output
+```
+
+Override keys replace derived defaults. Relative paths are resolved relative to the directory containing `config.yaml`.
+
+</details>
+
 ## Smoke test
 
 ```bash

ukam_os_builder-0.1.0.dev2/config.yaml → ukam_os_builder-0.1.0.dev4/config.example.yaml

@@ -5,15 +5,6 @@ paths:
   # Base working directory for all data
   work_dir: ./data
 
-  # Downloaded zip files from OS
-  downloads_dir: ./data/downloads
-
-  # Extracted CSV files and intermediate parquet
-  extracted_dir: ./data/extracted
-
-  # Final output parquet files
-  output_dir: ./data/output
-
 # OS Data Hub download settings
 # Given a datapackage at: https://osdatahub.os.uk/data/downloads/data-packages/16331
 # You can get versions from:

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ukam-os-builder"
-version = "0.1.0.dev2"
+version = "0.1.0.dev4"
 description = "Download, process and transform OS address data (NGD or ABP) for UK address matching"
 readme = "README.md"
 requires-python = ">=3.10"

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev4}/tests/test_api.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from textwrap import dedent
+from typing import Literal
 
 import pytest
 
@@ -39,6 +41,37 @@ def test_create_config_and_env_writes_expected_files(tmp_path: Path) -> None:
     assert "OS_PROJECT_API_SECRET=your_api_secret_here" in env_text
 
 
+def test_create_config_and_env_writes_supplied_api_credentials(tmp_path: Path) -> None:
+    config_path = tmp_path / "config.yaml"
+    env_path = tmp_path / ".env"
+
+    create_config_and_env(
+        config_out=config_path,
+        env_out=env_path,
+        source="ngd",
+        package_id="16331",
+        version_id="104444",
+        api_key="my-key",
+        api_secret="my-secret",
+    )
+
+    env_text = env_path.read_text()
+    assert "OS_PROJECT_API_KEY=my-key" in env_text
+    assert "OS_PROJECT_API_SECRET=my-secret" in env_text
+
+
+def test_create_config_and_env_rejects_partial_api_credentials(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="must be provided together"):
+        create_config_and_env(
+            config_out=tmp_path / "config.yaml",
+            env_out=tmp_path / ".env",
+            source="ngd",
+            package_id="16331",
+            version_id="104444",
+            api_key="my-key",
+        )
+
+
 def test_run_from_config_applies_overrides(
     monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
@@ -52,9 +85,6 @@ def test_run_from_config_applies_overrides(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -70,7 +100,9 @@ def test_run_from_config_applies_overrides(
     def fake_check_api(_settings: object) -> None:
         calls["checked_api"] = True
 
-    def fake_run_pipeline(step: str, settings: object, force: bool, list_only: bool) -> None:
+    def fake_run_pipeline(
+        step: Literal["all", "download"], settings: object, force: bool, list_only: bool
+    ) -> None:
         calls["step"] = step
         calls["force"] = force
         calls["list_only"] = list_only
@@ -94,6 +126,47 @@ def test_run_from_config_applies_overrides(
     assert calls["num_chunks"] == 5
 
 
+def test_run_from_config_accepts_api_key_secret_overrides(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    monkeypatch.delenv("OS_PROJECT_API_KEY", raising=False)
+    monkeypatch.delenv("OS_PROJECT_API_SECRET", raising=False)
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        source:
+          type: ngd
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", lambda _settings: None)
+    monkeypatch.setattr("ukam_os_builder.api.api.run_pipeline", lambda **_kwargs: None)
+
+    run_from_config(
+        config_path=config_path,
+        api_key="runtime-key",
+        api_secret="runtime-secret",
+    )
+
+    assert os.environ["OS_PROJECT_API_KEY"] == "runtime-key"
+    assert os.environ["OS_PROJECT_API_SECRET"] == "runtime-secret"
+
+
+def test_run_from_config_rejects_partial_api_credentials(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="must be provided together"):
+        run_from_config(
+            config_path=tmp_path / "config.yaml",
+            api_key="runtime-key",
+        )
+
+
 def test_run_from_config_validates_list_only_step(tmp_path: Path) -> None:
     with pytest.raises(ValueError, match="--list-only can only be used"):
         run_from_config(config_path=tmp_path / "config.yaml", step="extract", list_only=True)
@@ -126,7 +199,9 @@ def test_run_from_config_uses_source_override_for_pipeline_validation(
 
     monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", lambda _settings: None)
 
-    def fake_run_pipeline(step: str, settings: object, force: bool, list_only: bool) -> None:
+    def fake_run_pipeline(
+        step: Literal["all", "download"], settings: object, force: bool, list_only: bool
+    ) -> None:
         calls["step"] = step
         calls["source"] = settings.source.type
         calls["force"] = force
@@ -195,10 +270,6 @@ def test_run_from_config_applies_schema_path_override(
 
                 paths:
                     work_dir: ./data
-                    downloads_dir: ./data/downloads
-                    extracted_dir: ./data/extracted
-                    output_dir: ./data/output
-                    parquet_dir: ./data/parquet
 
                 os_downloads:
                     package_id: "16465"

ukam_os_builder-0.1.0.dev4/tests/test_cli.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from ukam_os_builder import cli
+
+
+def test_build_cli_passes_api_credentials_to_run_from_config(monkeypatch) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_run_from_config(**kwargs):
+        captured.update(kwargs)
+        return None
+
+    monkeypatch.setattr(cli, "run_from_config", fake_run_from_config)
+    monkeypatch.setattr(cli, "_configure_logging", lambda _verbose: None)
+
+    exit_code = cli.main(
+        [
+            "--config",
+            "config.yaml",
+            "--step",
+            "download",
+            "--list-only",
+            "--api-key",
+            "runtime-key",
+            "--api-secret",
+            "runtime-secret",
+        ]
+    )
+
+    assert exit_code == 0
+    assert captured["api_key"] == "runtime-key"
+    assert captured["api_secret"] == "runtime-secret"