ukam-os-builder 0.1.0.dev2__tar.gz → 0.1.0.dev3__tar.gz

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/.gitignore

@@ -5,6 +5,7 @@ data/
 !tests/data/**
 scripts/os_docs.md
 .env
+config.yaml
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[codz]

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/AGENTS.md

@@ -7,7 +7,7 @@ This project transforms NGD (National Geographic Database) data into a clean fla
 ## Repository Structure
 
 ```
-├── config.yaml         # Pipeline configuration
+├── config.example.yaml # Pipeline configuration template (copy to config.yaml)
 ├── script.py           # Main entry point
 ├── pyproject.toml      # Project metadata and dependencies
 ├── README.md           # User documentation

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ukam-os-builder
-Version: 0.1.0.dev2
+Version: 0.1.0.dev3
 Summary: Download, process and transform OS address data (NGD or ABP) for UK address matching
 Project-URL: Homepage, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
 Project-URL: Repository, https://github.com/moj-analytical-services/prepare_ngd_for_address_matching
@@ -140,7 +140,13 @@ result = inspect_flatfile_variants(config_path="config.yaml", top_offset=0, show
 <summary>Configure manually</summary>
 
 If you prefer not to use the setup wizard, edit `config.yaml` directly.
-Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, then adjust `paths` and `processing` as needed.
+Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`.
+
+Most users only need one path setting:
+
+- `paths.work_dir` (default `./data`, relative to the config file directory)
+
+The tool derives all other directories automatically under `work_dir`.
 
 </details>
 
@@ -153,7 +159,7 @@ Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, the
 
 ### Command notes
 
-- `--list-only` is only valid with `--step download` or `--step all`.
+- `step` only supports `download` and `all` to simplify usage. Use `--overwrite` to re-run a step with the same parameters.
 - CLI overrides take precedence over values in `config.yaml`.
 - By default, `ukam-os-build` loads `.env` from the same directory as your config, unless `--env-file` is supplied.
 
@@ -294,25 +300,6 @@ When the same UPRN and address combination appears in multiple sources, records
 4. Historic
 5. Demolished
 
-## Manual Download
-
-If you prefer to download manually:
-- Sign in to https://osdatahub.os.uk/
-- Create a datapackage with NGD address features
-- Download the zip file
-
-To run the pipeline from a manual download:
-
-1. Place the zip in the downloads directory configured in `config.yaml`
-   - By default this is `data/downloads/`
-   - The extract step looks for `*.zip` files in this folder
-
-2. Run the pipeline starting from extract:
-
-```bash
-ukam-os-build --config config.yaml --step extract
-ukam-os-build --config config.yaml --step flatfile
-```
 
 ## OS Downloads API
 
@@ -348,10 +335,6 @@ source:
 
 paths:
   work_dir: ./data
-  downloads_dir: ./data/downloads
-  extracted_dir: ./data/extracted
-  parquet_dir: ./data/parquet
-  output_dir: ./data/output
 
 os_downloads:
   package_id: "<your_package_id>"
@@ -366,6 +349,34 @@ processing:
   # duckdb_memory_limit: "8GB"
 ```
 
+By default, the tool creates these directories under `paths.work_dir`:
+
+- downloads: `<work_dir>/downloads`
+- extracted: `<work_dir>/extracted`
+- parquet: `<work_dir>/parquet`
+- output: `<work_dir>/output`
+
+<details>
+<summary>Advanced: override default directories</summary>
+
+Most users won’t need this.
+
+If you need to customize locations, use `paths.overrides`:
+
+```yaml
+paths:
+  work_dir: ./data
+  overrides:
+    downloads_dir: ./somewhere/downloads
+    extracted_dir: /mnt/fast/extracted
+    parquet_dir: ./data/parquet
+    output_dir: ./output
+```
+
+Override keys replace derived defaults. Relative paths are resolved relative to the directory containing `config.yaml`.
+
+</details>
+
 ## Smoke test
 
 ```bash

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/README.md

@@ -114,7 +114,13 @@ result = inspect_flatfile_variants(config_path="config.yaml", top_offset=0, show
 <summary>Configure manually</summary>
 
 If you prefer not to use the setup wizard, edit `config.yaml` directly.
-Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, then adjust `paths` and `processing` as needed.
+Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`.
+
+Most users only need one path setting:
+
+- `paths.work_dir` (default `./data`, relative to the config file directory)
+
+The tool derives all other directories automatically under `work_dir`.
 
 </details>
 
@@ -127,7 +133,7 @@ Set `source.type`, `os_downloads.package_id`, and `os_downloads.version_id`, the
 
 ### Command notes
 
-- `--list-only` is only valid with `--step download` or `--step all`.
+- `step` only supports `download` and `all` to simplify usage. Use `--overwrite` to re-run a step with the same parameters.
 - CLI overrides take precedence over values in `config.yaml`.
 - By default, `ukam-os-build` loads `.env` from the same directory as your config, unless `--env-file` is supplied.
 
@@ -268,25 +274,6 @@ When the same UPRN and address combination appears in multiple sources, records
 4. Historic
 5. Demolished
 
-## Manual Download
-
-If you prefer to download manually:
-- Sign in to https://osdatahub.os.uk/
-- Create a datapackage with NGD address features
-- Download the zip file
-
-To run the pipeline from a manual download:
-
-1. Place the zip in the downloads directory configured in `config.yaml`
-   - By default this is `data/downloads/`
-   - The extract step looks for `*.zip` files in this folder
-
-2. Run the pipeline starting from extract:
-
-```bash
-ukam-os-build --config config.yaml --step extract
-ukam-os-build --config config.yaml --step flatfile
-```
 
 ## OS Downloads API
 
@@ -322,10 +309,6 @@ source:
 
 paths:
   work_dir: ./data
-  downloads_dir: ./data/downloads
-  extracted_dir: ./data/extracted
-  parquet_dir: ./data/parquet
-  output_dir: ./data/output
 
 os_downloads:
   package_id: "<your_package_id>"
@@ -340,6 +323,34 @@ processing:
   # duckdb_memory_limit: "8GB"
 ```
 
+By default, the tool creates these directories under `paths.work_dir`:
+
+- downloads: `<work_dir>/downloads`
+- extracted: `<work_dir>/extracted`
+- parquet: `<work_dir>/parquet`
+- output: `<work_dir>/output`
+
+<details>
+<summary>Advanced: override default directories</summary>
+
+Most users won’t need this.
+
+If you need to customize locations, use `paths.overrides`:
+
+```yaml
+paths:
+  work_dir: ./data
+  overrides:
+    downloads_dir: ./somewhere/downloads
+    extracted_dir: /mnt/fast/extracted
+    parquet_dir: ./data/parquet
+    output_dir: ./output
+```
+
+Override keys replace derived defaults. Relative paths are resolved relative to the directory containing `config.yaml`.
+
+</details>
+
 ## Smoke test
 
 ```bash

ukam_os_builder-0.1.0.dev2/config.yaml → ukam_os_builder-0.1.0.dev3/config.example.yaml

@@ -5,15 +5,6 @@ paths:
   # Base working directory for all data
   work_dir: ./data
 
-  # Downloaded zip files from OS
-  downloads_dir: ./data/downloads
-
-  # Extracted CSV files and intermediate parquet
-  extracted_dir: ./data/extracted
-
-  # Final output parquet files
-  output_dir: ./data/output
-
 # OS Data Hub download settings
 # Given a datapackage at: https://osdatahub.os.uk/data/downloads/data-packages/16331
 # You can get versions from:

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ukam-os-builder"
-version = "0.1.0.dev2"
+version = "0.1.0.dev3"
 description = "Download, process and transform OS address data (NGD or ABP) for UK address matching"
 readme = "README.md"
 requires-python = ">=3.10"

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/tests/test_api.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from textwrap import dedent
+from typing import Literal
 
 import pytest
 
@@ -39,6 +41,37 @@ def test_create_config_and_env_writes_expected_files(tmp_path: Path) -> None:
     assert "OS_PROJECT_API_SECRET=your_api_secret_here" in env_text
 
 
+def test_create_config_and_env_writes_supplied_api_credentials(tmp_path: Path) -> None:
+    config_path = tmp_path / "config.yaml"
+    env_path = tmp_path / ".env"
+
+    create_config_and_env(
+        config_out=config_path,
+        env_out=env_path,
+        source="ngd",
+        package_id="16331",
+        version_id="104444",
+        api_key="my-key",
+        api_secret="my-secret",
+    )
+
+    env_text = env_path.read_text()
+    assert "OS_PROJECT_API_KEY=my-key" in env_text
+    assert "OS_PROJECT_API_SECRET=my-secret" in env_text
+
+
+def test_create_config_and_env_rejects_partial_api_credentials(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="must be provided together"):
+        create_config_and_env(
+            config_out=tmp_path / "config.yaml",
+            env_out=tmp_path / ".env",
+            source="ngd",
+            package_id="16331",
+            version_id="104444",
+            api_key="my-key",
+        )
+
+
 def test_run_from_config_applies_overrides(
     monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
@@ -52,9 +85,6 @@ def test_run_from_config_applies_overrides(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -70,7 +100,9 @@ def test_run_from_config_applies_overrides(
     def fake_check_api(_settings: object) -> None:
         calls["checked_api"] = True
 
-    def fake_run_pipeline(step: str, settings: object, force: bool, list_only: bool) -> None:
+    def fake_run_pipeline(
+        step: Literal["all", "download"], settings: object, force: bool, list_only: bool
+    ) -> None:
         calls["step"] = step
         calls["force"] = force
         calls["list_only"] = list_only
@@ -94,6 +126,47 @@ def test_run_from_config_applies_overrides(
     assert calls["num_chunks"] == 5
 
 
+def test_run_from_config_accepts_api_key_secret_overrides(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    monkeypatch.delenv("OS_PROJECT_API_KEY", raising=False)
+    monkeypatch.delenv("OS_PROJECT_API_SECRET", raising=False)
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        source:
+          type: ngd
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", lambda _settings: None)
+    monkeypatch.setattr("ukam_os_builder.api.api.run_pipeline", lambda **_kwargs: None)
+
+    run_from_config(
+        config_path=config_path,
+        api_key="runtime-key",
+        api_secret="runtime-secret",
+    )
+
+    assert os.environ["OS_PROJECT_API_KEY"] == "runtime-key"
+    assert os.environ["OS_PROJECT_API_SECRET"] == "runtime-secret"
+
+
+def test_run_from_config_rejects_partial_api_credentials(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="must be provided together"):
+        run_from_config(
+            config_path=tmp_path / "config.yaml",
+            api_key="runtime-key",
+        )
+
+
 def test_run_from_config_validates_list_only_step(tmp_path: Path) -> None:
     with pytest.raises(ValueError, match="--list-only can only be used"):
         run_from_config(config_path=tmp_path / "config.yaml", step="extract", list_only=True)
@@ -126,7 +199,9 @@ def test_run_from_config_uses_source_override_for_pipeline_validation(
 
     monkeypatch.setattr("ukam_os_builder.api.api.get_package_version", lambda _settings: None)
 
-    def fake_run_pipeline(step: str, settings: object, force: bool, list_only: bool) -> None:
+    def fake_run_pipeline(
+        step: Literal["all", "download"], settings: object, force: bool, list_only: bool
+    ) -> None:
         calls["step"] = step
         calls["source"] = settings.source.type
         calls["force"] = force
@@ -195,10 +270,6 @@ def test_run_from_config_applies_schema_path_override(
 
                 paths:
                     work_dir: ./data
-                    downloads_dir: ./data/downloads
-                    extracted_dir: ./data/extracted
-                    output_dir: ./data/output
-                    parquet_dir: ./data/parquet
 
                 os_downloads:
                     package_id: "16465"

ukam_os_builder-0.1.0.dev3/tests/test_cli.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from ukam_os_builder import cli
+
+
+def test_build_cli_passes_api_credentials_to_run_from_config(monkeypatch) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_run_from_config(**kwargs):
+        captured.update(kwargs)
+        return None
+
+    monkeypatch.setattr(cli, "run_from_config", fake_run_from_config)
+    monkeypatch.setattr(cli, "_configure_logging", lambda _verbose: None)
+
+    exit_code = cli.main(
+        [
+            "--config",
+            "config.yaml",
+            "--step",
+            "download",
+            "--list-only",
+            "--api-key",
+            "runtime-key",
+            "--api-secret",
+            "runtime-secret",
+        ]
+    )
+
+    assert exit_code == 0
+    assert captured["api_key"] == "runtime-key"
+    assert captured["api_secret"] == "runtime-secret"

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/tests/test_settings.py

@@ -25,9 +25,6 @@ def test_load_settings_resolves_paths_relative_to_config(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -57,9 +54,6 @@ def test_load_settings_rejects_unknown_config_key(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -87,9 +81,6 @@ def test_load_settings_missing_package_id_has_clear_message(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           version_id: "104444"
@@ -141,9 +132,6 @@ def test_load_settings_requires_env_vars(tmp_path: Path, monkeypatch: pytest.Mon
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -167,9 +155,6 @@ def test_load_settings_validates_positive_read_timeout(
         """
         paths:
           work_dir: ./data
-          downloads_dir: ./data/downloads
-          extracted_dir: ./data/extracted
-          output_dir: ./data/output
 
         os_downloads:
           package_id: "16465"
@@ -205,3 +190,59 @@ def test_load_settings_defaults_source_and_num_chunks(
 
     assert settings.source.type == "ngd"
     assert settings.processing.num_chunks == 20
+
+
+def test_load_settings_applies_path_overrides(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    monkeypatch.setenv("OS_PROJECT_API_KEY", "key")
+    monkeypatch.setenv("OS_PROJECT_API_SECRET", "secret")
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        paths:
+          work_dir: ./data
+          overrides:
+            downloads_dir: ./custom/downloads
+            extracted_dir: /tmp/extracted
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    settings = load_settings(config_path, load_env=False)
+
+    assert settings.paths.work_dir == (tmp_path / "data").resolve()
+    assert settings.paths.downloads_dir == (tmp_path / "custom/downloads").resolve()
+    assert str(settings.paths.extracted_dir).endswith("/tmp/extracted")
+    assert settings.paths.parquet_dir == (tmp_path / "data/parquet").resolve()
+    assert settings.paths.output_dir == (tmp_path / "data/output").resolve()
+
+
+def test_load_settings_rejects_legacy_path_keys(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    monkeypatch.setenv("OS_PROJECT_API_KEY", "key")
+    monkeypatch.setenv("OS_PROJECT_API_SECRET", "secret")
+
+    config_path = tmp_path / "config.yaml"
+    _write_config(
+        config_path,
+        """
+        paths:
+          work_dir: ./data
+          downloads_dir: ./legacy/downloads
+
+        os_downloads:
+          package_id: "16465"
+          version_id: "104444"
+        """,
+    )
+
+    with pytest.raises(SettingsError, match="no longer supported"):
+        load_settings(config_path, load_env=False)

ukam_os_builder-0.1.0.dev3/tests/test_setup_wizard.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from ukam_os_builder import setup_wizard
+
+
+def _input_feeder(values: list[str]):
+    iterator = iter(values)
+
+    def _fake_input(_prompt: str, markup: bool = False) -> str:  # noqa: ARG001
+        return next(iterator)
+
+    return _fake_input
+
+
+def test_setup_wizard_prompts_for_env_credentials_and_overwrites_existing(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    config_path = tmp_path / "config.yaml"
+    env_path = tmp_path / ".env"
+    env_path.write_text("OS_PROJECT_API_KEY=old\nOS_PROJECT_API_SECRET=old\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        setup_wizard.console,
+        "input",
+        _input_feeder(
+            [
+                "",  # source (default)
+                "pkg-1",
+                "ver-1",
+                "",
+                "n",  # advanced settings
+                "y",  # setup .env now
+                "y",  # overwrite existing .env
+                "new-key",
+                "new-secret",
+            ]
+        ),
+    )
+
+    captured: dict[str, object] = {}
+
+    def fake_write_config_and_env(**kwargs):
+        captured.update(kwargs)
+        return Path(kwargs["config_out"]).resolve(), Path(kwargs["env_out"]).resolve(), True
+
+    monkeypatch.setattr(setup_wizard, "write_config_and_env", fake_write_config_and_env)
+
+    exit_code = setup_wizard.main(["--config-out", str(config_path), "--env-out", str(env_path)])
+
+    assert exit_code == 0
+    assert captured["write_env"] is True
+    assert captured["overwrite_env"] is True
+    assert captured["api_key"] == "new-key"
+    assert captured["api_secret"] == "new-secret"
+
+
+def test_setup_wizard_skips_env_update_when_user_declines(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    config_path = tmp_path / "config.yaml"
+    env_path = tmp_path / ".env"
+
+    monkeypatch.setattr(
+        setup_wizard.console,
+        "input",
+        _input_feeder(
+            [
+                "",  # source (default)
+                "pkg-1",
+                "ver-1",
+                "",
+                "n",  # advanced settings
+                "n",  # setup .env now
+            ]
+        ),
+    )
+
+    captured: dict[str, object] = {}
+
+    def fake_write_config_and_env(**kwargs):
+        captured.update(kwargs)
+        return Path(kwargs["config_out"]).resolve(), Path(kwargs["env_out"]).resolve(), False
+
+    monkeypatch.setattr(setup_wizard, "write_config_and_env", fake_write_config_and_env)
+
+    exit_code = setup_wizard.main(["--config-out", str(config_path), "--env-out", str(env_path)])
+
+    assert exit_code == 0
+    assert captured["write_env"] is False
+    assert captured["api_key"] is None
+    assert captured["api_secret"] is None
+
+
+def test_setup_wizard_decline_overwrite_keeps_existing_env(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    config_path = tmp_path / "config.yaml"
+    env_path = tmp_path / ".env"
+    env_path.write_text("OS_PROJECT_API_KEY=old\nOS_PROJECT_API_SECRET=old\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        setup_wizard.console,
+        "input",
+        _input_feeder(
+            [
+                "",  # source (default)
+                "pkg-1",
+                "ver-1",
+                "",
+                "n",  # advanced settings
+                "y",  # setup .env now
+                "n",  # do not overwrite existing .env
+            ]
+        ),
+    )
+
+    captured: dict[str, object] = {}
+
+    def fake_write_config_and_env(**kwargs):
+        captured.update(kwargs)
+        return Path(kwargs["config_out"]).resolve(), Path(kwargs["env_out"]).resolve(), False
+
+    monkeypatch.setattr(setup_wizard, "write_config_and_env", fake_write_config_and_env)
+
+    exit_code = setup_wizard.main(["--config-out", str(config_path), "--env-out", str(env_path)])
+
+    assert exit_code == 0
+    assert captured["write_env"] is False
+    assert captured["overwrite_env"] is False

{ukam_os_builder-0.1.0.dev2 → ukam_os_builder-0.1.0.dev3}/ukam_os_builder/__init__.py

@@ -8,7 +8,7 @@ from ukam_os_builder.os_builder.inspect_results import (
     inspect_flatfile_variants,
 )
 
-__version__ = "0.1.0.dev2"
+__version__ = "0.1.0.dev3"
 
 __all__ = [
     "create_config_and_env",