portolan-python 0.1.2__tar.gz → 0.1.4__tar.gz

{portolan_python-0.1.2 → portolan_python-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: portolan-python
-Version: 0.1.2
+Version: 0.1.4
 Summary: A lightweight Python implementation of the Portolan specification.
 Author: Portolan contributors
 License: Apache-2.0

{portolan_python-0.1.2 → portolan_python-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "portolan-python"
-version = "0.1.2"
+version = "0.1.4"
 description = "A lightweight Python implementation of the Portolan specification."
 readme = "README.md"
 requires-python = ">=3.10"

portolan_python-0.1.4/src/portolan/registry.py

@@ -0,0 +1,297 @@
+"""Read and fetch Portolan registry exports."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+from collections.abc import Callable
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError
+from urllib.parse import urljoin, urlparse
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+JsonObject = dict[str, Any]
+
+DEFAULT_REGISTRY_URL = (
+    "https://raw.githubusercontent.com/portolan-sdi/portolan-registry/"
+    "refs/heads/main/exports/catalogs.json"
+)
+
+
+@dataclass(frozen=True)
+class RegistryCatalogEntry:
+    """One catalog entry from a Portolan registry export."""
+
+    id: str
+    url: str
+    title: str | None = None
+    status: str | None = None
+
+
+def load_registry_entries(
+    registry_url: str = DEFAULT_REGISTRY_URL,
+    *,
+    fetch_json: Callable[[str], JsonObject] | None = None,
+    catalog_ids: set[str] | None = None,
+    include_stale: bool = False,
+    limit: int | None = None,
+) -> list[RegistryCatalogEntry]:
+    """Load catalog entries from a Portolan registry export."""
+    fetch = fetch_json or _fetch_json
+    registry = fetch(registry_url)
+    entries: list[RegistryCatalogEntry] = []
+    for link in registry.get("links", []):
+        if not isinstance(link, dict) or link.get("rel") != "child":
+            continue
+        href = link.get("href")
+        registry_id = link.get("portolan_registry:id")
+        if not isinstance(href, str) or not isinstance(registry_id, str):
+            continue
+        status = link.get("portolan_registry:status")
+        if status != "valid" and not include_stale:
+            continue
+        if catalog_ids is not None and registry_id not in catalog_ids:
+            continue
+        title = link.get("title")
+        entries.append(
+            RegistryCatalogEntry(
+                id=registry_id,
+                url=urljoin(registry_url, href),
+                title=title if isinstance(title, str) else None,
+                status=status if isinstance(status, str) else None,
+            )
+        )
+        if limit is not None and len(entries) >= limit:
+            break
+    return entries
+
+
+def download_registry_catalog(
+    catalog_url: str,
+    output_dir: Path,
+    *,
+    expected_catalog_id: str | None = None,
+    fetch_json: Callable[[str], JsonObject] | None = None,
+) -> Path:
+    """Download a published catalog snapshot for local workflows."""
+    _validate_remote_url(catalog_url)
+    fetch = fetch_json or _fetch_json
+    catalog = fetch(catalog_url)
+    catalog_id = str(catalog.get("id") or _fallback_catalog_id(catalog_url))
+    _validate_catalog_id(catalog_id)
+    if expected_catalog_id is not None:
+        _validate_catalog_id(expected_catalog_id)
+        if catalog_id != expected_catalog_id:
+            raise ValueError(
+                f"Catalog id '{catalog_id}' does not match registry id '{expected_catalog_id}'"
+            )
+    catalog_root = output_dir / catalog_id
+    output_dir.mkdir(parents=True, exist_ok=True)
+    with _catalog_lock(output_dir, catalog_id):
+        _validate_catalog_root(output_dir, catalog_root)
+        staging_root = Path(
+            tempfile.mkdtemp(prefix=f".{catalog_id}.staging-", dir=output_dir.resolve())
+        )
+        try:
+            _write_catalog_tree(catalog_url, catalog, catalog_url, staging_root, fetch)
+            _publish_snapshot(staging_root, catalog_root, output_dir)
+        except BaseException:
+            shutil.rmtree(staging_root, ignore_errors=True)
+            raise
+    return catalog_root
+
+
+def _fetch_json(url: str) -> JsonObject:
+    _validate_remote_url(url)
+    request = Request(url, headers={"User-Agent": "portolan-python"})
+    opener = build_opener(_SameOriginRedirectHandler(url))
+    with opener.open(request, timeout=30) as response:
+        data = json.loads(response.read().decode("utf-8"))
+    if not isinstance(data, dict):
+        raise TypeError(f"Expected JSON object from {url}")
+    return data
+
+
+def _validate_remote_url(url: str) -> None:
+    parsed = urlparse(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError(f"Registry URL must use HTTP or HTTPS: {url}")
+
+
+class _SameOriginRedirectHandler(HTTPRedirectHandler):
+    def __init__(self, original_url: str) -> None:
+        self._origin = _url_origin(original_url)
+        super().__init__()
+
+    def redirect_request(
+        self,
+        req: Request,
+        fp: Any,
+        code: int,
+        msg: str,
+        headers: Any,
+        newurl: str,
+    ) -> Request | None:
+        if _url_origin(newurl) != self._origin:
+            raise HTTPError(newurl, code, f"Redirect changed origin: {newurl}", headers, fp)
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def _url_origin(url: str) -> tuple[str, str]:
+    parsed = urlparse(url)
+    return parsed.scheme.lower(), parsed.netloc.lower()
+
+
+def _validate_catalog_id(catalog_id: str) -> None:
+    if catalog_id in {"", ".", ".."} or Path(catalog_id).name != catalog_id or "\\" in catalog_id:
+        raise ValueError(f"Catalog id must be a safe directory name: {catalog_id}")
+
+
+@contextmanager
+def _catalog_lock(output_dir: Path, catalog_id: str) -> Any:
+    lock_path = output_dir.resolve() / f".{catalog_id}.lock"
+    try:
+        descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+    except FileExistsError as err:
+        raise RuntimeError(f"Catalog download is already in progress: {catalog_id}") from err
+    os.close(descriptor)
+    try:
+        yield
+    finally:
+        lock_path.unlink(missing_ok=True)
+
+
+def _validate_catalog_root(output_dir: Path, catalog_root: Path) -> None:
+    if catalog_root.is_symlink():
+        raise ValueError(f"Catalog directory must not be a symlink: {catalog_root}")
+    if not catalog_root.resolve().is_relative_to(output_dir.resolve()):
+        raise ValueError(f"Catalog directory escapes output directory: {catalog_root}")
+
+
+def _publish_snapshot(staging_root: Path, catalog_root: Path, output_dir: Path) -> None:
+    _validate_catalog_root(output_dir, catalog_root)
+    backup_root = Path(
+        tempfile.mkdtemp(prefix=f".{catalog_root.name}.backup-", dir=output_dir.resolve())
+    )
+    backup_root.rmdir()
+    had_previous = catalog_root.exists()
+    if had_previous:
+        catalog_root.rename(backup_root)
+    try:
+        staging_root.rename(catalog_root)
+    except BaseException:
+        if had_previous:
+            backup_root.rename(catalog_root)
+        raise
+    if had_previous:
+        shutil.rmtree(backup_root)
+
+
+def _write_catalog_tree(
+    document_url: str,
+    document: JsonObject,
+    root_url: str,
+    output_root: Path,
+    fetch_json: Callable[[str], JsonObject],
+    visited: set[str] | None = None,
+    targets: dict[Path, str] | None = None,
+) -> None:
+    if visited is None:
+        visited = set()
+    if targets is None:
+        targets = {}
+    visited.add(document_url)
+
+    target = _target_document_path(root_url, document_url, output_root)
+    owner = targets.get(target)
+    if owner is not None and owner != document_url:
+        raise ValueError(f"Registry documents map to the same local path: {owner}, {document_url}")
+    targets[target] = document_url
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if document.get("type") == "Collection":
+        document = _with_absolute_asset_hrefs(document_url, document)
+    target.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
+
+    for link in document.get("links", []):
+        if not isinstance(link, dict) or link.get("rel") != "child":
+            continue
+        href = link.get("href")
+        if not isinstance(href, str):
+            continue
+        child_url = urljoin(document_url, href)
+        if child_url in visited:
+            continue
+        child_target = _target_document_path(root_url, child_url, output_root)
+        owner = targets.get(child_target)
+        if owner is not None and owner != child_url:
+            raise ValueError(f"Registry documents map to the same local path: {owner}, {child_url}")
+        child = fetch_json(child_url)
+        if child.get("type") in {"Catalog", "Collection"}:
+            _write_catalog_tree(
+                child_url,
+                child,
+                root_url,
+                output_root,
+                fetch_json,
+                visited,
+                targets,
+            )
+
+
+def _with_absolute_asset_hrefs(document_url: str, collection: JsonObject) -> JsonObject:
+    assets = collection.get("assets")
+    if not isinstance(assets, dict):
+        return collection
+    updated = dict(collection)
+    updated_assets: dict[str, Any] = {}
+    for key, asset in assets.items():
+        if not isinstance(asset, dict):
+            updated_assets[key] = asset
+            continue
+        href = asset.get("href")
+        if isinstance(href, str):
+            rewritten = dict(asset)
+            rewritten["href"] = urljoin(document_url, href)
+            updated_assets[key] = rewritten
+        else:
+            updated_assets[key] = asset
+    updated["assets"] = updated_assets
+    return updated
+
+
+def _relative_document_path(root_url: str, document_url: str) -> Path:
+    root = urlparse(root_url)
+    document = urlparse(document_url)
+    if (document.scheme.lower(), document.netloc.lower()) != (
+        root.scheme.lower(),
+        root.netloc.lower(),
+    ):
+        raise ValueError(f"Child document has a different origin: {document_url}")
+    root_path = Path(root.path).parent
+    document_path = Path(document.path)
+    try:
+        relative_path = document_path.relative_to(root_path)
+    except ValueError as err:
+        raise ValueError(f"Child document is outside catalog root: {document_url}") from err
+    if ".." in relative_path.parts:
+        raise ValueError(f"Child document is outside catalog root: {document_url}")
+    return relative_path
+
+
+def _target_document_path(root_url: str, document_url: str, output_root: Path) -> Path:
+    relative_path = _relative_document_path(root_url, document_url)
+    resolved_root = output_root.resolve()
+    target = (output_root / relative_path).resolve()
+    if not target.is_relative_to(resolved_root):
+        raise ValueError(f"Child document escapes catalog root: {document_url}")
+    return target
+
+
+def _fallback_catalog_id(catalog_url: str) -> str:
+    parent = Path(urlparse(catalog_url).path).parent.name
+    return parent or "catalog"

{portolan_python-0.1.2 → portolan_python-0.1.4}/tests/test_registry_api.py

@@ -5,10 +5,13 @@ from __future__ import annotations
 import json
 from pathlib import Path
 from typing import Any
+from urllib.error import HTTPError
+from urllib.request import Request
 
 import pytest
 
 from portolan import download_registry_catalog, load_registry_entries
+from portolan.registry import _SameOriginRedirectHandler
 
 pytestmark = pytest.mark.unit
 
@@ -57,6 +60,26 @@ def test_load_registry_entries_filters_to_valid_children() -> None:
     ]
 
 
+def test_load_registry_entries_resolves_relative_child_urls() -> None:
+    registry = {
+        "links": [
+            {
+                "rel": "child",
+                "href": "../demo/catalog.json",
+                "portolan_registry:id": "demo",
+                "portolan_registry:status": "valid",
+            }
+        ]
+    }
+
+    entries = load_registry_entries(
+        "https://registry.test/exports/catalogs.json",
+        fetch_json=lambda url: registry,
+    )
+
+    assert entries[0].url == "https://registry.test/demo/catalog.json"
+
+
 def test_load_registry_entries_supports_catalog_ids_include_stale_and_limit() -> None:
     registry = {
         "links": [
@@ -196,6 +219,162 @@ def test_download_registry_catalog_skips_child_cycles(tmp_path: Path) -> None:
     assert fetched_urls == [root_url, child_url]
 
 
+@pytest.mark.parametrize(
+    ("child_url", "message"),
+    [
+        ("https://other.test/demo/collection.json", "different origin"),
+        ("https://example.test/demo/../../outside.json", "outside catalog root"),
+    ],
+)
+def test_download_registry_catalog_rejects_unsafe_child_urls_before_fetch(
+    child_url: str,
+    message: str,
+    tmp_path: Path,
+) -> None:
+    root_url = "https://example.test/demo/catalog.json"
+    catalog = {
+        "type": "Catalog",
+        "id": "demo",
+        "links": [{"rel": "child", "href": child_url}],
+    }
+    fetched_urls: list[str] = []
+
+    def fetch_json(url: str) -> dict[str, Any]:
+        fetched_urls.append(url)
+        if url != root_url:
+            raise AssertionError(f"Fetched unsafe child URL: {url}")
+        return catalog
+
+    with pytest.raises(ValueError, match=message):
+        download_registry_catalog(root_url, tmp_path, fetch_json=fetch_json)
+
+    assert fetched_urls == [root_url]
+
+
+def test_download_registry_catalog_rejects_symlink_escape(tmp_path: Path) -> None:
+    root_url = "https://example.test/demo/catalog.json"
+    child_url = "https://example.test/demo/linked/collection.json"
+    output_dir = tmp_path / "output"
+    outside_dir = tmp_path / "outside"
+    catalog_root = output_dir / "demo"
+    outside_dir.mkdir()
+    catalog_root.mkdir(parents=True)
+    (catalog_root / "linked").symlink_to(outside_dir, target_is_directory=True)
+    responses = {
+        root_url: {
+            "type": "Catalog",
+            "id": "demo",
+            "links": [{"rel": "child", "href": child_url}],
+        },
+        child_url: _collection("linked", {}),
+    }
+
+    download_registry_catalog(
+        root_url,
+        output_dir,
+        fetch_json=lambda url: responses[url],
+    )
+
+    assert not (outside_dir / "collection.json").exists()
+    assert (catalog_root / "linked" / "collection.json").exists()
+
+
+def test_download_registry_catalog_rejects_catalog_root_symlink(tmp_path: Path) -> None:
+    output_dir = tmp_path / "output"
+    outside_dir = tmp_path / "outside"
+    output_dir.mkdir()
+    outside_dir.mkdir()
+    (output_dir / "demo").symlink_to(outside_dir, target_is_directory=True)
+
+    with pytest.raises(ValueError, match="Catalog directory must not be a symlink"):
+        download_registry_catalog(
+            "https://example.test/demo/catalog.json",
+            output_dir,
+            fetch_json=lambda url: {"type": "Catalog", "id": "demo", "links": []},
+        )
+
+    assert list(outside_dir.iterdir()) == []
+
+
+def test_download_registry_catalog_rejects_registry_id_mismatch(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="does not match registry id"):
+        download_registry_catalog(
+            "https://example.test/demo/catalog.json",
+            tmp_path,
+            expected_catalog_id="selected-catalog",
+            fetch_json=lambda url: {"type": "Catalog", "id": "other-catalog", "links": []},
+        )
+
+    assert not (tmp_path / "selected-catalog").exists()
+
+
+def test_download_registry_catalog_preserves_snapshot_when_child_fetch_fails(
+    tmp_path: Path,
+) -> None:
+    catalog_root = tmp_path / "demo"
+    catalog_root.mkdir()
+    previous = catalog_root / "catalog.json"
+    previous.write_text('{"id": "previous"}\n', encoding="utf-8")
+    root_url = "https://example.test/demo/catalog.json"
+
+    def fetch_json(url: str) -> dict[str, Any]:
+        if url == root_url:
+            return {
+                "type": "Catalog",
+                "id": "demo",
+                "links": [{"rel": "child", "href": "./missing.json"}],
+            }
+        raise HTTPError(url, 503, "Unavailable", hdrs=None, fp=None)
+
+    with pytest.raises(HTTPError):
+        download_registry_catalog(root_url, tmp_path, fetch_json=fetch_json)
+
+    assert previous.read_text(encoding="utf-8") == '{"id": "previous"}\n'
+    assert not list(tmp_path.glob(".demo.staging-*"))
+
+
+def test_same_origin_redirect_handler_rejects_cross_origin_redirect() -> None:
+    handler = _SameOriginRedirectHandler("https://example.test/demo/catalog.json")
+
+    with pytest.raises(HTTPError, match="Redirect changed origin"):
+        handler.redirect_request(
+            Request("https://example.test/demo/child.json"),
+            fp=None,
+            code=302,
+            msg="Found",
+            headers={},
+            newurl="http://127.0.0.1/private",
+        )
+
+
+def test_download_registry_catalog_rejects_local_path_collisions(tmp_path: Path) -> None:
+    root_url = "https://example.test/demo/catalog.json"
+    first_url = "https://example.test/demo/collection.json?version=1"
+    second_url = "https://example.test/demo/collection.json?version=2"
+    responses = {
+        root_url: {
+            "type": "Catalog",
+            "id": "demo",
+            "links": [
+                {"rel": "child", "href": first_url},
+                {"rel": "child", "href": second_url},
+            ],
+        },
+        first_url: _collection("first", {}),
+        second_url: _collection("second", {}),
+    }
+    fetched_urls: list[str] = []
+
+    def fetch_json(url: str) -> dict[str, Any]:
+        fetched_urls.append(url)
+        return responses[url]
+
+    with pytest.raises(ValueError, match="same local path"):
+        download_registry_catalog(root_url, tmp_path, fetch_json=fetch_json)
+
+    assert fetched_urls == [root_url, first_url]
+
+
 def test_download_registry_catalog_recurses_nested_catalogs_and_uses_fallback_id(
     tmp_path: Path,
 ) -> None:

{portolan_python-0.1.2 → portolan_python-0.1.4}/uv.lock

@@ -468,7 +468,7 @@ wheels = [
 
 [[package]]
 name = "portolan-python"
-version = "0.1.2"
+version = "0.1.4"
 source = { editable = "." }
 
 [package.optional-dependencies]

portolan_python-0.1.2/src/portolan/registry.py

@@ -1,169 +0,0 @@
-"""Read and fetch Portolan registry exports."""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Callable
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-from urllib.parse import urljoin, urlparse
-from urllib.request import Request, urlopen
-
-JsonObject = dict[str, Any]
-
-DEFAULT_REGISTRY_URL = (
-    "https://raw.githubusercontent.com/portolan-sdi/portolan-registry/"
-    "refs/heads/main/exports/catalogs.json"
-)
-
-
-@dataclass(frozen=True)
-class RegistryCatalogEntry:
-    """One catalog entry from a Portolan registry export."""
-
-    id: str
-    url: str
-    title: str | None = None
-    status: str | None = None
-
-
-def load_registry_entries(
-    registry_url: str = DEFAULT_REGISTRY_URL,
-    *,
-    fetch_json: Callable[[str], JsonObject] | None = None,
-    catalog_ids: set[str] | None = None,
-    include_stale: bool = False,
-    limit: int | None = None,
-) -> list[RegistryCatalogEntry]:
-    """Load catalog entries from a Portolan registry export."""
-    fetch = fetch_json or _fetch_json
-    registry = fetch(registry_url)
-    entries: list[RegistryCatalogEntry] = []
-    for link in registry.get("links", []):
-        if not isinstance(link, dict) or link.get("rel") != "child":
-            continue
-        href = link.get("href")
-        registry_id = link.get("portolan_registry:id")
-        if not isinstance(href, str) or not isinstance(registry_id, str):
-            continue
-        status = link.get("portolan_registry:status")
-        if status != "valid" and not include_stale:
-            continue
-        if catalog_ids is not None and registry_id not in catalog_ids:
-            continue
-        title = link.get("title")
-        entries.append(
-            RegistryCatalogEntry(
-                id=registry_id,
-                url=href,
-                title=title if isinstance(title, str) else None,
-                status=status if isinstance(status, str) else None,
-            )
-        )
-        if limit is not None and len(entries) >= limit:
-            break
-    return entries
-
-
-def download_registry_catalog(
-    catalog_url: str,
-    output_dir: Path,
-    *,
-    fetch_json: Callable[[str], JsonObject] | None = None,
-) -> Path:
-    """Download a published catalog snapshot for local workflows."""
-    _validate_remote_url(catalog_url)
-    fetch = fetch_json or _fetch_json
-    catalog = fetch(catalog_url)
-    catalog_id = str(catalog.get("id") or _fallback_catalog_id(catalog_url))
-    if catalog_id in {"", ".", ".."} or Path(catalog_id).name != catalog_id or "\\" in catalog_id:
-        raise ValueError(f"Catalog id must be a safe directory name: {catalog_id}")
-    catalog_root = output_dir / catalog_id
-    _write_catalog_tree(catalog_url, catalog, catalog_url, catalog_root, fetch)
-    return catalog_root
-
-
-def _fetch_json(url: str) -> JsonObject:
-    _validate_remote_url(url)
-    request = Request(url, headers={"User-Agent": "portolan-python"})
-    with urlopen(request, timeout=30) as response:
-        data = json.loads(response.read().decode("utf-8"))
-    if not isinstance(data, dict):
-        raise TypeError(f"Expected JSON object from {url}")
-    return data
-
-
-def _validate_remote_url(url: str) -> None:
-    parsed = urlparse(url)
-    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
-        raise ValueError(f"Registry URL must use HTTP or HTTPS: {url}")
-
-
-def _write_catalog_tree(
-    document_url: str,
-    document: JsonObject,
-    root_url: str,
-    output_root: Path,
-    fetch_json: Callable[[str], JsonObject],
-    visited: set[str] | None = None,
-) -> None:
-    if visited is None:
-        visited = set()
-    visited.add(document_url)
-
-    relative_path = _relative_document_path(root_url, document_url)
-    target = output_root / relative_path
-    target.parent.mkdir(parents=True, exist_ok=True)
-    if document.get("type") == "Collection":
-        document = _with_absolute_asset_hrefs(document_url, document)
-    target.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
-
-    for link in document.get("links", []):
-        if not isinstance(link, dict) or link.get("rel") != "child":
-            continue
-        href = link.get("href")
-        if not isinstance(href, str):
-            continue
-        child_url = urljoin(document_url, href)
-        if child_url in visited:
-            continue
-        child = fetch_json(child_url)
-        if child.get("type") in {"Catalog", "Collection"}:
-            _write_catalog_tree(child_url, child, root_url, output_root, fetch_json, visited)
-
-
-def _with_absolute_asset_hrefs(document_url: str, collection: JsonObject) -> JsonObject:
-    assets = collection.get("assets")
-    if not isinstance(assets, dict):
-        return collection
-    updated = dict(collection)
-    updated_assets: dict[str, Any] = {}
-    for key, asset in assets.items():
-        if not isinstance(asset, dict):
-            updated_assets[key] = asset
-            continue
-        href = asset.get("href")
-        if isinstance(href, str):
-            rewritten = dict(asset)
-            rewritten["href"] = urljoin(document_url, href)
-            updated_assets[key] = rewritten
-        else:
-            updated_assets[key] = asset
-    updated["assets"] = updated_assets
-    return updated
-
-
-def _relative_document_path(root_url: str, document_url: str) -> Path:
-    root_path = Path(urlparse(root_url).path).parent
-    document_path = Path(urlparse(document_url).path)
-    try:
-        relative = document_path.relative_to(root_path)
-    except ValueError:
-        return Path(document_path.name or "catalog.json")
-    return relative
-
-
-def _fallback_catalog_id(catalog_url: str) -> str:
-    parent = Path(urlparse(catalog_url).path).parent.name
-    return parent or "catalog"