sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_data/import_/fetcher.py

@@ -0,0 +1,259 @@
+"""Source fetching: URL download and local file validation."""
+
+import tempfile
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Callable
+from urllib.parse import unquote, urlparse
+
+import httpx
+from sunstone.ssrf import is_public_url
+
+# Maximum download size: 512 MiB.
+MAX_DOWNLOAD_BYTES: int = 512 * 1024 * 1024
+
+# Maximum number of HTTP redirects to follow.
+MAX_REDIRECTS: int = 10
+
+
+class FileType(Enum):
+    """Supported file types for import."""
+
+    CSV = "csv"
+    EXCEL = "excel"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class SourceInfo:
+    """Information about a fetched source."""
+
+    local_path: Path
+    filename: str
+    file_type: FileType
+    encoding: str | None
+    is_temp: bool  # True if downloaded to temp file
+    source_url: str | None  # Original URL if downloaded
+
+
+class SSRFError(ValueError):
+    """Raised when a URL targets a disallowed network address."""
+
+
+class DownloadTooLargeError(ValueError):
+    """Raised when a download exceeds MAX_DOWNLOAD_BYTES."""
+
+
+# ---------------------------------------------------------------------------
+# Public helpers
+# ---------------------------------------------------------------------------
+
+
+def detect_file_type(filename: str) -> FileType:
+    """Detect file type from filename extension."""
+    lower = filename.lower()
+    if lower.endswith(".csv"):
+        return FileType.CSV
+    elif lower.endswith((".xlsx", ".xls")):
+        return FileType.EXCEL
+    return FileType.UNKNOWN
+
+
+def fetch_source(
+    source: str,
+    progress_callback: Callable[[int, int | None], None] | None = None,
+) -> SourceInfo:
+    """Fetch source from URL or validate local file.
+
+    Args:
+        source: Local file path or URL to fetch.
+        progress_callback: Optional callback(downloaded_bytes, total_bytes).
+
+    Returns:
+        SourceInfo with local path and metadata.
+
+    Raises:
+        FileNotFoundError: If local file doesn't exist.
+        httpx.HTTPError: If URL download fails.
+        ValueError: If file type is not supported or URL scheme is invalid.
+        SSRFError: If URL targets a disallowed address.
+        DownloadTooLargeError: If the response body exceeds MAX_DOWNLOAD_BYTES.
+    """
+    if source.startswith(("http://", "https://")):
+        return _download_url(source, progress_callback)
+
+    # Reject non-HTTP URL schemes (file://, ftp://, gopher://, etc.)
+    if "://" in source:
+        scheme = source.split("://", 1)[0].lower()
+        if scheme not in ("http", "https"):
+            raise ValueError(f"Unsupported URL scheme: {scheme}://. Only http and https are allowed.")
+
+    return _validate_local(source)
+
+
+# ---------------------------------------------------------------------------
+# Local file handling
+# ---------------------------------------------------------------------------
+
+
+def _validate_local(path: str) -> SourceInfo:
+    """Validate local file exists and determine type."""
+    local_path = Path(path).expanduser().resolve()
+
+    if not local_path.exists():
+        raise FileNotFoundError(f"File not found: {local_path}")
+
+    if not local_path.is_file():
+        raise ValueError(f"Not a file: {local_path}")
+
+    file_type = detect_file_type(local_path.name)
+    if file_type == FileType.UNKNOWN:
+        raise ValueError(f"Unsupported file type: {local_path.suffix}. Supported: .csv, .xlsx, .xls")
+
+    return SourceInfo(
+        local_path=local_path,
+        filename=local_path.name,
+        file_type=file_type,
+        encoding=None,  # Will be detected during parsing
+        is_temp=False,
+        source_url=None,
+    )
+
+
+# ---------------------------------------------------------------------------
+# URL validation
+# ---------------------------------------------------------------------------
+
+
+def _validate_url(url: str) -> None:
+    """Validate a URL's scheme and resolved IP addresses.
+
+    Delegates to ``sunstone.ssrf.is_public_url`` for scheme, DNS resolution,
+    and IP blocklist checks.  Raises :class:`SSRFError` on rejection.
+    """
+    if not is_public_url(url):
+        raise SSRFError(f"URL blocked by SSRF protection: {url}")
+
+
+def _strip_sensitive_headers(
+    headers: dict[str, str],
+    original_url: str,
+    redirect_url: str,
+) -> dict[str, str]:
+    """Strip sensitive headers when redirecting across origins."""
+    orig = urlparse(original_url)
+    redir = urlparse(redirect_url)
+
+    same_origin = orig.scheme == redir.scheme and orig.hostname == redir.hostname and orig.port == redir.port
+
+    if same_origin:
+        return headers
+
+    sensitive = {"authorization", "cookie", "proxy-authorization"}
+    return {k: v for k, v in headers.items() if k.lower() not in sensitive}
+
+
+# ---------------------------------------------------------------------------
+# Download with SSRF protection
+# ---------------------------------------------------------------------------
+
+
+def _download_url(
+    url: str,
+    progress_callback: Callable[[int, int | None], None] | None = None,
+) -> SourceInfo:
+    """Download URL to temp file with SSRF protection and size limits."""
+    # Validate initial URL
+    _validate_url(url)
+
+    # Extract filename from URL
+    parsed = urlparse(url)
+    url_path = unquote(parsed.path)
+    filename = Path(url_path).name or "download"
+
+    # Determine file type
+    file_type = detect_file_type(filename)
+    if file_type == FileType.UNKNOWN:
+        raise ValueError(f"Cannot determine file type from URL: {url}. URL must end with .csv, .xlsx, or .xls")
+
+    # Create temp file with appropriate extension
+    suffix = Path(filename).suffix
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
+    tmp_path = Path(tmp.name)
+
+    try:
+        current_url = url
+        headers: dict[str, str] = {}
+
+        for _redirect_num in range(MAX_REDIRECTS + 1):
+            with httpx.stream(
+                "GET",
+                current_url,
+                follow_redirects=False,
+                timeout=60.0,
+                headers=headers,
+            ) as response:
+                # Handle redirects manually
+                if response.is_redirect:
+                    redirect_url = str(response.next_request.url) if response.next_request else None
+                    if not redirect_url:
+                        raise SSRFError(f"Redirect with no valid target from {current_url}")
+
+                    # Validate redirect target
+                    _validate_url(redirect_url)
+                    headers = _strip_sensitive_headers(headers, current_url, redirect_url)
+                    current_url = redirect_url
+                    continue
+
+                response.raise_for_status()
+
+                # Check Content-Length before downloading
+                content_length_str = response.headers.get("content-length")
+                if content_length_str:
+                    try:
+                        content_length = int(content_length_str)
+                    except ValueError:
+                        content_length = None  # Ignore malformed
+                    else:
+                        if content_length > MAX_DOWNLOAD_BYTES:
+                            raise DownloadTooLargeError(
+                                f"Content-Length {content_length} exceeds maximum {MAX_DOWNLOAD_BYTES} bytes."
+                            )
+
+                # Check content type for HTML (error page) detection
+                content_type = response.headers.get("content-type", "")
+                if "text/html" in content_type and file_type != FileType.CSV:
+                    raise ValueError(f"URL returned HTML instead of expected file. Content-Type: {content_type}")
+
+                total = int(content_length_str) if content_length_str else None
+                downloaded = 0
+
+                for chunk in response.iter_bytes(chunk_size=8192):
+                    downloaded += len(chunk)
+                    if downloaded > MAX_DOWNLOAD_BYTES:
+                        raise DownloadTooLargeError(f"Download exceeded maximum {MAX_DOWNLOAD_BYTES} bytes.")
+                    tmp.write(chunk)
+                    if progress_callback:
+                        progress_callback(downloaded, total)
+
+                # Successful download — break out of redirect loop
+                break
+        else:
+            raise SSRFError(f"Too many redirects ({MAX_REDIRECTS}) following URL: {url}")
+
+        tmp.close()
+
+        return SourceInfo(
+            local_path=tmp_path,
+            filename=filename,
+            file_type=file_type,
+            encoding=None,
+            is_temp=True,
+            source_url=url,
+        )
+
+    except Exception:
+        tmp.close()
+        tmp_path.unlink(missing_ok=True)
+        raise

sunstone_data/import_/owner.py

@@ -0,0 +1,130 @@
+"""Resolve the data-owner organization for an import.
+
+The owner is a path-structured organization identifier (e.g.
+``un/who/emissions-div``). It is resolved at import time, and the resolved
+path drives three things:
+
+- the Iceberg namespace root (governance anchor), via :func:`ensure_root_prefix`;
+- the ``sunstone.owner.org`` table property (full org URI), via :func:`org_uri`;
+- the RDF org tree emitted by the DCAT builder (Phase 2).
+
+Resolution precedence is strict — there is no silent default:
+
+1. an explicit ``--owner`` flag;
+2. the first matching rule in the ordered source-URL rules file;
+3. otherwise :class:`OwnerUnresolved` is raised.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+class OwnerError(ValueError):
+    """Base class for owner-resolution failures.
+
+    Subclasses ``ValueError`` so the import CLI's existing ``except ValueError``
+    handler renders them as clean ``Invalid input`` errors.
+    """
+
+
+class OwnerUnresolved(OwnerError):
+    """No owner could be resolved (no --owner and no matching rule)."""
+
+
+class OwnerInvalid(OwnerError):
+    """The resolved owner path is malformed or its root is not allow-listed."""
+
+
+@dataclass(frozen=True)
+class OwnerRule:
+    """One ordered source-URL rule: a regex and the owner path it assigns."""
+
+    pattern: str
+    owner: str
+
+
+def load_rules(rules_path: Path | None) -> list[OwnerRule]:
+    """Load ordered owner rules from a YAML file.
+
+    Returns an empty list when ``rules_path`` is None or does not exist, so a
+    deployment without a rules file simply has no rules (and must rely on
+    ``--owner``).
+    """
+    if rules_path is None or not rules_path.exists():
+        return []
+    data = yaml.safe_load(rules_path.read_text(encoding="utf-8")) or {}
+    return [OwnerRule(pattern=r["pattern"], owner=r["owner"]) for r in data.get("rules", [])]
+
+
+def _validate(owner: str, org_roots: tuple[str, ...]) -> str:
+    """Normalize and validate an owner path.
+
+    Strips surrounding slashes/whitespace, rejects empty segments, and (when
+    ``org_roots`` is non-empty) requires the root segment to be allow-listed.
+    An empty ``org_roots`` disables root checking (permissive dev default).
+    """
+    stripped = owner.strip().strip("/")
+    parts = stripped.split("/")
+    if not stripped or any(not p for p in parts):
+        raise OwnerInvalid(f"malformed owner path: {owner!r}")
+    root = parts[0]
+    if org_roots and root not in org_roots:
+        raise OwnerInvalid(f"owner root {root!r} not in configured org_roots {list(org_roots)}")
+    return "/".join(parts)
+
+
+def resolve_owner(
+    source_url: str,
+    owner_flag: str | None,
+    *,
+    rules: list[OwnerRule],
+    org_roots: tuple[str, ...],
+) -> str:
+    """Resolve the owner path for an import. Strict: never returns ownerless.
+
+    Args:
+        source_url: The import's fetch URL (``source_info.source_url``); empty
+            for local-file imports, which therefore never match a URL rule.
+        owner_flag: The explicit ``--owner`` value, if any.
+        rules: Ordered owner rules (first regex match wins).
+        org_roots: Allowed root segments; empty disables the check.
+
+    Returns:
+        A normalized owner path, e.g. ``"un/who/emissions-div"``.
+
+    Raises:
+        OwnerInvalid: If the chosen owner path is malformed or root-disallowed.
+        OwnerUnresolved: If no ``--owner`` and no rule matches ``source_url``.
+    """
+    if owner_flag:
+        return _validate(owner_flag, org_roots)
+    for rule in rules:
+        if re.search(rule.pattern, source_url):
+            return _validate(rule.owner, org_roots)
+    raise OwnerUnresolved(f"no owner resolved for {source_url!r}: pass --owner or add a matching source-URL rule")
+
+
+def owner_root(owner: str) -> str:
+    """Return the root (level-0) segment of an owner path."""
+    return owner.strip("/").split("/")[0]
+
+
+def ensure_root_prefix(namespace: str, root: str) -> str:
+    """Guarantee ``namespace``'s level-0 segment is the owner ``root``.
+
+    Already-rooted namespaces are returned unchanged (no ``un.un``);
+    otherwise the root is prepended as the governance anchor.
+    """
+    if namespace.split(".")[0] == root:
+        return namespace
+    return f"{root}.{namespace}"
+
+
+def org_uri(owner: str, base_uri: str) -> str:
+    """Build the full, stable org URI for an owner path."""
+    return f"{base_uri.rstrip('/')}/{owner.strip('/')}"