pkdb 0.11.0__py3-none-any.whl

pkdb/__init__.py

@@ -0,0 +1,22 @@
+"""PK-DB Python client and shared scientific preparation engine."""
+
+from typing import TYPE_CHECKING
+
+__version__ = "0.11.0"
+
+from pkdb.domain.vocabulary import Vocabulary
+from pkdb.preparation import PreparedBundle, prepare
+
+if TYPE_CHECKING:
+    from pkdb.client import Client
+
+
+def __getattr__(name):
+    if name == "Client":
+        from pkdb.client import Client
+
+        return Client
+    raise AttributeError(name)
+
+
+__all__ = ["Client", "PreparedBundle", "Vocabulary", "__version__", "prepare"]

pkdb/__main__.py

@@ -0,0 +1,3 @@
+from pkdb.cli import main
+
+raise SystemExit(main())

pkdb/cache.py

@@ -0,0 +1,132 @@
+"""Explicit, content-addressed offline vocabulary snapshots."""
+
+import hashlib
+import json
+import os
+import sys
+from importlib.resources import files
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from urllib.parse import urlsplit, urlunsplit
+
+from pkdb.domain.vocabulary import Vocabulary, vocabulary_hash
+
+
+def endpoint_root(value: str) -> str:
+    url = urlsplit(value)
+    if (
+        url.scheme not in {"http", "https"}
+        or not url.hostname
+        or url.username
+        or url.password
+        or url.query
+        or url.fragment
+    ):
+        raise ValueError(
+            "Endpoint must be an HTTP(S) URL without credentials, query, or fragment"
+        )
+    # Reject invalid ports here, before any request is attempted.
+    _ = url.port
+    path = url.path.rstrip("/")
+    if path.endswith(("/api/v1", "/api/v2")):
+        path = path[:-7]
+    return urlunsplit((url.scheme.lower(), url.netloc.lower(), path, "", ""))
+
+
+def vocabulary_envelope(vocabulary: Vocabulary) -> dict:
+    return {
+        "schema_version": 1,
+        "vocabulary": vocabulary.model_dump(mode="json"),
+        "vocabulary_hash": vocabulary_hash(vocabulary),
+    }
+
+
+def parse_vocabulary(value: dict) -> Vocabulary:
+    if value.get("schema_version") != 1:
+        raise ValueError("Unsupported vocabulary snapshot schema")
+    vocabulary = Vocabulary.model_validate(value.get("vocabulary"))
+    if value.get("vocabulary_hash") != vocabulary_hash(vocabulary):
+        raise ValueError("Vocabulary snapshot hash does not match its contents")
+    return vocabulary
+
+
+def atomic_json(path: Path, value: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with NamedTemporaryFile(
+            mode="w", encoding="utf-8", dir=path.parent, delete=False
+        ) as handle:
+            temporary = Path(handle.name)
+            json.dump(value, handle, ensure_ascii=False, allow_nan=False, indent=2)
+            handle.write("\n")
+        temporary.replace(path)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def load_vocabulary(path: str | Path) -> Vocabulary:
+    value = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError("Expected a vocabulary snapshot object")
+    return parse_vocabulary(value)
+
+
+def bundled_vocabulary() -> Vocabulary:
+    return parse_vocabulary(
+        json.loads(
+            files("pkdb").joinpath("data/vocabulary.json").read_text(encoding="utf-8")
+        )
+    )
+
+
+def cache_directory() -> Path:
+    if value := os.environ.get("PKDB_CACHE_DIR"):
+        return Path(value)
+    if sys.platform == "win32":
+        root = Path(os.environ.get("LOCALAPPDATA", str(Path.home() / "AppData/Local")))
+    elif sys.platform == "darwin":
+        root = Path.home() / "Library/Caches"
+    else:
+        root = Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache")))
+    return root / "pkdb"
+
+
+class VocabularyCache:
+    def __init__(self, directory: str | Path | None = None):
+        self.directory = Path(directory) if directory is not None else cache_directory()
+
+    def _pointer(self, endpoint: str) -> Path:
+        key = hashlib.sha256(endpoint_root(endpoint).encode()).hexdigest()
+        return self.directory / "endpoints" / f"{key}.json"
+
+    def store(self, endpoint: str, vocabulary: Vocabulary) -> Path:
+        digest = vocabulary_hash(vocabulary)
+        path = self.directory / "snapshots" / f"{digest}.json"
+        atomic_json(path, vocabulary_envelope(vocabulary))
+        atomic_json(self._pointer(endpoint), {"vocabulary_hash": digest})
+        return path
+
+    def load(self, endpoint: str) -> Vocabulary:
+        value = json.loads(self._pointer(endpoint).read_text(encoding="utf-8"))
+        if not isinstance(value, dict):
+            raise ValueError("Invalid vocabulary cache index")
+        digest = value.get("vocabulary_hash")
+        if (
+            not isinstance(digest, str)
+            or len(digest) != 64
+            or any(c not in "0123456789abcdef" for c in digest)
+        ):
+            raise ValueError("Invalid vocabulary cache index")
+        try:
+            vocabulary = load_vocabulary(
+                self.directory / "snapshots" / f"{digest}.json"
+            )
+        except FileNotFoundError:
+            raise ValueError(
+                "Cached vocabulary snapshot is missing; synchronize again"
+            ) from None
+        if vocabulary_hash(vocabulary) != digest:
+            raise ValueError("Vocabulary cache index does not match snapshot")
+        return vocabulary

pkdb/cli.py

@@ -0,0 +1,158 @@
+"""Prepare, validate, and upload existing PK-DB study folders."""
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from pkdb import __version__
+from pkdb.cache import VocabularyCache, atomic_json, bundled_vocabulary
+from pkdb.client import Client
+from pkdb.domain.vocabulary import Vocabulary
+from pkdb.errors import ClientError
+from pkdb.preparation import prepare, study_folders
+from pkdb.schemas.validation import StudyValidationError
+
+
+def _redact(value, token):
+    if isinstance(value, str) and token:
+        return value.replace(token, "[redacted]")
+    if isinstance(value, dict):
+        return {key: _redact(item, token) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_redact(item, token) for item in value]
+    return value
+
+
+def main(argv=None, *, client=None) -> int:
+    parser = argparse.ArgumentParser(prog="pkdb", description=__doc__)
+    parser.add_argument("--version", action="version", version=f"pkdb {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+    for name in ("prepare", "validate", "upload"):
+        command = commands.add_parser(name)
+        command.add_argument(
+            "folder", type=Path, help="Existing study folder or parent directory"
+        )
+        command.add_argument(
+            "--endpoint", "--api-url", default=os.environ.get("PKDB_ENDPOINT")
+        )
+        command.add_argument(
+            "--vocabulary", type=Path, help="Pinned vocabulary snapshot JSON"
+        )
+        command.add_argument("--cache-dir", type=Path)
+        command.add_argument(
+            "--offline",
+            action="store_true",
+            help="Never contact the server (prepare/validate only)",
+        )
+        command.add_argument(
+            "--output",
+            type=Path,
+            help="JSON output file outside the study folder (one study only)",
+        )
+    vocabulary = commands.add_parser("vocabulary")
+    actions = vocabulary.add_subparsers(dest="action", required=True)
+    sync = actions.add_parser(
+        "sync", help="Explicitly retrieve and cache the endpoint's validation rules"
+    )
+    sync.add_argument("--endpoint", default=os.environ.get("PKDB_ENDPOINT"))
+    sync.add_argument("--cache-dir", type=Path)
+    sync.add_argument(
+        "--output", type=Path, help="Portable project vocabulary lock file"
+    )
+    args = parser.parse_args(argv)
+    if args.command == "upload" and args.offline:
+        parser.error("upload requires network access; use validate --offline")
+    token = os.environ.get("PKDB_API_KEY")
+    cache = VocabularyCache(args.cache_dir)
+    try:
+        if args.command == "vocabulary":
+            with Client(
+                args.endpoint, api_key=token, transport=client, cache=cache
+            ) as api:
+                snapshot = api.vocabulary()
+                if args.output:
+                    snapshot.save(args.output)
+                from pkdb.domain.vocabulary import vocabulary_hash
+
+                print(
+                    json.dumps(
+                        {
+                            "ok": True,
+                            "vocabulary_version": snapshot.version,
+                            "vocabulary_hash": vocabulary_hash(snapshot),
+                        }
+                    )
+                )
+            return 0
+        folders = study_folders(args.folder)
+        if args.output:
+            if len(folders) != 1:
+                raise ValueError("--output requires a single study folder")
+            if args.output.resolve().is_relative_to(folders[0].resolve()):
+                raise ValueError(
+                    "Write --output outside the study folder to keep source files unchanged"
+                )
+        if args.vocabulary:
+            snapshot = Vocabulary.load(args.vocabulary)
+        elif args.endpoint:
+            try:
+                snapshot = cache.load(args.endpoint)
+            except FileNotFoundError:
+                snapshot = bundled_vocabulary()
+        else:
+            snapshot = bundled_vocabulary()
+        if args.command == "upload" and (not args.endpoint or not token):
+            raise ValueError("Upload requires an endpoint and PKDB_API_KEY")
+    except (ValueError, OSError, ClientError) as error:
+        print(
+            json.dumps(_redact({"ok": False, "error": str(error)}, token)),
+            file=sys.stderr,
+        )
+        return 1
+    failed = False
+    for folder in folders:
+        result = {"path": str(folder), "ok": False}
+        try:
+            prepared = prepare(folder, vocabulary=snapshot)
+            result.update(sid=prepared.study.sid)
+            if args.command == "upload":
+                with Client(
+                    args.endpoint, api_key=token, transport=client, cache=cache
+                ) as api:
+                    result.update(api.upload(prepared).model_dump(mode="json"))
+            elif args.command == "prepare":
+                result.update(prepared.model_dump())
+            else:
+                result.update(
+                    report=prepared.report.model_dump(mode="json"),
+                    vocabulary_version=prepared.prepared.vocabulary_version,
+                    vocabulary_hash=prepared.vocabulary_hash,
+                    processing_version=prepared.prepared.processing_version,
+                )
+            result["ok"] = True
+        except StudyValidationError as error:
+            result.update(
+                error="Study validation failed",
+                report=error.report.model_dump(mode="json"),
+            )
+        except ClientError as error:
+            result.update(error=str(error), status_code=error.status_code)
+            if error.report:
+                result["report"] = error.report.model_dump(mode="json")
+        except (ValueError, OSError) as error:
+            result["error"] = str(error)
+        result = _redact(result, token)
+        if args.output:
+            try:
+                atomic_json(args.output, result)
+            except OSError as error:
+                result.update(ok=False, error=str(error))
+        print(json.dumps(result, ensure_ascii=False, allow_nan=False), flush=True)
+        failed |= not result["ok"]
+    return int(failed)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pkdb/client.py

@@ -0,0 +1,369 @@
+"""Synchronous, typed HTTP API with explicit offline preparation."""
+
+import json
+import os
+from contextlib import ExitStack
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import BinaryIO
+from urllib.parse import quote
+
+import httpx2
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from pkdb import __version__
+from pkdb.cache import (
+    VocabularyCache,
+    bundled_vocabulary,
+    endpoint_root,
+    parse_vocabulary,
+)
+from pkdb.domain.validation import PROCESSING_VERSION, prepare_study
+from pkdb.domain.vocabulary import Vocabulary
+from pkdb.errors import ClientError, CompatibilityError
+from pkdb.importers.folder import parse_bundle
+from pkdb.preparation import PreparedBundle, prepare
+from pkdb.schemas.replacement import ReplacementResult
+from pkdb.schemas.responses import (
+    GroupResponse,
+    IndividualResponse,
+    InterventionResponse,
+    OutputResponse,
+    ReferenceResponse,
+    StudyResponse,
+)
+from pkdb.schemas.study import CanonicalStudy
+from pkdb.schemas.validation import ValidationReport, fail
+
+
+class ResultPage[T: BaseModel](BaseModel):
+    items: list[T]
+    count: int = Field(ge=0)
+    page: int = Field(ge=1)
+    pages: int = Field(ge=1)
+
+
+class UploadLimits(BaseModel):
+    max_rows: int = Field(gt=0)
+    max_files: int = Field(gt=0)
+    max_upload_bytes: int = Field(gt=0)
+    max_attachment_bytes: int = Field(gt=0)
+
+
+class Capabilities(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+    schema_version: int
+    server_version: str
+    processing_version: str
+    vocabulary_version: str
+    vocabulary_hash: str
+    upload_limits: UploadLimits
+
+
+class Studies:
+    def __init__(self, client: Client):
+        self.client = client
+
+    def get(self, sid: str) -> CanonicalStudy:
+        return self.client._model(
+            CanonicalStudy,
+            self.client._request("GET", f"/api/v2/studies/{quote(str(sid), safe='')}"),
+        )
+
+    def list(self, **filters) -> ResultPage[StudyResponse]:
+        return self.client._page("studies", StudyResponse, filters)
+
+
+class Client:
+    """An endpoint-scoped client. Construction performs no network requests.
+
+    A caller-supplied HTTP transport remains owned by the caller. Requests never
+    follow redirects and writes are never retried automatically.
+    """
+
+    def __init__(
+        self,
+        endpoint: str | None = None,
+        api_key: str | None = None,
+        *,
+        transport=None,
+        cache: VocabularyCache | None = None,
+    ):
+        endpoint = endpoint or os.environ.get("PKDB_ENDPOINT")
+        if not endpoint:
+            raise ValueError("Set PKDB_ENDPOINT or pass endpoint")
+        self.endpoint = endpoint_root(endpoint)
+        self._api_key = (
+            api_key if api_key is not None else os.environ.get("PKDB_API_KEY")
+        )
+        self._owns_transport = transport is None
+        self._transport = (
+            transport
+            if transport is not None
+            else httpx2.Client(
+                timeout=httpx2.Timeout(600, connect=10), follow_redirects=False
+            )
+        )
+        self.cache = cache if cache is not None else VocabularyCache()
+        self._vocabulary: Vocabulary | None = None
+        self.studies = Studies(self)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def close(self):
+        if self._owns_transport:
+            self._transport.close()
+
+    def _headers(self, *, required=False) -> dict[str, str]:
+        if required and not self._api_key:
+            raise ClientError(
+                "Set PKDB_API_KEY or pass api_key for uploads and downloads"
+            )
+        headers = {"User-Agent": f"pkdb/{__version__}"}
+        if self._api_key:
+            kind = "Bearer" if self._api_key.startswith("pkdb_live_") else "Token"
+            headers["Authorization"] = f"{kind} {self._api_key}"
+        return headers
+
+    def _check(self, response):
+        if 200 <= response.status_code < 300:
+            return
+        report = None
+        body = None
+        try:
+            body = response.json()
+            if isinstance(body, dict) and isinstance(body.get("issues"), list):
+                report = ValidationReport.model_validate(body)
+        except ValueError, ValidationError:
+            pass
+        compatibility_messages = {
+            "vocabulary_mismatch": "Server vocabulary changed; synchronize vocabulary and prepare again",
+            "vocabulary_changed": "Server vocabulary changed during upload; synchronize vocabulary and prepare again",
+            "processing_version_mismatch": "Server processing rules differ; install the matching pkdb release and prepare again",
+        }
+        detail = body.get("detail") if isinstance(body, dict) else None
+        message = f"PK-DB request rejected (HTTP {response.status_code})"
+        error = ClientError
+        if (
+            response.status_code == 409
+            and isinstance(detail, str)
+            and detail in compatibility_messages
+        ):
+            error = CompatibilityError
+            message = compatibility_messages[detail]
+        raise error(
+            message,
+            status_code=response.status_code,
+            report=report,
+        )
+
+    def _request(self, method, path, *, headers=None, **kwargs):
+        request_headers = {**self._headers(), **(headers or {})}
+        try:
+            response = self._transport.request(
+                method,
+                self.endpoint + path,
+                headers=request_headers,
+                follow_redirects=False,
+                **kwargs,
+            )
+        except httpx2.RequestError:
+            message = "PK-DB request failed"
+            if method != "GET":
+                message += (
+                    "; upload outcome may be unknown. Inspect the study before retrying"
+                )
+            raise ClientError(message) from None
+        self._check(response)
+        return response
+
+    @staticmethod
+    def _model[T: BaseModel](model: type[T], response: httpx2.Response) -> T:
+        try:
+            return model.model_validate(response.json())
+        except ValueError, ValidationError:
+            raise ClientError("Server returned an invalid response contract") from None
+
+    def capabilities(self) -> Capabilities:
+        value = self._model(Capabilities, self._request("GET", "/api/v2/capabilities"))
+        if value.schema_version != 1:
+            raise CompatibilityError(
+                "Unsupported server client-protocol version; update pkdb"
+            )
+        return value
+
+    def vocabulary(self, *, refresh: bool = True) -> Vocabulary:
+        if refresh:
+            try:
+                value = parse_vocabulary(
+                    self._request("GET", "/api/v2/vocabulary").json()
+                )
+            except ValueError, ValidationError, AttributeError:
+                raise ClientError(
+                    "Server returned an invalid vocabulary snapshot"
+                ) from None
+            self.cache.store(self.endpoint, value)
+        else:
+            value = self.cache.load(self.endpoint)
+        self._vocabulary = value
+        return value
+
+    def _local_vocabulary(self) -> Vocabulary:
+        if self._vocabulary is not None:
+            return self._vocabulary
+        try:
+            return self.cache.load(self.endpoint)
+        except FileNotFoundError:
+            return bundled_vocabulary()
+
+    def upload(self, study: PreparedBundle | str | Path) -> ReplacementResult:
+        prepared = (
+            study
+            if isinstance(study, PreparedBundle)
+            else prepare(study, vocabulary=self._local_vocabulary())
+        )
+        headers = self._headers(required=True)
+        # Detect file changes and validate again against the captured vocabulary
+        # before contacting the server. Never trust mutated Pydantic objects.
+        with prepared.source() as source:
+            canonical = parse_bundle(source, max_rows=prepared.max_rows)
+            checked = prepare_study(canonical, prepared.vocabulary)
+            capabilities = self.capabilities()
+            if capabilities.processing_version != PROCESSING_VERSION:
+                raise CompatibilityError(
+                    "Server processing rules differ; install the matching pkdb release and prepare again"
+                )
+            if capabilities.vocabulary_hash != prepared.vocabulary_hash:
+                raise CompatibilityError(
+                    "Server vocabulary differs; run pkdb vocabulary sync and prepare again"
+                )
+            limits = capabilities.upload_limits
+            if len(source.files) > limits.max_files:
+                fail("file_limit", "Too many source files for this server")
+            if limits.max_rows < prepared.max_rows:
+                parse_bundle(source, max_rows=limits.max_rows)
+            sizes = [path.stat().st_size for path in source.files.values()]
+            if any(size > limits.max_attachment_bytes for size in sizes):
+                fail("file_limit", "Attachment exceeds the server's byte limit")
+            payloads = {
+                "study": json.dumps(source.study),
+                "reference": json.dumps(source.reference),
+            }
+            if (
+                sum(sizes) + sum(len(value.encode()) for value in payloads.values())
+                > limits.max_upload_bytes
+            ):
+                fail(
+                    "file_limit", "Study bundle exceeds the server's upload byte limit"
+                )
+            headers.update(
+                {
+                    "X-PKDB-Vocabulary-Hash": prepared.vocabulary_hash,
+                    "X-PKDB-Processing-Version": PROCESSING_VERSION,
+                }
+            )
+            with ExitStack() as stack:
+                parts: list[
+                    tuple[str, tuple[None, str] | tuple[str, BinaryIO, str]]
+                ] = [(name, (None, value)) for name, value in payloads.items()]
+                parts.extend(
+                    (
+                        "files",
+                        (
+                            name,
+                            stack.enter_context(path.open("rb")),
+                            "application/octet-stream",
+                        ),
+                    )
+                    for name, path in source.files.items()
+                )
+                response = self._request(
+                    "PUT",
+                    f"/api/v2/studies/{quote(checked.study.sid, safe='')}",
+                    headers=headers,
+                    files=parts,
+                )
+            result = self._model(ReplacementResult, response)
+            if result.sid != checked.study.sid:
+                raise ClientError(
+                    "Server did not confirm the uploaded study identifier"
+                )
+            return result
+
+    @staticmethod
+    def _params(filters: dict) -> dict:
+        params = {}
+        for key, value in filters.items():
+            if isinstance(value, (list, tuple)):
+                value = ",".join(str(item) for item in value)
+            elif isinstance(value, bool):
+                value = str(value).lower()
+            if value is not None:
+                params[key] = value
+        return params
+
+    def _page[T: BaseModel](
+        self, entity: str, model: type[T], filters: dict
+    ) -> ResultPage[T]:
+        params = self._params(filters)
+        response = self._request("GET", f"/api/v1/{entity}/", params=params)
+        try:
+            body = response.json()
+            return ResultPage(
+                items=[model.model_validate(item) for item in body["data"]["data"]],
+                count=body["data"]["count"],
+                page=body["current_page"],
+                pages=body["last_page"],
+            )
+        except ValueError, KeyError, TypeError, ValidationError:
+            raise ClientError("Server returned an invalid result page") from None
+
+    def query(self, entity: str, **filters) -> ResultPage:
+        models = {
+            "studies": StudyResponse,
+            "outputs": OutputResponse,
+            "groups": GroupResponse,
+            "individuals": IndividualResponse,
+            "interventions": InterventionResponse,
+            "references": ReferenceResponse,
+        }
+        if entity not in models:
+            raise ValueError(f"Unsupported query entity: {entity}")
+        return self._page(entity, models[entity], filters)
+
+    def download(self, destination: str | Path, **filters) -> Path:
+        """Stream an authenticated dataset ZIP to an atomic destination file."""
+        headers = self._headers(required=True)
+        path = Path(destination)
+        temporary = None
+        try:
+            with self._transport.stream(
+                "GET",
+                self.endpoint + "/api/v1/filter/",
+                params={**self._params(filters), "download": "true"},
+                headers=headers,
+                follow_redirects=False,
+            ) as response:
+                if not 200 <= response.status_code < 300:
+                    response.read()
+                    self._check(response)
+                if "zip" not in response.headers.get("content-type", ""):
+                    raise ClientError("Server did not return a dataset ZIP")
+                path.parent.mkdir(parents=True, exist_ok=True)
+                with NamedTemporaryFile(dir=path.parent, delete=False) as handle:
+                    temporary = Path(handle.name)
+                    for chunk in response.iter_bytes():
+                        handle.write(chunk)
+                temporary.replace(path)
+        except httpx2.RequestError:
+            raise ClientError(
+                "Dataset download failed; destination was not replaced"
+            ) from None
+        finally:
+            if temporary is not None:
+                temporary.unlink(missing_ok=True)
+        return path