netrias_client 0.1.0__py3-none-any.whl

netrias_client/_gateway_bypass.py

@@ -0,0 +1,217 @@
+"""Temporary gateway bypass helpers for direct Lambda invocation.
+
+'why': mitigate API Gateway timeouts by calling the CDE recommendation alias directly
+
+# TODO: remove this module once API Gateway latency is resolved and direct Lambda
+# calls are no longer necessary.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Mapping, Sequence
+from typing import Callable, IO, Protocol, cast
+
+
+class GatewayBypassError(RuntimeError):
+    """Raised when the direct Lambda invocation fails."""
+
+
+class _LambdaClient(Protocol):
+    def invoke(
+        self,
+        FunctionName: str,
+        Qualifier: str,
+        Payload: bytes,
+    ) -> Mapping[str, object]:
+        ...
+
+
+class _ClientFactory(Protocol):
+    def __call__(self, service_name: str, **kwargs: object) -> object:
+        ...
+
+
+class _SessionProtocol(Protocol):
+    def client(self, service_name: str, **kwargs: object) -> object:
+        ...
+
+
+def invoke_cde_recommendation_alias(
+    target_schema: str,
+    target_version: str,
+    columns: Mapping[str, Sequence[object]],
+    function_name: str = "cde-recommendation",
+    alias: str = "prod",
+    region_name: str = "us-east-2",
+    timeout_seconds: float | None = None,
+    profile_name: str | None = None,
+    logger: logging.Logger | None = None,
+    top_k: int | None = None,
+) -> Mapping[str, object]:
+    """Call the CDE recommendation Lambda alias directly and return its parsed payload.
+
+    NOTE: This bypass is temporary. Prefer the public API once API Gateway limits are addressed.
+    """
+
+    client = _build_lambda_client(
+        region_name=region_name,
+        profile_name=profile_name,
+        timeout_seconds=timeout_seconds,
+    )
+    normalized_columns = _normalized_columns(columns)
+    body_dict: dict[str, object] = {
+        "target_schema": target_schema,
+        "target_version": target_version,
+        "data": normalized_columns,
+    }
+    if top_k is not None:
+        body_dict["top_k"] = top_k
+    body = json.dumps(body_dict)
+    event = {"body": body, "isBase64Encoded": False}
+
+    active_logger = logger or logging.getLogger("netrias_client")
+
+    active_logger.info(
+        "gateway bypass invoke start: function=%s alias=%s schema=%s columns=%s",
+        function_name,
+        alias,
+        target_schema,
+        len(columns),
+    )
+
+    try:
+        response = client.invoke(
+            FunctionName=function_name,
+            Qualifier=alias,
+            Payload=json.dumps(event).encode("utf-8"),
+        )
+    except Exception as exc:  # pragma: no cover - boto3 specific
+        active_logger.error(
+            "gateway bypass invoke failed: function=%s alias=%s err=%s",
+            function_name,
+            alias,
+            exc,
+        )
+        raise GatewayBypassError(f"lambda invoke failed: {exc}") from exc
+
+    status_code = response.get("StatusCode")
+    payload_stream = cast(IO[bytes] | None, response.get("Payload"))
+    raw_payload = _read_lambda_payload(payload_stream)
+    payload = _json_payload(raw_payload)
+
+    active_logger.info(
+        "gateway bypass invoke complete: function=%s alias=%s status=%s",
+        function_name,
+        alias,
+        status_code,
+    )
+
+    return _extract_body_mapping(payload)
+
+
+def _build_lambda_client(
+    region_name: str,
+    profile_name: str | None,
+    timeout_seconds: float | None,
+) -> _LambdaClient:
+    boto3, Config = _load_boto_dependencies()
+    config = (
+        Config(
+            read_timeout=timeout_seconds,
+            connect_timeout=min(timeout_seconds, 10.0),
+        )
+        if timeout_seconds is not None
+        else None
+    )
+
+    if profile_name:
+        session_factory = cast(
+            Callable[..., object],
+            getattr(boto3, "Session"),
+        )
+        session = cast(
+            _SessionProtocol,
+            session_factory(profile_name=profile_name, region_name=region_name),
+        )
+        factory = cast(_ClientFactory, session.client)
+    else:
+        factory = cast(_ClientFactory, getattr(boto3, "client"))
+
+    return _lambda_client_from_factory(factory, region_name=region_name, config=config)
+
+
+def _load_boto_dependencies():
+    try:
+        import boto3  # pyright: ignore[reportMissingTypeStubs]
+        from botocore.config import Config  # pyright: ignore[reportMissingTypeStubs]
+    except ImportError as exc:  # pragma: no cover - optional dependency
+        raise GatewayBypassError(
+            "boto3 is required for the gateway bypass helper; install netrias-client[aws] or boto3 explicitly"
+        ) from exc
+    return boto3, Config
+
+
+def _lambda_client_from_factory(
+    factory: _ClientFactory,
+    region_name: str,
+    config: object | None,
+) -> _LambdaClient:
+    kwargs: dict[str, object] = {"region_name": region_name}
+    if config is not None:
+        kwargs["config"] = config
+    client_obj = factory("lambda", **kwargs)
+    return cast(_LambdaClient, client_obj)
+
+
+def _read_lambda_payload(stream: IO[bytes] | None) -> bytes:
+    if stream is None:
+        return b""
+    return stream.read()
+
+
+def _json_payload(raw_payload: bytes) -> Mapping[str, object]:
+    if not raw_payload:
+        return {}
+    try:
+        return cast(Mapping[str, object], json.loads(raw_payload.decode("utf-8")))
+    except json.JSONDecodeError as exc:  # pragma: no cover - unexpected lambda output
+        raise GatewayBypassError(f"lambda returned non-JSON payload: {exc}") from exc
+
+
+def _extract_body_mapping(payload: Mapping[str, object]) -> Mapping[str, object]:
+    body = payload.get("body")
+    if isinstance(body, str):
+        try:
+            return cast(Mapping[str, object], json.loads(body))
+        except json.JSONDecodeError as exc:  # pragma: no cover - unexpected lambda output
+            raise GatewayBypassError(f"lambda body was not valid JSON: {exc}") from exc
+    return payload
+
+
+def _normalized_columns(columns: Mapping[str, Sequence[object]]) -> dict[str, list[str]]:
+    normalized: dict[str, list[str]] = {}
+    for key, values in columns.items():
+        name = _normalized_column_key(key)
+        if name is None:
+            continue
+        cleaned = _normalized_column_values(values)
+        if cleaned:
+            normalized[name] = cleaned
+    return normalized
+
+
+def _normalized_column_key(raw: str) -> str | None:
+    text = raw.strip()
+    return text or None
+
+
+def _normalized_column_values(values: Sequence[object]) -> list[str]:
+    return [text for text in (_normalized_column_value(value) for value in values) if text]
+
+
+def _normalized_column_value(value: object) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None

netrias_client/_http.py

@@ -0,0 +1,234 @@
+"""HTTP helpers for harmonization and discovery."""
+from __future__ import annotations
+
+import csv
+import gzip
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Final
+from urllib.parse import quote
+
+import httpx
+
+from ._adapter import normalize_manifest_mapping
+
+SCHEMA_VERSION: Final[str] = "1.0"
+DEFAULT_MODEL_VERSION: Final[str] = "v1"
+MAX_COMPRESSED_BYTES: Final[int] = 10 * 1024 * 1024
+
+def build_harmonize_payload(
+    csv_path: Path,
+    manifest: Path | Mapping[str, object] | None,
+    model_version: str = DEFAULT_MODEL_VERSION,
+) -> bytes:
+    """Return gzip-compressed harmonization payload for the given CSV and manifest."""
+
+    rows = _read_tabular(csv_path)
+    header = rows[0] if rows else []
+    data_rows = rows[1:] if len(rows) > 1 else []
+
+    envelope: dict[str, object] = {
+        "schemaVersion": SCHEMA_VERSION,
+        "modelVersion": model_version,
+        "document": {
+            "name": csv_path.name,
+            "sheetName": None,
+            "header": header,
+            "rows": data_rows,
+        },
+    }
+
+    mapping = normalize_manifest_mapping(manifest)
+    if mapping:
+        envelope["mapping"] = mapping
+
+    raw = json.dumps(envelope, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    compressed = gzip.compress(raw)
+    if len(compressed) > MAX_COMPRESSED_BYTES:
+        raise ValueError("compressed harmonization payload exceeds 10 MiB")
+    return compressed
+
+async def submit_harmonize_job(
+    base_url: str,
+    api_key: str,
+    payload_gz: bytes,
+    timeout: float,
+    idempotency_key: str | None = None,
+) -> httpx.Response:
+    """Submit a harmonization job request and return the raw response."""
+
+    url = _build_job_submit_url(base_url)
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "Content-Encoding": "gzip",
+    }
+    if idempotency_key:
+        headers["Idempotency-Key"] = idempotency_key
+
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.post(url, content=payload_gz, headers=headers)
+
+async def fetch_job_status(
+    base_url: str,
+    api_key: str,
+    job_id: str,
+    timeout: float,
+) -> httpx.Response:
+    """Return the status response for a previously submitted harmonization job."""
+
+    url = _build_job_status_url(base_url, job_id)
+    headers = {"Authorization": f"Bearer {api_key}"}
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.get(url, headers=headers)
+
+async def request_mapping_discovery(
+    base_url: str,
+    api_key: str,
+    timeout: float,
+    schema: str,
+    version: str,
+    columns: Mapping[str, Sequence[str]],
+    top_k: int | None = None,
+) -> httpx.Response:
+    """Submit column samples for mapping recommendations."""
+
+    url = _build_discovery_url(base_url)
+    headers = {
+        "Content-Type": "application/json",
+        "x-api-key": api_key,
+    }
+    body: dict[str, object] = {
+        "target_schema": schema,
+        "target_version": version,
+        "data": columns,
+    }
+    if top_k is not None:
+        body["top_k"] = top_k
+    payload = {"body": json.dumps(body)}
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.post(url, headers=headers, json=payload)
+
+
+async def fetch_data_models(
+    base_url: str,
+    api_key: str,
+    timeout: float,
+    query: str | None = None,
+    include_versions: bool = False,
+    include_counts: bool = False,
+    limit: int | None = None,
+    offset: int = 0,
+) -> httpx.Response:
+    """Fetch data models from the Data Model Store."""
+
+    url = f"{base_url.rstrip('/')}/data-models"
+    headers = {"x-api-key": api_key}
+    params = _build_data_models_params(query, include_versions, include_counts, limit, offset)
+
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.get(url, headers=headers, params=params)
+
+
+def _build_data_models_params(
+    query: str | None,
+    include_versions: bool,
+    include_counts: bool,
+    limit: int | None,
+    offset: int,
+) -> dict[str, str | int]:
+    """Build query parameters for data models endpoint."""
+
+    candidates: list[tuple[str, str | int | None]] = [
+        ("offset", offset),
+        ("q", query),
+        ("include_versions", "true" if include_versions else None),
+        ("include_counts", "true" if include_counts else None),
+        ("limit", limit),
+    ]
+    return {k: v for k, v in candidates if v is not None}
+
+
+async def fetch_cdes(
+    base_url: str,
+    api_key: str,
+    timeout: float,
+    model_key: str,
+    version: str,
+    include_description: bool = False,
+    query: str | None = None,
+    limit: int | None = None,
+    offset: int = 0,
+) -> httpx.Response:
+    """Fetch CDEs for a data model version from the Data Model Store."""
+
+    url = f"{base_url.rstrip('/')}/data-models/{quote(model_key, safe='')}/versions/{quote(version, safe='')}/cdes"
+    headers = {"x-api-key": api_key}
+    params: dict[str, str | int] = {"offset": offset}
+    if include_description:
+        params["include_description"] = "true"
+    if query:
+        params["q"] = query
+    if limit is not None:
+        params["limit"] = limit
+
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.get(url, headers=headers, params=params)
+
+
+async def fetch_pvs(
+    base_url: str,
+    api_key: str,
+    timeout: float,
+    model_key: str,
+    version: str,
+    cde_key: str,
+    include_inactive: bool = False,
+    query: str | None = None,
+    limit: int | None = None,
+    offset: int = 0,
+) -> httpx.Response:
+    """Fetch permissible values for a CDE from the Data Model Store."""
+
+    path = (
+        f"/data-models/{quote(model_key, safe='')}"
+        f"/versions/{quote(version, safe='')}"
+        f"/cdes/{quote(cde_key, safe='')}/pvs"
+    )
+    url = f"{base_url.rstrip('/')}{path}"
+    headers = {"x-api-key": api_key}
+    params: dict[str, str | int] = {"offset": offset}
+    if include_inactive:
+        params["include_inactive"] = "true"
+    if query:
+        params["q"] = query
+    if limit is not None:
+        params["limit"] = limit
+
+    async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+        return await client.get(url, headers=headers, params=params)
+
+def _build_job_submit_url(base_url: str) -> str:
+    base = base_url.rstrip("/")
+    return f"{base}/v1/jobs/harmonize"
+
+def _build_job_status_url(base_url: str, job_id: str) -> str:
+    base = base_url.rstrip("/")
+    return f"{base}/v1/jobs/{job_id}"
+
+def _build_discovery_url(base_url: str) -> str:
+    base = base_url.rstrip("/")
+    return f"{base}/cde-recommendation"
+
+def _read_tabular(path: Path) -> list[list[str]]:
+    if not path.exists():
+        raise FileNotFoundError(path)
+    ext = path.suffix.lower()
+    if ext not in {".csv", ".tsv"}:
+        raise ValueError("harmonization only supports CSV or TSV inputs")
+    delimiter = "," if ext == ".csv" else "\t"
+    with path.open("r", encoding="utf-8", newline="") as handle:
+        reader = csv.reader(handle, delimiter=delimiter)
+        return [list(row) for row in reader]
+

netrias_client/_io.py

@@ -0,0 +1,28 @@
+"""I/O helpers for streaming responses.
+
+'why': keep file operations small and testable; avoid partial outputs
+"""
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+import httpx
+
+
+async def stream_download_to_file(response: httpx.Response, dest_path: Path) -> Path:
+    """Stream an HTTP response body to `dest_path` atomically.
+
+    Writes to a temporary file in the destination directory and then renames.
+    """
+
+    dest_path = Path(dest_path)
+    tmp_dir = dest_path.parent
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=tmp_dir, delete=False, suffix=".partial") as tmp:
+        async for chunk in response.aiter_bytes():
+            _ = tmp.write(chunk)
+        tmp_path = Path(tmp.name)
+    _ = tmp_path.replace(dest_path)
+    return dest_path
+

netrias_client/_logging.py

@@ -0,0 +1,46 @@
+"""Logger helpers for the Netrias client."""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Final
+
+from ._models import LogLevel
+
+
+_FORMAT: Final[str] = "%(asctime)s %(levelname)s netrias_client: %(message)s"
+
+
+def configure_logger(
+    name: str,
+    level: LogLevel,
+    log_directory: Path | None,
+) -> logging.Logger:
+    """Configure and return a logger dedicated to a Netrias client instance."""
+
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.propagate = False
+
+    formatter = logging.Formatter(fmt=_FORMAT)
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    if log_directory is not None:
+        log_directory.mkdir(parents=True, exist_ok=True)
+        file_path = log_directory / f"{name.replace('.', '_')}.log"
+        file_handler = logging.FileHandler(file_path, encoding="utf-8")
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    mapping = {
+        LogLevel.CRITICAL: logging.CRITICAL,
+        LogLevel.ERROR: logging.ERROR,
+        LogLevel.WARNING: logging.WARNING,
+        LogLevel.INFO: logging.INFO,
+        LogLevel.DEBUG: logging.DEBUG,
+    }
+    logger.setLevel(mapping[level])
+    return logger

netrias_client/_models.py

@@ -0,0 +1,115 @@
+"""Define dataclasses and types for the client.
+
+'why': capture configuration and results in typed, testable shapes
+"""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Literal
+
+
+class LogLevel(str, Enum):
+    """Enumerate supported logging levels for the client."""
+
+    CRITICAL = "CRITICAL"
+    ERROR = "ERROR"
+    WARNING = "WARNING"
+    INFO = "INFO"
+    DEBUG = "DEBUG"
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Capture runtime settings for API calls."""
+
+    api_key: str
+    discovery_url: str
+    harmonization_url: str
+    timeout: float
+    log_level: LogLevel
+    confidence_threshold: float
+    discovery_use_gateway_bypass: bool
+    log_directory: Path | None
+    data_model_store_endpoints: DataModelStoreEndpoints | None = None
+
+
+@dataclass(frozen=True)
+class HarmonizationResult:
+    """Communicate harmonization outcome in a consistent shape."""
+
+    file_path: Path
+    status: Literal["succeeded", "failed", "timeout"]
+    description: str
+    mapping_id: str | None = None
+
+
+@dataclass(frozen=True)
+class MappingRecommendationOption:
+    """Capture a single recommended target for a source column."""
+
+    target: str | None
+    confidence: float | None
+    target_cde_id: int | None = None
+    raw: Mapping[str, object] | None = None
+
+
+@dataclass(frozen=True)
+class MappingSuggestion:
+    """Group recommendation options for a single source column."""
+
+    source_column: str
+    options: tuple[MappingRecommendationOption, ...]
+    raw: Mapping[str, object] | None = None
+
+
+@dataclass(frozen=True)
+class MappingDiscoveryResult:
+    """Communicate column mapping recommendations for a dataset."""
+
+    schema: str
+    suggestions: tuple[MappingSuggestion, ...]
+    raw: Mapping[str, object]
+
+
+@dataclass(frozen=True)
+class DataModelStoreEndpoints:
+    """Encapsulate Data Model Store endpoint URLs for swappability.
+
+    'why': endpoints may change; grouping them enables single-point override
+    """
+
+    base_url: str
+
+
+@dataclass(frozen=True)
+class DataModel:
+    """Represent a data commons/model from the Data Model Store."""
+
+    data_commons_id: int
+    key: str
+    name: str
+    description: str | None
+    is_active: bool
+
+
+@dataclass(frozen=True)
+class CDE:
+    """Represent a Common Data Element within a data model version."""
+
+    cde_key: str
+    cde_id: int
+    cde_version_id: int
+    description: str | None = None
+
+
+@dataclass(frozen=True)
+class PermissibleValue:
+    """Represent a permissible value for a CDE."""
+
+    pv_id: int
+    value: str
+    description: str | None
+    is_active: bool