netrias_client 0.0.1__py3-none-any.whl

netrias_client/_discovery.py

@@ -0,0 +1,437 @@
+"""Mapping discovery workflow functions.
+
+'why': call the recommendation service and normalize responses for callers
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import csv
+import time
+from pathlib import Path
+from collections.abc import Mapping, Sequence
+from typing import cast
+
+import httpx
+import logging
+
+from ._adapter import build_column_mapping_payload
+from ._config import BYPASS_ALIAS, BYPASS_FUNCTION, BYPASS_REGION
+from ._errors import MappingDiscoveryError, NetriasAPIUnavailable
+from ._gateway_bypass import GatewayBypassError, invoke_cde_recommendation_alias
+from ._http import request_mapping_discovery
+from ._models import MappingDiscoveryResult, MappingRecommendationOption, MappingSuggestion, Settings
+from ._validators import validate_column_samples, validate_target_schema, validate_source_path
+
+
+ManifestPayload = dict[str, dict[str, dict[str, object]]]
+
+
+async def _discover_mapping_async(
+    settings: Settings,
+    target_schema: str,
+    column_samples: Mapping[str, Sequence[object]],
+    logger: logging.Logger,
+) -> ManifestPayload:
+    """Perform mapping discovery via the recommendation endpoint."""
+
+    schema = validate_target_schema(target_schema)
+    samples: dict[str, list[str]] = validate_column_samples(column_samples)
+    started = time.perf_counter()
+    logger.info("discover mapping start: schema=%s columns=%s", schema, len(samples))
+
+    try:
+        result = await _discover_with_backend(settings, schema, samples, logger)
+    except (httpx.TimeoutException, httpx.HTTPError, GatewayBypassError) as exc:
+        _handle_discovery_error(schema, started, exc, logger)
+        raise AssertionError("_handle_discovery_error should raise") from exc
+
+    manifest = build_column_mapping_payload(
+        result,
+        threshold=settings.confidence_threshold,
+        logger=logger,
+    )
+    elapsed = time.perf_counter() - started
+    logger.info(
+        "discover mapping complete: schema=%s columns=%s duration=%.2fs",
+        schema,
+        len(manifest.get("column_mappings", {})),
+        elapsed,
+    )
+    return manifest
+
+
+def discover_mapping(
+    settings: Settings,
+    target_schema: str,
+    column_samples: Mapping[str, Sequence[object]],
+    logger: logging.Logger,
+) -> ManifestPayload:
+    """Sync wrapper around `_discover_mapping_async`."""
+
+    return asyncio.run(
+        _discover_mapping_async(
+            settings=settings,
+            target_schema=target_schema,
+            column_samples=column_samples,
+            logger=logger,
+        )
+    )
+
+
+async def discover_mapping_async(
+    settings: Settings,
+    target_schema: str,
+    column_samples: Mapping[str, Sequence[object]],
+    logger: logging.Logger,
+) -> ManifestPayload:
+    """Async entry point mirroring `discover_mapping` semantics."""
+
+    return await _discover_mapping_async(
+        settings=settings,
+        target_schema=target_schema,
+        column_samples=column_samples,
+        logger=logger,
+    )
+
+
+def discover_cde_mapping(
+    settings: Settings,
+    source_csv: Path,
+    target_schema: str,
+    sample_limit: int,
+    logger: logging.Logger,
+) -> ManifestPayload:
+    """Convenience wrapper that derives column samples from a CSV file."""
+
+    samples = _samples_from_csv(source_csv, sample_limit)
+    return discover_mapping(
+        settings=settings,
+        target_schema=target_schema,
+        column_samples=samples,
+        logger=logger,
+    )
+
+
+async def discover_mapping_from_csv_async(
+    settings: Settings,
+    source_csv: Path,
+    target_schema: str,
+    sample_limit: int,
+    logger: logging.Logger,
+) -> ManifestPayload:
+    """Async variant of `discover_mapping_from_csv`."""
+
+    samples = _samples_from_csv(source_csv, sample_limit)
+    return await discover_mapping_async(
+        settings=settings,
+        target_schema=target_schema,
+        column_samples=samples,
+        logger=logger,
+    )
+
+
+async def _discover_with_backend(
+    settings: Settings,
+    schema: str,
+    samples: Mapping[str, Sequence[str]],
+    logger: logging.Logger,
+) -> MappingDiscoveryResult:
+    if settings.discovery_use_gateway_bypass:
+        logger.debug("discover backend via bypass alias")
+        payload = invoke_cde_recommendation_alias(
+            target_schema=schema,
+            columns=samples,
+            function_name=BYPASS_FUNCTION,
+            alias=BYPASS_ALIAS,
+            region_name=BYPASS_REGION,
+            timeout_seconds=settings.timeout,
+            logger=logger,
+        )
+        return _result_from_payload(payload, schema)
+
+    logger.debug("discover backend via HTTP API")
+    response = await request_mapping_discovery(
+        base_url=settings.discovery_url,
+        api_key=settings.api_key,
+        timeout=settings.timeout,
+        schema=schema,
+        columns=samples,
+    )
+    return _interpret_discovery_response(response, schema)
+
+
+def _handle_discovery_error(
+    schema: str,
+    started: float,
+    exc: Exception,
+    logger: logging.Logger,
+) -> None:
+    elapsed = time.perf_counter() - started
+    if isinstance(exc, httpx.TimeoutException):  # pragma: no cover - exercised via integration tests
+        logger.error("discover mapping timeout: schema=%s duration=%.2fs err=%s", schema, elapsed, exc)
+        raise NetriasAPIUnavailable("mapping discovery timed out") from exc
+    if isinstance(exc, GatewayBypassError):
+        logger.error(
+            "discover mapping bypass error: schema=%s duration=%.2fs err=%s",
+            schema,
+            elapsed,
+            exc,
+        )
+        raise NetriasAPIUnavailable(f"gateway bypass error: {exc}") from exc
+
+    logger.error(
+        "discover mapping transport error: schema=%s duration=%.2fs err=%s",
+        schema,
+        elapsed,
+        exc,
+    )
+    raise NetriasAPIUnavailable(f"mapping discovery transport error: {exc}") from exc
+
+
+def _interpret_discovery_response(response: httpx.Response, requested_schema: str) -> MappingDiscoveryResult:
+    if response.status_code >= 500:
+        message = _error_message(response)
+        raise NetriasAPIUnavailable(message)
+    if response.status_code >= 400:
+        message = _error_message(response)
+        raise MappingDiscoveryError(message)
+
+    payload = _load_payload(response)
+    return _result_from_payload(payload, requested_schema)
+
+
+def _result_from_payload(payload: Mapping[str, object], requested_schema: str) -> MappingDiscoveryResult:
+    schema = _resolved_schema(payload, requested_schema)
+    suggestions = _suggestions_from_payload(payload)
+    return MappingDiscoveryResult(schema=schema, suggestions=suggestions, raw=payload)
+
+
+def _error_message(response: httpx.Response) -> str:
+    mapping = _mapping_or_none(_safe_json(response))
+    message = _message_from_mapping(mapping)
+    if message:
+        return message
+    return _default_error(response)
+
+
+def _extract_message(payload: Mapping[str, object]) -> str | None:
+    for key in ("message", "error", "detail"):
+        value = payload.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return None
+
+
+def _message_from_mapping(payload: Mapping[str, object] | None) -> str | None:
+    if payload is None:
+        return None
+    direct = _extract_message(payload)
+    if direct:
+        return direct
+    nested = _resolve_body_optional(payload)
+    if nested:
+        return _extract_message(nested)
+    return None
+
+
+def _mapping_or_none(data: object) -> Mapping[str, object] | None:
+    if isinstance(data, Mapping):
+        return cast(Mapping[str, object], data)
+    return None
+
+
+def _safe_json(response: httpx.Response) -> object:
+    try:
+        return cast(object, response.json())
+    except json.JSONDecodeError:
+        return None
+
+
+def _default_error(response: httpx.Response) -> str:
+    return f"mapping discovery failed (HTTP {response.status_code})"
+
+
+def _resolve_body_optional(container: Mapping[str, object]) -> dict[str, object] | None:
+    body = container.get("body")
+    if body is None:
+        return None
+    parsed = _decode_body(body, strict=False)
+    if isinstance(parsed, dict):
+        return _coerce_mapping(cast(Mapping[object, object], parsed), strict=False)
+    return None
+
+
+def _expect_mapping(data: object) -> dict[str, object]:
+    if isinstance(data, dict):
+        mapping = _coerce_mapping(cast(Mapping[object, object], data), strict=True)
+        if mapping is not None:
+            return mapping
+    raise MappingDiscoveryError("mapping discovery response body must be a JSON object")
+
+
+def _extract_body_object(container: Mapping[str, object]) -> dict[str, object] | None:
+    if "body" not in container:
+        return None
+    parsed = _decode_body(container["body"], strict=True)
+    if isinstance(parsed, dict):
+        mapping = _coerce_mapping(cast(Mapping[object, object], parsed), strict=True)
+        if mapping is not None:
+            return mapping
+    raise MappingDiscoveryError("mapping discovery response body must be a JSON object")
+
+
+def _entries_from_value(value: object) -> tuple[Mapping[str, object], ...]:
+    if not isinstance(value, list):
+        return ()
+    collected: list[Mapping[str, object]] = []
+    items = cast(list[object], value)
+    for item in items:
+        if isinstance(item, Mapping):
+            collected.append(cast(Mapping[str, object], item))
+    return tuple(collected)
+
+
+def _coerce_mapping(obj: Mapping[object, object], strict: bool) -> dict[str, object] | None:
+    result: dict[str, object] = {}
+    for key, value in obj.items():
+        if not isinstance(key, str):
+            if strict:
+                raise MappingDiscoveryError("mapping discovery response body must be a JSON object")
+            return None
+        result[key] = value
+    return result
+
+
+def _samples_from_csv(csv_path: Path, sample_limit: int) -> dict[str, list[str]]:
+    dataset = validate_source_path(csv_path)
+    headers, rows = _read_limited_rows(dataset, sample_limit)
+    samples: dict[str, list[str]] = {header: [] for header in headers}
+    _fill_samples(samples, rows)
+    return {key: value for key, value in samples.items() if value}
+
+
+def _read_limited_rows(dataset: Path, sample_limit: int) -> tuple[list[str], list[dict[str, str | None]]]:
+    headers: list[str] = []
+    rows: list[dict[str, str | None]] = []
+    with dataset.open("r", encoding="utf-8", newline="") as handle:
+        reader = csv.DictReader(handle)
+        headers = [header for header in reader.fieldnames or [] if header]
+        for index, row in enumerate(reader):
+            if index >= sample_limit:
+                break
+            rows.append(row)
+    return headers, rows
+
+
+def _fill_samples(samples: dict[str, list[str]], rows: list[dict[str, str | None]]) -> None:
+    for row in rows:
+        _append_row(samples, row)
+
+
+def _append_row(samples: dict[str, list[str]], row: dict[str, str | None]) -> None:
+    for header, raw_value in row.items():
+        if header not in samples or raw_value is None:
+            continue
+        value = raw_value.strip()
+        if value:
+            samples[header].append(value)
+
+
+def _decode_body(body: object, strict: bool) -> object:
+    if not isinstance(body, str):
+        return body
+    try:
+        return cast(object, json.loads(body))
+    except json.JSONDecodeError as exc:
+        if strict:
+            raise MappingDiscoveryError("mapping discovery body was not valid JSON") from exc
+        return None
+
+
+def _load_payload(response: httpx.Response) -> dict[str, object]:
+    data = _safe_json(response)
+    mapping = _expect_mapping(data)
+    body = _extract_body_object(mapping)
+    if body is not None:
+        return body
+    return mapping
+
+
+def _resolved_schema(payload: Mapping[str, object], requested_schema: str) -> str:
+    for key in ("target_schema", "schema", "recommended_schema"):
+        value = payload.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return requested_schema
+
+
+def _suggestions_from_payload(payload: Mapping[str, object]) -> tuple[MappingSuggestion, ...]:
+    raw_entries = _candidate_entries(payload)
+    suggestions: list[MappingSuggestion] = []
+    for entry in raw_entries:
+        source = _source_column(entry)
+        if not source:
+            continue
+        options = _options_from_entry(entry)
+        suggestions.append(
+            MappingSuggestion(source_column=source, options=options, raw=entry)
+        )
+    return tuple(suggestions)
+
+
+def _candidate_entries(payload: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    for key in ("recommendations", "columns", "suggestions"):
+        entries = _entries_from_value(payload.get(key))
+        if entries:
+            return entries
+    return ()
+
+
+def _source_column(entry: Mapping[str, object]) -> str | None:
+    candidates = (
+        entry.get("column"),
+        entry.get("source_column"),
+        entry.get("name"),
+        entry.get("field"),
+    )
+    for candidate in candidates:
+        if isinstance(candidate, str):
+            name = candidate.strip()
+            if name:
+                return name
+    return None
+
+
+def _options_from_entry(entry: Mapping[str, object]) -> tuple[MappingRecommendationOption, ...]:
+    raw_options = entry.get("suggestions") or entry.get("options") or entry.get("targets")
+    if not isinstance(raw_options, list):
+        return ()
+    options: list[MappingRecommendationOption] = []
+    items = cast(list[object], raw_options)
+    for item in items:
+        if not isinstance(item, Mapping):
+            continue
+        mapping = cast(Mapping[str, object], item)
+        target = _option_target(mapping)
+        confidence = _option_confidence(mapping)
+        options.append(MappingRecommendationOption(target=target, confidence=confidence, raw=mapping))
+    return tuple(options)
+
+
+def _option_target(option: Mapping[str, object]) -> str | None:
+    for key in ("target", "cde", "field", "name", "qualified_name"):
+        value = option.get(key)
+        if isinstance(value, str):
+            candidate = value.strip()
+            if candidate:
+                return candidate
+    return None
+
+
+def _option_confidence(option: Mapping[str, object]) -> float | None:
+    for key in ("confidence", "score", "probability"):
+        value = option.get(key)
+        if isinstance(value, (int, float)):
+            return float(value)
+    return None

netrias_client/_errors.py

@@ -0,0 +1,33 @@
+"""Define client-specific exceptions.
+
+'why': keep error taxonomy explicit and lightweight
+"""
+from __future__ import annotations
+
+
+class NetriasClientError(Exception):
+    """Base class for all client-specific exceptions."""
+
+
+class ClientConfigurationError(NetriasClientError):
+    """Raised when configuration is incomplete or malformed."""
+
+
+class FileValidationError(NetriasClientError):
+    """Raised for unreadable files, unsupported extensions, or size violations."""
+
+
+class MappingValidationError(NetriasClientError):
+    """Raised when mapping discovery inputs fail validation."""
+
+
+class OutputLocationError(NetriasClientError):
+    """Raised when the output path is unwritable or collides with an existing directory."""
+
+
+class NetriasAPIUnavailable(NetriasClientError):
+    """Raised for timeouts or network failures."""
+
+
+class MappingDiscoveryError(NetriasClientError):
+    """Raised when the mapping discovery API returns an error payload."""

netrias_client/_gateway_bypass.py

@@ -0,0 +1,208 @@
+"""Temporary gateway bypass helpers for direct Lambda invocation.
+
+'why': mitigate API Gateway timeouts by calling the CDE recommendation alias directly
+
+# TODO: remove this module once API Gateway latency is resolved and direct Lambda
+# calls are no longer necessary.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Mapping, Sequence
+from typing import Callable, IO, Protocol, cast
+
+
+class GatewayBypassError(RuntimeError):
+    """Raised when the direct Lambda invocation fails."""
+
+
+class _LambdaClient(Protocol):
+    def invoke(
+        self,
+        FunctionName: str,
+        Qualifier: str,
+        Payload: bytes,
+    ) -> Mapping[str, object]:
+        ...
+
+
+class _ClientFactory(Protocol):
+    def __call__(self, service_name: str, **kwargs: object) -> object:
+        ...
+
+
+class _SessionProtocol(Protocol):
+    def client(self, service_name: str, **kwargs: object) -> object:
+        ...
+
+
+def invoke_cde_recommendation_alias(
+    target_schema: str,
+    columns: Mapping[str, Sequence[object]],
+    function_name: str = "cde-recommendation",
+    alias: str = "prod",
+    region_name: str = "us-east-2",
+    timeout_seconds: float | None = None,
+    profile_name: str | None = None,
+    logger: logging.Logger | None = None,
+) -> Mapping[str, object]:
+    """Call the CDE recommendation Lambda alias directly and return its parsed payload.
+
+    NOTE: This bypass is temporary. Prefer the public API once API Gateway limits are addressed.
+    """
+
+    client = _build_lambda_client(
+        region_name=region_name,
+        profile_name=profile_name,
+        timeout_seconds=timeout_seconds,
+    )
+    normalized_columns = _normalized_columns(columns)
+    body = json.dumps({"target_schema": target_schema, "data": normalized_columns})
+    event = {"body": body, "isBase64Encoded": False}
+
+    active_logger = logger or logging.getLogger("netrias_client")
+
+    active_logger.info(
+        "gateway bypass invoke start: function=%s alias=%s schema=%s columns=%s",
+        function_name,
+        alias,
+        target_schema,
+        len(columns),
+    )
+
+    try:
+        response = client.invoke(
+            FunctionName=function_name,
+            Qualifier=alias,
+            Payload=json.dumps(event).encode("utf-8"),
+        )
+    except Exception as exc:  # pragma: no cover - boto3 specific
+        active_logger.error(
+            "gateway bypass invoke failed: function=%s alias=%s err=%s",
+            function_name,
+            alias,
+            exc,
+        )
+        raise GatewayBypassError(f"lambda invoke failed: {exc}") from exc
+
+    status_code = response.get("StatusCode")
+    payload_stream = cast(IO[bytes] | None, response.get("Payload"))
+    raw_payload = _read_lambda_payload(payload_stream)
+    payload = _json_payload(raw_payload)
+
+    active_logger.info(
+        "gateway bypass invoke complete: function=%s alias=%s status=%s",
+        function_name,
+        alias,
+        status_code,
+    )
+
+    return _extract_body_mapping(payload)
+
+
+def _build_lambda_client(
+    region_name: str,
+    profile_name: str | None,
+    timeout_seconds: float | None,
+) -> _LambdaClient:
+    boto3, Config = _load_boto_dependencies()
+    config = (
+        Config(
+            read_timeout=timeout_seconds,
+            connect_timeout=min(timeout_seconds, 10.0),
+        )
+        if timeout_seconds is not None
+        else None
+    )
+
+    if profile_name:
+        session_factory = cast(
+            Callable[..., object],
+            getattr(boto3, "Session"),
+        )
+        session = cast(
+            _SessionProtocol,
+            session_factory(profile_name=profile_name, region_name=region_name),
+        )
+        factory = cast(_ClientFactory, session.client)
+    else:
+        factory = cast(_ClientFactory, getattr(boto3, "client"))
+
+    return _lambda_client_from_factory(factory, region_name=region_name, config=config)
+
+
+def _load_boto_dependencies():
+    try:
+        import boto3  # pyright: ignore[reportMissingTypeStubs]
+        from botocore.config import Config  # pyright: ignore[reportMissingTypeStubs]
+    except ImportError as exc:  # pragma: no cover - optional dependency
+        raise GatewayBypassError(
+            "boto3 is required for the gateway bypass helper; install netrias-client[aws] or boto3 explicitly"
+        ) from exc
+    return boto3, Config
+
+
+def _lambda_client_from_factory(
+    factory: _ClientFactory,
+    region_name: str,
+    config: object | None,
+) -> _LambdaClient:
+    kwargs: dict[str, object] = {"region_name": region_name}
+    if config is not None:
+        kwargs["config"] = config
+    client_obj = factory("lambda", **kwargs)
+    return cast(_LambdaClient, client_obj)
+
+
+def _read_lambda_payload(stream: IO[bytes] | None) -> bytes:
+    if stream is None:
+        return b""
+    return stream.read()
+
+
+def _json_payload(raw_payload: bytes) -> Mapping[str, object]:
+    if not raw_payload:
+        return {}
+    try:
+        return cast(Mapping[str, object], json.loads(raw_payload.decode("utf-8")))
+    except json.JSONDecodeError as exc:  # pragma: no cover - unexpected lambda output
+        raise GatewayBypassError(f"lambda returned non-JSON payload: {exc}") from exc
+
+
+def _extract_body_mapping(payload: Mapping[str, object]) -> Mapping[str, object]:
+    body = payload.get("body")
+    if isinstance(body, str):
+        try:
+            return cast(Mapping[str, object], json.loads(body))
+        except json.JSONDecodeError as exc:  # pragma: no cover - unexpected lambda output
+            raise GatewayBypassError(f"lambda body was not valid JSON: {exc}") from exc
+    return payload
+
+
+def _normalized_columns(columns: Mapping[str, Sequence[object]]) -> dict[str, list[str]]:
+    normalized: dict[str, list[str]] = {}
+    for key, values in columns.items():
+        name = _normalized_column_key(key)
+        if name is None:
+            continue
+        cleaned = _normalized_column_values(values)
+        if cleaned:
+            normalized[name] = cleaned
+    return normalized
+
+
+def _normalized_column_key(raw: str) -> str | None:
+    text = raw.strip()
+    return text or None
+
+
+def _normalized_column_values(values: Sequence[object]) -> list[str]:
+    return [text for text in (_normalized_column_value(value) for value in values) if text]
+
+
+def _normalized_column_value(value: object) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None