acceldata-sdk-python 26.7.0__py3-none-any.whl

acceldata/services/datasource_service.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from acceldata.client.http_client import BaseHttpClient
+from acceldata.client.transient_retry import (
+    RetryConfig,
+    effective_transient_retry,
+    transient_retry_call,
+)
+from acceldata.models.api.catalog.assembly import Assembly
+from acceldata.models.api.catalog.crawler_action_response import CrawlerActionResponse
+from acceldata.models.api.catalog.crawler_status_response import CrawlerStatusResponse
+from acceldata.models.sdk.catalog.asset_source_type import AssetSourceType
+from acceldata.models.sdk.catalog.datasource_resource import DatasourceResource
+
+logger = logging.getLogger("sdk")
+
+
+class DataSourceService:
+    def __init__(self, http_client: BaseHttpClient) -> None:
+        self._client = http_client
+
+    def start_crawler(
+        self,
+        datasource_name: str,
+        *,
+        transient_retry: RetryConfig | None = None,
+    ) -> CrawlerActionResponse:
+        """POST to start a crawler run (single request)."""
+        cfg = effective_transient_retry(transient_retry, None)
+
+        def _do() -> CrawlerActionResponse:
+            if datasource_name is None or datasource_name == "":
+                raise ValueError("datasource name is required")
+            url = self._client.catalog_urls.crawl_datasource(datasource_name=datasource_name)
+            response = self._client._post(url=url, payload={})
+            if isinstance(response, dict):
+                data = response.get("data")
+                if isinstance(data, dict):
+                    started = CrawlerActionResponse.from_dict(data)
+                    if started is None:
+                        raise TypeError("crawler start returned empty data")
+                    return started
+            raise TypeError(f"Expected dict from crawler start, got {type(response).__name__}")
+
+        return transient_retry_call(_do, cfg, logger=logger, operation_name="start_crawler")
+
+    def get_crawler_status(
+        self,
+        datasource_name: str,
+        *,
+        transient_retry: RetryConfig | None = None,
+    ) -> CrawlerStatusResponse:
+        cfg = effective_transient_retry(transient_retry, None)
+
+        def _do() -> CrawlerStatusResponse:
+            if datasource_name is None or datasource_name == "":
+                raise ValueError("datasource name is required")
+            url = self._client.catalog_urls.crawl_datasource(datasource_name=datasource_name)
+            response = self._client._get(url=url)
+            logger.debug("Crawler response: %s", response)
+            if isinstance(response, dict):
+                data = response.get("data")
+                if isinstance(data, dict):
+                    return CrawlerStatusResponse.from_dict(data)
+            raise TypeError(f"Expected dict from crawler status, got {type(response).__name__}")
+
+        return transient_retry_call(_do, cfg, logger=logger, operation_name="get_crawler_status")
+
+    def get_datasource(
+        self,
+        assembly_identifier: Any,
+        properties: bool = False,
+    ) -> DatasourceResource:
+        """
+        Find a datasource by name or numeric id in the catalog.
+
+        :param assembly_identifier: Datasource name or id in the catalog.
+        :param properties: When True, include assembly property templates in the response.
+        """
+        url = self._client.catalog_urls.assembly(assembly_identifier, properties=properties)
+        response = self._client._get(url=url)
+        if not isinstance(response, dict):
+            raise TypeError(f"Expected dict from assemblies GET, got {type(response).__name__}")
+        raw = response.get("data")
+        logger.debug("assemblies response: %s", raw)
+        if raw is None:
+            raise ValueError("Datasource not found.")
+        model = Assembly.from_dict(raw)
+        return DatasourceResource(self, model)
+
+    def get_datasources(
+        self,
+        source_type: AssetSourceType | None = None,
+    ) -> list[DatasourceResource]:
+        """
+        Find datasources by type in the catalog.
+
+        :param source_type: Catalog source type filter (optional). Passed as ``sourceType`` query parameter.
+        :return: List of datasource resources.
+        """
+        url = self._client.catalog_urls.datasources()
+        params: dict[str, str] | None = None
+        if source_type is not None:
+            params = {"sourceType": source_type.name}
+        response = self._client._get(url=url, params=params)
+        if not isinstance(response, dict):
+            raise TypeError(f"Expected dict from assemblies list, got {type(response).__name__}")
+        raw = response.get("data")
+        if raw is None:
+            raw = []
+        if not isinstance(raw, list):
+            raise TypeError(f"Expected list in data, got {type(raw).__name__}")
+        return [
+            DatasourceResource(self, Assembly.from_dict(item))
+            for item in raw
+            if isinstance(item, dict)
+        ]

acceldata/services/generic_service.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from acceldata.client.http_client import BaseHttpClient
+from acceldata.models.api.catalog.analytics_pipeline import AnalyticsPipeline
+from acceldata.models.api.catalog.asset_type import AssetType
+from acceldata.models.api.catalog.connection_type import ConnectionType
+from acceldata.models.api.catalog.source_type import SourceType
+
+logger = logging.getLogger("generic_service")
+
+
+def _asset_types_list_from_response(response: dict[str, Any]) -> list[Any]:
+    for key in ("data", "assetTypes"):
+        raw = response.get(key)
+        if raw is None:
+            continue
+        if isinstance(raw, (list, tuple)):
+            return list(raw)
+    raise KeyError(
+        "asset-types response must include 'data' or 'assetTypes' with a list"
+    )
+
+
+def _connection_types_list_from_response(response: dict[str, Any]) -> list[Any]:
+    for key in ("data", "connectionTypes"):
+        raw = response.get(key)
+        if raw is None:
+            continue
+        if isinstance(raw, (list, tuple)):
+            return list(raw)
+    raise KeyError(
+        "connection-types response must include 'data' or 'connectionTypes' with a list"
+    )
+
+
+def _source_types_list_from_response(response: dict[str, Any]) -> list[Any]:
+    for key in ("data", "sourceTypes"):
+        raw = response.get(key)
+        if raw is None:
+            continue
+        if isinstance(raw, (list, tuple)):
+            return list(raw)
+    raise KeyError(
+        "source-types response must include 'data' or 'sourceTypes' with a list"
+    )
+
+
+class GenericService:
+    def __init__(self, http_client: BaseHttpClient) -> None:
+        self._client = http_client
+
+    def get_asset_types(self) -> list[AssetType]:
+        url = self._client.catalog_urls.asset_types()
+        response = self._client._get(url=url)
+        logger.debug("get_asset_types response: %s", response)
+        if not isinstance(response, dict):
+            raise TypeError(
+                "asset-types response must be a JSON object, "
+                f"got {type(response).__name__}"
+            )
+        raw = _asset_types_list_from_response(response)
+        result: list[AssetType] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                raise TypeError(
+                    "each asset type must be a JSON object, "
+                    f"got {type(item).__name__}"
+                )
+            result.append(AssetType.from_dict(item))
+        return result
+
+    def get_connection_types(self) -> list[ConnectionType]:
+        url = self._client.catalog_urls.connection_types()
+        response = self._client._get(url=url)
+        if not isinstance(response, dict):
+            raise TypeError(
+                "connection-types response must be a JSON object, "
+                f"got {type(response).__name__}"
+            )
+        raw = _connection_types_list_from_response(response)
+        result: list[ConnectionType] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                raise TypeError(
+                    "each connection type must be a JSON object, "
+                    f"got {type(item).__name__}"
+                )
+            result.append(ConnectionType.from_dict(item))
+        return result
+
+    def get_source_types(self) -> list[SourceType]:
+        """
+        Description:
+            list all sourcetypes in ADOC catalog
+        :return: (SourceType) list of sourcetype
+        """
+        url = self._client.catalog_urls.source_types()
+        response = self._client._get(url=url)
+        raw = _source_types_list_from_response(response)
+        result: list[SourceType] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                raise TypeError(
+                    "each source type must be a JSON object, "
+                    f"got {type(item).__name__}"
+                )
+            result.append(SourceType.from_dict(item))
+        return result
+
+    def get_analysis_pipeline(self, pipeline_identifier: str | int) -> AnalyticsPipeline:
+        """GET catalog ``/analytics-pipelines/{id|name}`` and parse the ``pipeline`` object."""
+        url = self._client.catalog_urls.analytics_pipeline(pipeline_identifier)
+        response = self._client._get(url=url)
+        if not isinstance(response, dict):
+            raise TypeError(
+                "analytics-pipelines response must be a JSON object, "
+                f"got {type(response).__name__}"
+            )
+        inner = response.get("pipeline")
+        if inner is None:
+            raise KeyError(
+                "analytics-pipelines response must include a 'pipeline' object"
+            )
+        if not isinstance(inner, dict):
+            raise TypeError(
+                "pipeline must be a JSON object, "
+                f"got {type(inner).__name__}"
+            )
+        return AnalyticsPipeline.from_dict(inner)

acceldata/services/marker_config.py

@@ -0,0 +1,232 @@
+"""Catalog ``markerConfig`` wire ``type`` values and helpers (service / SDK layer).
+
+The OpenAPI ``MarkerConfig`` schema is a discriminated union. Each branch payload
+must include a ``type`` string matching
+``#/components/schemas/MarkerConfig`` discriminator mapping. Use the
+``*_marker_config`` functions (names mirror the generated classes, e.g.
+:func:`bounds_date_time_marker_config` for :class:`BoundsDateTimeMarkerConfig`) so
+callers do not set that field by hand; for low-level access see
+:func:`marker_type_for` and :data:`API_TYPE_BY_MARKER_CLASS`, and
+:func:`as_marker_config` to normalize values before they are sent on API requests.
+"""
+
+from __future__ import annotations
+
+from types import MappingProxyType
+from typing import Any, Optional, Type, TypeVar, Union, cast
+
+from acceldata.models.api.catalog.bounds_date_time_marker_config import (
+    BoundsDateTimeMarkerConfig,
+)
+from acceldata.models.api.catalog.bounds_file_event_marker_config import (
+    BoundsFileEventMarkerConfig,
+)
+from acceldata.models.api.catalog.bounds_id_marker_config import BoundsIdMarkerConfig
+from acceldata.models.api.catalog.date_partitioning_marker_config import (
+    DatePartitioningMarkerConfig,
+)
+from acceldata.models.api.catalog.date_time_marker_config import DateTimeMarkerConfig
+from acceldata.models.api.catalog.file_marker_config import FileMarkerConfig
+from acceldata.models.api.catalog.id_marker_config import IdMarkerConfig
+from acceldata.models.api.catalog.marker_config import MarkerConfig
+from acceldata.models.api.catalog.marker_config_fallback import MarkerConfigFallback
+from acceldata.models.api.catalog.offset_based_marker_config import (
+    OffsetBasedMarkerConfig,
+)
+from acceldata.models.api.catalog.pub_sub_lookback_marker_config import (
+    PubSubLookbackMarkerConfig,
+)
+from acceldata.models.api.catalog.pub_sub_timestamp_marker_config import (
+    PubSubTimestampMarkerConfig,
+)
+from acceldata.models.api.catalog.timestamp_based_marker_config import (
+    TimestampBasedMarkerConfig,
+)
+
+# Catalog MarkerConfig oneOf: discriminator propertyName "type" -> $ref
+# (see components/schemas/MarkerConfig in the cleaned spec).
+_WIRE: dict[Type[object], str] = {
+    BoundsDateTimeMarkerConfig: "boundDateTime",
+    BoundsFileEventMarkerConfig: "boundFileEvent",
+    BoundsIdMarkerConfig: "bound",
+    DatePartitioningMarkerConfig: "date_partitioned",
+    DateTimeMarkerConfig: "datetime",
+    FileMarkerConfig: "file",
+    IdMarkerConfig: "id",
+    OffsetBasedMarkerConfig: "offset",
+    PubSubLookbackMarkerConfig: "pubsub_lookback",
+    PubSubTimestampMarkerConfig: "pubsub_timestamp",
+    TimestampBasedMarkerConfig: "timestamp",
+    MarkerConfigFallback: "MarkerConfigFallback",
+}
+
+# Read-only: generated class -> wire ``type`` (discriminator) string
+API_TYPE_BY_MARKER_CLASS: Any = MappingProxyType(_WIRE)
+
+MMarker = TypeVar("MMarker", bound=object)
+
+# Union of all branch models + wrapper for inputs (profiling, and other call sites).
+MarkerConfigInput = Union[
+    MarkerConfig,
+    BoundsDateTimeMarkerConfig,
+    BoundsFileEventMarkerConfig,
+    BoundsIdMarkerConfig,
+    DatePartitioningMarkerConfig,
+    DateTimeMarkerConfig,
+    FileMarkerConfig,
+    IdMarkerConfig,
+    OffsetBasedMarkerConfig,
+    PubSubLookbackMarkerConfig,
+    PubSubTimestampMarkerConfig,
+    TimestampBasedMarkerConfig,
+    MarkerConfigFallback,
+]
+
+
+def marker_type_for(model_type: type) -> str:
+    """Return the API ``type`` string for a given generated *MarkerConfig* class."""
+    try:
+        return _WIRE[model_type]
+    except KeyError as e:  # pragma: no cover - guard for SDK misuse
+        raise TypeError(
+            f"Not a known MarkerConfig branch class: {model_type!r}"
+        ) from e
+
+
+def _build_marker(model_cls: type[MMarker], /, **kwargs: Any) -> MMarker:
+    if model_cls not in _WIRE:
+        raise TypeError(
+            f"Unknown marker model class: {model_cls!r}. "
+            "Use a known *MarkerConfig* branch (see API_TYPE_BY_MARKER_CLASS keys)."
+        )
+    if model_cls is MarkerConfigFallback:
+        if "type" not in kwargs:
+            raise TypeError(
+                "MarkerConfigFallback requires an explicit 'type' "
+                "(unknown backend name or the mapping key \"MarkerConfigFallback\")."
+            )
+        return model_cls(**kwargs)  # type: ignore[call-arg, no-any-return]
+    wire = _WIRE[cast(Type[object], model_cls)]
+    return model_cls(**{**kwargs, "type": wire})  # type: ignore[call-arg, no-any-return]
+
+
+def date_time_marker_config(**kwargs: Any) -> DateTimeMarkerConfig:
+    """Build a :class:`DateTimeMarkerConfig` with the correct API ``type``."""
+    return _build_marker(DateTimeMarkerConfig, **kwargs)
+
+
+def bounds_date_time_marker_config(**kwargs: Any) -> BoundsDateTimeMarkerConfig:
+    """Build a :class:`BoundsDateTimeMarkerConfig` with the correct API ``type``."""
+    return _build_marker(BoundsDateTimeMarkerConfig, **kwargs)
+
+
+def bounds_file_event_marker_config(**kwargs: Any) -> BoundsFileEventMarkerConfig:
+    """Build a :class:`BoundsFileEventMarkerConfig` with the correct API ``type``."""
+    return _build_marker(BoundsFileEventMarkerConfig, **kwargs)
+
+
+def date_partitioning_marker_config(**kwargs: Any) -> DatePartitioningMarkerConfig:
+    """Build a :class:`DatePartitioningMarkerConfig` with the correct API ``type``."""
+    return _build_marker(DatePartitioningMarkerConfig, **kwargs)
+
+
+def file_marker_config(**kwargs: Any) -> FileMarkerConfig:
+    """Build a :class:`FileMarkerConfig` with the correct API ``type``."""
+    return _build_marker(FileMarkerConfig, **kwargs)
+
+
+def offset_based_marker_config(**kwargs: Any) -> OffsetBasedMarkerConfig:
+    """Build an :class:`OffsetBasedMarkerConfig` with the correct API ``type``."""
+    return _build_marker(OffsetBasedMarkerConfig, **kwargs)
+
+
+def pub_sub_lookback_marker_config(**kwargs: Any) -> PubSubLookbackMarkerConfig:
+    """Build a :class:`PubSubLookbackMarkerConfig` with the correct API ``type``."""
+    return _build_marker(PubSubLookbackMarkerConfig, **kwargs)
+
+
+def pub_sub_timestamp_marker_config(**kwargs: Any) -> PubSubTimestampMarkerConfig:
+    """Build a :class:`PubSubTimestampMarkerConfig` with the correct API ``type``."""
+    return _build_marker(PubSubTimestampMarkerConfig, **kwargs)
+
+
+def timestamp_based_marker_config(**kwargs: Any) -> TimestampBasedMarkerConfig:
+    """Build a :class:`TimestampBasedMarkerConfig` with the correct API ``type``."""
+    return _build_marker(TimestampBasedMarkerConfig, **kwargs)
+
+
+def marker_config_fallback(**kwargs: Any) -> MarkerConfigFallback:
+    """Build a :class:`MarkerConfigFallback`; you must pass ``type=`` in ``kwargs``."""
+    return _build_marker(MarkerConfigFallback, **kwargs)
+
+
+def id_marker_config(
+    *,
+    id_column_name: str,
+    base_asset_id: int | None = None,
+    initial_offset: int | None = None,
+) -> IdMarkerConfig:
+    """Build an :class:`IdMarkerConfig` with the correct API ``type`` (``"id"``)."""
+    mk: dict[str, Any] = {"id_column_name": id_column_name}
+    if base_asset_id is not None:
+        mk["base_asset_id"] = base_asset_id
+    if initial_offset is not None:
+        mk["initial_offset"] = initial_offset
+    return _build_marker(IdMarkerConfig, **mk)
+
+
+def bounds_id_marker_config(
+    *,
+    id_column_name: str,
+    from_id: int | None = None,
+    to_id: int | None = None,
+    base_asset_id: int | None = None,
+) -> BoundsIdMarkerConfig:
+    """Build a :class:`BoundsIdMarkerConfig` with the correct API ``type`` (``"bound"``).
+
+    For **incremental** profiling on an id column, use :func:`id_marker_config` instead.
+    """
+    mk: dict[str, Any] = {"id_column_name": id_column_name}
+    if base_asset_id is not None:
+        mk["base_asset_id"] = base_asset_id
+    if from_id is not None:
+        mk["from_id"] = from_id
+    if to_id is not None:
+        mk["to_id"] = to_id
+    return _build_marker(BoundsIdMarkerConfig, **mk)
+
+
+def _apply_canonical_type_if_known(instance: object) -> object:
+    t = type(instance)
+    if t is MarkerConfigFallback:
+        return instance
+    wire = _WIRE.get(t)
+    if wire is None or getattr(instance, "type", None) == wire:
+        return instance
+    m = instance
+    return m.model_copy(update={"type": wire})
+
+
+def as_marker_config(value: MarkerConfigInput, /) -> MarkerConfig:
+    """Normalize a payload and wrap in :class:`MarkerConfig` (public catalog union type)."""
+    if isinstance(value, MarkerConfig):
+        inner = value.actual_instance
+        if inner is None:
+            return value
+        updated = _apply_canonical_type_if_known(inner)
+        if updated is inner:
+            return value
+        return MarkerConfig(actual_instance=updated)
+    if type(value) in _WIRE and value is not None:
+        updated = _apply_canonical_type_if_known(value)
+        return MarkerConfig(actual_instance=updated)  # type: ignore[arg-type]
+    raise TypeError(f"Unsupported marker config value type: {type(value)!r}")
+
+
+def prepare_marker_config_for_request(
+    value: Optional[MarkerConfigInput],
+) -> Optional[MarkerConfig]:
+    """Apply discriminator defaults and return the catalog :class:`MarkerConfig` for HTTP bodies."""
+    if value is None:
+        return None
+    return as_marker_config(value)

acceldata/services/pipeline_big_decimal_api.py

@@ -0,0 +1,102 @@
+"""Parse pipeline API BigDecimal payloads: structured dicts or plain JSON numbers (non-generated helper)."""
+
+from __future__ import annotations
+
+from decimal import Decimal, InvalidOperation
+from typing import TYPE_CHECKING, Any, Optional
+
+from acceldata.models.api.pipeline.big_integer import BigInteger
+
+if TYPE_CHECKING:
+    from acceldata.models.api.pipeline.big_decimal import BigDecimal
+
+
+def _decimal_to_unscaled_scale_unsigned(abs_d: Decimal) -> tuple[int, int]:
+    if abs_d == 0:
+        return 0, 0
+    t = abs_d.as_tuple()
+    digits_int = int("".join(str(x) for x in t.digits) or "0")
+    exp = t.exponent
+    if exp >= 0:
+        return digits_int * (10**exp), 0
+    return digits_int, -exp
+
+
+def big_decimal_from_json_scalar(val: Any) -> BigDecimal:
+    """Build pipeline BigDecimal when the API returns a JSON number or decimal string."""
+    from acceldata.models.api.pipeline.big_decimal import BigDecimal
+
+    if isinstance(val, str):
+        try:
+            d = Decimal(val.strip())
+        except InvalidOperation as exc:
+            raise TypeError(f"BigDecimal value is not a valid decimal string: {val!r}") from exc
+    elif isinstance(val, bool):
+        raise TypeError("BigDecimal value must be a number, not bool")
+    elif isinstance(val, int):
+        d = Decimal(val)
+    elif isinstance(val, float):
+        d = Decimal(str(val))
+    else:
+        raise TypeError(f"BigDecimal value has unsupported type {type(val).__name__}")
+
+    if not d.is_finite():
+        raise TypeError("BigDecimal value must be a finite decimal")
+
+    sign = -1 if d < 0 else (0 if d == 0 else 1)
+    d_abs = abs(d)
+    unscaled_mag, scale = _decimal_to_unscaled_scale_unsigned(d_abs)
+    int_compact = sign * unscaled_mag
+
+    if sign == 0:
+        bi = BigInteger.model_validate(
+            {
+                "signum": 0,
+                "mag": [],
+                "bitCountPlusOne": 1,
+                "bitLengthPlusOne": 1,
+                "lowestSetBitPlusTwo": 2,
+                "firstNonzeroIntNumPlusTwo": 2,
+            }
+        )
+        prec = 1
+        s_cache = "0"
+    else:
+        bl = max(abs(unscaled_mag).bit_length(), 1)
+        bi = BigInteger.model_validate(
+            {
+                "signum": sign,
+                "mag": [{"value": unscaled_mag}],
+                "bitCountPlusOne": bl + 1,
+                "bitLengthPlusOne": bl + 1,
+                "lowestSetBitPlusTwo": 2,
+                "firstNonzeroIntNumPlusTwo": 2,
+            }
+        )
+        prec = max(len(str(abs(int_compact))), scale + 1 if scale else 1)
+        s_cache = format(d, "f")
+
+    return BigDecimal.model_validate(
+        {
+            "intVal": bi,
+            "scale": scale,
+            "precision": prec,
+            "stringCache": s_cache,
+            "intCompact": int_compact,
+        }
+    )
+
+
+def parse_optional_big_decimal_from_api(raw: Any) -> Optional[BigDecimal]:
+    """Accept structured BigDecimal JSON, or a plain number/string from the API."""
+    from acceldata.models.api.pipeline.big_decimal import BigDecimal
+
+    if raw is None:
+        return None
+    if isinstance(raw, BigDecimal):
+        return raw
+    if isinstance(raw, dict):
+        return BigDecimal.from_dict(raw)
+    if isinstance(raw, (int, float, str)):
+        return big_decimal_from_json_scalar(raw)
+    return BigDecimal.from_dict(raw)