acceldata-sdk-python 26.7.0__py3-none-any.whl

acceldata/models/sdk/pipeline/__init__.py

@@ -0,0 +1,62 @@
+from acceldata.models.sdk.pipeline.create_pipeline_input_request import CreatePipelineInputRequest
+from acceldata.models.sdk.pipeline.create_job_input import CreateJobInput, JobInputOutputRef, build_create_job_payload
+from acceldata.models.sdk.pipeline.pipeline_source_type import PipelineSourceType
+from acceldata.models.sdk.pipeline.pipeline_run_input import (
+    PipelineRunInput,
+    PipelineRunResult,
+    PipelineRunStatus,
+)
+
+# Do not import PipelineResource eagerly: it imports PipelineService, while PipelineService
+# imports this package's CreatePipelineInputRequest — eager import would circularly load
+# pipeline_service before PipelineService is defined.
+
+__all__ = [
+    "CreatePipelineInputRequest",
+    "PipelineSourceType",
+    "CreateJobInput",
+    "PipelineRunInput",
+    "PipelineRunResult",
+    "PipelineRunStatus",
+    "JobInputOutputRef",
+    "build_create_job_payload",
+    "PipelineResource",
+    "PipelineRunResource",
+    "SpanContext",
+    "SpanResource",
+    "SpanEvent",
+    "LogEvent",
+    "GenericEvent",
+]
+
+
+def __getattr__(name: str):
+    if name == "PipelineResource":
+        from acceldata.models.sdk.pipeline.pipeline_resource import PipelineResource
+
+        return PipelineResource
+    if name == "PipelineRunResource":
+        from acceldata.models.sdk.pipeline.pipeline_run_resource import PipelineRunResource
+
+        return PipelineRunResource
+    if name == "SpanContext":
+        from acceldata.models.sdk.pipeline.span_resource import SpanResource
+
+        return SpanResource
+    if name == "SpanResource":
+        from acceldata.models.sdk.pipeline.span_resource import SpanResource
+
+        return SpanResource
+    if name == "SpanEvent":
+        from acceldata.models.sdk.pipeline.events.span_event import SpanEvent
+
+        return SpanEvent
+    if name == "LogEvent":
+        from acceldata.models.sdk.pipeline.events.log_event import LogEvent
+
+        return LogEvent
+    if name == "GenericEvent":
+        from acceldata.models.sdk.pipeline.events.generic_event import GenericEvent
+
+        return GenericEvent
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

acceldata/models/sdk/pipeline/create_job_input.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from acceldata.models.api.pipeline.meta import Meta
+
+
+@dataclass
+class JobInputOutputRef:
+    """
+    Input/output dataset or job reference for a pipeline job (maps to API ``Identifier``).
+
+    Specify either ``asset_uid`` (``source.path`` form) or ``job_uid``, not both — same rules as the
+    earlier ``Node`` helper.
+    """
+
+    asset_uid: Optional[str] = None
+    job_uid: Optional[str] = None
+
+    def to_identifier_dict(self) -> Dict[str, Any]:
+        # Same rules as earlier ``Node`` (asset vs job reference).
+        if self.asset_uid is not None and self.job_uid is not None:
+            raise ValueError("Please do not specify both asset_uid and job_uid")
+        if self.asset_uid is not None:
+            source, asset_uid = self.asset_uid.split(".", 1)
+            return {"source": source, "asset_uid": asset_uid}
+        if self.job_uid is not None:
+            return {"jobUid": self.job_uid}
+        raise ValueError("Please specify atleast one of asset_uid or job_uid")
+
+
+@dataclass
+class CreateJobInput:
+    """
+        Customer-facing description of a job to create on a pipeline (replaces earlier ``CreateJob``).
+
+    Use with :class:`~acceldata.models.sdk.pipeline.pipeline_resource.PipelineResource`. It is not an
+    Not a wire DTO type; build API payloads via :func:`build_create_job_payload`.
+
+    When ``bounded_by_span`` is True, ``span_created_at`` / ``span_updated_at`` are sent on the
+    span create request (``createdAt`` / ``updatedAt``).
+    """
+
+    uid: str
+    name: str
+    description: Optional[str] = None
+    inputs: List[JobInputOutputRef] = field(default_factory=list)
+    outputs: List[JobInputOutputRef] = field(default_factory=list)
+    meta: Optional[Meta] = None
+    pipeline_run_id: Optional[int] = None
+    bounded_by_span: bool = False
+    span_uid: Optional[str] = None
+    with_explicit_time: bool = False
+    # When bounded_by_span is True: optional span row timestamps on create (createdAt / updatedAt).
+    span_created_at: Optional[datetime] = None
+    span_updated_at: Optional[datetime] = None
+    context: Optional[Dict[str, Any]] = None
+
+
+def _strip_none_entries(d: Dict[str, Any]) -> Dict[str, Any]:
+    """Like earlier ``asdict(..., dict_factory=lambda x: {k: v for k, v in x if v is not None})``."""
+    return {k: v for k, v in d.items() if v is not None}
+
+
+def build_create_job_payload(
+        job: CreateJobInput,
+        *,
+        version_id: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Serialize ``CreateJobInput`` to the JSON body for ``PUT .../pipelines/{id}/jobs``.
+
+    Mirrors earlier ``_convert_job_to_dict``: base fields, ``meta`` as a dict, and each input/output
+    with ``None`` omitted (earlier code used ``asdict`` + ``dict_factory`` on I/O rows). Identifier rows
+    use :meth:`JobInputOutputRef.to_identifier_dict` (``Node`` semantics), then stripped.
+
+    Sends ``pipeLineRunId`` from ``pipeline_run_id`` when set, and ``version`` when ``version_id``
+    is provided (callers obtain it from :meth:`~acceldata.services.pipeline_service.PipelineService.get_pipeline_run`,
+    matching earlier behavior).
+    """
+    job_dict: Dict[str, Any] = {
+        "uid": job.uid,
+        "name": job.name,
+    }
+    if job.description is not None:
+        job_dict["description"] = job.description
+    if job.context is not None:
+        job_dict["context"] = job.context
+
+    if hasattr(job, "meta") and job.meta is not None:
+        job_dict["meta"] = job.meta.to_dict()
+
+    if job.pipeline_run_id is not None:
+        job_dict["pipeLineRunId"] = job.pipeline_run_id
+    if version_id is not None:
+        job_dict["version"] = version_id
+
+    input_dict: List[Dict[str, Any]] = []
+    for ds_in in job.inputs:
+        input_dict.append(_strip_none_entries(ds_in.to_identifier_dict()))
+    output_dict: List[Dict[str, Any]] = []
+    for ds_out in job.outputs:
+        output_dict.append(_strip_none_entries(ds_out.to_identifier_dict()))
+
+    job_dict["inputs"] = input_dict
+    job_dict["outputs"] = output_dict
+    return job_dict

acceldata/models/sdk/pipeline/create_pipeline_input_request.py

@@ -0,0 +1,72 @@
+# coding: utf-8
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from acceldata.models.api.pipeline.pipeline import Meta
+from acceldata.models.api.pipeline.pipeline import Pipeline
+from acceldata.models.api.pipeline.pipeline_input_request import PipelineInputRequest
+from acceldata.models.api.pipeline.tag import Tag
+from acceldata.models.sdk.pipeline.pipeline_source_type import PipelineSourceType
+
+_DEFAULT_SCHEDULER_TYPE = "EXTERNAL"
+_DEFAULT_SOURCE_TYPE = PipelineSourceType.OTHERS
+
+
+def _resolve_source_type(source_type: PipelineSourceType | str | None) -> str:
+    if source_type is None:
+        return _DEFAULT_SOURCE_TYPE.value
+    if isinstance(source_type, PipelineSourceType):
+        return source_type.value
+    PipelineSourceType(source_type)  # fail on unknown wire values
+    return str(source_type)
+
+
+def _meta_from_user(meta: Optional[Any]) -> Optional[Meta]:
+    if meta is None:
+        return None
+    owner = getattr(meta, "owner", None)
+    team = getattr(meta, "team", None)
+    code_location = getattr(meta, "codeLocation", None)
+    if code_location is None:
+        code_location = getattr(meta, "code_location", None)
+    return Meta(owner=owner, team=team, code_location=code_location)
+
+
+class CreatePipelineInputRequest(PipelineInputRequest):
+    """
+    :class:`PipelineInputRequest` with a ``CreatePipeline``-style constructor: only the
+    fields below are exposed; other required ``Pipeline`` fields use SDK defaults.
+    """
+
+    def __init__(
+            self,
+            uid: str,
+            name: str,
+            description: Optional[str] = None,
+            meta: Optional[Meta] = None,
+            createdAt: Optional[datetime] = None,
+            updatedAt: Optional[datetime] = None,
+            sourceType: PipelineSourceType = PipelineSourceType.OTHERS,
+            context: Optional[Dict[str, Any]] = None,
+            tags: Optional[list[Tag]] = None,
+    ) -> None:
+        resolved_source_type = _resolve_source_type(sourceType)
+        pipeline = Pipeline(
+            uid=uid,
+            name=name,
+            description=description,
+            meta=_meta_from_user(meta),
+            createdAt=createdAt,
+            updatedAt=updatedAt,
+            schedulerType=_DEFAULT_SCHEDULER_TYPE,
+            sourceType=resolved_source_type,
+            context=context,
+            enabled=True,
+            interrupt=False,
+            scheduled=False,
+            tags=list(tags) if tags is not None else [],
+        )
+        super().__init__(pipeline=pipeline)

acceldata/models/sdk/pipeline/events/__init__.py

@@ -0,0 +1,5 @@
+from acceldata.models.sdk.pipeline.events.generic_event import GenericEvent
+from acceldata.models.sdk.pipeline.events.log_event import LogEvent
+from acceldata.models.sdk.pipeline.events.span_event import SpanEvent
+
+__all__ = ["GenericEvent", "LogEvent", "SpanEvent"]

acceldata/models/sdk/pipeline/events/generic_event.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Optional
+
+from acceldata.models.sdk.pipeline.events.span_event import SpanEvent
+from acceldata.models.sdk.pipeline.span_resource import SpanEventType
+
+
+class GenericEvent(SpanEvent):
+    """
+    Custom span event payload for :meth:`~acceldata.models.sdk.pipeline.span_resource.SpanResource.send_event`.
+
+    Use this for **custom** ``event_uid`` values plus optional ``context_data`` (and optional
+    ``log_data`` when the uid is ``LOG``). This is a **separate** type from
+    :class:`~acceldata.models.sdk.pipeline.events.log_event.LogEvent`, which is only for
+    structured log lines with ``eventUid`` ``LOG``.
+    """
+
+    def __init__(
+        self,
+        context_data: Optional[dict[str, Any]] = None,
+        *,
+        event_uid: str = SpanEventType.LOG.value,
+        log_data: Optional[str] = None,
+        created_at: Optional[datetime] = None,
+    ) -> None:
+        self.context_data = context_data
+        self.event_uid = event_uid
+        self.log_data = log_data
+        self.created_at = created_at

acceldata/models/sdk/pipeline/events/log_event.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Optional
+
+from acceldata.models.sdk.pipeline.events.span_event import SpanEvent
+from acceldata.models.sdk.pipeline.span_resource import SpanEventType
+
+
+class LogEvent(SpanEvent):
+    """
+    Log payload for a span (``eventUid`` ``LOG``).
+
+    Distinct from :class:`~acceldata.models.sdk.pipeline.events.generic_event.GenericEvent`:
+    use ``LogEvent`` for log text + context; use ``GenericEvent`` for arbitrary
+    ``event_uid`` and context.
+    """
+
+    def __init__(
+        self,
+        log_data: str,
+        context_data: Optional[dict[str, Any]] = None,
+        created_at: Optional[datetime] = None,
+    ) -> None:
+        """
+        :param log_data: Log body for the event.
+        :param context_data: Optional structured context for the event.
+        :param created_at: Optional explicit time for the event; if omitted, the server
+            typically uses receive time.
+        """
+        self.log_data = log_data
+        self.context_data = context_data
+        self.event_uid = SpanEventType.LOG.value
+        self.created_at = created_at

acceldata/models/sdk/pipeline/events/span_event.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+
+class SpanEvent:
+    """
+    Base class for span events sent to the control plane via
+    :meth:`~acceldata.models.sdk.pipeline.span_resource.SpanResource.send_event`.
+    """

acceldata/models/sdk/pipeline/pipeline_resource.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from acceldata.models.api.pipeline.node import Node
+from acceldata.models.api.pipeline.pipeline import Pipeline
+from acceldata.models.api.pipeline.tag import Tag
+from acceldata.models.api.pipeline.pipeline_response import PipelineResponse
+from acceldata.models.sdk.pipeline.create_job_input import CreateJobInput, build_create_job_payload
+from acceldata.models.sdk.pipeline.pipeline_run_input import PipelineRunInput
+from acceldata.models.sdk.pipeline.pipeline_run_resource import PipelineRunResource
+from acceldata.models.sdk.pipeline.span_resource import SpanResource
+from acceldata.utils.span_helpers import ensure_span_for_job
+from acceldata.services.pipeline_service import PipelineService
+from acceldata.models.api.pipeline.pipeline_runs_response import PipelineRunsResponse
+
+
+class PipelineResource:
+    """
+    Customer-facing pipeline operations that wrap :class:`Pipeline` without modifying it.
+
+    Holds the underlying :class:`Pipeline` and delegates HTTP calls to
+    :class:`~acceldata.services.pipeline_service.PipelineService` (:attr:`pipeline`);
+    use :meth:`to_dict` for a wire representation.
+    """
+
+    def __init__(self, model: Pipeline, service: PipelineService) -> None:
+        self._model = model
+        self._service = service
+
+    @property
+    def pipeline(self) -> Pipeline:
+        return self._model
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self._model.to_dict()
+
+    @classmethod
+    def from_response(cls, response: PipelineResponse, service: PipelineService) -> PipelineResource:
+        return cls(response.pipeline, service)
+
+    def _require_pipeline_id(self) -> int:
+        if self._model.id is None:
+            raise ValueError("Pipeline id is required for this operation")
+        return int(self._model.id)
+
+    # --- identity (read-through to generated ``Pipeline``) ---
+
+    @property
+    def uid(self) -> str:
+        return self._model.uid
+
+    @property
+    def name(self) -> str:
+        return self._model.name
+
+    @property
+    def id(self) -> Optional[int]:
+        return self._model.id
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, PipelineResource):
+            return NotImplemented
+        return self._model.uid == other._model.uid
+
+    def __repr__(self) -> str:
+        return f"PipelineResource(model={self._model!r})"
+
+    # --- API methods (``Pipeline`` parity with the earlier SDK) ---
+
+    def delete(self) -> Any:
+        return self._service.delete_pipeline(self._require_pipeline_id())
+
+    def update(self, pipeline: Pipeline) -> PipelineResource:
+        """PUT the full pipeline (e.g. after editing tags); ``pipeline.id`` must be set."""
+        resp = self._service.update_pipeline(pipeline)
+        self._model = resp.pipeline
+        return self
+
+    def replace_tags(self, tags: list[Tag]) -> PipelineResource:
+        """Replace the pipeline's tag list via GET + PUT (see :meth:`~acceldata.services.pipeline_service.PipelineService.replace_pipeline_tags`)."""
+        ident = self._model.uid if self._model.uid else self._require_pipeline_id()
+        resp = self._service.replace_pipeline_tags(ident, tags)
+        self._model = resp.pipeline
+        return self
+
+    def create_job(self, job: CreateJobInput) -> Node:
+        if not job.uid or not job.name or job.pipeline_run_id is None:
+            raise ValueError(
+                "To create a job, uid, name, and pipeline_run_id are required"
+            )
+        pipeline_id = self._require_pipeline_id()
+        pr = self._service.get_pipeline_run(pipeline_run_id=job.pipeline_run_id)
+        vid = pr.version_id
+        payload = build_create_job_payload(
+            job,
+            version_id=int(vid) if vid is not None else None,
+        )
+        node_response = self._service.create_job(payload, pipeline_id)
+
+        if job.bounded_by_span:
+            if job.span_uid is None:
+                raise ValueError(
+                    "When bounded_by_span is True, span_uid is required"
+                )
+            ensure_span_for_job(
+                self._service,
+                pipeline_run_id=job.pipeline_run_id,
+                span_uid=job.span_uid,
+                job_uid=job.uid,
+                created_at=job.span_created_at,
+                updated_at=job.span_updated_at,
+            )
+        return node_response.node
+
+    def create_pipeline_run(
+            self,
+            request: PipelineRunInput | None = None,
+    ) -> PipelineRunResource:
+        if request is None:
+            request = PipelineRunInput()
+        run = self._service.create_pipeline_run(
+            self._require_pipeline_id(),
+            context_data=request.context_data,
+            continuation_id=request.continuation_id,
+            started_at=request.started_at,
+        )
+        return PipelineRunResource(run, self._service)
+
+    def get_latest_pipeline_run(self) -> PipelineRunResource:
+        run = self._service.get_latest_pipeline_run(self._require_pipeline_id())
+        return PipelineRunResource(run, self._service)
+
+    def get_run(
+            self,
+            pipeline_run_id: Optional[int] = None,
+            continuation_id: Optional[str] = None,
+    ) -> PipelineRunResource:
+        run = self._service.get_pipeline_run(
+            pipeline_run_id=pipeline_run_id,
+            continuation_id=continuation_id,
+            pipeline_id=self._require_pipeline_id(),
+        )
+        return PipelineRunResource(run, self._service)
+
+    def get_runs(self) -> PipelineRunsResponse:
+        return self._service.get_pipeline_runs(self._require_pipeline_id())
+
+    def get_root_span(
+            self,
+            pipeline_run_id: Optional[int] = None,
+            *,
+            continuation_id: Optional[str] = None,
+    ) -> Optional[SpanResource]:
+        """
+        Resolve the root span for a run as :class:`~acceldata.models.sdk.pipeline.span_resource.SpanResource`.
+
+        If neither ``pipeline_run_id`` nor ``continuation_id`` is given, the latest run for this
+        pipeline is used.
+        """
+        if pipeline_run_id is None and continuation_id is None:
+            run = self.get_latest_pipeline_run()
+        else:
+            run = self.get_run(
+                pipeline_run_id=pipeline_run_id,
+                continuation_id=continuation_id,
+            )
+        rid = run.id
+        if rid is None:
+            raise ValueError("Pipeline run id is required to get root span")
+        return self._service.get_root_span(int(rid))

acceldata/models/sdk/pipeline/pipeline_run_input.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+from acceldata.models.api.pipeline.pipeline_run import PipelineRun
+from acceldata.models.api.pipeline.pipeline_run_request import PipelineRunRequest
+
+
+class PipelineRunResult(str, Enum):
+    """
+    Pipeline run outcome.
+
+    ``PipelineRun.result`` (wire schema)::
+
+        "result": {
+          "type": "string",
+          "description": "Allowed values at generation time: RUNNING, SUCCESS, FAILURE, CANCELLED."
+        }
+    """
+
+    RUNNING = "RUNNING"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+    CANCELLED = "CANCELLED"
+
+
+class PipelineRunStatus(str, Enum):
+    """
+    Pipeline run lifecycle status.
+
+    ``PipelineRun.status`` (wire schema)::
+
+        "status": {
+          "type": "string",
+          "description": "Allowed values at generation time: STARTED, COMPLETED, FAILED, ABORTED."
+        }
+    """
+
+    STARTED = "STARTED"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    ABORTED = "ABORTED"
+
+
+@dataclass
+class PipelineRunInput:
+    """
+    Fields you can set when creating or updating a pipeline run.
+
+    For :meth:`~acceldata.models.sdk.pipeline.pipeline_resource.PipelineResource.create_pipeline_run`,
+    only ``context_data``, ``continuation_id``, and ``started_at`` apply; the API always starts a run
+    with ``status`` ``STARTED``.
+
+    For :meth:`~acceldata.models.sdk.pipeline.pipeline_run_resource.PipelineRunResource.update_pipeline_run`,
+    ``result``, ``status``, and ``finished_at`` are sent when provided. Method parameters on
+    ``update_pipeline_run`` default to :attr:`PipelineRunResult.SUCCESS` and
+    :attr:`PipelineRunStatus.COMPLETED` when you omit them.
+
+    For create, build the wire body with :meth:`to_pipeline_run_request`.
+    """
+
+    context_data: Optional[dict[str, Any]] = None
+    continuation_id: Optional[str] = None
+    started_at: Optional[datetime] = None
+    finished_at: Optional[datetime] = None
+    result: Optional[PipelineRunResult] = None
+    status: Optional[PipelineRunStatus] = None
+
+    def to_pipeline_run_request(self, pipeline_id: int) -> PipelineRunRequest:
+        fields: dict[str, Any] = {
+            "pipeline_id": pipeline_id,
+            "status": PipelineRunStatus.STARTED.value,
+        }
+        if self.context_data is not None:
+            fields["args"] = self.context_data
+        if self.continuation_id is not None:
+            fields["continuation_id"] = self.continuation_id
+        if self.started_at is not None:
+            fields["started_at"] = self.started_at
+        run = PipelineRun.model_construct(**fields)
+        return PipelineRunRequest(run=run)