cognite-toolkit 0.7.42__py3-none-any.whl → 0.7.44__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/data_classes/dataset.py

@@ -0,0 +1,35 @@
+from typing import ClassVar
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
+
+from .identifiers import ExternalId
+
+
+class DataSet(BaseModelObject):
+    external_id: str | None = None
+    name: str | None = None
+    description: str | None = None
+    metadata: dict[str, str] | None = None
+    write_protected: bool = False
+
+    def as_id(self) -> ExternalId:
+        if self.external_id is None:
+            raise ValueError("Cannot convert DataSet to ExternalId when external_id is None")
+        return ExternalId(external_id=self.external_id)
+
+
+class DataSetRequest(DataSet, RequestUpdateable):
+    container_fields: ClassVar[frozenset[str]] = frozenset({"metadata"})
+
+
+class DataSetResponse(DataSet, ResponseResource[DataSetRequest]):
+    id: int
+    created_time: int
+    last_updated_time: int
+
+    def as_request_resource(self) -> DataSetRequest:
+        return DataSetRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/event.py

@@ -1,12 +1,15 @@
 from typing import ClassVar
 
-from cognite_toolkit._cdf_tk.client.data_classes.base import RequestUpdateable, ResponseResource
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
 
 from .identifiers import ExternalId, InternalOrExternalId
 
 
-class EventRequest(RequestUpdateable):
-    container_fields: ClassVar[frozenset[str]] = frozenset({"metadata", "asset_ids"})
+class Event(BaseModelObject):
     external_id: str | None = None
     data_set_id: int | None = None
     start_time: int | None = None
@@ -24,20 +27,14 @@ class EventRequest(RequestUpdateable):
         return ExternalId(external_id=self.external_id)
 
 
-class EventResponse(ResponseResource[EventRequest]):
-    external_id: str | None = None
-    data_set_id: int | None = None
-    start_time: int | None = None
-    end_time: int | None = None
-    type: str | None = None
-    subtype: str | None = None
-    description: str | None = None
-    metadata: dict[str, str] | None = None
-    asset_ids: list[int] | None = None
-    source: str | None = None
+class EventRequest(Event, RequestUpdateable):
+    container_fields: ClassVar[frozenset[str]] = frozenset({"metadata", "asset_ids"})
+
+
+class EventResponse(Event, ResponseResource[EventRequest]):
     id: int
     created_time: int
     last_updated_time: int
 
     def as_request_resource(self) -> EventRequest:
-        return EventRequest.model_validate(self.dump())
+        return EventRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/extraction_pipeline.py

@@ -0,0 +1,59 @@
+from typing import ClassVar
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
+
+from .identifiers import ExternalId
+
+
+class RawTable(BaseModelObject):
+    db_name: str
+    table_name: str
+
+
+class Contact(BaseModelObject):
+    name: str | None = None
+    email: str | None = None
+    role: str | None = None
+    send_notification: bool | None = None
+
+
+class NotificationConfig(BaseModelObject):
+    allowed_not_seen_range_in_minutes: int | None = None
+
+
+class ExtractionPipeline(BaseModelObject):
+    external_id: str
+    name: str
+    description: str | None = None
+    data_set_id: int
+    raw_tables: list[RawTable] | None = None
+    schedule: str | None = None
+    contacts: list[Contact] | None = None
+    metadata: dict[str, str] | None = None
+    source: str | None = None
+    documentation: str | None = None
+    notification_config: NotificationConfig | None = None
+    created_by: str | None = None
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+
+class ExtractionPipelineRequest(ExtractionPipeline, RequestUpdateable):
+    container_fields: ClassVar[frozenset[str]] = frozenset({"raw_tables", "contacts", "metadata"})
+    non_nullable_fields: ClassVar[frozenset[str]] = frozenset(
+        {"documentation", "source", "notification_config", "schedule", "description"}
+    )
+
+
+class ExtractionPipelineResponse(ExtractionPipeline, ResponseResource[ExtractionPipelineRequest]):
+    id: int
+    created_time: int
+    last_updated_time: int
+
+    def as_request_resource(self) -> ExtractionPipelineRequest:
+        return ExtractionPipelineRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/filemetadata.py

@@ -1,16 +1,14 @@
 from typing import ClassVar, Literal
 
-from pydantic import JsonValue
+from pydantic import Field, JsonValue
 
-from cognite_toolkit._cdf_tk.client.data_classes.base import RequestUpdateable, ResponseResource
+from cognite_toolkit._cdf_tk.client.data_classes.base import BaseModelObject, RequestUpdateable, ResponseResource
 
 from .identifiers import ExternalId
 from .instance_api import NodeReference
 
 
-class FileMetadataRequest(RequestUpdateable):
-    container_fields: ClassVar[frozenset[str]] = frozenset({"metadata", "labels", "asset_ids", "security_categories"})
-    non_nullable_fields: ClassVar[frozenset[str]] = frozenset({"asset_ids", "security_categories"})
+class FileMetadata(BaseModelObject):
     external_id: str | None = None
     name: str
     directory: str | None = None
@@ -31,26 +29,24 @@ class FileMetadataRequest(RequestUpdateable):
         return ExternalId(external_id=self.external_id)
 
 
-class FileMetadataResponse(ResponseResource[FileMetadataRequest]):
+class FileMetadataRequest(FileMetadata, RequestUpdateable):
+    container_fields: ClassVar[frozenset[str]] = frozenset({"metadata", "labels", "asset_ids", "security_categories"})
+    non_nullable_fields: ClassVar[frozenset[str]] = frozenset({"asset_ids", "security_categories"})
+    # This field is not part of the request when creating or updating a resource
+    # but we added it here for convenience so that it is available when converting
+    # from response to request.
+    instance_id: NodeReference | None = Field(default=None, exclude=True)
+
+
+class FileMetadataResponse(FileMetadata, ResponseResource[FileMetadataRequest]):
     created_time: int
     last_updated_time: int
     uploaded_time: int | None = None
     uploaded: bool
     id: int
-    external_id: str | None = None
-    name: str
-    directory: str | None = None
     instance_id: NodeReference | None = None
-    source: str | None = None
-    mime_type: str | None = None
-    metadata: dict[str, str] | None = None
-    asset_ids: list[int] | None = None
-    data_set_id: int | None = None
-    labels: list[dict[Literal["externalId"], str]] | None = None
-    geo_location: JsonValue | None = None
-    source_created_time: int | None = None
-    source_modified_time: int | None = None
-    security_categories: list[int] | None = None
+    # This field is required in the upload endpoint response, but not in any other file metadata response
+    upload_url: str | None = None
 
     def as_request_resource(self) -> FileMetadataRequest:
         return FileMetadataRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_destination.py

@@ -0,0 +1,34 @@
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
+
+from .identifiers import ExternalId
+
+
+class Credentials(BaseModelObject):
+    nonce: str
+
+
+class HostedExtractorDestination(BaseModelObject):
+    external_id: str
+    target_data_set_id: int | None = None
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+
+class HostedExtractorDestinationRequest(HostedExtractorDestination, RequestUpdateable):
+    credentials: Credentials | None = None
+
+
+class HostedExtractorDestinationResponse(
+    HostedExtractorDestination, ResponseResource[HostedExtractorDestinationRequest]
+):
+    session_id: int | None = None
+    created_time: int
+    last_updated_time: int
+
+    def as_request_resource(self) -> HostedExtractorDestinationRequest:
+        return HostedExtractorDestinationRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_job.py

@@ -0,0 +1,134 @@
+from typing import Annotated, Literal
+
+from pydantic import Field, JsonValue
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
+
+from .identifiers import ExternalId
+
+
+class JobFormatDefinition(BaseModelObject):
+    type: str
+
+
+class PrefixConfig(BaseModelObject):
+    from_topic: bool | None = None
+    prefix: str | None = None
+
+
+class SpaceRef(BaseModelObject):
+    space: str
+
+
+class CogniteFormat(JobFormatDefinition):
+    type: Literal["cognite"] = "cognite"
+    encoding: str | None = None
+    compression: str | None = None
+    prefix: PrefixConfig | None = None
+    data_models: list[SpaceRef] | None = None
+
+
+class CustomFormat(JobFormatDefinition):
+    type: Literal["custom"] = "custom"
+    encoding: str | None = None
+    compression: str | None = None
+    mapping_id: str
+
+
+class RockwellFormat(JobFormatDefinition):
+    type: Literal["rockwell"] = "rockwell"
+    encoding: str | None = None
+    compression: str | None = None
+    prefix: PrefixConfig | None = None
+    data_models: list[SpaceRef] | None = None
+
+
+class ValueFormat(JobFormatDefinition):
+    type: Literal["value"] = "value"
+    encoding: str | None = None
+    compression: str | None = None
+    prefix: PrefixConfig | None = None
+    data_models: list[SpaceRef] | None = None
+
+
+JobFormat = Annotated[
+    CogniteFormat | CustomFormat | RockwellFormat | ValueFormat,
+    Field(discriminator="type"),
+]
+
+
+class IncrementalLoadDefinition(BaseModelObject):
+    type: str
+
+
+class BodyIncrementalLoad(IncrementalLoadDefinition):
+    type: Literal["body"] = "body"
+    value: str
+
+
+class NextUrlIncrementalLoad(IncrementalLoadDefinition):
+    type: Literal["nextUrl"] = "nextUrl"
+    value: str
+
+
+class HeaderIncrementalLoad(IncrementalLoadDefinition):
+    type: Literal["headerValue"] = "headerValue"
+    key: str
+    value: str
+
+
+class QueryParamIncrementalLoad(IncrementalLoadDefinition):
+    type: Literal["queryParam"] = "queryParam"
+    key: str
+    value: str
+
+
+class MQTTConfig(BaseModelObject):
+    topic_filter: str
+
+
+class KafkaConfig(BaseModelObject):
+    topic: str
+    partitions: int | None = None
+
+
+class RestConfig(BaseModelObject):
+    interval: Literal["5m", "15m", "1h", "6h", "12h", "1d"]
+    path: str
+    method: Literal["get", "post"] | None = None
+    body: JsonValue | None = None
+    query: dict[str, str] | None = None
+    headers: dict[str, str] | None = None
+    incremental_load: BodyIncrementalLoad | HeaderIncrementalLoad | QueryParamIncrementalLoad | None = Field(
+        None, discriminator="type"
+    )
+    pagination: (
+        BodyIncrementalLoad | NextUrlIncrementalLoad | HeaderIncrementalLoad | QueryParamIncrementalLoad | None
+    ) = Field(None, discriminator="type")
+
+
+class HostedExtractorJob(BaseModelObject):
+    external_id: str
+    destination_id: str
+    source_id: str
+    format: JobFormat
+    config: MQTTConfig | KafkaConfig | RestConfig
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+
+class HostedExtractorJobRequest(HostedExtractorJob, RequestUpdateable): ...
+
+
+class HostedExtractorJobResponse(HostedExtractorJob, ResponseResource[HostedExtractorJobRequest]):
+    status: str | None = None
+    created_time: int
+    last_updated_time: int
+
+    def as_request_resource(self) -> HostedExtractorJobRequest:
+        return HostedExtractorJobRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_mapping.py

@@ -0,0 +1,72 @@
+from typing import Annotated, Literal
+
+from pydantic import Field
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+    ResponseResource,
+)
+
+from .identifiers import ExternalId
+
+
+class Mapping(BaseModelObject):
+    expression: str
+
+
+class MappingInputDefinition(BaseModelObject):
+    type: str
+
+
+class ProtobufFile(BaseModelObject):
+    file_name: str
+    content: str
+
+
+class ProtobufInput(BaseModelObject):
+    type: Literal["protobuf"] = "protobuf"
+    message_name: str
+    files: list[ProtobufFile]
+
+
+class CSVInput(BaseModelObject):
+    type: Literal["csv"] = "csv"
+    delimiter: str = ","
+    custom_keys: list[str] | None = None
+
+
+class XMLInput(BaseModelObject):
+    type: Literal["xml"] = "xml"
+
+
+class JSONInput(BaseModelObject):
+    type: Literal["json"] = "json"
+
+
+MappingInput = Annotated[
+    ProtobufInput | CSVInput | XMLInput | JSONInput,
+    Field(discriminator="type"),
+]
+
+
+class HostedExtractorMapping(BaseModelObject):
+    external_id: str
+    mapping: Mapping
+    published: bool
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+
+class HostedExtractorMappingRequest(HostedExtractorMapping, RequestUpdateable):
+    input: MappingInput | None = None
+
+
+class HostedExtractorMappingResponse(HostedExtractorMapping, ResponseResource[HostedExtractorMappingRequest]):
+    input: MappingInput
+    created_time: int
+    last_updated_time: int
+
+    def as_request_resource(self) -> HostedExtractorMappingRequest:
+        return HostedExtractorMappingRequest.model_validate(self.dump(), extra="ignore")

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/__init__.py

@@ -0,0 +1,63 @@
+from typing import Annotated
+
+from pydantic import Field, TypeAdapter
+
+from ._auth import (
+    BasicAuthenticationRequest,
+    BasicAuthenticationResponse,
+    ClientCredentialAuthenticationRequest,
+    ClientCredentialAuthenticationResponse,
+    HTTPBasicAuthenticationRequest,
+    HTTPBasicAuthenticationResponse,
+    ScramShaAuthenticationRequest,
+    ScramShaAuthenticationResponse,
+)
+from ._certificate import AuthCertificateRequest, CACertificateRequest, CertificateResponse
+from ._eventhub import EventHubSourceRequest, EventHubSourceResponse
+from ._kafka import KafkaBroker, KafkaSourceRequest, KafkaSourceResponse
+from ._mqtt import MQTTSourceRequest, MQTTSourceResponse
+from ._rest import RESTSourceRequest, RESTSourceResponse
+
+HostedExtractorSourceRequestUnion = Annotated[
+    KafkaSourceRequest | EventHubSourceRequest | MQTTSourceRequest | RESTSourceRequest,
+    Field(discriminator="type"),
+]
+
+HostedExtractorSourceRequest: TypeAdapter[HostedExtractorSourceRequestUnion] = TypeAdapter(
+    HostedExtractorSourceRequestUnion
+)
+
+HostedExtractorSourceResponseUnion = Annotated[
+    KafkaSourceResponse | EventHubSourceResponse | MQTTSourceResponse | RESTSourceResponse,
+    Field(discriminator="type"),
+]
+
+HostedExtractorSourceResponse: TypeAdapter[HostedExtractorSourceResponseUnion] = TypeAdapter(
+    HostedExtractorSourceResponseUnion
+)
+
+
+__all__ = [
+    "AuthCertificateRequest",
+    "BasicAuthenticationRequest",
+    "BasicAuthenticationResponse",
+    "CACertificateRequest",
+    "CertificateResponse",
+    "ClientCredentialAuthenticationRequest",
+    "ClientCredentialAuthenticationResponse",
+    "EventHubSourceRequest",
+    "EventHubSourceResponse",
+    "HTTPBasicAuthenticationRequest",
+    "HTTPBasicAuthenticationResponse",
+    "HostedExtractorSourceRequest",
+    "HostedExtractorSourceResponse",
+    "KafkaBroker",
+    "KafkaSourceRequest",
+    "KafkaSourceResponse",
+    "MQTTSourceRequest",
+    "MQTTSourceResponse",
+    "RESTSourceRequest",
+    "RESTSourceResponse",
+    "ScramShaAuthenticationRequest",
+    "ScramShaAuthenticationResponse",
+]

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_auth.py

@@ -0,0 +1,63 @@
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+)
+
+
+class AuthenticationRequestDefinition(BaseModelObject):
+    type: str
+
+
+class AuthenticationResponseDefinition(BaseModelObject):
+    type: str
+
+
+class BasicAuthentication(BaseModelObject):
+    type: Literal["basic"] = "basic"
+    username: str
+
+
+class BasicAuthenticationRequest(BasicAuthentication, AuthenticationRequestDefinition):
+    password: str | None = None
+
+
+class BasicAuthenticationResponse(BasicAuthentication, AuthenticationResponseDefinition): ...
+
+
+class ClientCredentialAuthentication(BaseModelObject):
+    type: Literal["clientCredential"] = "clientCredential"
+    client_id: str
+    token_url: str
+    scopes: str
+    default_expires_in: str | None = None
+
+
+class ClientCredentialAuthenticationRequest(ClientCredentialAuthentication, AuthenticationRequestDefinition):
+    client_secret: str
+
+
+class ClientCredentialAuthenticationResponse(ClientCredentialAuthentication, AuthenticationResponseDefinition): ...
+
+
+class ScramShaAuthentication(BaseModelObject):
+    type: Literal["scramSha256", "scramSha512"]
+    username: str
+
+
+class ScramShaAuthenticationRequest(ScramShaAuthentication, AuthenticationRequestDefinition):
+    password: str
+
+
+class ScramShaAuthenticationResponse(ScramShaAuthentication, AuthenticationResponseDefinition): ...
+
+
+class HTTPBasicAuthenticationRequest(AuthenticationRequestDefinition):
+    type: Literal["header", "query"]
+    key: str
+    value: str
+
+
+class HTTPBasicAuthenticationResponse(AuthenticationResponseDefinition):
+    type: Literal["header", "query"]
+    key: str

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_base.py

@@ -0,0 +1,26 @@
+from typing import Any, Literal
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    RequestUpdateable,
+)
+from cognite_toolkit._cdf_tk.client.data_classes.identifiers import ExternalId
+
+
+class SourceRequestDefinition(RequestUpdateable):
+    type: str
+    external_id: str
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+    def as_update(self, mode: Literal["patch", "replace"]) -> dict[str, Any]:
+        output = super().as_update(mode)
+        output["type"] = self.type
+        return output
+
+
+class SourceResponseDefinition(BaseModelObject):
+    external_id: str
+    created_time: int
+    last_updated_time: int

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_certificate.py

@@ -0,0 +1,20 @@
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+)
+
+
+class CACertificateRequest(BaseModelObject):
+    type: str
+    certificate: str
+
+
+class AuthCertificateRequest(BaseModelObject):
+    key: str
+    key_password: str | None = None
+    type: str
+    certificate: str
+
+
+class CertificateResponse(BaseModelObject):
+    thumbprint: str
+    expires_at: int

cognite_toolkit/_cdf_tk/client/data_classes/hosted_extractor_source/_eventhub.py

@@ -0,0 +1,31 @@
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.client.data_classes.base import (
+    BaseModelObject,
+    ResponseResource,
+)
+
+from ._auth import BasicAuthenticationRequest, BasicAuthenticationResponse
+from ._base import SourceRequestDefinition, SourceResponseDefinition
+
+
+class EventHubSource(BaseModelObject):
+    type: Literal["eventHub"] = "eventHub"
+    host: str
+    event_hub_name: str
+    consumer_group: str | None = None
+
+
+class EventHubSourceRequest(EventHubSource, SourceRequestDefinition):
+    authentication: BasicAuthenticationRequest | None = None
+
+
+class EventHubSourceResponse(
+    SourceResponseDefinition,
+    EventHubSource,
+    ResponseResource[EventHubSourceRequest],
+):
+    authentication: BasicAuthenticationResponse | None = None
+
+    def as_request_resource(self) -> EventHubSourceRequest:
+        return EventHubSourceRequest.model_validate(self.dump(), extra="ignore")