cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_mapping.py

@@ -0,0 +1,117 @@
+import sys
+from types import MappingProxyType
+from typing import Any, ClassVar, cast
+
+from pydantic import Field, ModelWrapValidatorHandler, model_serializer, model_validator
+from pydantic_core.core_schema import SerializationInfo, SerializerFunctionWrapHandler
+
+from cognite_toolkit._cdf_tk.utils import humanize_collection
+
+from .base import BaseModelResource, ToolkitResource
+
+if sys.version_info < (3, 11):
+    from typing_extensions import Self
+else:
+    from typing import Self
+
+
+class Mapping(BaseModelResource):
+    expression: str = Field(
+        description="Custom transform expression written in the Cognite transformation language.", max_length=2000
+    )
+
+
+class MappingInput(BaseModelResource):
+    type: ClassVar[str]
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def find_type(cls, data: "dict[str, Any] | MappingInput", handler: ModelWrapValidatorHandler[Self]) -> Self:
+        if isinstance(data, MappingInput):
+            return cast(Self, data)
+        if not isinstance(data, dict):
+            raise ValueError(f"Invalid input type data '{type(data)}' expected dict")
+
+        if cls is not MappingInput:
+            # We are already in a subclass, so just validate as usual
+            return handler(data)
+        # If not we need to find the right subclass based on the type field.
+        if "type" not in data:
+            raise ValueError("Invalid input data missing 'type' key")
+        type_ = data["type"]
+        if type_ not in _MAPPING_INPUT_CLS_BY_TYPE:
+            raise ValueError(
+                f"invalid type '{type_}'. Expected one of {humanize_collection(_MAPPING_INPUT_CLS_BY_TYPE.keys(), bind_word='or')}"
+            )
+        cls_ = _MAPPING_INPUT_CLS_BY_TYPE[type_]
+        return cast(Self, cls_.model_validate({k: v for k, v in data.items() if k != "type"}))
+
+    @model_serializer(mode="wrap", when_used="always", return_type=dict)
+    def include_type(self, handler: SerializerFunctionWrapHandler) -> dict:
+        serialized_data = handler(self)
+        serialized_data["type"] = self.type
+        return serialized_data
+
+
+class JsonMappingInput(MappingInput):
+    type = "json"
+
+
+class XMLMappingInput(MappingInput):
+    type = "xml"
+
+
+class CSVMappingInput(MappingInput):
+    type = "csv"
+    delimiter: str = Field(
+        description="A single ASCII character used as the separator in the CSV file.",
+        min_length=1,
+        max_length=1,
+        default=",",
+    )
+    custom_keys: list[str] | None = Field(
+        None,
+        description="List of headers. If this is not set, the headers will be retrieved from the CSV file.",
+        min_length=1,
+        max_length=20,
+    )
+
+
+class ProtobufFile(BaseModelResource):
+    file_name: str = Field(
+        description="Name of protobuf file. Must contain only letters, numbers, underscores, and hyphens, and must end with '.proto'.",
+        max_length=128,
+        pattern=r"[a-zA-Z0-9_-]+\.proto",
+    )
+    content: str = Field(description="Protobuf file content. Must be a valid protobuf file.", max_length=10000)
+
+
+class ProtobufMappingInput(MappingInput):
+    type = "protobuf"
+    message_name: str = Field(description="Name of root message in the protobuf files.", max_length=128)
+    files: list[ProtobufFile] = Field(description="The protobuf schema in text format.", max_length=5000)
+
+
+class HostedExtractorMappingYAML(ToolkitResource):
+    external_id: str = Field(
+        description="The external ID provided by the client. Must be unique for the resource type.", max_length=255
+    )
+    mapping: Mapping
+    input: MappingInput | None = Field(None, description="The input format of the data to be transformed.")
+    published: bool = Field(description="Whether this mapping is published and should be available to be used in jobs.")
+
+    @model_serializer(mode="wrap")
+    def serialize_input(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> dict:
+        # Input is serialized as empty dict
+        # This issue arises because Pydantic's serialization mechanism doesn't automatically
+        # handle polymorphic serialization for subclasses of MappingInput.
+        # To address this, we include the below to explicitly calling model dump on the input
+        serialized_data = handler(self)
+        if self.input is not None:
+            serialized_data["input"] = self.input.model_dump(**vars(info))
+        return serialized_data
+
+
+_MAPPING_INPUT_CLS_BY_TYPE: MappingProxyType[str, type[MappingInput]] = MappingProxyType(
+    {s.type: s for s in MappingInput.__subclasses__()}
+)

cognite_toolkit/_cdf_tk/resource_classes/hosted_extractor_source.py

@@ -0,0 +1,390 @@
+import sys
+from abc import ABC
+from types import MappingProxyType
+from typing import Any, ClassVar, Literal, cast
+
+from pydantic import (
+    Field,
+    ModelWrapValidatorHandler,
+    SecretStr,
+    field_serializer,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+from pydantic_core.core_schema import SerializationInfo, SerializerFunctionWrapHandler
+
+from cognite_toolkit._cdf_tk.utils import humanize_collection
+from cognite_toolkit._cdf_tk.utils._auxiliary import get_concrete_subclasses
+
+from .base import BaseModelResource, ToolkitResource
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+class CACertificate(BaseModelResource):
+    type: Literal["der", "pem"] = Field(
+        description="Type of certificate in the certificate field.",
+    )
+    certificate: str = Field(
+        description="Base 64 encoded der certificate, or a pem certificate with headers.",
+        max_length=100000,
+    )
+
+
+class AuthCertificate(BaseModelResource):
+    key: str = Field(
+        description="The key for the certificate",
+        max_length=100000,
+    )
+    key_password: SecretStr | None = Field(
+        None,
+        description="The password for the certificate key",
+        min_length=1,
+        max_length=255,
+    )
+    type: Literal["der", "pem"] = Field(
+        description="Type of certificate in the certificate field.",
+    )
+    certificate: str = Field(
+        description="Base 64 encoded der certificate, or a pem certificate with headers.",
+        max_length=100000,
+    )
+
+    @field_serializer("key_password", when_used="json")
+    def dump_key_password(self, v: SecretStr | None) -> str | None:
+        return v.get_secret_value() if v else None
+
+
+class Authentication(BaseModelResource):
+    type: ClassVar[str]
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def find_source_type(
+        cls, data: "dict[str, Any] | Authentication", handler: ModelWrapValidatorHandler[Self]
+    ) -> Self:
+        if isinstance(data, Authentication):
+            return cast(Self, data)
+        if not isinstance(data, dict):
+            raise ValueError(f"Invalid authentication data '{type(data)}' expected dict")
+
+        if cls is not Authentication:
+            # We are already in a subclass, so just validate as usual
+            return handler(data)
+        # If not we need to find the right subclass based on the type field.
+        if "type" not in data:
+            raise ValueError("Invalid authentication data missing 'type' key")
+        type_ = data["type"]
+        if type_ not in _AUTHENTICATION_CLS_BY_TYPE:
+            raise ValueError(
+                f"invalid authentication type '{type_}'. Expected one of {humanize_collection(_AUTHENTICATION_CLS_BY_TYPE.keys(), bind_word='or')}"
+            )
+        cls_ = _AUTHENTICATION_CLS_BY_TYPE[type_]
+        return cast(Self, cls_.model_validate({k: v for k, v in data.items() if k != "type"}))
+
+    @model_serializer(mode="wrap", when_used="always", return_type=dict)
+    def include_type(self, handler: SerializerFunctionWrapHandler) -> dict:
+        if self.type is None:
+            raise ValueError("Type is not set")
+        serialized_data = handler(self)
+        serialized_data["type"] = self.type
+        return serialized_data
+
+
+class BasicAuthentication(Authentication):
+    type: ClassVar[str] = "basic"
+    username: str = Field(
+        description="Username used for basic authentication.",
+        max_length=200,
+    )
+    password: SecretStr = Field(
+        description="Password used for basic authentication.",
+        max_length=200,
+    )
+
+    @field_serializer("password", when_used="json")
+    def dump_password(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class ClientCredentials(Authentication):
+    type: ClassVar[str] = "clientCredentials"
+    client_id: str = Field(
+        description="Client ID for for the service principal used by the extractor",
+    )
+    client_secret: SecretStr = Field(description="Client secret for for the service principal used by the extractor")
+    token_url: str = Field(
+        description="URL to fetch authentication tokens from",
+    )
+    scope: str = Field(
+        description="A space separated list of scopes",
+    )
+    default_expires_in: str | None = Field(
+        None,
+        description="Default value for the expires_in OAuth 2.0 parameter. If the identity provider does not return expires_in in token requests, this parameter must be set or the request will fail.",
+    )
+
+    @field_serializer("client_secret", when_used="json")
+    def dump_client_secret(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class QueryCredentials(Authentication):
+    type: ClassVar[str] = "query"
+    key: str = Field(
+        description="Key for the query parameter to place the authentication token in.",
+    )
+    value: SecretStr = Field(description="Value of the authentication token")
+
+    @field_serializer("value", when_used="json")
+    def dump_value(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class HeaderCredentials(Authentication):
+    type: ClassVar[str] = "header"
+    key: str = Field(
+        description="Key for the header to place the authentication token in",
+    )
+    value: SecretStr = Field(description="Value of the authentication token")
+
+    @field_serializer("value", when_used="json")
+    def dump_value(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class ScramSha(Authentication, ABC):
+    username: str = Field(
+        description="Username for authentication",
+        max_length=200,
+    )
+    password: SecretStr = Field(
+        description="Password for authentication",
+        max_length=200,
+    )
+
+    @field_serializer("password", when_used="json")
+    def dump_password(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class ScramSha256(ScramSha):
+    type: ClassVar[str] = "scramSha256"
+
+
+class ScramSha512(ScramSha):
+    type: ClassVar[str] = "scramSha512"
+
+
+class HostedExtractorSourceYAML(ToolkitResource):
+    type: ClassVar[str]
+    external_id: str = Field(
+        description="The external ID provided by the client. Must be unique for the resource type.",
+        max_length=255,
+    )
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def find_source_type(
+        cls, data: "dict[str, Any] | HostedExtractorSourceYAML", handler: ModelWrapValidatorHandler[Self]
+    ) -> Self:
+        if isinstance(data, HostedExtractorSourceYAML):
+            return cast(Self, data)
+        if not isinstance(data, dict):
+            raise ValueError(f"Invalid hosted extractor source data '{type(data)}' expected dict")
+
+        if cls is not HostedExtractorSourceYAML:
+            # We are already in a subclass, so just validate as usual
+            return handler(data)
+        # If not we need to find the right subclass based on the type field.
+        if "type" not in data:
+            raise ValueError("Invalid hosted extractor source data missing 'type' key")
+        type_ = data["type"]
+        if type_ not in _SOURCE_CLS_BY_TYPE:
+            raise ValueError(
+                f"Invalid hosted extractor source type='{type_}'. Expected one of {humanize_collection(_SOURCE_CLS_BY_TYPE.keys(), bind_word='or')}"
+            )
+        cls_ = _SOURCE_CLS_BY_TYPE[type_]
+        return cast(Self, cls_.model_validate({k: v for k, v in data.items() if k != "type"}))
+
+    @model_serializer(mode="wrap", when_used="always", return_type=dict)
+    def include_type(self, handler: SerializerFunctionWrapHandler) -> dict:
+        if self.type is None:
+            raise ValueError("Type is not set")
+        serialized_data = handler(self)
+        serialized_data["type"] = self.type
+        return serialized_data
+
+
+class EventHubSource(HostedExtractorSourceYAML):
+    type: ClassVar[str] = "eventhub"
+    host: str = Field(
+        description="Host name or IP address of the event hub consumer endpoint.",
+        max_length=200,
+    )
+    event_hub_name: str = Field(
+        description="Name of the event hub",
+        max_length=200,
+    )
+    key_name: str = Field(
+        description="The name of the Event Hub key to use for authentication.",
+        max_length=200,
+    )
+    key_value: SecretStr = Field(
+        description="Value of the Event Hub key to use for authentication.",
+        max_length=200,
+    )
+    consumer_group: str | None = Field(
+        None,
+        description="The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group.",
+        max_length=200,
+    )
+
+    @field_serializer("key_value", when_used="json")
+    def dump_secret(self, v: SecretStr) -> str:
+        return v.get_secret_value()
+
+
+class RESTSource(HostedExtractorSourceYAML):
+    type: ClassVar[str] = "rest"
+    host: str = Field(
+        description="Host or IP address to connect to.",
+        max_length=200,
+    )
+    scheme: Literal["http", "https"] = Field(
+        "https",
+        description="Type of connection to establish",
+    )
+    port: int | None = Field(
+        None,
+        description="Port on server to connect to. Uses default ports based on the scheme if omitted.",
+        ge=1,
+        le=65535,
+    )
+    ca_certificate: str | CACertificate | None = Field(
+        None,
+        description="Custom certificate authority certificate to let the source use a self signed certificate.",
+    )
+    authentication: Authentication | None = Field(None, description="Authentication details for source")
+
+    @field_validator("authentication", mode="after")
+    def validate_authentication(cls, value: Authentication | None) -> Authentication | None:
+        valid_auth = (BasicAuthentication, ClientCredentials, QueryCredentials, HeaderCredentials)
+        if value is not None and not isinstance(value, valid_auth):
+            raise ValueError(
+                f"Invalid authentication type '{value.type}' for REST source. Expected one of {humanize_collection([a.type for a in valid_auth], bind_word='or')}"
+            )
+        return value
+
+    @model_serializer(mode="wrap")
+    def serialize_authentication(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> dict:
+        # Authentication are serialized as dict {}
+        # This issue arises because Pydantic's serialization mechanism doesn't automatically
+        # handle polymorphic serialization for subclasses of Authentication.
+        # To address this, we include the below to explicitly calling model dump on the authentication
+        serialized_data = handler(self)
+        if self.authentication:
+            serialized_data["authentication"] = self.authentication.model_dump(**vars(info))
+        return serialized_data
+
+
+class MQTTSource(HostedExtractorSourceYAML, ABC):
+    host: str = Field(
+        description="Host or IP address of the MQTT broker to connect to.",
+        max_length=200,
+    )
+    port: int | None = Field(
+        None,
+        description="Port on the MQTT broker to connect to.",
+        ge=1,
+        le=65535,
+    )
+    authentication: BasicAuthentication | None = Field(
+        None,
+        description="Method used for authenticating with the mqtt broker. This may be used together with auth certificate.",
+    )
+    use_tls: bool | None = Field(
+        None,
+        description="If true, use TLS when connecting to the broker.",
+    )
+    ca_certificate: CACertificate | None = Field(
+        None,
+        description="Custom certificate authority certificate to let the source use a self signed certificate.",
+    )
+    auth_certificate: AuthCertificate | None = Field(
+        None,
+        description="Authentication certificate (if configured) used to authenticate to source.",
+    )
+
+
+class MQTT3Source(MQTTSource):
+    type: ClassVar[str] = "mqtt3"
+
+
+class MQTT5Source(MQTTSource):
+    type: ClassVar[str] = "mqtt5"
+
+
+class KafkaBroker(BaseModelResource):
+    host: str = Field(
+        description="Host name or IP address of the bootstrap broker.",
+        max_length=200,
+    )
+    port: int = Field(
+        description="Port on the bootstrap broker to connect to.",
+        ge=1,
+        le=65535,
+    )
+
+
+class KafkaSource(HostedExtractorSourceYAML):
+    type: ClassVar[str] = "kafka"
+    bootstrap_brokers: list[KafkaBroker] = Field(
+        description="List of redundant kafka brokers to connect to.", min_length=1, max_length=8
+    )
+    authentication: Authentication | None = Field(None, description="Authentication details for source")
+    use_tls: bool | None = Field(
+        None,
+        description="If true, use TLS when connecting to the broker",
+    )
+    ca_certificate: CACertificate | None = Field(
+        None,
+        description="Custom certificate authority certificate to let the source use a self signed certificate.",
+    )
+    auth_certificate: AuthCertificate | None = Field(
+        None,
+        description="Authentication certificate (if configured) used to authenticate to source.",
+    )
+
+    @field_validator("authentication", mode="after")
+    def validate_authentication(cls, value: Authentication | None) -> Authentication | None:
+        valid_auth = (BasicAuthentication, ClientCredentials, ScramSha256, ScramSha512)
+        if value is not None and not isinstance(value, valid_auth):
+            raise ValueError(
+                f"Invalid authentication type '{value.type}' for Kafka source. Expected one of {humanize_collection([a.type for a in valid_auth], bind_word='or')}"
+            )
+        return value
+
+    @model_serializer(mode="wrap")
+    def serialize_authentication(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> dict:
+        # Authentication are serialized as dict {}
+        # This issue arises because Pydantic's serialization mechanism doesn't automatically
+        # handle polymorphic serialization for subclasses of Authentication.
+        # To address this, we include the below to explicitly calling model dump on the authentication
+        serialized_data = handler(self)
+        if self.authentication:
+            serialized_data["authentication"] = self.authentication.model_dump(**vars(info))
+        return serialized_data
+
+
+_SOURCE_CLS_BY_TYPE: MappingProxyType[str, type[HostedExtractorSourceYAML]] = MappingProxyType(
+    {source.type: source for source in get_concrete_subclasses(HostedExtractorSourceYAML)}
+)
+
+_AUTHENTICATION_CLS_BY_TYPE: MappingProxyType[str, type[Authentication]] = MappingProxyType(
+    {auth.type: auth for auth in get_concrete_subclasses(Authentication)}
+)

cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py

@@ -0,0 +1,109 @@
+from typing import Any
+
+from .base import BaseModelResource, ToolkitResource
+
+
+class FeatureToggles(BaseModelResource):
+    """Feature toggles for InField location configuration."""
+
+    three_d: bool | None = None
+    trends: bool | None = None
+    documents: bool | None = None
+    workorders: bool | None = None
+    notifications: bool | None = None
+    media: bool | None = None
+    template_checklist_flow: bool | None = None
+
+
+class AccessManagement(BaseModelResource):
+    """Access management configuration."""
+
+    template_admins: list[str] | None = None  # list of CDF group external IDs
+    checklist_admins: list[str] | None = None  # list of CDF group external IDs
+
+
+class DataFilterPath(BaseModelResource):
+    """Path reference for data filters."""
+
+    space: str
+    external_id: str
+
+
+class DataFilter(BaseModelResource):
+    """Data filter configuration for a resource type."""
+
+    path: DataFilterPath | None = None
+    instance_spaces: list[str] | None = None
+
+
+class DataFilters(BaseModelResource):
+    """Data filters configuration."""
+
+    files: DataFilter | None = None
+    assets: DataFilter | None = None
+    operations: DataFilter | None = None
+    time_series: DataFilter | None = None
+    notifications: DataFilter | None = None
+    maintenance_orders: DataFilter | None = None
+
+
+class DataStorage(BaseModelResource):
+    """Data storage configuration."""
+
+    root_location: DataFilterPath | None = None
+    app_instance_space: str | None = None
+
+
+class ViewMapping(BaseModelResource):
+    """View mapping configuration."""
+
+    space: str
+    version: str
+    external_id: str
+
+
+class ViewMappings(BaseModelResource):
+    """View mappings configuration."""
+
+    asset: ViewMapping | None = None
+    operation: ViewMapping | None = None
+    notification: ViewMapping | None = None
+    maintenance_order: ViewMapping | None = None
+
+
+class Discipline(BaseModelResource):
+    """Discipline configuration."""
+
+    name: str
+    external_id: str
+
+
+class InFieldCDMLocationConfigYAML(ToolkitResource):
+    """Properties for InFieldCDMLocationConfig node.
+
+    Fields:
+    - space: The space of the InField CDM location configuration
+    - external_id: The external ID of the InField CDM location configuration
+    - name: The name of the location configuration
+    - description: The description of the location configuration
+    - feature_toggles: Feature toggles configuration
+    - access_management: Access management configuration
+    - data_filters: Data filters configuration for various resource types
+    - data_storage: Data storage configuration
+    - view_mappings: View mappings configuration
+    - disciplines: List of disciplines
+    - data_exploration_config: Data exploration configuration
+    """
+
+    space: str
+    external_id: str
+
+    name: str | None = None
+    description: str | None = None
+    feature_toggles: FeatureToggles | None = None
+    access_management: AccessManagement | None = None
+    data_filters: DataFilters | None = None
+    data_storage: DataStorage | None = None
+    view_mappings: ViewMappings | None = None
+    disciplines: list[Discipline] | None = None
+    data_exploration_config: dict[str, Any] | None = None