cognite-toolkit 0.6.39__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/connection/operation_to_maintenance_order.Transformation.sql

@@ -1,32 +0,0 @@
-/* The WORKORDER_NUMBER is not unique in the workorder table,
-so use the following query to select the first match. */
-with unique_workitem as (
-  select
-    *,
-    row_number() over (partition by `sourceId` order by `sourceId`) as rn
-  from
-    {{ rawSourceDatabase }}.`workitem`
-),
-worder_unique as (
-  select
-    *,
-    row_number() over (partition by `WORKORDER_NUMBER` order by `sourceId`) as rn
-  from
-    {{ rawSourceDatabase }}.`workorder`
-)
-select
-  cast(task.`sourceId` as STRING) as externalId,
-  node_reference('{{ instanceSpace }}', cast(worder.`sourceId` as STRING)) as maintenanceOrder
-from
-  unique_workitem as task
-join
-  worder_unique as worder
-on
-  task.`WORKORDER_NUMBER` = worder.`WORKORDER_NUMBER`
-where
-  isnotnull(task.`sourceId`) AND
-  isnotnull(task.`WORKORDER_NUMBER`) AND
-  isnotnull(worder.`WORKORDER_NUMBER`) AND
-  isnotnull(worder.`sourceId`) AND
-  worder.rn = 1 AND
-  task.rn = 1

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/connection/operation_to_maintenance_order.Transformation.yaml

@@ -1,13 +0,0 @@
-externalId: {{ operationToMaintenanceOrderTransformationExternalId }}
-dataSetExternalId: {{ dataset }}
-name: Operation to Maintenance Order for {{ sourceName }}
-destination:
-  type: nodes
-  view:
-    space: {{ schemaSpace }}
-    externalId: {{ organization }}Operation
-    version: v1
-  instanceSpace: {{ instanceSpace }}
-ignoreNullFields: true
-isPublic: true
-conflictMode: upsert

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/maintenanceOrder.Transformation.sql

@@ -1,18 +0,0 @@
-/* MAPPING_MODE_ENABLED: false */
-/* {"version":1,"sourceType":"raw","mappings":[{"from":"externalId","to":"externalId","asType":"STRING"},{"from":"","to":"aliases","asType":"ARRAY<STRING>"},{"from":"name","to":"name","asType":"STRING"},{"from":"","to":"source","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"from":"","to":"path","asType":"ARRAY<STRUCT<`space`:STRING, `externalId`:STRING>>"},{"from":"","to":"root","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"from":"","to":"assetClass","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"from":"","to":"sourceUpdatedUser","asType":"STRING"},{"from":"","to":"pathLastUpdatedTime","asType":"TIMESTAMP"},{"from":"","to":"sourceUpdatedTime","asType":"TIMESTAMP"},{"from":"","to":"type","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"from":"","to":"sourceCreatedUser","asType":"STRING"},{"from":"","to":"parent","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"},{"from":"description","to":"description","asType":"STRING"},{"from":"","to":"tags","asType":"ARRAY<STRING>"},{"from":"","to":"sourceCreatedTime","asType":"TIMESTAMP"},{"from":"","to":"sourceContext","asType":"STRING"},{"from":"","to":"sourceId","asType":"STRING"},{"from":"","to":"object3D","asType":"STRUCT<`space`:STRING, `externalId`:STRING>"}],"sourceLevel1":"sap_staging","sourceLevel2":"dump"} */
-select
-  cast(`sourceId` as STRING) as externalId,
-  cast(`WORKORDER_DESC` as STRING) as description,
-  cast(`WORKORDER_TITLE` as STRING) as name,
-  cast(`WORKORDER_STATUS` as STRING) as status,
-  cast(`WORKORDER_SCHEDULEDSTART` as TIMESTAMP) as scheduledStartTime,
-  cast(`WORKORDER_DUEDATE` as TIMESTAMP) as scheduledEndTime,
-  cast(`WORKORDER_PLANNEDSTART` as TIMESTAMP) as startTime,
-  cast(`WORKORDER_COMPLETIONDATE` as TIMESTAMP) as endTime,
-  cast(`WORKORDER_CREATEDDATE` as TIMESTAMP) as sourceCreatedTime,
-  cast(`WORKORDER_MAITENANCETYPE` as STRING) as type,
-  cast(`WORKORDER_PRIORITYDESC` as STRING) as priorityDescription
-from
-  `{{ rawSourceDatabase }}`.`workorder`
-where
-  isnotnull(`sourceId`)

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/maintenanceOrder.Transformation.yaml

@@ -1,13 +0,0 @@
-externalId: {{ maintenanceOrderTransformationExternalId }}
-dataSetExternalId: {{ dataset }}
-name: Maintenance Order Transformations for {{ sourceName }}
-destination:
-  type: nodes
-  view:
-    space: {{ schemaSpace}}
-    externalId: {{ organization }}MaintenanceOrder
-    version: v1
-  instanceSpace: {{ instanceSpace }}
-ignoreNullFields: true
-isPublic: true
-conflictMode: upsert

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/operation.Transformation.sql

@@ -1,18 +0,0 @@
--- The source data has duplicates this filters them out.
-with unique_workitem as (
-  select
-    *,
-    row_number() over (partition by `sourceId` order by `sourceId`) as rn
-  from
-    `{{ rawSourceDatabase }}`.`workitem`
-)
-select
-  cast(`sourceId` as STRING) as externalId,
-  cast(`WORKORDER_TASKNAME` as STRING) as name,
-  cast(`WORKORDER_STATUS` as STRING) as status,
-  array(cast(`WORKORDER_ITEMNAME` as STRING)) as tags
-from
-  unique_workitem
-where
-  isnotnull(`sourceId`) and
-  rn = 1

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/transformations/population/operation.Transformation.yaml

@@ -1,13 +0,0 @@
-externalId: {{ operationTransformationExternalId }}
-dataSetExternalId: {{ dataset }}
-name: Operation Transformations for {{ sourceName }}
-destination:
-  type: nodes
-  view:
-    space: {{ schemaSpace}}
-    externalId: {{ organization }}Operation
-    version: v1
-  instanceSpace: {{ instanceSpace }}
-ignoreNullFields: true
-isPublic: true
-conflictMode: upsert

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/population.Workflow.yaml

@@ -1,3 +0,0 @@
-externalId: {{ workflow }}
-description: This workflow populates Cognite Data Fusion with data from {{ sourceName }}.
-dataSetExternalId: {{ dataset }}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/trigger.WorkflowTrigger.yaml

@@ -1,9 +0,0 @@
-externalId: {{ workflow }}_trigger
-triggerRule:
-  triggerType: schedule
-  cronExpression: "0 4 * * MON"
-workflowExternalId: {{ workflow }}
-workflowVersion: v1
-authentication:
-  clientId: {{ workflowClientId }}
-  clientSecret: {{ workflowClientSecret }}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sap_events/workflows/v1.WorkflowVersion.yaml

@@ -1,28 +0,0 @@
-workflowExternalId: {{ workflow }}
-version: 'v1'
-workflowDefinition:
-  tasks:
-    - externalId: {{ maintenanceOrderTransformationExternalId }}
-      type: transformation
-      parameters:
-        transformation:
-          externalId: {{ maintenanceOrderTransformationExternalId }}
-          concurrencyPolicy: fail
-      onFailure: abortWorkflow
-    - externalId: {{ operationTransformationExternalId }}
-      type: transformation
-      parameters:
-        transformation:
-          externalId: {{ operationTransformationExternalId }}
-          concurrencyPolicy: fail
-      onFailure: abortWorkflow
-    - externalId: {{ operationToMaintenanceOrderTransformationExternalId}}
-      type: transformation
-      parameters:
-        transformation:
-          externalId: {{ operationToMaintenanceOrderTransformationExternalId }}
-          concurrencyPolicy: fail
-      onFailure: abortWorkflow
-      dependsOn:
-        - externalId: {{ maintenanceOrderTransformationExternalId }}
-        - externalId: {{ operationTransformationExternalId }}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/auth/workflow.Group.yaml

@@ -1,47 +0,0 @@
-name: gp_{{ workflow }}
-sourceId: {{ groupSourceId }}
-metadata:
-  origin: cognite-toolkit
-capabilities:
-  # minimum runtime capabilities
-  - sessionsAcl:
-      actions:
-        - CREATE
-      scope:
-        all: {}
-  - projectsAcl:
-      actions:
-        - READ
-        - LIST
-      scope:
-        all: {}
-  - groupsAcl:
-      actions:
-        - LIST
-      scope:
-        all: {}
-  # transformation-specific capabilities
-  - rawAcl:
-      actions:
-        - READ
-      scope:
-        tableScope:
-          dbsToTables:
-            {{ rawSourceDatabase }}: []
-  - dataModelInstancesAcl:
-      actions:
-        - READ
-        - WRITE
-        - WRITE_PROPERTIES
-      scope:
-        spaceIdScope:
-          spaceIds:
-            - {{ instanceSpace }}
-  - dataModelsAcl:
-      actions:
-        - READ
-      scope:
-        spaceIdScope:
-          spaceIds:
-            - {{ schemaSpace }}
-  

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/default.config.yaml

@@ -1,13 +0,0 @@
-sourceName: Springfield Sharepoint
-sourceId: springfield_sharepoint
-fileTransformationExternalId: files_metadata_springfield
-dataset: ingestion
-rawSourceDatabase: ingestion
-instanceSpace: springfield_instances
-groupSourceId: <not set>
-organization: YourOrg
-schemaSpace: sp_enterprise_process_industry
-
-workflow: files_metadata_population # can also contain location files_metadata_springfield_population
-workflowClientId: ${SHAREPOINT_SERVICE_PRINCIPAL_CLIENT_ID}
-workflowClientSecret: ${SHAREPOINT_SERVICE_PRINCIPAL_CLIENT_SECRET}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/files/upload.CogniteFile.yaml

@@ -1,4 +0,0 @@
-- externalId: VAL_$FILENAME
-  space: {{ instanceSpace }}
-  name: $FILENAME
-  mimeType: application/pdf

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/module.toml

@@ -1,26 +0,0 @@
-[module]
-title = "Sharepoint Files"
-
-[[data]]
-repoType = "http"
-repo = "https://apps-cdn.cogniteapp.com/toolkit/publicdata/files"
-source = "PH-25578-P-4110006-001.pdf;PH-25578-P-4110010-001.pdf;PH-25578-P-4110119-001.pdf;PH-ME-P-0003-001.pdf;PH-ME-P-0004-001.pdf;PH-ME-P-0151-001.pdf;PH-ME-P-0152-001.pdf;PH-ME-P-0153-001.pdf;PH-ME-P-0156-001.pdf;PH-ME-P-0156-002.pdf;PH-ME-P-0160-001.pdf"
-destination = "files/"
-
-[[data]]
-repoType = "http"
-repo = "https://apps-cdn.cogniteapp.com/toolkit"
-source = "publicdata/valhall_file_metadata.csv"
-destination = "raw/files.Table.csv"
-
-[[extra_resources]]
-location = "cdf_common/data_sets/demo.DataSet.yaml"
-
-[[extra_resources]]
-location = "cdf_common/data_models/demo.Space.yaml"
-
-[[extra_resources]]
-location = "cdf_common/raw/source.DataBase.yaml"
-
-[[extra_resources]]
-location = "models/cdf_process_industry_extension/data_models"

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/raw/files.Table.yaml

@@ -1,2 +0,0 @@
-dbName: {{ rawSourceDatabase }}
-tableName: files_metadata

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/transformations/population/files.Transformation.sql

@@ -1,9 +0,0 @@
-select
-	concat('VAL_', name) as externalId,
- 	name,
-	source as sourceId,
-	mime_type as mimeType
-from `{{ rawSourceDatabase }}`.`files_metadata`
-where
-	isnotnull(mime_type) and
-	mime_type = 'application/pdf'

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/transformations/population/files.Transformation.yaml

@@ -1,13 +0,0 @@
-externalId: {{ fileTransformationExternalId }}
-dataSetExternalId: {{ dataset }}
-name: File Transformations for {{ sourceName }}
-destination:
-  type: nodes
-  view:
-    space: {{ schemaSpace }}
-    externalId: {{ organization }}File
-    version: v1
-  instanceSpace: {{ instanceSpace }}
-ignoreNullFields: true
-isPublic: true
-conflictMode: upsert

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/population.Workflow.yaml

@@ -1,3 +0,0 @@
-externalId: {{ workflow }}
-description: This workflow populates Cognite Data Fusion with data from {{ sourceName }}.
-dataSetExternalId: {{ dataset }}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/trigger.WorkflowTrigger.yaml

@@ -1,9 +0,0 @@
-externalId: {{ workflow }}_trigger
-triggerRule:
-  triggerType: schedule
-  cronExpression: "0 4 * * MON"
-workflowExternalId: {{ workflow }}
-workflowVersion: v1
-authentication:
-  clientId: {{ workflowClientId }}
-  clientSecret: {{ workflowClientSecret }}

cognite_toolkit/_builtin_modules/sourcesystem/cdf_sharepoint/workflows/v1.WorkflowVersion.yaml

@@ -1,11 +0,0 @@
-workflowExternalId: {{ workflow }}
-version: 'v1'
-workflowDefinition:
-  tasks:
-    - externalId: {{ fileTransformationExternalId }}
-      type: transformation
-      parameters:
-        transformation:
-          externalId: {{ fileTransformationExternalId }}
-          concurrencyPolicy: fail
-      onFailure: abortWorkflow

cognite_toolkit/_cdf_tk/_parameters/__init__.py

@@ -1,18 +0,0 @@
-"""This is an internal subpackage used to check the parameters of a class."""
-
-from .constants import ANY_INT, ANY_STR, ANYTHING
-from .data_classes import ParameterSet, ParameterSpec, ParameterSpecSet, ParameterValue
-from .functions import read_parameter_from_init_type_hints, read_parameters_from_dict
-
-__all__ = [
-    "ANYTHING",
-    "ANY_INT",
-    "ANY_STR",
-    "ParameterSet",
-    "ParameterSpec",
-    "ParameterSpecSet",
-    "ParameterSpecSet",
-    "ParameterValue",
-    "read_parameter_from_init_type_hints",
-    "read_parameters_from_dict",
-]

cognite_toolkit/_cdf_tk/_parameters/constants.py

@@ -1,82 +0,0 @@
-from functools import total_ordering
-
-BASE_TYPES = {t.__name__ for t in (str, int, float, bool)}
-CONTAINER_TYPES = {t.__name__ for t in (list, dict)}
-TYPES = BASE_TYPES | CONTAINER_TYPES
-
-
-@total_ordering
-class _AnyInt(str):
-    def __eq__(self, other: object) -> bool:
-        if isinstance(other, int):
-            return True
-        elif isinstance(other, str):
-            return False
-        return NotImplemented
-
-    def __lt__(self, other: object) -> bool:
-        if isinstance(other, int):
-            return True
-        elif isinstance(other, str):
-            return True
-        return NotImplemented
-
-    def __hash__(self) -> int:
-        return id(ANY_INT)
-
-    def __str__(self) -> str:
-        return "AnyInt"
-
-    def __repr__(self) -> str:
-        return "AnyInt"
-
-
-@total_ordering
-class _AnyStr(str):
-    def __eq__(self, other: object) -> bool:
-        if isinstance(other, str):
-            return True
-        elif isinstance(other, int):
-            return False
-        return NotImplemented
-
-    def __lt__(self, other: object) -> bool:
-        if isinstance(other, str):
-            return True
-        elif isinstance(other, int):
-            return False
-        return NotImplemented
-
-    def __hash__(self) -> int:
-        return id(ANY_STR)
-
-    def __str__(self) -> str:
-        return "AnyStr"
-
-    def __repr__(self) -> str:
-        return "AnyStr"
-
-
-@total_ordering
-class _Anything(str):
-    def __eq__(self, other: object) -> bool:
-        return True
-
-    def __lt__(self, other: object) -> bool:
-        return True
-
-    def __hash__(self) -> int:
-        return id(ANYTHING)
-
-    def __str__(self) -> str:
-        return "Anything"
-
-    def __repr__(self) -> str:
-        return "Anything"
-
-
-ANY_INT = _AnyInt()
-ANY_STR = _AnyStr()
-ANYTHING = _Anything()
-
-SINGLETONS = {ANY_STR, ANY_INT, ANYTHING}

cognite_toolkit/_cdf_tk/_parameters/data_classes.py

@@ -1,200 +0,0 @@
-import dataclasses
-import sys
-import typing
-from collections.abc import Hashable, Iterable, MutableSet
-from dataclasses import dataclass
-from functools import total_ordering
-
-# Ruff wants to get AbstractSet from collections.abc, but mypy does not allow it.
-from typing import AbstractSet, Generic, TypeVar, final  # noqa: UP035
-
-from cognite.client.utils._text import to_camel_case
-
-if sys.version_info >= (3, 11):
-    from typing import Self
-else:
-    from typing_extensions import Self
-from .constants import ANYTHING, SINGLETONS
-
-
-@total_ordering
-@dataclass(frozen=True)
-class Parameter:
-    path: tuple[str | int, ...]
-
-    @property
-    def key(self) -> str:
-        if isinstance(self.path[-1], str):
-            return self.path[-1]
-        else:
-            return str(self.path[-1])
-
-    def __lt__(self, other: Self) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path < other.path
-
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path == other.path
-
-    @property
-    def has_any_type(self) -> bool:
-        return any(name in SINGLETONS for name in self.path)
-
-
-@final
-@dataclass(frozen=True)
-class ParameterSpec(Parameter):
-    types: frozenset[str]
-    is_required: bool
-    _is_nullable: bool | None = None
-
-    @property
-    def is_nullable(self) -> bool:
-        return self._is_nullable or not self.is_required
-
-    # The eq, ne, and hash methods must be implemented for each subclass to get the set operations to work correctly.
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path == other.path
-
-    def __ne__(self, other: object) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path != other.path
-
-    def __hash__(self) -> int:
-        return hash(self.path)
-
-
-@final
-@dataclass(frozen=True)
-class ParameterValue(Parameter):
-    type: str
-    value: str | int | float | bool | None
-
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path == other.path
-
-    def __ne__(self, other: object) -> bool:
-        if not isinstance(other, Parameter):
-            return NotImplemented
-        return self.path != other.path
-
-    def __hash__(self) -> int:
-        return hash(self.path)
-
-
-T_Parameter = TypeVar("T_Parameter", bound=Parameter)
-
-
-class ParameterSet(Hashable, MutableSet, Generic[T_Parameter]):
-    def __init__(self, iterable: Iterable[T_Parameter] = ()) -> None:
-        self.data: set[T_Parameter] = set(iterable)
-
-    def as_camel_case(self) -> "ParameterSet[T_Parameter]":
-        output = type(self)()
-        for parameter in self:
-            new_path = tuple(
-                to_camel_case(name) if name not in SINGLETONS and isinstance(name, str) else name
-                for name in parameter.path
-            )
-            # This is because parameters are immutable
-            new_param_copy = dataclasses.replace(parameter, path=new_path)
-            output.add(new_param_copy)
-        return output
-
-    @property
-    def has_any_type(self) -> bool:
-        return any(parameter.has_any_type for parameter in self)
-
-    def subset_any_type_paths(self) -> "ParameterSet[T_Parameter]":
-        return type(self)(parameter for parameter in self if parameter.has_any_type)
-
-    def subset_anything_paths(self) -> "ParameterSet[T_Parameter]":
-        return type(self)(parameter for parameter in self if parameter.path[-1] is ANYTHING)
-
-    def subset(self, path: tuple[str | int, ...] | int) -> "ParameterSet[T_Parameter]":
-        if isinstance(path, tuple):
-            return type(self)(parameter for parameter in self if parameter.path[: len(path)] == path)
-        elif isinstance(path, int):
-            return type(self)(parameter for parameter in self if len(parameter.path) <= path)
-        raise TypeError(f"Expected tuple or int, got {type(path)}")
-
-    def __hash__(self) -> int:
-        return hash(self.data)
-
-    def __contains__(self, value: object) -> bool:
-        return value in self.data
-
-    def __iter__(self) -> typing.Iterator[T_Parameter]:
-        return iter(self.data)
-
-    def __len__(self) -> int:
-        return len(self.data)
-
-    def __repr__(self) -> str:
-        return repr(self.data)
-
-    def add(self, item: T_Parameter) -> None:
-        self.data.add(item)
-
-    def discard(self, item: T_Parameter) -> None:
-        self.data.discard(item)
-
-    def update(self, other: "ParameterSet[T_Parameter]") -> None:
-        self.data.update(other.data)
-
-    def difference(self, other: "ParameterSet[T_Parameter]") -> "ParameterSet[T_Parameter]":
-        output = type(self)(self.data.difference(other.data))
-        other_any = other.subset_any_type_paths()
-        other_anything = other.subset_anything_paths()
-        if not other_any:
-            # Operation done using hashes
-            return output
-        # Operation done using equality as ANY_STR and ANY_INT does not match on hash
-        # Note that this a Parameter with ANY_STR in the path can remove multiple items.
-        discard = set()
-        for item in output:
-            for any_ in other_any:
-                if item == any_:
-                    discard.add(item)
-            for anything in other_anything:
-                if item.path[: len(anything.path)] == anything.path:
-                    discard.add(item)
-        for d in discard:
-            output.discard(d)
-        return output
-
-    def __sub__(self, other: AbstractSet) -> "ParameterSet[T_Parameter]":
-        return self.difference(type(self)(other))
-
-
-class ParameterSpecSet(ParameterSet[ParameterSpec]):
-    def __init__(self, iterable: Iterable[ParameterSpec] = (), spec_name: str | None = None) -> None:
-        super().__init__(iterable)
-        self.is_complete = True
-        self.spec_name = spec_name
-
-    def required(self, level: int | None = None) -> ParameterSet[ParameterSpec]:
-        if level is None:
-            return ParameterSet[ParameterSpec](parameter for parameter in self if parameter.is_required)
-        else:
-            return ParameterSet[ParameterSpec](
-                parameter for parameter in self if len(parameter.path) <= level and parameter.is_required
-            )
-
-    def as_camel_case(self) -> "ParameterSpecSet":
-        output = typing.cast(ParameterSpecSet, super().as_camel_case())
-        output.is_complete = self.is_complete
-        return output
-
-    def update(self, other: ParameterSet[ParameterSpec]) -> None:
-        if isinstance(other, ParameterSpecSet):
-            self.is_complete &= other.is_complete
-        super().update(other)