cognite-toolkit 0.6.82__py3-none-any.whl → 0.6.83__py3-none-any.whl

cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/handler.py

@@ -1,168 +0,0 @@
-from datetime import datetime, timedelta, timezone
-from itertools import islice
-from timeit import default_timer
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes import ExtractionPipelineRun
-from cognite.client.data_classes.data_modeling import NodeId, ViewId
-from cognite.client.data_classes.data_modeling.cdm.v1 import CogniteAsset, CogniteTimeSeries
-from cognite.client.data_classes.filters import Prefix, ContainsAny
-
-from ice_cream_factory_api import IceCreamFactoryAPI
-
-from cognite.client.config import global_config
-global_config.disable_pypi_version_check = True
-
-from itertools import islice
-
-
-def batcher(iterable, batch_size):
-    iterator = iter(iterable)
-    while batch := list(islice(iterator, batch_size)):
-        yield batch
-
-
-def get_time_series_for_site(client: CogniteClient, site):
-    this_site = site.lower()
-    sub_tree_root = client.data_modeling.instances.retrieve_nodes(
-        NodeId("icapi_dm_space", this_site),
-        node_cls=CogniteAsset
-    )
-
-    if not sub_tree_root:
-        print(
-            f"----No CogniteAssets in CDF for {site}!----\n"
-            f"    Run the 'Create Cognite Asset Hierarchy' transformation!"
-        )
-        return []
-
-    sub_tree_nodes = client.data_modeling.instances.list(
-        instance_type=CogniteAsset,
-        filter=Prefix(property=["cdf_cdm", "CogniteAsset/v1", "path"], value=sub_tree_root.path),
-        limit=None
-    )
-
-    if not sub_tree_nodes:
-        print(
-            f"----No CogniteTimeSeries in CDF for {site}!----\n"
-            f"    Run the 'Contextualize Timeseries and Assets' transformation!"
-        )
-        return []
-
-    value_list = [{"space": node.space, "externalId": node.external_id} for node in sub_tree_nodes]
-
-    time_series = [
-        client.data_modeling.instances.search(
-            view=ViewId("cdf_cdm", "CogniteTimeSeries", "v1"),
-            instance_type=CogniteTimeSeries,
-            filter=ContainsAny(property=["cdf_cdm", "CogniteTimeSeries/v1", "assets"], values=batch),
-            limit=None
-        )
-        for batch in batcher(value_list, 20)
-    ]
-
-    # Combine list of batch results into a single NodeList
-    time_series = [node for nodelist in time_series for node in nodelist]
-
-    if not time_series:
-        print("No CogniteTimeSeries in the CogniteCore Data Model (cdf_cdm Space)")
-
-    time_series = [
-        item for item in time_series
-        if any(substring in item.external_id for substring in ["planned_status", "good"])
-    ]
-
-    return time_series
-
-
-def report_ext_pipe(client: CogniteClient, status, message=None):
-    ext_pipe_run = ExtractionPipelineRun(
-        extpipe_external_id="ep_icapi_datapoints",
-        status=status,
-        message=message
-    )
-
-    client.extraction_pipelines.runs.create(run=ext_pipe_run)
-
-def handle(client: CogniteClient = None, data=None):
-    report_ext_pipe(client, "seen")
-    
-    sites = None
-    backfill = None
-    hours = None
-    max_hours = 336
-
-    if data:
-        sites = data.get("sites")
-        backfill = data.get("backfill")
-        hours = data.get("hours")
-
-        if hours and hours > max_hours:
-            print(f"{hours} > {max_hours}! The Ice Cream API can't serve more than {max_hours} hours of datapoints, setting hours to max")
-            hours = max_hours
-
-    all_sites = [
-        "Houston",
-        "Oslo",
-        "Kuala_Lumpur",
-        "Hannover",
-        "Nuremberg",
-        "Marseille",
-        "Sao_Paulo",
-        "Chicago",
-        "Rotterdam",
-        "London",
-    ]
-
-    sites = sites or all_sites
-    backfill = backfill or True
-    hours = hours or max_hours
-
-    now = datetime.now(timezone.utc).timestamp() * 1000
-    increment = timedelta(hours=hours).total_seconds() * 1000
-
-    ice_cream_api = IceCreamFactoryAPI(base_url="https://ice-cream-factory.inso-internal.cognite.ai")
-
-    try:
-        for site in sites:
-            print(f"Getting Data Points for {site}")
-            big_start = default_timer()
-
-            time_series = get_time_series_for_site(client, site)
-
-            latest_dps = {
-                dp.external_id: dp.timestamp
-                for dp in client.time_series.data.retrieve_latest(
-                    external_id=[ts.external_id for ts in time_series],
-                    ignore_unknown_ids=True
-                )
-            } if not backfill else None
-
-            to_insert = []
-            for ts in time_series:
-                # figure out the window of datapoints to pull for this Time Series
-                latest = latest_dps[ts.external_id][0] if not backfill and latest_dps.get(ts.external_id) else None
-
-                start = latest if latest else now - increment
-                end = now
-            
-                dps_list = ice_cream_api.get_datapoints(timeseries_ext_id=ts.external_id, start=start, end=end)
-
-                for dp_dict in dps_list:
-                    dp_dict["instance_id"] = NodeId(space="icapi_dm_space", external_id=dp_dict["instance_id"])
-
-                to_insert.extend(dps_list)
-
-                if len(to_insert) > 50:
-                    client.time_series.data.insert_multiple(datapoints=to_insert)
-                    to_insert = []
-
-            if to_insert:
-                client.time_series.data.insert_multiple(datapoints=to_insert)
-                print(f"  {hours}h of Datapoints took {default_timer() - big_start:.2f} seconds")
-            else:
-                print(f"  No TimeSeries, for {hours}h of Datapoints took {default_timer() - big_start:.2f} seconds")
-
-        report_ext_pipe(client, "success")
-    except Exception as e:
-        report_ext_pipe(client, "fail", e)

cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/ice_cream_factory_api.py

@@ -1,61 +0,0 @@
-from typing import Dict, Union
-
-import orjson
-from requests import Response, Session, adapters
-
-
-class IceCreamFactoryAPI:
-    """Class for Ice Cream Factory API."""
-
-    def __init__(self, base_url: str):
-        self.base_url = base_url
-        self.adapter = adapters.HTTPAdapter(max_retries=3)
-        self.session = Session()
-        self.session.mount("https://", self.adapter)
-
-    def get_response(
-        self, headers: Dict[str, str], url_suffix: str, params: Dict[str, Union[str, int, float]] = {}
-    ) -> Response:
-        """
-        Get response from API.
-
-        Args:
-            headers: request header
-            url_suffix: string to add to base url
-            params: query parameters
-        """
-
-        response = self.session.get(f"{self.base_url}/{url_suffix}", headers=headers, timeout=40, params=params)
-        response.raise_for_status()
-        return response
-
-    def get_datapoints(self, timeseries_ext_id: str, start: Union[str, int, float], end: Union[str, int, float]):
-        """
-        Get datapoints for a timeseries external id. This will also return datapoints for an associated timeseries
-
-        (e.g. request for external id "HPM2C561:planned_status" will return datapoints for "HPM2C561:planned_status" AND
-        "HPM2C561:status". Similar, request for timeseries with external id "HPM2C561:count" will return datapoints for
-        "HPM2C561:count" AND ""HPM2C561:good").
-
-        Args:
-            timeseries_ext_id: external id of timeseries to get datapoints for
-            start: start for datapoints (UNIX timestamp (int, float) or string with format 'YYYY-MM-DD HH:MM')
-            end: end for datapoints (UNIX timestamp (int, float) or string with format 'YYYY-MM-DD HH:MM')
-        """
-        params = {"start": start, "end": end, "external_id": timeseries_ext_id}
-        response = self.get_response(headers={}, url_suffix="datapoints/oee", params=params)
-
-        response_dict = orjson.loads(response.content)
-
-        response_dict = [
-            {
-                "instance_id": ts,
-                "datapoints": [
-                    # convert timestamp to ms (*1000) for CDF uploads
-                    {"timestamp": dp[0] * 1000, "value": dp[1]}
-                    for dp in dps
-                ]
-            } for ts, dps in response_dict.items() if len(dps) > 1
-        ]
-
-        return response_dict

cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/functions/icapi_datapoints_extractor/requirements.txt

@@ -1,3 +0,0 @@
-cognite-sdk
-orjson >= 3.10.5
-guppy3

cognite_toolkit/_builtin_modules/bootcamp/ice_cream_api/module.toml

@@ -1,3 +0,0 @@
-[module]
-title = "Bootcamp"
-is_selected_by_default = true

cognite_toolkit/_builtin_modules/bootcamp/module.toml

@@ -1,3 +0,0 @@
-[module]
-title = "Bootcamp"
-is_selected_by_default = false

cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/default.config.yaml

@@ -1,6 +0,0 @@
-data_pipeline_oee_source_id: <OBJECT ID FOR DATA PIPELINE OEE ENTRA GROUP>
-
-uc_oee_ds_external_id: ds_uc_oee
-oee_space_name: oee_ts_space
-data_pipeline_oee_client_id: ${DATA_PIPELINE_OEE_CLIENT_ID}
-data_pipeline_oee_client_secret: ${DATA_PIPELINE_OEE_CLIENT_SECRET}

cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/functions/oee_timeseries/handler.py

@@ -1,229 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-from itertools import islice
-from datetime import timedelta
-from typing import Any, Dict
-
-from cognite.client import CogniteClient
-from cognite.client.data_classes.data_modeling import NodeId, ViewId
-from cognite.client.data_classes.data_modeling.cdm.v1 import CogniteAsset, CogniteTimeSeries, CogniteTimeSeriesApply
-from cognite.client.data_classes.filters import Prefix, ContainsAny
-from cognite.client.exceptions import CogniteNotFoundError
-
-import numpy as np
-
-from cognite.client.config import global_config
-global_config.disable_pypi_version_check = True
-
-
-def batcher(iterable, batch_size):
-    iterator = iter(iterable)
-    while batch := list(islice(iterator, batch_size)):
-        yield batch
-
-
-def get_time_series_for_site(client: CogniteClient, site, space):
-    this_site = site.lower()
-    sub_tree_root = client.data_modeling.instances.retrieve_nodes(
-        NodeId(space, this_site),
-        node_cls=CogniteAsset
-    )
-
-    if not sub_tree_root:
-        print(
-            f"----No CogniteAssets in CDF for {site}!----\n"
-            f"    Run the 'Create Cognite Asset Hierarchy' transformation!"
-        )
-        return
-
-    sub_tree_nodes = client.data_modeling.instances.list(
-        instance_type=CogniteAsset,
-        filter=Prefix(property=["cdf_cdm", "CogniteAsset/v1", "path"], value=sub_tree_root.path),
-        limit=None
-    )
-
-    if not sub_tree_nodes:
-        print(
-            f"----No CogniteTimeSeries in CDF for {site}!----\n"
-            f"    Run the 'Contextualize Timeseries and Assets' transformation!"
-        )
-        return
-
-    value_list = [{"space": node.space, "externalId": node.external_id} for node in sub_tree_nodes]
-
-    time_series = [
-        client.data_modeling.instances.search(
-            view=ViewId("cdf_cdm", "CogniteTimeSeries", "v1"),
-            instance_type=CogniteTimeSeries,
-            space=space,
-            filter=ContainsAny(property=["cdf_cdm", "CogniteTimeSeries/v1", "assets"], values=batch),
-            limit=None
-        )
-        for batch in batcher(value_list, 20)
-    ]
-
-    # Combine list of batch results into a single NodeList
-    time_series = [node for nodelist in time_series for node in nodelist]
-
-    if not time_series:
-        print("No CogniteTimeSeries in the CogniteCore Data Model (cdf_cdm Space)")
-
-    return time_series
-
-def handle(client: CogniteClient, data: Dict[str, Any] = {}) -> None:
-    lookback_minutes = None
-    sites = None
-
-    if data:
-        lookback_minutes = timedelta(minutes=data.get("lookback_minutes", 60)).total_seconds() * 1000
-        sites = data.get("sites")
-
-    all_sites = [
-        "Houston",
-        "Oslo",
-        "Kuala_Lumpur",
-        "Hannover",
-        "Nuremberg",
-        "Marseille",
-        "Sao_Paulo",
-        "Chicago",
-        "Rotterdam",
-        "London",
-    ]
-
-    lookback_minutes = lookback_minutes or timedelta(minutes=60).total_seconds() * 1000
-    sites = sites or all_sites
-
-    print(f"Processing datapoints for these sites: {sites}")
-    with ThreadPoolExecutor(max_workers=10) as executor:
-        futures = [executor.submit(process_site, client, lookback_minutes, site) for site in sites]
-        for f in futures:
-            f.result()
-
-def process_site(client, lookback_minutes, site):
-    oee_space = "oee_ts_space"
-    source_space = "icapi_dm_space"
-
-    timeseries = get_time_series_for_site(client, site, source_space)
-    asset_eids = list(set([item.external_id.split(sep=":")[0] for item in timeseries]))
-    instance_ids = [NodeId(space=source_space, external_id=ts.external_id) for ts in timeseries]
-    all_latest_dps = client.time_series.data.retrieve_latest(instance_id=instance_ids)
-
-    # Organize latest datapoints by equipment for alignment
-    assets_dps = {
-        external_id: [latest_dp for latest_dp in all_latest_dps if external_id in latest_dp.instance_id.external_id]
-        for external_id in asset_eids
-    }
-
-    for asset, latest_dps in assets_dps.items():
-        print(f"Calculating OEE for {asset}")
-        count_node = f"NodeId({source_space}, {asset}:count)"
-        good_node = f"NodeId({source_space}, {asset}:good)"
-        status_node = f"NodeId({source_space}, {asset}:status)"
-        planned_status_node = f"NodeId({source_space}, {asset}:planned_status)"
-
-        end = min([dp.timestamp[0] for dp in latest_dps if latest_dps and dp.timestamp], default=None)
-
-        if end:
-            dps_df = client.time_series.data.retrieve_dataframe(
-                instance_id=[dp.instance_id for dp in latest_dps],
-                start=end - lookback_minutes,
-                end=end,
-                aggregates=["sum"],
-                granularity="1m",
-                include_aggregate_name=False,
-                limit=None
-            )
-
-            # Frontfill because "planned_status" and "status" only have datapoints when the value changes
-            dps_df = dps_df.ffill()
-
-            # Fill the rest with the opposite
-            try:
-                first_valid_value = dps_df[planned_status_node].loc[dps_df[planned_status_node].first_valid_index()]
-            except Exception as e:
-                print(f"Failed to find datapoints for {planned_status_node}, {e}")
-                continue
-
-            backfill_value = 1.0 if first_valid_value == 0.0 else 0.0
-            dps_df[planned_status_node] = dps_df[planned_status_node].fillna(value=backfill_value)
-
-            # Same for status
-            first_valid_value = dps_df[status_node].loc[dps_df[status_node].first_valid_index()]
-            backfill_value = 1.0 if first_valid_value == 0.0 else 0.0
-            dps_df[status_node] = dps_df[status_node].fillna(value=backfill_value)
-
-            count_dps = dps_df[count_node]
-            good_dps = dps_df[good_node]
-            status_dps = dps_df[status_node]
-            planned_status_dps = dps_df[planned_status_node]
-
-            total_items = len(count_dps)
-
-            if (
-                total_items != len(good_dps)
-                or total_items != len(status_dps)
-                or total_items != len(planned_status_dps)
-            ):
-                # We expect ALL dependent timeseries to have the exact same number of datapoints
-                # for the specified time range for the calculation to execute.
-                print(
-                    f"""{asset}: Unable to retrieve datapoints for all required OEE timeseries (count, good, status, planned_status)
-                    between {end - lookback_minutes} and {end}. Ensure that data is available for the time range specified."""
-                )
-
-            # Calculate the components of OEE
-            off_spec_node = f"{asset}:off_spec"
-            quality_node = f"{asset}:quality"
-            performance_node = f"{asset}:performance"
-            availability_node = f"{asset}:availability"
-            oee_node = f"{asset}:oee"
-
-            dps_df[off_spec_node] = count_dps - good_dps
-            dps_df[quality_node] = good_dps / count_dps
-            dps_df[performance_node] = (count_dps / status_dps) / (60.0 / 3.0)
-            dps_df[availability_node] = status_dps / planned_status_dps
-
-            dps_df[oee_node] = dps_df[quality_node] * dps_df[performance_node] * dps_df[availability_node]
-
-            # Fill in the divide by zeros
-            dps_df = dps_df.fillna(value=0.0)
-            dps_df = dps_df.replace([np.inf, -np.inf], 0.0)
-
-            # Drop input timeseries
-            dps_df = dps_df.drop(columns=[count_node, good_node, status_node, planned_status_node])
-
-            to_insert = [
-                {
-                    "instance_id": NodeId(space="oee_ts_space", external_id=external_id),
-                    "datapoints": list(zip(dps_df[external_id].index, dps_df[external_id]))
-                }
-                for external_id in dps_df.columns
-            ]
-
-            try:
-                client.time_series.data.insert_multiple(to_insert)
-            except CogniteNotFoundError as e:
-                # Create the missing oee timeseries since they don't exist
-                ts_to_create = []
-                for node_id in e.not_found:
-                    print(f"Creating CogniteTimeSeries {node_id}")
-
-                    external_id = node_id["instanceId"]["externalId"]
-
-                    # change external_id to a readable name
-                    # Ex: "OSLPROFILTRASYS185:off_spec" to "OSLPROFILTRASYS185 Off Spec"
-                    name = external_id.split(":")
-                    name[-1] = name[-1].replace("_", " ").title()
-
-                    ts_to_create.append(
-                        CogniteTimeSeriesApply(
-                            space=oee_space,
-                            external_id=external_id,
-                            name=" ".join(name),
-                            is_step=False,
-                            time_series_type="numeric",
-                        )
-                    )
-
-                client.data_modeling.instances.apply(ts_to_create)
-                client.time_series.data.insert_multiple(to_insert)

cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/functions/oee_timeseries/requirements.txt

@@ -1,2 +0,0 @@
-pandas
-cognite-sdk

cognite_toolkit/_builtin_modules/bootcamp/use_cases/oee/module.toml

@@ -1,3 +0,0 @@
-[module]
-title = "Bootcamp"
-is_selected_by_default = true

cognite_toolkit/_builtin_modules/cdf.toml

@@ -1,13 +0,0 @@
-[cdf]#<PLACEHOLDER>
-default_env = "<DEFAULT_ENV_PLACEHOLDER>"
-
-[modules]
-# This is the version of the modules. It should not be changed manually.
-# It will be updated by the 'cdf modules upgrade' command.
-version = "0.6.82"
-
-
-[plugins]
-run = true
-pull = false
-dump = false

cognite_toolkit/_builtin_modules/cdf_common/data_models/demo.Space.yaml

@@ -1,3 +0,0 @@
-space: '{{ instanceSpace }}'
-name: 'Instance space'
-description: This space contains data

cognite_toolkit/_builtin_modules/cdf_common/data_sets/demo.DataSet.yaml

@@ -1,3 +0,0 @@
-name: 'Ingestion'
-externalId: {{ dataset }}
-description: This dataset contains Transformations, Functions, and Workflows for ingesting data into Cognite Data Fusion.

cognite_toolkit/_builtin_modules/cdf_common/default.config.yaml

@@ -1,8 +0,0 @@
-dataset: ingestion
-instanceSpace: springfield_instances
-directRelationJob: ctx_files_direct_relation_write
-rawSourceDatabase: ingestion
-rawStateDatabase: contextualizationState
-rawStateTable: diagramParsing
-schemaSpace: cdf_cdm
-annotationSpace: springfield_instances

cognite_toolkit/_builtin_modules/cdf_common/extraction_pipelines/ctx_files_direct_relation_write.ExtractionPipeline.yaml

@@ -1,54 +0,0 @@
-externalId: '{{ directRelationJob }}'
-name: 'Direct Relation Writer'
-dataSetExternalId: {{ dataset }}
-description: Writes direct relations based on annotations.
-documentation: >
-  # Direct Relation Writer
-
-  This extraction pipeline is responsible for writing the direct relations into 
-  a data model based on annotations.
-
-  ## Annotations
-  Annotations are found based on three criteria:
-
-  1. It has to have data in the (`cdf_cdm", "CogniteAnnotation`, `v1`) view.
-  2. It is stored in the space configured in the `annotationSpace`.
-  3. It has to have a `sourceContext` that contains the views of the start and end nodes of the relations.
-
-  ```json
-  {"start" : {
-    "space": "cdf_cdm",
-    "externalId": "CogniteFile",
-    "version": "v1"
-  },
-  "end" : {
-    "space": "cdf_cdm",
-    "externalId": "CogniteAsset",
-    "version": "v1"
-  }}
-  ```
-
-  ## Configuration
-
-  The configuration of the direct relation writer:
-
-  ```yaml
-  state: # The Annotation writer uses the instances/sync endpoint to fetch annotations. 
-    # The state stores the last cursor.
-    rawDatabase: <>
-    rawTable: <>
-  data:
-    annotationSpace: <> # The space the edges for the annotations are stored.
-    directRelationMapping: # How to go from Annotation to DirectRelation
-      - startNodeView: # The view of the start node.
-         space: <> .
-         externalId: <>
-         version: <> 
-         directRelationProperty: <> # The property that contains the direct relation.
-          # Must be set either on the start or end node.
-        endNodeView: # The view of the end node.
-          space: <>
-          externalId: <>
-          version: <>
-          # directRelationProperty: <> # Must be set either on the start or end node.
-  ```

cognite_toolkit/_builtin_modules/cdf_common/extraction_pipelines/ctx_files_direct_relation_write.config.yaml

@@ -1,28 +0,0 @@
-externalId: '{{ directRelationJob }}'
-config:
-  state:
-    rawDatabase: {{ rawStateDatabase }}
-    rawTable: {{ rawStateTable }}
-  data:
-    annotationSpace: '{{ annotationSpace }}'
-    directRelationMappings:
-      - startNodeView:
-          space: {{ schemaSpace }}
-          externalId: CogniteFile
-          version: v1
-          directRelationProperty: assets
-        endNodeView:
-          space: {{ schemaSpace }}
-          externalId: CogniteAsset
-          version: v1
-      - startNodeView:
-          type: view
-          space: {{ schemaSpace }}
-          externalId: CogniteFile
-          version: v1
-        endNodeView:
-          type: viewProperty
-          space: {{ schemaSpace }}
-          externalId: CogniteEquipment
-          version: v1
-          directRelationProperty: files