cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.87.4"
+__version__ = "0.88.0"

cognite/neat/app/api/data_classes/rest.py

@@ -16,25 +16,6 @@ class RuleRequest(BaseModel):
     rule: str
 
 
-class UploadToCdfRequest(BaseModel):
-    file_name: str = ""
-    file_type: str = "workflow"
-    comments: str = ""
-    author: str = ""
-    tag: str = ""
-
-
-class DownloadFromCdfRequest(BaseModel):
-    file_name: str = ""
-    file_type: str = "workflow"
-    version: str = ""
-
-
-class LoadGraphRequest(BaseModel):
-    graph_source_template_name: str
-    source_location: str
-
-
 class RunWorkflowRequest(BaseModel):
     name: str
     config: dict

cognite/neat/app/api/explorer.py

@@ -10,7 +10,12 @@ from cognite import neat
 from cognite.neat.app.api.asgi.metrics import prometheus_app
 from cognite.neat.app.api.configuration import NEAT_APP, UI_PATH
 from cognite.neat.app.api.context_manager import lifespan
-from cognite.neat.app.api.routers import configuration, core, crud, data_exploration, metrics, rules, workflows
+from cognite.neat.app.api.routers import (
+    configuration,
+    crud,
+    metrics,
+    workflows,
+)
 from cognite.neat.app.api.utils.logging import EndpointFilter
 
 app = FastAPI(title="Neat", lifespan=lifespan)
@@ -45,10 +50,7 @@ app.mount("/data", StaticFiles(directory=NEAT_APP.config.data_store_path), name=
 app.include_router(configuration.router)
 app.include_router(metrics.router)
 app.include_router(workflows.router)
-app.include_router(rules.router)
 app.include_router(crud.router)
-app.include_router(data_exploration.router)
-app.include_router(core.router)
 
 
 # General routes

cognite/neat/app/api/routers/crud.py

@@ -14,28 +14,13 @@ from cognite.neat.workflows.utils import get_file_hash
 router = APIRouter()
 
 
-@router.get("/api/cdf/neat-resources")
-def get_neat_resources(resource_type: str | None = None):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    if resource_type is None:
-        return {"error": "Resource type is not specified"}
-    result = NEAT_APP.cdf_store.get_list_of_resources_from_cdf(resource_type=resource_type)
-    logging.debug(f"Got {len(result)} resources")
-    return {"result": result}
-
-
-@router.post("/api/cdf/init-neat-resources")
-def init_neat_cdf_resources(resource_type: str | None = None):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    NEAT_APP.cdf_store.init_cdf_resources(resource_type=resource_type)
-    return {"result": "ok"}
-
-
 @router.post("/api/file/upload/{workflow_name}/{file_type}/{step_id}/{action}")
 async def file_upload_handler(
-    files: list[UploadFile], workflow_name: str, file_type: str, step_id: str, action: str
+    files: list[UploadFile],
+    workflow_name: str,
+    file_type: str,
+    step_id: str,
+    action: str,
 ) -> dict[str, str]:
     if NEAT_APP.cdf_store is None or NEAT_APP.workflow_manager is None:
         return {"error": "NeatApp is not initialized"}
@@ -91,7 +76,12 @@ async def file_upload_handler(
         if workflow is None:
             return {"error": f"Workflow {workflow_name} not found"}
         flow_msg = FlowMessage(
-            payload={"file_name": file_name, "hash": file_version, "full_path": full_path, "file_type": file_type}
+            payload={
+                "file_name": file_name,
+                "hash": file_version,
+                "full_path": full_path,
+                "file_type": file_type,
+            }
         )
         start_step_id = None if step_id == "none" else step_id
 

cognite/neat/app/api/routers/workflows.py

@@ -1,20 +1,19 @@
 import logging
 import shutil
-from pathlib import Path
 from typing import cast
 
 from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile
 from fastapi.responses import FileResponse, JSONResponse
 
 from cognite.neat.app.api.configuration import NEAT_APP
-from cognite.neat.app.api.data_classes.rest import DownloadFromCdfRequest, RunWorkflowRequest, UploadToCdfRequest
+from cognite.neat.app.api.data_classes.rest import (
+    RunWorkflowRequest,
+)
 from cognite.neat.workflows import WorkflowFullStateReport
 from cognite.neat.workflows.base import WorkflowDefinition
-from cognite.neat.workflows.migration.wf_manifests import migrate_wf_manifest
 from cognite.neat.workflows.model import FlowMessage
-from cognite.neat.workflows.steps.data_contracts import SolutionGraph, SourceGraph
+from cognite.neat.workflows.steps.data_contracts import NeatGraph
 from cognite.neat.workflows.steps.step_model import DataContract
-from cognite.neat.workflows.utils import get_file_hash
 
 router = APIRouter()
 
@@ -27,7 +26,11 @@ def start_workflow(request: RunWorkflowRequest):
     start_status = NEAT_APP.workflow_manager.start_workflow_instance(
         request.name, sync=request.sync, flow_msg=FlowMessage()
     )
-    result = {"workflow_instance": None, "is_success": start_status.is_success, "status_text": start_status.status_text}
+    result = {
+        "workflow_instance": None,
+        "is_success": start_status.is_success,
+        "status_text": start_status.status_text,
+    }
     return {"result": result}
 
 
@@ -58,25 +61,6 @@ def get_workflow_files(workflow_name: str):
     return {"files": workflow.get_list_of_workflow_artifacts()}
 
 
-@router.post("/api/workflow/package/{workflow_name}")
-def package_workflow(workflow_name: str):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    package_file = NEAT_APP.cdf_store.package_workflow(workflow_name)
-    hash_ = get_file_hash(NEAT_APP.config.workflows_store_path / package_file)
-    return {"package": package_file, "hash": hash_}
-
-
-@router.post("/api/workflow/context-cleanup/{workflow_name}")
-def cleanup_workflow_data(workflow_name: str):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
-    if workflow is not None:
-        workflow.cleanup_workflow_context()
-    return {"result": "ok"}
-
-
 @router.post("/api/workflow/create")
 def create_new_workflow(request: WorkflowDefinition):
     if NEAT_APP.workflow_manager is None:
@@ -95,30 +79,24 @@ def delete_workflow(workflow_name: str):
     return {"result": "ok"}
 
 
-@router.get("/api/workflow/executions")
-def get_list_of_workflow_executions():
-    return {"executions": NEAT_APP.cdf_store.get_list_of_workflow_executions_from_cdf()}
-
-
-@router.get("/api/workflow/detailed-execution-report/{execution_id}")
-def get_detailed_execution(execution_id: str):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    return {"report": NEAT_APP.cdf_store.get_detailed_workflow_execution_report_from_cdf(execution_id)}
-
-
 @router.post("/api/workflow/reload-workflows")
 def reload_workflows():
     NEAT_APP.workflow_manager.load_workflows_from_storage()
     NEAT_APP.triggers_manager.reload_all_triggers()
-    return {"result": "ok", "workflows": NEAT_APP.workflow_manager.get_list_of_workflows()}
+    return {
+        "result": "ok",
+        "workflows": NEAT_APP.workflow_manager.get_list_of_workflows(),
+    }
 
 
 @router.post("/api/workflow/reload-single-workflow/{workflow_name}")
 def reload_single_workflows(workflow_name: str):
     NEAT_APP.workflow_manager.load_single_workflow_from_storage(workflow_name)
     NEAT_APP.triggers_manager.reload_all_triggers()
-    return {"result": "ok", "workflows": NEAT_APP.workflow_manager.get_list_of_workflows()}
+    return {
+        "result": "ok",
+        "workflows": NEAT_APP.workflow_manager.get_list_of_workflows(),
+    }
 
 
 @router.get("/api/workflow/workflow-definition/{workflow_name}")
@@ -152,58 +130,6 @@ def update_workflow_definition(workflow_name: str, request: WorkflowDefinition):
     return {"result": "ok"}
 
 
-@router.post("/api/workflow/upload-wf-to-cdf/{workflow_name}")
-def upload_workflow_to_cdf(workflow_name: str, request: UploadToCdfRequest):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    NEAT_APP.cdf_store.save_workflow_to_cdf(
-        workflow_name, changed_by=request.author, comments=request.comments, tag=request.tag
-    )
-    return {"result": "ok"}
-
-
-@router.post("/api/workflow/upload-rules-cdf/{workflow_name}")
-def upload_rules_to_cdf(workflow_name: str, request: UploadToCdfRequest):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    file_path = Path(NEAT_APP.config.rules_store_path, request.file_name)
-    NEAT_APP.cdf_store.save_resource_to_cdf(
-        workflow_name, "neat-wf-rules", file_path, changed_by=request.author, comments=request.comments
-    )
-    return {"result": "ok"}
-
-
-@router.post("/api/workflow/download-wf-from-cdf")
-def download_wf_from_cdf(request: DownloadFromCdfRequest):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    NEAT_APP.cdf_store.load_workflows_from_cdf(request.file_name, request.version)
-    return {"result": "ok"}
-
-
-@router.post("/api/workflow/download-rules-from-cdf")
-def download_rules_to_cdf(request: DownloadFromCdfRequest):
-    if NEAT_APP.cdf_store is None:
-        return {"error": "NeatApp is not initialized"}
-    NEAT_APP.cdf_store.load_rules_file_from_cdf(request.file_name, request.version)
-    return {"file_name": request.file_name, "hash": request.version}
-
-
-@router.post("/api/workflow/migrate-workflow")
-def migrate_workflow():
-    return migrate_wf_manifest(NEAT_APP.config.data_store_path)
-
-
-@router.get("/api/workflow/pre-cdf-assets/{workflow_name}")
-def get_pre_cdf_assets(workflow_name: str):
-    if NEAT_APP.workflow_manager is None:
-        return {"error": "Workflow Manager is not initialized"}
-    workflow = NEAT_APP.workflow_manager.get_workflow(workflow_name)
-    if workflow is None:
-        return {"assets": []}
-    return {"assets": workflow.data["CategorizedAssets"]}
-
-
 @router.get("/api/workflow/context/{workflow_name}")
 def get_context(workflow_name: str):
     if NEAT_APP.workflow_manager is None:
@@ -228,8 +154,8 @@ def get_context_object(workflow_name: str, object_name: str):
     if object_name not in context:
         return {"error": f"Item {object_name} is not found in workflow context"}
 
-    if object_name == "SourceGraph" or object_name == "SolutionGraph":
-        return {"object": cast(SourceGraph | SolutionGraph, context[object_name]).graph.diagnostic_report()}
+    if object_name == "NeatGraph":
+        return {"object": cast(NeatGraph, context[object_name]).graph.type_}
 
     cobject = context[object_name]
     if isinstance(cobject, DataContract):
@@ -285,7 +211,11 @@ def http_trigger_start_workflow(workflow_name: str, step_id: str, request: Reque
 
 @router.post("/api/workflow/{workflow_name}/resume/{step_id}/{instance_id}")
 def http_trigger_resume_workflow(
-    workflow_name: str, step_id: str, instance_id: str, request: Request, body: bytes = fast_api_depends
+    workflow_name: str,
+    step_id: str,
+    instance_id: str,
+    request: Request,
+    body: bytes = fast_api_depends,
 ):
     if NEAT_APP.triggers_manager is None:
         return JSONResponse(content={"error": "Triggers Manager is not initialized"}, status_code=400)

cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -30,6 +30,8 @@ class AssetsExtractor(BaseExtractor):
             limit the extraction to 1000 assets to test the setup.
         unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
             a JSON string.
+        skip_metadata_values (set[str] | frozenset[str] | None, optional): A set of values to skip when unpacking
+            metadata. Defaults to frozenset({"nan", "null", "none", ""}).
     """
 
     _SPACE_PATTERN = re.compile(r"\s+")
@@ -42,6 +44,7 @@ class AssetsExtractor(BaseExtractor):
         total: int | None = None,
         limit: int | None = None,
         unpack_metadata: bool = True,
+        skip_metadata_values: set[str] | frozenset[str] | None = frozenset({"nan", "null", "none", ""}),
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
         self.assets = assets
@@ -49,6 +52,7 @@ class AssetsExtractor(BaseExtractor):
         self.total = total
         self.limit = min(limit, total) if limit and total else limit
         self.unpack_metadata = unpack_metadata
+        self.skip_metadata_values = skip_metadata_values
 
     @classmethod
     def from_dataset(
@@ -162,7 +166,7 @@ class AssetsExtractor(BaseExtractor):
         if asset.source:
             triples.append((id_, self.namespace.source, Literal(asset.source)))
 
-        # properties ref creation and update
+        # properties' ref creation and update
         triples.append(
             (
                 id_,
@@ -193,7 +197,9 @@ class AssetsExtractor(BaseExtractor):
         if asset.metadata:
             if self.unpack_metadata:
                 for key, value in asset.metadata.items():
-                    if value:
+                    if value and (
+                        self.skip_metadata_values is None or value.casefold() not in self.skip_metadata_values
+                    ):
                         triples.append(
                             (
                                 id_,

cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -250,9 +250,9 @@ def _generate_mock_data_property_triples(
     python_type = value_type.python
     triples = []
     for id_ in instance_ids:
-        if python_type == int:
+        if python_type is int:
             triples.append((id_, URIRef(namespace[property_]), Literal(random.randint(1, 1983))))
-        elif python_type == float:
+        elif python_type is float:
             triples.append((id_, URIRef(namespace[property_]), Literal(numpy.float32(random.uniform(1, 1983)))))
         # generate string
         else:

cognite/neat/graph/loaders/_base.py

@@ -48,18 +48,23 @@ class BaseLoader(ABC, Generic[T_Output]):
 class CDFLoader(BaseLoader[T_Output]):
     _UPLOAD_BATCH_SIZE: ClassVar[int] = 1000
 
-    def load_into_cdf(self, client: CogniteClient, dry_run: bool = False) -> UploadResultList:
-        return UploadResultList(self.load_into_cdf_iterable(client, dry_run))
-
-    def load_into_cdf_iterable(self, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
-        missing_capabilities = client.iam.verify_capabilities(self._get_required_capabilities())
-        if missing_capabilities:
-            upload_result = UploadResult[Hashable](name=type(self).__name__)
-            upload_result.issues.append(
-                FailedAuthorizationError(action="Upload to CDF", reason=str(missing_capabilities))
-            )
-            yield upload_result
-            return
+    def load_into_cdf(
+        self, client: CogniteClient, dry_run: bool = False, check_client: bool = True
+    ) -> UploadResultList:
+        return UploadResultList(self.load_into_cdf_iterable(client, dry_run, check_client))
+
+    def load_into_cdf_iterable(
+        self, client: CogniteClient, dry_run: bool = False, check_client: bool = True
+    ) -> Iterable[UploadResult]:
+        if check_client:
+            missing_capabilities = client.iam.verify_capabilities(self._get_required_capabilities())
+            if missing_capabilities:
+                upload_result = UploadResult[Hashable](name=type(self).__name__)
+                upload_result.issues.append(
+                    FailedAuthorizationError(action="Upload to CDF", reason=str(missing_capabilities))
+                )
+                yield upload_result
+                return
 
         issues = NeatIssueList[NeatIssue]()
         items: list[T_Output] = []