cmem-plugin-validation 1.1.2__tar.gz → 1.3.0__tar.gz

{cmem_plugin_validation-1.1.2 → cmem_plugin_validation-1.3.0}/PKG-INFO

@@ -1,22 +1,24 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: cmem-plugin-validation
-Version: 1.1.2
+Version: 1.3.0
 Summary: Validate graph and data structures.
-Home-page: https://github.com/eccenca/cmem-plugin-validation
 License: Apache-2.0
+License-File: LICENSE
 Keywords: eccenca Corporate Memory,plugin
 Author: eccenca GmbH
 Author-email: cmempy-developer@eccenca.com
-Requires-Python: >=3.11,<4.0
+Requires-Python: >=3.13,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Plugins
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: cmem-plugin-base (>=4.7.0,<5.0.0)
-Requires-Dist: jsonschema (>=4.23.0,<5.0.0)
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: cmem-client (>=1.0.0,<2.0.0)
+Requires-Dist: cmem-plugin-base (>=4.19.0,<5.0.0)
+Requires-Dist: httpx (>=0.27.0,<0.28.0)
+Requires-Dist: jsonschema (>=4.25.1,<5.0.0)
+Project-URL: Homepage, https://github.com/eccenca/cmem-plugin-validation
 Description-Content-Type: text/markdown
 
 # cmem-plugin-validation
@@ -25,7 +27,7 @@ Validate graph and data structures.
 
 [![eccenca Corporate Memory][cmem-shield]][cmem-link]
 
-This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line clients like this:
+This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
 
 ```
 cmemc admin workspace python install cmem-plugin-validation

{cmem_plugin_validation-1.1.2 → cmem_plugin_validation-1.3.0}/README-public.md

@@ -4,7 +4,7 @@ Validate graph and data structures.
 
 [![eccenca Corporate Memory][cmem-shield]][cmem-link]
 
-This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line clients like this:
+This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
 
 ```
 cmemc admin workspace python install cmem-plugin-validation

{cmem_plugin_validation-1.1.2 → cmem_plugin_validation-1.3.0}/cmem_plugin_validation/validate_entities/task.py

@@ -7,12 +7,10 @@ from collections.abc import Generator, Sequence
 from types import SimpleNamespace
 from typing import Any
 
-from cmem.cmempy.workspace.projects.resources.resource import get_resource_response
-from cmem.cmempy.workspace.tasks import get_task
+from cmem_client.client import Client
 from cmem_plugin_base.dataintegration.context import (
     ExecutionContext,
     ExecutionReport,
-    UserContext,
 )
 from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
 from cmem_plugin_base.dataintegration.entity import Entities
@@ -24,11 +22,6 @@ from cmem_plugin_base.dataintegration.ports import (
     FlexibleNumberOfInputs,
     UnknownSchemaPort,
 )
-from cmem_plugin_base.dataintegration.utils import (
-    setup_cmempy_user_access,
-    split_task_id,
-    write_to_dataset,
-)
 from cmem_plugin_base.dataintegration.utils.entity_builder import build_entities_from_data
 from jsonschema import validate
 from jsonschema.exceptions import ValidationError
@@ -45,9 +38,11 @@ JSON Schema specification.
 The used JSON Schema needs to be provided as a JSON Dataset in the project.
 
 ### Input Modes
+
 The plugin supports two input modes for validation:
+
 1. **Validate Entities**: Validates entities received from the input port in the workflow.
-2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project. 
+2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project.
    - If the JSON dataset is a JSON array, the schema will validate each object inside the array.
    - If the JSON dataset is a JSON object, it will be validated against the schema directly.
 
@@ -55,28 +50,26 @@ Validated data objects can be sent to an output port for further processing in t
 or saved in a JSON dataset in the project.
 
 ### Output Modes
-1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities to the output port.
-2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset in the project.
+
+1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities
+   to the output port.
+2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset
+   in the project.
 
 ### Error Handling
+
 The task can either:
+
 - Fail instantly if there is a data violation, halting the workflow.
-- Provide warnings in the workflow report, allowing follow-up tasks to run based on the validated data.
+- Provide warnings in the workflow report, allowing follow-up tasks to run based on the
+  validated data.
 
 The error handling behavior is configurable through the `Fail on violations` parameter.
-
 """
 
 
 DEFAULT_FAIL_ON_VIOLATION = False
 
-
-def get_task_metadata(project: str, task: str, context: UserContext) -> dict:
-    """Get metadata information of a task"""
-    setup_cmempy_user_access(context=context)
-    return dict(get_task(project=project, task=task))
-
-
 SOURCE = SimpleNamespace()
 SOURCE.entities = "entities"
 SOURCE.dataset = "dataset"
@@ -165,7 +158,7 @@ class ValidateEntity(WorkflowPlugin):
     inputs: Sequence[Entities]
     execution_context: ExecutionContext
 
-    def __init__(  #  noqa: PLR0913
+    def __init__(  #  noqa: PLR0913 PLR0917
         self,
         source_mode: str,
         target_mode: str,
@@ -290,10 +283,10 @@ class ValidateEntity(WorkflowPlugin):
             )
         )
         if self.target_mode == TARGET.dataset:
-            write_to_dataset(
-                dataset_id=f"{context.task.project_id()}:{self.target_dataset}",
-                file_resource=io.StringIO(json.dumps(valid_json_objects)),
-                context=context.user,
+            Client.from_context(context=context).datasets.post_file_resource(
+                project_id=context.task.project_id(),
+                dataset_id=self.target_dataset,
+                file_resource=io.BytesIO(json.dumps(valid_json_objects).encode("utf-8")),
             )
             return None
 
@@ -312,16 +305,18 @@ class ValidateEntity(WorkflowPlugin):
     @staticmethod
     def _get_json_dataset_content(context: ExecutionContext, dataset: str) -> dict | list[dict]:
         """Get json dataset content"""
-        dataset_id = f"{context.task.project_id()}:{dataset}"
-        project_id, task_id = split_task_id(dataset_id)
-        task_meta_data = get_task_metadata(project_id, task_id, context.user)
-        resource_name = str(task_meta_data["data"]["parameters"]["file"]["value"])
-        response = get_resource_response(project_id, resource_name)
-        return response.json()  # type: ignore[no-any-return]
+        client = Client.from_context(context=context)
+        project_id = context.task.project_id()
+        resource_name = str(
+            client.datasets.get_item(project_id=project_id, dataset_id=dataset).data.parameters[
+                "file"
+            ]
+        )
+        return json.loads(client.files.read(f"{project_id}:{resource_name}"))  # type: ignore[no-any-return]
 
     def _convert_entities_to_json(
         self, inputs: Sequence[Entities], path_to_entities: dict[str, Entities], path: str = ""
-    ) -> Generator[dict[str, Any], None, None]:
+    ) -> Generator[dict[str, Any]]:
         """Convert a sequence of Entities into JSON-like dictionaries using recursive traversal."""
         for entities in inputs:
             # Initialize path-to-entities map for the root level

cmem_plugin_validation-1.3.0/cmem_plugin_validation/validate_graph/state.py

@@ -0,0 +1,31 @@
+"""Graph validation process state"""
+
+from cmem_client.client import Client
+
+
+class State:
+    """State of a validation process"""
+
+    client: Client
+    id_: str
+    data: dict
+    status: str
+    completed: int
+    total: int
+    with_violations: int
+    violations: int
+
+    def __init__(self, client: Client, id_: str):
+        self.client = client
+        self.id_ = id_
+        self.refresh()
+
+    def refresh(self) -> None:
+        """Refresh state of validation process"""
+        aggregation = self.client.validations.get_aggregation(batch_id=self.id_)
+        self.data = aggregation.model_dump(by_alias=True, exclude_none=True)
+        self.status = aggregation.state
+        self.completed = aggregation.resource_processed_count
+        self.total = aggregation.resource_count
+        self.with_violations = aggregation.resources_with_violations_count
+        self.violations = aggregation.violations_count

{cmem_plugin_validation-1.1.2 → cmem_plugin_validation-1.3.0}/cmem_plugin_validation/validate_graph/task.py

@@ -4,9 +4,9 @@ import json
 from collections.abc import Sequence
 from time import sleep
 
-from cmem.cmempy.dp.proxy import graph as graph_api
-from cmem.cmempy.dp.shacl import validation
-from cmem.cmempy.queries import SparqlQuery
+from cmem_client.client import Client
+from cmem_client.models.query_catalog import Query
+from cmem_client.models.validation import STATUS_RUNNING, STATUS_SCHEDULED, ValidationViolation
 from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
 from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
 from cmem_plugin_base.dataintegration.entity import (
@@ -21,9 +21,8 @@ from cmem_plugin_base.dataintegration.ports import (
     FixedNumberOfInputs,
     FixedSchemaPort,
 )
-from cmem_plugin_base.dataintegration.utils import setup_cmempy_user_access
 from cmem_plugin_base.dataintegration.utils.entity_builder import build_entities_from_data
-from requests import HTTPError
+from httpx import HTTPStatusError
 
 from cmem_plugin_validation.validate_graph.state import State
 
@@ -111,7 +110,7 @@ WHERE { ?resource a ?class . FILTER isIRI(?resource) }
 class ValidateGraph(WorkflowPlugin):
     """Validate resources in a graph"""
 
-    def __init__(  # noqa: PLR0913
+    def __init__(  # noqa: PLR0913 PLR0917
         self,
         context_graph: str,
         shape_graph: str = DEFAULT_SHAPE_GRAPH,
@@ -156,33 +155,29 @@ class ValidateGraph(WorkflowPlugin):
     ) -> Entities | None:
         """Run the workflow operator."""
         self.log.info("Start validation task.")
-        setup_cmempy_user_access(context=context.user)
+        client = Client.from_context(context=context)
         if self.clear_result_graph and self.result_graph:
-            graph_api.delete(graph=self.result_graph)
-        query = SparqlQuery(text=self.sparql_query).get_filled_text(
-            placeholder={"context_graph": self.context_graph}
+            client.graphs.delete_item(key=self.result_graph, skip_if_missing=True)
+        query = Query(text=self.sparql_query).fill_placeholders(
+            placeholders={"context_graph": self.context_graph}
         )
         try:
-            process_id = validation.start(
+            process_id = client.validations.start(
                 context_graph=self.context_graph,
                 shape_graph=self.shape_graph,
-                result_graph=self.result_graph if self.result_graph else None,
+                result_graph=self.result_graph or None,
                 query=query,
             )
-        except HTTPError as error:
-            context.report.update(
-                ExecutionReport(
-                    error=json.loads(error.response.text)["detail"],
-                )
-            )
-            raise RuntimeError(json.loads(error.response.text)["detail"]) from error
-        state = State(id_=process_id)
+        except HTTPStatusError as error:
+            detail = json.loads(error.response.text)["detail"]
+            context.report.update(ExecutionReport(error=detail))
+            raise RuntimeError(detail) from error
+        state = State(client=client, id_=process_id)
         while True:
             sleep(1)
-            setup_cmempy_user_access(context=context.user)
             state.refresh()
             if context.workflow and context.workflow.status() != "Running":
-                validation.cancel(batch_id=process_id)
+                client.validations.cancel(batch_id=process_id)
                 context.report.update(
                     ExecutionReport(
                         entity_count=state.completed,
@@ -192,7 +187,7 @@ class ValidateGraph(WorkflowPlugin):
                 )
                 self.log.info("End validation task (Cancelled Workflow).")
                 return None
-            if state.status in (validation.STATUS_SCHEDULED, validation.STATUS_RUNNING):
+            if state.status in (STATUS_SCHEDULED, STATUS_RUNNING):
                 # when reported as running or scheduled, start another loop
                 context.report.update(
                     ExecutionReport(
@@ -224,11 +219,22 @@ class ValidateGraph(WorkflowPlugin):
         if not self.output_results:
             return None
 
-        violations = []
-        for result in list(validation.get(batch_id=process_id)["results"]):
-            resource_iri = result.get("resourceIri")
-            for _ in result["violations"]:
-                violation = dict(_)
-                violation["resourceIri"] = resource_iri
-                violations.append(violation)
+        violations = [
+            self._as_violation_data(violation, resource_iri=result.resource_iri)
+            for result in client.validations.get_result(batch_id=process_id).results
+            for violation in result.violations
+        ]
         return build_entities_from_data(data=violations)
+
+    def _as_violation_data(self, violation: ValidationViolation, resource_iri: str) -> dict:
+        """Turn a violation into a plain dict ordered like the output schema
+
+        The entity paths of the returned entities are derived from the key order of these
+        dicts, so the keys of the output schema come first, followed by any additional key
+        the API delivers and the IRI of the validated resource.
+        """
+        data: dict = violation.model_dump(by_alias=True, exclude_none=True)
+        ordered: dict = {
+            _.path: data.pop(_.path) for _ in self.output_schema.paths if _.path in data
+        }
+        return ordered | data | {"resourceIri": resource_iri}

{cmem_plugin_validation-1.1.2 → cmem_plugin_validation-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cmem-plugin-validation"
-version = "1.1.2"
+version = "1.3.0"
 license = "Apache-2.0"
 description = "Validate graph and data structures."
 authors = ["eccenca GmbH <cmempy-developer@eccenca.com>"]
@@ -13,29 +13,36 @@ keywords = [
 ]
 homepage = "https://github.com/eccenca/cmem-plugin-validation"
 
+[tool.poetry.requires-plugins]
+poetry-plugin-export = "^1.10.0"
+poetry-plugin-shell = "^1.0.1"
+poetry-dynamic-versioning = "^1.10.0"
+
 [tool.poetry.dependencies]# if you need to change python version here, change it also in .python-version
-python = "^3.11"
-jsonschema = "^4.23.0"
+python = "^3.13"
+jsonschema = "^4.25.1"
+cmem-client = "^1.0.0"
+httpx = "^0.27.0"
 
 [tool.poetry.dependencies.cmem-plugin-base]
-version = "^4.7.0"
+version = "^4.19.0"
 allow-prereleases = false
 
 [tool.poetry.group.dev.dependencies.cmem-cmemc]
-version = "^24.2.0"
+version = ">=24.2.0"
 
 [tool.poetry.group.dev.dependencies]
-genbadge = {extras = ["coverage"], version = "^1.1.1"}
-mypy = "^1.11.1"
-pip = "^24"
-pytest = "^8.3.2"
-pytest-cov = "^5.0.0"
+deptry = "^0.25.1"
+genbadge = {extras = ["coverage"], version = "^1.1.3"}
+mypy = "^2.3.0"
+pip = "^26"
+pytest = "^9.1.1"
+pytest-cov = "^7.1.0"
 pytest-dotenv = "^0.5.2"
-pytest-html = "^4.1.1"
-pytest-memray = { version = "^1.7.0",  markers = "platform_system != 'Windows'" }
-ruff = "^0.6.1"
-safety = "^1.10.3"
-types-requests = "^2.31.0.20240406"
+pytest-html = "^4.2.0"
+pytest-memray = { version = "^1.10.0",  markers = "platform_system != 'Windows'" }
+ruff = "^0.16.2"
+trivy-py-ecc = "^0.73.0.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
@@ -68,7 +75,7 @@ exclude_also = [
 
 [tool.ruff]
 line-length = 100
-target-version = "py311"
+target-version = "py313"
 
 [tool.ruff.format]
 line-ending = "lf" # Use `\n` line endings for all files
@@ -76,7 +83,6 @@ line-ending = "lf" # Use `\n` line endings for all files
 [tool.ruff.lint]
 select = ["ALL"]
 ignore = [
-    "ANN101",  # Missing type annotation for self in method
     "ANN204",  # Missing return type annotation for special method `__init__`
     "COM812",  # missing-trailing-comma
     "D107",    # Missing docstring in __init__
@@ -93,5 +99,5 @@ ignore = [
     "PD",      # opinionated linting for pandas code
     "S101",    # use of assert detected
     "TRY003",  # Avoid specifying long messages outside the exception class
+    "CPY001",  # Missing copyright notice
 ]
-

cmem_plugin_validation-1.1.2/cmem_plugin_validation/validate_graph/state.py

@@ -1,28 +0,0 @@
-"""Graph validation process state"""
-
-from cmem.cmempy.dp.shacl import validation
-
-
-class State:
-    """State of a validation process"""
-
-    id_: str
-    data: dict
-    status: str
-    completed: int
-    total: int
-    with_violations: int
-    violations: int
-
-    def __init__(self, id_: str):
-        self.id_ = id_
-        self.refresh()
-
-    def refresh(self) -> None:
-        """Refresh state of validation process"""
-        self.data = validation.get_aggregation(batch_id=self.id_)
-        self.status = self.data.get("state", "UNKNOWN")
-        self.completed = self.data.get("resourceProcessedCount", 0)
-        self.total = self.data.get("resourceCount", 0)
-        self.with_violations = self.data.get("resourcesWithViolationsCount", 0)
-        self.violations = self.data.get("violationsCount", 0)