cmem-plugin-validation 1.2.0__tar.gz → 1.3.1__tar.gz

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2021 CMEM
+   Copyright eccenca GmbH
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmem-plugin-validation
-Version: 1.2.0
+Version: 1.3.1
 Summary: Validate graph and data structures.
 License: Apache-2.0
 License-File: LICENSE
@@ -14,10 +14,10 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Requires-Dist: cmem-cmempy (>=25.3.0)
-Requires-Dist: cmem-plugin-base (>=4.15.0,<5.0.0)
+Requires-Dist: cmem-client (>=1.0.0,<2.0.0)
+Requires-Dist: cmem-plugin-base (>=4.20.0,<5.0.0)
+Requires-Dist: httpx (>=0.27.0,<0.28.0)
 Requires-Dist: jsonschema (>=4.25.1,<5.0.0)
-Requires-Dist: requests (>=2.0.1)
 Project-URL: Homepage, https://github.com/eccenca/cmem-plugin-validation
 Description-Content-Type: text/markdown
 
@@ -27,7 +27,7 @@ Validate graph and data structures.
 
 [![eccenca Corporate Memory][cmem-shield]][cmem-link]
 
-This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
+This is a plugin for [eccenca Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
 
 ```
 cmemc admin workspace python install cmem-plugin-validation
@@ -36,7 +36,7 @@ cmemc admin workspace python install cmem-plugin-validation
 [![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] 
 
 [cmem-link]: https://documentation.eccenca.com
-[cmem-shield]: https://img.shields.io/endpoint?url=https://dev.documentation.eccenca.com/badge.json
+[cmem-shield]: https://img.shields.io/endpoint?url=https://documentation.eccenca.com/latest/badge.json
 [poetry-link]: https://python-poetry.org/
 [poetry-shield]: https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json
 [ruff-link]: https://docs.astral.sh/ruff/

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/README-public.md

@@ -4,7 +4,7 @@ Validate graph and data structures.
 
 [![eccenca Corporate Memory][cmem-shield]][cmem-link]
 
-This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
+This is a plugin for [eccenca Corporate Memory](https://documentation.eccenca.com). You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line client like this:
 
 ```
 cmemc admin workspace python install cmem-plugin-validation
@@ -13,7 +13,7 @@ cmemc admin workspace python install cmem-plugin-validation
 [![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] 
 
 [cmem-link]: https://documentation.eccenca.com
-[cmem-shield]: https://img.shields.io/endpoint?url=https://dev.documentation.eccenca.com/badge.json
+[cmem-shield]: https://img.shields.io/endpoint?url=https://documentation.eccenca.com/latest/badge.json
 [poetry-link]: https://python-poetry.org/
 [poetry-shield]: https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json
 [ruff-link]: https://docs.astral.sh/ruff/

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/cmem_plugin_validation/validate_entities/task.py

@@ -7,12 +7,10 @@ from collections.abc import Generator, Sequence
 from types import SimpleNamespace
 from typing import Any
 
-from cmem.cmempy.workspace.projects.resources.resource import get_resource_response
-from cmem.cmempy.workspace.tasks import get_task
+from cmem_client.client import Client
 from cmem_plugin_base.dataintegration.context import (
     ExecutionContext,
     ExecutionReport,
-    UserContext,
 )
 from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
 from cmem_plugin_base.dataintegration.entity import Entities
@@ -24,11 +22,6 @@ from cmem_plugin_base.dataintegration.ports import (
     FlexibleNumberOfInputs,
     UnknownSchemaPort,
 )
-from cmem_plugin_base.dataintegration.utils import (
-    setup_cmempy_user_access,
-    split_task_id,
-    write_to_dataset,
-)
 from cmem_plugin_base.dataintegration.utils.entity_builder import build_entities_from_data
 from jsonschema import validate
 from jsonschema.exceptions import ValidationError
@@ -77,13 +70,6 @@ The error handling behavior is configurable through the `Fail on violations` par
 
 DEFAULT_FAIL_ON_VIOLATION = False
 
-
-def get_task_metadata(project: str, task: str, context: UserContext) -> dict:
-    """Get metadata information of a task"""
-    setup_cmempy_user_access(context=context)
-    return dict(get_task(project=project, task=task))
-
-
 SOURCE = SimpleNamespace()
 SOURCE.entities = "entities"
 SOURCE.dataset = "dataset"
@@ -172,7 +158,7 @@ class ValidateEntity(WorkflowPlugin):
     inputs: Sequence[Entities]
     execution_context: ExecutionContext
 
-    def __init__(  #  noqa: PLR0913
+    def __init__(  #  noqa: PLR0913 PLR0917
         self,
         source_mode: str,
         target_mode: str,
@@ -297,10 +283,10 @@ class ValidateEntity(WorkflowPlugin):
             )
         )
         if self.target_mode == TARGET.dataset:
-            write_to_dataset(
-                dataset_id=f"{context.task.project_id()}:{self.target_dataset}",
-                file_resource=io.StringIO(json.dumps(valid_json_objects)),
-                context=context.user,
+            Client.from_context(context=context).datasets.post_file_resource(
+                project_id=context.task.project_id(),
+                dataset_id=self.target_dataset,
+                file_resource=io.BytesIO(json.dumps(valid_json_objects).encode("utf-8")),
             )
             return None
 
@@ -319,16 +305,18 @@ class ValidateEntity(WorkflowPlugin):
     @staticmethod
     def _get_json_dataset_content(context: ExecutionContext, dataset: str) -> dict | list[dict]:
         """Get json dataset content"""
-        dataset_id = f"{context.task.project_id()}:{dataset}"
-        project_id, task_id = split_task_id(dataset_id)
-        task_meta_data = get_task_metadata(project_id, task_id, context.user)
-        resource_name = str(task_meta_data["data"]["parameters"]["file"]["value"])
-        response = get_resource_response(project_id, resource_name)
-        return response.json()  # type: ignore[no-any-return]
+        client = Client.from_context(context=context)
+        project_id = context.task.project_id()
+        resource_name = str(
+            client.datasets.get_item(project_id=project_id, dataset_id=dataset).data.parameters[
+                "file"
+            ]
+        )
+        return json.loads(client.files.read(f"{project_id}:{resource_name}"))  # type: ignore[no-any-return]
 
     def _convert_entities_to_json(
         self, inputs: Sequence[Entities], path_to_entities: dict[str, Entities], path: str = ""
-    ) -> Generator[dict[str, Any], None, None]:
+    ) -> Generator[dict[str, Any]]:
         """Convert a sequence of Entities into JSON-like dictionaries using recursive traversal."""
         for entities in inputs:
             # Initialize path-to-entities map for the root level

cmem_plugin_validation-1.3.1/cmem_plugin_validation/validate_graph/state.py

@@ -0,0 +1,31 @@
+"""Graph validation process state"""
+
+from cmem_client.client import Client
+
+
+class State:
+    """State of a validation process"""
+
+    client: Client
+    id_: str
+    data: dict
+    status: str
+    completed: int
+    total: int
+    with_violations: int
+    violations: int
+
+    def __init__(self, client: Client, id_: str):
+        self.client = client
+        self.id_ = id_
+        self.refresh()
+
+    def refresh(self) -> None:
+        """Refresh state of validation process"""
+        aggregation = self.client.validations.get_aggregation(batch_id=self.id_)
+        self.data = aggregation.model_dump(by_alias=True, exclude_none=True)
+        self.status = aggregation.state
+        self.completed = aggregation.resource_processed_count
+        self.total = aggregation.resource_count
+        self.with_violations = aggregation.resources_with_violations_count
+        self.violations = aggregation.violations_count

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/cmem_plugin_validation/validate_graph/task.py

@@ -4,9 +4,9 @@ import json
 from collections.abc import Sequence
 from time import sleep
 
-from cmem.cmempy.dp.proxy import graph as graph_api
-from cmem.cmempy.dp.shacl import validation
-from cmem.cmempy.queries import SparqlQuery
+from cmem_client.client import Client
+from cmem_client.models.query_catalog import Query
+from cmem_client.models.validation import STATUS_RUNNING, STATUS_SCHEDULED, ValidationViolation
 from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
 from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
 from cmem_plugin_base.dataintegration.entity import (
@@ -21,9 +21,8 @@ from cmem_plugin_base.dataintegration.ports import (
     FixedNumberOfInputs,
     FixedSchemaPort,
 )
-from cmem_plugin_base.dataintegration.utils import setup_cmempy_user_access
 from cmem_plugin_base.dataintegration.utils.entity_builder import build_entities_from_data
-from requests import HTTPError
+from httpx import HTTPStatusError
 
 from cmem_plugin_validation.validate_graph.state import State
 
@@ -32,6 +31,12 @@ Start a graph validation process which verifies, that resources in a specific gr
 according to the node shapes in a shape catalog graph.
 """
 
+CONTEXT_GRAPH_CLASSES = [
+    "https://vocab.eccenca.com/di/Dataset",
+    "http://rdfs.org/ns/void#Dataset",
+    "https://vocab.eccenca.com/shui/QueryCatalog",
+    "http://www.w3.org/2002/07/owl#Ontology",
+]
 DEFAULT_SHAPE_GRAPH = "https://vocab.eccenca.com/shacl/"
 DEFAULT_RESULT_GRAPH = ""
 DEFAULT_CLEAR_RESULT_GRAPH = False
@@ -55,6 +60,7 @@ WHERE { ?resource a ?class . FILTER isIRI(?resource) }
             label="Context Graph",
             description="This graph holds the resources you want to validate.",
             param_type=GraphParameterType(
+                classes=CONTEXT_GRAPH_CLASSES,
                 show_di_graphs=False,
                 show_graphs_without_class=True,
                 show_system_graphs=True,
@@ -111,7 +117,7 @@ WHERE { ?resource a ?class . FILTER isIRI(?resource) }
 class ValidateGraph(WorkflowPlugin):
     """Validate resources in a graph"""
 
-    def __init__(  # noqa: PLR0913
+    def __init__(  # noqa: PLR0913 PLR0917
         self,
         context_graph: str,
         shape_graph: str = DEFAULT_SHAPE_GRAPH,
@@ -156,33 +162,29 @@ class ValidateGraph(WorkflowPlugin):
     ) -> Entities | None:
         """Run the workflow operator."""
         self.log.info("Start validation task.")
-        setup_cmempy_user_access(context=context.user)
+        client = Client.from_context(context=context)
         if self.clear_result_graph and self.result_graph:
-            graph_api.delete(graph=self.result_graph)
-        query = SparqlQuery(text=self.sparql_query).get_filled_text(
-            placeholder={"context_graph": self.context_graph}
+            client.graphs.delete_item(key=self.result_graph, skip_if_missing=True)
+        query = Query(text=self.sparql_query).fill_placeholders(
+            placeholders={"context_graph": self.context_graph}
         )
         try:
-            process_id = validation.start(
+            process_id = client.validations.start(
                 context_graph=self.context_graph,
                 shape_graph=self.shape_graph,
-                result_graph=self.result_graph if self.result_graph else None,
+                result_graph=self.result_graph or None,
                 query=query,
             )
-        except HTTPError as error:
-            context.report.update(
-                ExecutionReport(
-                    error=json.loads(error.response.text)["detail"],
-                )
-            )
-            raise RuntimeError(json.loads(error.response.text)["detail"]) from error
-        state = State(id_=process_id)
+        except HTTPStatusError as error:
+            detail = json.loads(error.response.text)["detail"]
+            context.report.update(ExecutionReport(error=detail))
+            raise RuntimeError(detail) from error
+        state = State(client=client, id_=process_id)
         while True:
             sleep(1)
-            setup_cmempy_user_access(context=context.user)
             state.refresh()
             if context.workflow and context.workflow.status() != "Running":
-                validation.cancel(batch_id=process_id)
+                client.validations.cancel(batch_id=process_id)
                 context.report.update(
                     ExecutionReport(
                         entity_count=state.completed,
@@ -192,7 +194,7 @@ class ValidateGraph(WorkflowPlugin):
                 )
                 self.log.info("End validation task (Cancelled Workflow).")
                 return None
-            if state.status in (validation.STATUS_SCHEDULED, validation.STATUS_RUNNING):
+            if state.status in (STATUS_SCHEDULED, STATUS_RUNNING):
                 # when reported as running or scheduled, start another loop
                 context.report.update(
                     ExecutionReport(
@@ -224,11 +226,22 @@ class ValidateGraph(WorkflowPlugin):
         if not self.output_results:
             return None
 
-        violations = []
-        for result in list(validation.get(batch_id=process_id)["results"]):
-            resource_iri = result.get("resourceIri")
-            for _ in result["violations"]:
-                violation = dict(_)
-                violation["resourceIri"] = resource_iri
-                violations.append(violation)
+        violations = [
+            self._as_violation_data(violation, resource_iri=result.resource_iri)
+            for result in client.validations.get_result(batch_id=process_id).results
+            for violation in result.violations
+        ]
         return build_entities_from_data(data=violations)
+
+    def _as_violation_data(self, violation: ValidationViolation, resource_iri: str) -> dict:
+        """Turn a violation into a plain dict ordered like the output schema
+
+        The entity paths of the returned entities are derived from the key order of these
+        dicts, so the keys of the output schema come first, followed by any additional key
+        the API delivers and the IRI of the validated resource.
+        """
+        data: dict = violation.model_dump(by_alias=True, exclude_none=True)
+        ordered: dict = {
+            _.path: data.pop(_.path) for _ in self.output_schema.paths if _.path in data
+        }
+        return ordered | data | {"resourceIri": resource_iri}

{cmem_plugin_validation-1.2.0 → cmem_plugin_validation-1.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cmem-plugin-validation"
-version = "1.2.0"
+version = "1.3.1"
 license = "Apache-2.0"
 description = "Validate graph and data structures."
 authors = ["eccenca GmbH <cmempy-developer@eccenca.com>"]
@@ -13,32 +13,36 @@ keywords = [
 ]
 homepage = "https://github.com/eccenca/cmem-plugin-validation"
 
+[tool.poetry.requires-plugins]
+poetry-plugin-export = "^1.10.0"
+poetry-plugin-shell = "^1.0.1"
+poetry-dynamic-versioning = "^1.10.0"
+
 [tool.poetry.dependencies]# if you need to change python version here, change it also in .python-version
 python = "^3.13"
 jsonschema = "^4.25.1"
-cmem-cmempy = ">=25.3.0"
-requests = ">=2.0.1"
+cmem-client = "^1.0.0"
+httpx = "^0.27.0"
 
 [tool.poetry.dependencies.cmem-plugin-base]
-version = "^4.15.0"
+version = "^4.20.0"
 allow-prereleases = false
 
 [tool.poetry.group.dev.dependencies.cmem-cmemc]
 version = ">=24.2.0"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.23.1"
-genbadge = {extras = ["coverage"], version = "^1.1.2"}
-mypy = "^1.18.2"
-pip = "^25.2"
-pytest = "^8.4.2"
-pytest-cov = "^7.0.0"
+deptry = "^0.25.1"
+genbadge = {extras = ["coverage"], version = "^1.1.3"}
+mypy = "^2.3.0"
+pip = "^26"
+pytest = "^9.1.1"
+pytest-cov = "^7.1.0"
 pytest-dotenv = "^0.5.2"
-pytest-html = "^4.1.1"
-pytest-memray = { version = "^1.8.0",  markers = "platform_system != 'Windows'" }
-ruff = "^0.13.3"
-safety = "^1.10.3"
-types-requests = "^2.31.0.20240406"
+pytest-html = "^4.2.0"
+pytest-memray = { version = "^1.10.0",  markers = "platform_system != 'Windows'" }
+ruff = "^0.16.2"
+trivy-py-ecc = "^0.73.0.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
@@ -54,8 +58,12 @@ bump = true
 warn_return_any = true
 ignore_missing_imports = true
 
+# Deliberately empty. pytest walks upward until a file *loads* a configuration,
+# and only the presence of this table makes pyproject.toml one - without it the
+# walk carries on into the parent directories, where any pytest.ini, tox.ini or
+# setup.cfg, even an empty one, would set this project's rootdir and could push
+# its own addopts into this test suite. Do not tidy it away.
 [tool.pytest.ini_options]
-addopts = ""
 
 [tool.coverage.report]
 exclude_also = [
@@ -71,7 +79,7 @@ exclude_also = [
 
 [tool.ruff]
 line-length = 100
-target-version = "py311"
+target-version = "py313"
 
 [tool.ruff.format]
 line-ending = "lf" # Use `\n` line endings for all files
@@ -81,6 +89,7 @@ select = ["ALL"]
 ignore = [
     "ANN204",  # Missing return type annotation for special method `__init__`
     "COM812",  # missing-trailing-comma
+    "CPY001",  # Missing copyright notice
     "D107",    # Missing docstring in __init__
     "D203",    # [*] 1 blank line required before class docstring
     "D211",    # No blank lines allowed before class docstring
@@ -93,6 +102,11 @@ ignore = [
     "G004",    # Logging statement uses f-string
     "ISC001",  # single-line-implicit-string-concatenation
     "PD",      # opinionated linting for pandas code
-    "S101",    # use of assert detected
     "TRY003",  # Avoid specifying long messages outside the exception class
 ]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "S101",    # asserts allowed in tests
+    "ARG",     # unused function args (pytest fixtures)
+]

cmem_plugin_validation-1.2.0/cmem_plugin_validation/validate_graph/state.py

@@ -1,28 +0,0 @@
-"""Graph validation process state"""
-
-from cmem.cmempy.dp.shacl import validation
-
-
-class State:
-    """State of a validation process"""
-
-    id_: str
-    data: dict
-    status: str
-    completed: int
-    total: int
-    with_violations: int
-    violations: int
-
-    def __init__(self, id_: str):
-        self.id_ = id_
-        self.refresh()
-
-    def refresh(self) -> None:
-        """Refresh state of validation process"""
-        self.data = validation.get_aggregation(batch_id=self.id_)
-        self.status = self.data.get("state", "UNKNOWN")
-        self.completed = self.data.get("resourceProcessedCount", 0)
-        self.total = self.data.get("resourceCount", 0)
-        self.with_violations = self.data.get("resourcesWithViolationsCount", 0)
-        self.violations = self.data.get("violationsCount", 0)