cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc2__py3-none-any.whl

cmem_cmemc/constants.py

@@ -1,4 +1,5 @@
 """cmemc constants"""
+
 NAMESPACES = {
     "void": "http://rdfs.org/ns/void#",
     "di": "https://vocab.eccenca.com/di/",
@@ -11,3 +12,4 @@ NAMESPACES = {
 NS_ACL = "http://eccenca.com/ac/"
 NS_USER = "http://eccenca.com/"
 NS_GROUP = "http://eccenca.com/"
+NS_ACTION = "https://vocab.eccenca.com/auth/Action/"

cmem_cmemc/context.py

@@ -1,11 +1,12 @@
 """The main command line interface."""
+
 import ast
 import configparser
 import json
 import os
 import re
 import subprocess  # nosec
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from os import environ as env
 from os import getenv
 from pathlib import Path
@@ -14,19 +15,21 @@ from shutil import which
 import click
 import cmem.cmempy.config as cmempy_config
 import urllib3
-from cmem.cmempy.health import get_di_version, get_dp_version
+from cmem.cmempy.health import get_di_version, get_explore_version
 from pygments import highlight
 from pygments.formatters import get_formatter_by_name
 from pygments.lexers import get_lexer_by_name
 from rich import box
 from rich.console import Console
 from rich.table import Table
+from urllib3.exceptions import InsecureRequestWarning
 
 from cmem_cmemc.exceptions import InvalidConfigurationError
+from cmem_cmemc.string_processor import StringProcessor, process_row
 
-DI_TARGET_VERSION = "v24.1.0"
+DI_TARGET_VERSION = "v24.2.0"
 
-DP_TARGET_VERSION = "v24.1.0"
+EXPLORE_TARGET_VERSION = "v24.2.0"
 
 KNOWN_CONFIG_KEYS = {
     "CMEM_BASE_URI": cmempy_config.get_cmem_base_uri,
@@ -47,6 +50,8 @@ KNOWN_CONFIG_KEYS = {
 
 KNOWN_SECRET_KEYS = ("OAUTH_PASSWORD", "OAUTH_CLIENT_SECRET", "OAUTH_ACCESS_TOKEN")
 
+SSL_VERIFY_WARNING = "SSL verification is disabled (SSL_VERIFY=False)."
+
 
 class ApplicationContext:
     """Context of the command line interface."""
@@ -90,7 +95,7 @@ class ApplicationContext:
     def get_template_data(self) -> dict[str, str]:
         """Get the template data dict with vars from the context."""
         data: dict[str, str] = {}
-        today = str(datetime.now(tz=UTC).date())
+        today = str(datetime.now(tz=timezone.utc).date())
         data.update(date=today)
         if self.connection is not None:
             data.update(connection=self.connection.name)
@@ -108,8 +113,11 @@ class ApplicationContext:
 
     def set_config_dir(self) -> None:
         """Set the configuration directory"""
-        self.config_dir = Path(click.get_app_dir(self.app_name))
-        self.config_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            self.config_dir = Path(click.get_app_dir(self.app_name))
+            self.config_dir.mkdir(parents=True, exist_ok=True)
+        except (OSError, PermissionError, FileNotFoundError):
+            self.echo_debug(f"Could not create config directory {self.config_dir}")
 
     def set_config_file(self, config_file: str | None = None) -> None:
         """Set and return the context config file"""
@@ -117,7 +125,7 @@ class ApplicationContext:
             self.config_file = Path(config_file)
         else:
             self.config_file = self.config_dir / "config.ini"
-        self.echo_debug(f"Set config to {self.config_file.absolute().name}")
+        self.echo_debug(f"Set config to {self.config_file.absolute()}")
 
     def configure_cmempy(self, config: configparser.SectionProxy | None = None) -> None:
         """Configure the cmempy API to use a new connection."""
@@ -142,12 +150,6 @@ class ApplicationContext:
             self.set_credential_from_process(_, _ + "_PROCESS", config)
 
         self.echo_debug(f"CA bundle loaded from {cmempy_config.get_requests_ca_bundle()}")
-        # If cert validation is disabled, output a warning
-        # Also disable library warnings:
-        # https://urllib3.readthedocs.io/en/latest/advanced-usage.html
-        if not cmempy_config.get_ssl_verify():
-            self.echo_warning("SSL verification is disabled (SSL_VERIFY=False).")
-            urllib3.disable_warnings()
         return
 
     def set_connection_from_params(self, params: dict) -> None:
@@ -192,7 +194,7 @@ class ApplicationContext:
         self.config = self.get_config()
         self.connection = None
         if section_string is None or section_string == "":
-            self.echo_debug("No config given, use API default connection.")
+            self.echo_debug("No config given, use API defaults or environment connection.")
         elif section_string not in self.config:
             raise InvalidConfigurationError(
                 f"There is no connection '{section_string}' configured in "
@@ -202,6 +204,14 @@ class ApplicationContext:
             self.echo_debug(f"Use connection config: {section_string}")
             self.connection = self.config[section_string]
         self.configure_cmempy(self.connection)
+
+        # If cert validation is disabled, output a warning
+        # Also disable library warnings:
+        # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
+        if not cmempy_config.get_ssl_verify():
+            self.echo_warning(SSL_VERIFY_WARNING)
+            urllib3.disable_warnings(category=InsecureRequestWarning)
+
         return self.connection
 
     def get_config_file(self) -> Path:
@@ -214,17 +224,18 @@ class ApplicationContext:
 
     def get_config(self) -> configparser.RawConfigParser:
         """Parse the configuration"""
+        config = configparser.RawConfigParser()
         try:
-            config = configparser.RawConfigParser()
-            config_file = self.get_config_file()
             # https://stackoverflow.com/questions/1648517/
-            config.read(config_file, encoding="utf-8")
+            config.read(self.get_config_file(), encoding="utf-8")
         except configparser.Error as error:
             raise InvalidConfigurationError(
                 "The following config parser error needs to be fixed with your config file:\n"
                 f"{error!s}\n"
                 "You can use the 'config edit' command to fix this."
             ) from error
+        except Exception as error:  # noqa: BLE001
+            self.echo_debug(f"Could not read config file - provide empty config: {error!s}")
         return config
 
     @staticmethod
@@ -241,33 +252,59 @@ class ApplicationContext:
         cmemc_complete = os.getenv("_CMEMC_COMPLETE", default=None)
         if comp_words is not None:
             return True
-        if cmemc_complete is not None:
-            return True
-        return False
+        return cmemc_complete is not None
 
     @staticmethod
-    def echo_warning(message: str, nl: bool = True) -> None:
+    def echo_warning(message: str, nl: bool = True, condition: bool = True) -> None:
         """Output a warning message."""
+        if not condition:
+            return
         if ApplicationContext.is_completing():
             return
         click.secho(message, fg="yellow", err=True, nl=nl)
 
     @staticmethod
-    def echo_error(message: str, nl: bool = True, err: bool = True) -> None:
-        """Output an error message."""
-        # pylint: disable=invalid-name
-        click.secho(message, fg="red", err=err, nl=nl)
+    def echo_error(
+        message: str | list[str], nl: bool = True, err: bool = True, prepend_line: bool = False
+    ) -> None:
+        """Output an error message.
 
-    def echo_debug(self, message: str) -> None:
-        """Output a debug message if --debug is enabled."""
+        2024-05-17: also allows list of strings now
+        2024-05-17: new prepend_line parameter
+        """
         # pylint: disable=invalid-name
+        click.echo("") if prepend_line is True else None
+        messages: list[str] = [message] if isinstance(message, str) else message
+        for _ in messages:
+            click.secho(_, fg="red", err=err, nl=nl)
+
+    def echo_debug(self, message: str | list[str]) -> None:
+        """Output a debug message if --debug is enabled.
+
+        2024-05-17: also allows list of strings now
+        """
+        messages: list[str] = [message] if isinstance(message, str) else message
         if self.debug:
-            now = datetime.now(tz=UTC)
-            click.secho(f"[{now!s}] {message}", err=True, dim=True)
+            now = datetime.now(tz=timezone.utc)
+            for _ in messages:
+                click.secho(f"[{now!s}] {_}", err=True, dim=True)
 
-    def echo_info(self, message: str | list[str] | set[str], nl: bool = True, fg: str = "") -> None:
-        """Output one or more info messages, if not suppressed by --quiet."""
-        # pylint: disable=invalid-name
+    def echo_info(
+        self,
+        message: str | list[str] | set[str],
+        nl: bool = True,
+        fg: str = "",
+        condition: bool = True,
+    ) -> None:
+        """Output one or more info messages, if not suppressed by --quiet.
+
+        message: the string to output
+        nl: True if newlines need to be added (default)
+        fg: color
+        condition: do nothing if False
+        """
+        if not condition:
+            return
         if self.quiet:
             return
         if isinstance(message, str):
@@ -279,7 +316,6 @@ class ApplicationContext:
 
     def echo_info_json(self, object_: object) -> None:
         """Output a formatted and highlighted json as info message."""
-        # pylint: disable=invalid-name
         message = highlight(
             json.dumps(object_, indent=2),
             get_lexer_by_name("json"),
@@ -289,7 +325,6 @@ class ApplicationContext:
 
     def echo_info_xml(self, document: str) -> None:
         """Output a formatted and highlighted XML as info message."""
-        # pylint: disable=invalid-name
         message = highlight(
             document,
             get_lexer_by_name("xml"),
@@ -297,17 +332,28 @@ class ApplicationContext:
         )
         self.echo_info(message)
 
-    def echo_info_table(
+    def echo_info_table(  # noqa: PLR0913
         self,
         rows: list,
         headers: list[str],
         sort_column: int | None = None,
         caption: str | None = None,
+        cell_processing: dict[int, StringProcessor] | None = None,
+        empty_table_message: str | None = None,
     ) -> None:
         """Output a formatted and highlighted table as info message."""
         self.update_console_width()
+
+        if len(rows) == 0 and empty_table_message:
+            self.echo_warning(empty_table_message)
+            return
+
         if sort_column is not None:
             rows = sorted(rows, key=lambda k: k[sort_column].lower())
+
+        if cell_processing:
+            rows = [process_row(row, cell_processing) for row in rows]
+
         table = Table(
             box=box.HEAVY,
             row_styles=["bold", ""],
@@ -330,10 +376,10 @@ class ApplicationContext:
         )
         self.echo_info(message)
 
-    def echo_success(self, message: str, nl: bool = True) -> None:
+    def echo_success(self, message: str, nl: bool = True, condition: bool = True) -> None:
         """Output success message, if not suppressed by --quiet."""
         # pylint: disable=invalid-name
-        self.echo_info(message, fg="green", nl=nl)
+        self.echo_info(message, fg="green", nl=nl, condition=condition)
 
     @staticmethod
     def echo_result(message: str, nl: bool = True) -> None:
@@ -372,9 +418,9 @@ class ApplicationContext:
 
     def check_versions(self) -> None:
         """Check server versions against supported versions."""
-        # check DP version
+        # check Explore version
         self.check_concrete_version(
-            name="DataPlatform", version=get_dp_version(), target_version=DP_TARGET_VERSION
+            name="Explore", version=get_explore_version(), target_version=EXPLORE_TARGET_VERSION
         )
         self.check_concrete_version(
             name="DataIntegration", version=get_di_version(), target_version=DI_TARGET_VERSION
@@ -429,8 +475,8 @@ class ApplicationContext:
             )
         self.echo_debug(f"External credential process started {checked_command}")
         split_output = (
-            subprocess.run(  # nosec
-                checked_command,  # noqa: S603
+            subprocess.run(  # nosec  # noqa: S603
+                checked_command,
                 capture_output=True,
                 check=True,
             )

cmem_cmemc/manual_helper/graph.py

@@ -1,4 +1,5 @@
 """Generate a help text and command structure graph."""
+
 import contextlib
 
 import click.core

cmem_cmemc/manual_helper/multi_page.py

@@ -1,4 +1,5 @@
 """Generate a multi page documentation for documentation.eccenca.com."""
+
 import re
 from pathlib import Path
 
@@ -16,10 +17,11 @@ def get_icon_for_command_group(full_name: str) -> str:
     return {
         "admin": "material/key-link",
         "admin acl": "material/server-security",
+        "admin client": "material/account-cog",
         "admin metrics": "material/chart-line-variant",
+        "admin migration": "material/database-arrow-up-outline",
         "admin store": "material/database-outline",
         "admin user": "material/account-cog",
-        "admin client": "material/account-cog",
         "admin workspace": "material/folder-multiple-outline",
         "admin workspace python": "material/language-python",
         "config": "material/cog-outline",

cmem_cmemc/migrations/__init__.py

@@ -0,0 +1 @@
+"""Migration Recipes"""

cmem_cmemc/migrations/abc.py

@@ -0,0 +1,84 @@
+"""Migration recipe abstract base class"""
+
+import re
+from abc import ABC, abstractmethod
+from typing import ClassVar, Literal
+
+from cmem.cmempy.health import get_complete_status_info
+from cmem.cmempy.queries import SparqlQuery
+from packaging.version import Version
+
+components = Literal["explore", "build"]
+
+
+class MigrationRecipe(ABC):
+    """Tests and migration functions to migrate a specific type of resource between versions"""
+
+    id: str
+    description: str
+    component: components = "explore"
+    first_version: str | None = None
+    last_version: str | None = None
+    tags: ClassVar[list[str]] = []
+    default_prefixes: ClassVar[dict[str, str]] = {
+        "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+        "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+        "sh": "http://www.w3.org/ns/shacl#",
+        "auth": "https://vocab.eccenca.com/auth/",
+        "shui": "https://vocab.eccenca.com/shui/",
+    }
+
+    @abstractmethod
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+
+    @abstractmethod
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+
+    def version_matches(self, component_version: str = "") -> bool:
+        """Test if the component version matches the version specification of the recipe"""
+        status_info = get_complete_status_info()
+        first_version = self.first_version if self.first_version else "0.0.0"
+        last_version = self.last_version if self.last_version else "1000.0.0"
+        if component_version == "":
+            if self.component == "explore":
+                component_version = str(status_info["explore"]["version"])
+            if self.component == "build":
+                component_version = str(status_info["build"]["version"])
+        if component_version.startswith("v"):
+            component_version = component_version[1:]
+        # remove git describe stuff (24.2.1-501-gb0ca19b14)
+        component_version = re.sub(r"-.*", "", component_version)
+        if component_version == "":
+            return False
+        return Version(first_version) <= Version(component_version) <= Version(last_version)
+
+    def _get_default_prefixes(self) -> str:
+        """Get default prefixes as SPARQL Snippet"""
+        snippet = ""
+        for prefix, namespace in self.default_prefixes.items():
+            snippet += f"PREFIX {prefix}: <{namespace}>\n"
+        return snippet
+
+    def _select(self, query_text: str, placeholder: dict[str, str] | None = None) -> list[dict]:
+        """Get the bindings of a SPARQL Query result"""
+        if not placeholder:
+            placeholder = {"DEFAULT_PREFIXES": self._get_default_prefixes()}
+        else:
+            placeholder["DEFAULT_PREFIXES"] = self._get_default_prefixes()
+        query = SparqlQuery(query_text)
+        response: dict = query.get_json_results(placeholder=placeholder)
+        results: dict = response.get("results", {"bindings": []})
+        bindings: list[dict] = results.get("bindings")  # type: ignore[assignment]
+        return bindings
+
+    def _update(self, query_text: str, placeholder: dict[str, str] | None = None) -> None:
+        """Send an update query"""
+        if not placeholder:
+            placeholder = {"DEFAULT_PREFIXES": self._get_default_prefixes()}
+        else:
+            placeholder["DEFAULT_PREFIXES"] = self._get_default_prefixes()
+        query = SparqlQuery(query_text)
+        query.query_type = "UPDATE"
+        query.get_results(placeholder=placeholder)

cmem_cmemc/migrations/access_conditions_243.py

@@ -0,0 +1,122 @@
+"""Access Conditions migration recipes from before 24.3"""
+
+from typing import ClassVar
+
+from cmem.cmempy.dp.authorization.refresh import get as refresh_acls
+
+from cmem_cmemc.migrations.abc import MigrationRecipe, components
+
+
+class MoveAccessConditionsToNewGraph(MigrationRecipe):
+    """24.4 Access Conditions Migration"""
+
+    id = "acl-graph-24.3"
+    description = "Move access conditions and used queries to new ACL graph"
+    component: components = "explore"
+    first_version = "24.3"
+    tags: ClassVar[list[str]] = ["system", "acl"]
+    check_query = """{{DEFAULT_PREFIXES}}
+SELECT ?subject
+FROM <urn:elds-backend-access-conditions-graph>
+WHERE {
+  ?subject a ?class .
+  FILTER NOT EXISTS { ?subject shui:isSystemResource true }
+  FILTER (?class IN (auth:AccessCondition, shui:SparqlQuery) ) .
+}
+    """
+    move_query = """{{DEFAULT_PREFIXES}}
+DELETE { GRAPH <urn:elds-backend-access-conditions-graph> { ?s ?p ?o . } }
+INSERT { GRAPH <https://ns.eccenca.com/data/ac/> { ?s ?p ?o . } }
+WHERE {
+  GRAPH <urn:elds-backend-access-conditions-graph> {
+    ?s ?p ?o .
+    ?s a ?class .
+    FILTER (?class IN (auth:AccessCondition, shui:SparqlQuery) ) .
+    FILTER NOT EXISTS { ?s shui:isSystemResource true }
+  }
+}
+    """
+
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+        old_acls = self._select(self.check_query)
+        return len(old_acls) > 0
+
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+        self._update(self.move_query)
+        refresh_acls()
+
+
+class RenameAuthVocabularyResources(MigrationRecipe):
+    """24.4 Access Conditions Migration"""
+
+    id = "acl-vocab-24.3"
+    description = "Migrate auth vocabulary terms (actions and other grants)"
+    component: components = "explore"
+    first_version = "24.3"
+    tags: ClassVar[list[str]] = ["system", "acl"]
+    check_query = """{{DEFAULT_PREFIXES}}
+SELECT DISTINCT ?acl_id
+WHERE {
+  GRAPH ?acl_graph {
+	?acl_id a auth:AccessCondition .
+    FILTER NOT EXISTS { ?acl_id shui:isSystemResource true }
+    ?acl_id ?property ?old_term .
+    FILTER (?old_term in (
+        <urn:elds-backend-all-actions>,
+        <urn:elds-backend-actions-auth-access-control>,
+        <urn:eccenca:di>,
+        <urn:eccenca:ThesaurusUserInterface>,
+        <urn:eccenca:AccessInternalGraphs>,
+        <urn:eccenca:QueryUserInterface>,
+        <urn:eccenca:VocabularyUserInterface>,
+        <urn:eccenca:ExploreUserInterface>,
+        <https://vocab.eccenca.com/auth/Action/Viz/Manage>,
+        <https://vocab.eccenca.com/auth/Action/Viz/Read>,
+        <urn:elds-backend-anonymous-user>,
+        <urn:elds-backend-public-group>,
+        <urn:elds-backend-all-graphs>
+      ))
+  }
+  FILTER (?acl_graph IN (<urn:elds-backend-access-conditions-graph>, <https://ns.eccenca.com/data/ac/>))
+}
+    """
+    move_query = """{{DEFAULT_PREFIXES}}
+DELETE { GRAPH ?acl_graph { ?s ?p <{{OLD_IRI}}> . } }
+INSERT { GRAPH ?acl_graph { ?s ?p <{{NEW_IRI}}> . } }
+WHERE {
+  GRAPH ?acl_graph {
+    ?s ?p ?o .
+    FILTER NOT EXISTS { ?s shui:isSystemResource true }
+    FILTER (?o = <{{OLD_IRI}}> )
+  }
+  FILTER (?acl_graph IN (<urn:elds-backend-access-conditions-graph>, <https://ns.eccenca.com/data/ac/>))
+}
+    """
+    terms: ClassVar[dict[str, str]] = {
+        "urn:elds-backend-all-actions": "https://vocab.eccenca.com/auth/Action/AllActions",
+        "urn:elds-backend-actions-auth-access-control": "https://vocab.eccenca.com/auth/Action/ChangeAccessConditions",
+        "urn:eccenca:di": "https://vocab.eccenca.com/auth/Action/ChangeAccessConditions",
+        "urn:eccenca:ThesaurusUserInterface": "https://vocab.eccenca.com/auth/Action/Explore-ThesaurusCatalog",
+        "urn:eccenca:AccessInternalGraphs": "https://vocab.eccenca.com/auth/Action/Explore-ListSystemGraphs",
+        "urn:eccenca:QueryUserInterface": "https://vocab.eccenca.com/auth/Action/Explore-QueryCatalog",
+        "urn:eccenca:VocabularyUserInterface": "https://vocab.eccenca.com/auth/Action/Explore-VocabularyCatalog",
+        "urn:eccenca:ExploreUserInterface": "https://vocab.eccenca.com/auth/Action/Explore-KnowledgeGraphs",
+        "https://vocab.eccenca.com/auth/Action/Viz/Manage": "https://vocab.eccenca.com/auth/Action/Explore-BKE-Manage",
+        "https://vocab.eccenca.com/auth/Action/Viz/Read": "https://vocab.eccenca.com/auth/Action/Explore-BKE-Read",
+        "urn:elds-backend-anonymous-user": "https://vocab.eccenca.com/auth/AnonymousUser",
+        "urn:elds-backend-public-group": "https://vocab.eccenca.com/auth/PublicGroup",
+        "urn:elds-backend-all-graphs": "https://vocab.eccenca.com/auth/AllGraphs",
+    }
+
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+        acls_with_vocabs_to_change = self._select(self.check_query)
+        return len(acls_with_vocabs_to_change) > 0
+
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+        for old_iri, new_iri in self.terms.items():
+            self._update(self.move_query, placeholder={"OLD_IRI": old_iri, "NEW_IRI": new_iri})
+        refresh_acls()

cmem_cmemc/migrations/bootstrap_data.py

@@ -0,0 +1,28 @@
+"""Bootstrap Data migration recipe"""
+
+from typing import ClassVar
+
+from cmem.cmempy.dp.admin import import_bootstrap_data
+from cmem.cmempy.health import get_complete_status_info
+
+from cmem_cmemc.migrations.abc import MigrationRecipe, components
+
+
+class MigrateBootstrapData(MigrationRecipe):
+    """Update Bootstrap Data to current version"""
+
+    id = "bootstrap-data"
+    description = "Migrate bootstrap system data (same as admin store bootstrap --import)"
+    component: components = "explore"
+    first_version = "20.12"
+    tags: ClassVar[list[str]] = ["system", "shapes"]
+
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+        status_info = get_complete_status_info()
+        shapes_status = status_info["shapes"]["version"]
+        return bool(shapes_status != status_info["explore"]["version"])
+
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+        import_bootstrap_data()