cmem-cmemc 24.3.3__py3-none-any.whl → 25.1.0__py3-none-any.whl

cmem_cmemc/context.py

@@ -24,8 +24,10 @@ from rich.console import Console
 from rich.table import Table
 from urllib3.exceptions import InsecureRequestWarning
 
+from cmem_cmemc.config_parser import PureSectionConfigParser
 from cmem_cmemc.exceptions import InvalidConfigurationError
 from cmem_cmemc.string_processor import StringProcessor, process_row
+from cmem_cmemc.utils import is_enabled, str_to_bool
 
 DI_TARGET_VERSION = "v24.3.0"
 
@@ -56,34 +58,77 @@ SSL_VERIFY_WARNING = "SSL verification is disabled (SSL_VERIFY=False)."
 class ApplicationContext:
     """Context of the command line interface."""
 
+    APP_NAME: str = "cmemc"
+    DEFAULT_CONFIG_FILE: str = str(Path(click.get_app_dir(APP_NAME)) / "config.ini")
+    DEFAULT_EXTERNAL_HTTP_TIMEOUT: int = 10
+
     debug: bool
     quiet: bool
+    external_http_timeout: int
     config_dir: Path
     config_file: Path
-    config: configparser.RawConfigParser
-    connection: configparser.SectionProxy | None
+    config: PureSectionConfigParser
+    connection: str | None
     console: Console
     console_width: int | None = None
 
     # pylint: disable=too-many-instance-attributes
     # pylint: disable=too-many-public-methods
 
-    def __init__(self, debug: bool = False, quiet: bool = False):
+    def __init__(
+        self,
+        config_file: str,
+        connection: str | None = None,
+        debug: bool = False,
+        quiet: bool = False,
+    ):
         """Initialize main context."""
+        self.config_file = Path(config_file)
         self.app_name = "cmemc"
         self.set_debug(debug)
         self.set_quiet(quiet)
-        self.set_config_dir()
-        self.set_config_file()
-        if "CMEMC_CONFIG_FILE" in env:
-            self.config_file_default = env["CMEMC_CONFIG_FILE"]
-        else:
-            self.config_file_default = str(self.config_dir / "config.ini")
-        # since CMEM-3199, we do not initialize the connection on init
-        self.connection = None
+        self.ensure_app_config_dir()
+        self.set_connection(connection)
+        self.set_external_http_timeout(self.DEFAULT_EXTERNAL_HTTP_TIMEOUT)
         self.console = Console(markup=True, emoji_variant="emoji")
         self.update_console_width()
 
+    @staticmethod
+    def from_params(params: dict) -> "ApplicationContext":
+        """Create an ApplicationContext instance from a dictionary.
+
+        Expects 'config_file' key to be present in the dictionary.
+        """
+        config_file = params.get("config_file")
+        debug = str_to_bool(str(params.get("debug")))
+        quiet = str_to_bool(str(params.get("quiet")))
+        external_http_timeout = (
+            int(str(params.get("external_http_timeout")))
+            if "external_http_timeout" in params
+            else ApplicationContext.DEFAULT_EXTERNAL_HTTP_TIMEOUT
+        )
+        connection = str(params.get("connection")) if params.get("connection") else None
+        if not config_file:
+            raise ValueError("Missing required key: 'config_file' in config dictionary")
+        app = ApplicationContext(
+            config_file=config_file, connection=connection, debug=debug, quiet=quiet
+        )
+        app.set_external_http_timeout(external_http_timeout)
+        if not app.debug:
+            app.set_debug(is_enabled(params, app.get_config(), "debug"))
+        if not app.quiet:
+            app.set_quiet(is_enabled(params, app.get_config(), "quiet"))
+        app.echo_debug(f"use config file: {app.config_file}")
+        if app.connection:
+            app.echo_debug(f"set connection: {app.connection}")
+        defaults = app.get_defaults()
+        app.apply_config(defaults)
+        section_config = app.get_connection_config()
+        app.clear_known_keys()
+        app.apply_config(section_config)
+        app.configure_cmempy()
+        return app
+
     def update_console_width(self) -> None:
         """Update console width from environment variable."""
         console_width_env = getenv("CMEMC_CONSOLE_WIDTH", None)
@@ -98,7 +143,7 @@ class ApplicationContext:
         today = str(datetime.now(tz=timezone.utc).date())
         data.update(date=today)
         if self.connection is not None:
-            data.update(connection=self.connection.name)
+            data.update(connection=self.connection)
         else:
             data.update(connection="unnamed")
         return data
@@ -111,8 +156,13 @@ class ApplicationContext:
         """Set quiets state"""
         self.quiet = quiet
 
-    def set_config_dir(self) -> None:
-        """Set the configuration directory"""
+    def set_external_http_timeout(self, timeout: int) -> None:
+        """Set external http timeout"""
+        self.external_http_timeout = timeout
+        os.environ["CMEMC_EXTERNAL_HTTP_TIMEOUT"] = str(timeout)
+
+    def ensure_app_config_dir(self) -> None:
+        """Ensure the application's configuration directory exists."""
         try:
             self.config_dir = Path(click.get_app_dir(self.app_name))
             self.config_dir.mkdir(parents=True, exist_ok=True)
@@ -127,84 +177,77 @@ class ApplicationContext:
             self.config_file = self.config_dir / "config.ini"
         self.echo_debug(f"Set config to {self.config_file.absolute()}")
 
-    def configure_cmempy(self, config: configparser.SectionProxy | None = None) -> None:
-        """Configure the cmempy API to use a new connection."""
-        # With or without connection config, we do not want API stdout prints
-        env["CMEMPY_IS_CHATTY"] = "False"
-        if config is None:
-            return
-        for key in KNOWN_CONFIG_KEYS:
-            # with a loaded config, we delete all known keys from outside
-            if key in env:
-                env.pop(key)
-            if key in config:
-                value = str(config[key.lower()])
-                env[key] = value
+    def get_defaults(self) -> dict[str, str]:
+        """Populate missing values in `params` using defaults from the configuration."""
+        config = self.get_config()
+        defaults = dict(config.defaults())
+        if "CMEMC_CONFIG_FILE" in defaults:
+            defaults.pop("CMEMC_CONFIG_FILE")
+        if "CMEMC_DEBUG" in defaults and self.debug:
+            defaults.pop("CMEMC_DEBUG")
+        if "CMEMC_QUIET" in defaults and self.quiet:
+            defaults.pop("CMEMC_QUIET")
+        if "CMEMC_CONNECTION" in defaults and self.connection:
+            defaults.pop("CMEMC_CONNECTION")
+        if (
+            "CMEMC_EXTERNAL_HTTP_TIMEOUT" in defaults
+            and self.external_http_timeout != self.DEFAULT_EXTERNAL_HTTP_TIMEOUT
+        ):
+            defaults.pop("CMEMC_EXTERNAL_HTTP_TIMEOUT")
+        for key in defaults:
+            if key in os.environ:
+                defaults.pop(key)
+        return defaults
+
+    def get_connection_config(self) -> dict[str, str]:
+        """Get connection section config"""
+        if not self.connection:
+            return {}
+        config = dict(self.get_config()[self.connection])
+        if "CMEMC_CONFIG_FILE" in config:
+            config.pop("CMEMC_CONFIG_FILE")
+        if "CMEMC_CONNECTION" in config:
+            config.pop("CMEMC_CONNECTION")
+        return config
+
+    def apply_config(self, config: dict[str, str]) -> None:
+        """Apply the configuration to the context."""
+        for key, value in config.items():
+            if key == "CMEMC_DEBUG":
+                self.set_debug(str_to_bool(value))
+            elif key == "CMEMC_QUIET":
+                self.set_quiet(str_to_bool(value))
+            elif key == "CMEMC_CONNECTION":
+                self.echo_debug(f"set connection to {value}")
+                self.set_connection(value)
+            elif key == "CMEMC_EXTERNAL_HTTP_TIMEOUT":
+                self.set_external_http_timeout(int(value))
+            else:
                 if key in KNOWN_SECRET_KEYS:
                     self.echo_debug(key + " set by config")
                 else:
                     self.echo_debug(key + " set by config to " + value)
+                env[key] = value
 
-        # allow to fetch all secrets from an external process
-        for _ in KNOWN_SECRET_KEYS:
-            self.set_credential_from_process(_, _ + "_PROCESS", config)
-
-        self.echo_debug(f"CA bundle loaded from {cmempy_config.get_requests_ca_bundle()}")
-        return
+    def clear_known_keys(self) -> None:
+        """Clear know keys if the connection is set."""
+        if self.connection:
+            # clear all known keys
+            for key in KNOWN_CONFIG_KEYS:
+                if key in env:
+                    env.pop(key)
 
-    def set_connection_from_params(self, params: dict) -> None:
-        """Set connection and config by manually checking params (completion)."""
-        self.set_config_file(getenv("CMEMC_CONFIG_FILE", str(self.config_file)))
-        if params["config_file"]:
-            self.set_config_file(params["config_file"])
-
-        self.config = self.get_config()
-        # look for connection in environment and set connection
-        self.set_connection(getenv("CMEMC_CONNECTION", ""))
-        if params["connection"]:
-            self.set_connection(params["connection"])
-
-    def set_connection_from_args(self, args: dict) -> None:
-        """Set connection and config by manually checking param (completion)."""
-        # look for environment and load config
-        self.set_config_file(getenv("CMEMC_CONFIG_FILE", str(self.config_file)))
-        # look for config file in arguments and load config
-        found_config_file = False
-        for arg in args:
-            if found_config_file is True:
-                self.set_config_file(arg)
-                break
-            if arg == "--config-file":
-                found_config_file = True
-        self.config = self.get_config()
-        # look for connection in environment and set connection
-        self.set_connection(getenv("CMEMC_CONNECTION", ""))
-        # look for connection in arguments and set connection
-        found_connection = False
-        for arg in args:
-            if found_connection is True:
-                self.set_connection(arg)
-                return
-            if arg in ("-c", "--connection"):
-                found_connection = True
-        return
-
-    def set_connection(self, section_string: str) -> configparser.SectionProxy | None:
-        """Set connection config section based on section string."""
-        self.config = self.get_config()
-        self.connection = None
-        if section_string is None or section_string == "":
-            self.echo_debug("No config given, use API defaults or environment connection.")
-        elif section_string not in self.config:
-            raise InvalidConfigurationError(
-                f"There is no connection '{section_string}' configured in "
-                f"config '{self.config_file}'."
-            )
-        else:
-            self.echo_debug(f"Use connection config: {section_string}")
-            self.connection = self.config[section_string]
-        self.configure_cmempy(self.connection)
+    def configure_cmempy(self) -> None:
+        """Configure the cmempy API to use a new connection."""
+        # With or without connection config, we do not want API stdout prints
+        env["CMEMPY_IS_CHATTY"] = "False"
+        if self.connection:
+            config = self.get_config()[self.connection]
+            # allow to fetch all secrets from an external process
+            for _ in KNOWN_SECRET_KEYS:
+                self.set_credential_from_process(_, _ + "_PROCESS", config)
 
+            self.echo_debug(f"CA bundle loaded from {cmempy_config.get_requests_ca_bundle()}")
         # If cert validation is disabled, output a warning
         # Also disable library warnings:
         # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
@@ -212,27 +255,49 @@ class ApplicationContext:
             self.echo_warning(SSL_VERIFY_WARNING)
             urllib3.disable_warnings(category=InsecureRequestWarning)
 
-        return self.connection
+    @staticmethod
+    def set_connection_from_params(params: dict) -> None:
+        """Set connection and config by manually checking params (completion)."""
+        app = ApplicationContext.from_params(params)
+        defaults = app.get_defaults()
+        app.apply_config(defaults)
+        section_config = app.get_connection_config()
+        app.clear_known_keys()
+        app.apply_config(section_config)
+        app.configure_cmempy()
+
+    def set_connection(self, section_string: str | None = None) -> None:
+        """Set connection config section based on section string."""
+        self.connection = None
+        if section_string is None or section_string == "":
+            return
+        if section_string not in self.get_config():
+            raise InvalidConfigurationError(
+                self,
+                f"There is no connection '{section_string}' configured in "
+                f"config '{self.config_file}'.",
+            )
+        self.connection = section_string
 
     def get_config_file(self) -> Path:
         """Check the connection config file."""
         if not self.config_file.exists():
             with self.config_file.open(mode="a", encoding="UTF-8"):
                 self.echo_warning(f"Empty config created: {self.config_file.name}")
-        self.echo_debug("Config loaded: " + self.config_file.name)
         return self.config_file
 
-    def get_config(self) -> configparser.RawConfigParser:
+    def get_config(self) -> PureSectionConfigParser:
         """Parse the configuration"""
-        config = configparser.RawConfigParser()
+        config = PureSectionConfigParser()
         try:
             # https://stackoverflow.com/questions/1648517/
             config.read(self.get_config_file(), encoding="utf-8")
         except configparser.Error as error:
             raise InvalidConfigurationError(
+                self,
                 "The following config parser error needs to be fixed with your config file:\n"
                 f"{error!s}\n"
-                "You can use the 'config edit' command to fix this."
+                "You can use the 'config edit' command to fix this.",
             ) from error
         except Exception as error:  # noqa: BLE001
             self.echo_debug(f"Could not read config file - provide empty config: {error!s}")
@@ -282,12 +347,19 @@ class ApplicationContext:
         """Output a debug message if --debug is enabled.
 
         2024-05-17: also allows list of strings now
+        2025-04-07: extract output function
         """
         messages: list[str] = [message] if isinstance(message, str) else message
-        if self.debug:
-            now = datetime.now(tz=timezone.utc)
-            for _ in messages:
-                click.secho(f"[{now!s}] {_}", err=True, dim=True)
+        if self.debug and not ApplicationContext.is_completing():
+            self.echo_debug_string(messages)
+
+    @staticmethod
+    def echo_debug_string(message: str | list[str]) -> None:
+        """Output a debug message"""
+        messages: list[str] = [message] if isinstance(message, str) else message
+        now = datetime.now(tz=timezone.utc)
+        for _ in messages:
+            click.secho(f"[{now!s}] {_}", err=True, dim=True)
 
     def echo_info(
         self,
@@ -491,6 +563,3 @@ class ApplicationContext:
         pw_value = split_output[0]
         # set the password environment variable
         env[pw_key] = pw_value
-
-
-CONTEXT = ApplicationContext()

cmem_cmemc/exceptions.py

@@ -1,9 +1,22 @@
 """Declares all cli exceptions."""
 
+from typing import TYPE_CHECKING
 
-class InvalidConfigurationError(ValueError):
+if TYPE_CHECKING:
+    from cmem_cmemc.context import ApplicationContext
+
+
+class CmemcError(ValueError):
+    """Base exception for CMEM-CMEMC-related errors."""
+
+    def __init__(self, app: "ApplicationContext", *args: str):
+        super().__init__(*args)
+        self.app = app
+
+
+class InvalidConfigurationError(CmemcError):
     """The configuration given was not found or is broken."""
 
 
-class ServerError(ValueError):
+class ServerError(CmemcError):
     """The server reported an error with a process."""

cmem_cmemc/manual_helper/graph.py

@@ -43,6 +43,8 @@ def print_group_manual_graph_recursive(
     """Output documentation graph (recursive)."""
     commands = command_group.commands  # type: ignore[union-attr]
     for key in commands:
+        if key == "manual":
+            continue
         item = commands[key]
         iri = f":{prefix}{key}"
         ctx.obj.echo_info(f"{iri} skos:notation '{key}' .")

cmem_cmemc/manual_helper/multi_page.py

@@ -205,7 +205,7 @@ tags:
 
 
 def create_multi_page_documentation(ctx: click.core.Context, directory: str) -> None:
-    """Create a multi page reference manual for documentation.eccenca.com.
+    """Create a multipage reference manual for documentation.eccenca.com.
 
     Returns: None
     """

cmem_cmemc/manual_helper/single_page.py

@@ -24,6 +24,8 @@ def print_group_manual_recursive(
     """Output the help text of a command group (recursive)."""
     commands = command_group.commands  # type: ignore[union-attr]
     for key in commands:
+        if key == "manual":
+            continue
         command = commands[key]
         formatter = ctx.make_formatter()
         if isinstance(command, click.Group):

cmem_cmemc/migrations/remove_noop_triple_251.py

@@ -0,0 +1,50 @@
+"""RemoveHideHeaderFooterStatements migration"""
+
+from typing import ClassVar
+
+from cmem_cmemc.migrations.abc import MigrationRecipe, components
+
+
+class RemoveHideHeaderFooterStatements(MigrationRecipe):
+    """25.1 Remove Non-Operational shui:valueQueryHideHeader|Footer Triple"""
+
+    id = "hide-header-footer-25.1"
+    description = "Remove triples using deprecated shui:valueQueryHideHeader|Footer terms"
+    component: components = "explore"
+    first_version = "24.3"  # needs to changed right before release
+    tags: ClassVar[list[str]] = ["shapes", "user"]
+    check_query = """{{DEFAULT_PREFIXES}}
+SELECT ?shape
+WHERE {
+  GRAPH ?shapeCatalog {
+    ?shapeCatalog a shui:ShapeCatalog .
+    ?shape ?deprecatedProperty ?value .
+  }
+  VALUES ?deprecatedProperty {
+    shui:valueQueryHideHeader shui:valueQueryHideFooter
+  }
+} LIMIT 1
+    """
+    delete_query = """{{DEFAULT_PREFIXES}}
+DELETE {
+  GRAPH ?shapeCatalog { ?shape ?deprecatedProperty ?value . }
+}
+WHERE {
+  GRAPH ?shapeCatalog {
+    ?shapeCatalog a shui:ShapeCatalog .
+    ?shape ?deprecatedProperty ?value .
+  }
+  VALUES ?deprecatedProperty {
+    shui:valueQueryHideHeader shui:valueQueryHideFooter
+  }
+}
+    """
+
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+        existing_triples = self._select(self.check_query)
+        return len(existing_triples) == 1
+
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+        self._update(self.delete_query)

cmem_cmemc/migrations/shapes_widget_integrations_243.py

@@ -27,11 +27,6 @@ WHERE {
 }
     """
     move_query = """{{DEFAULT_PREFIXES}}
-PREFIX  rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-PREFIX  rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX shui: <https://vocab.eccenca.com/shui/>
-PREFIX sh: <http://www.w3.org/ns/shacl#>
-
 DELETE {
   GRAPH ?shapeCatalog {
     ?nodeShape shui:provideChartVisualization ?shuiChart .

cmem_cmemc/object_list.py

@@ -10,7 +10,7 @@ from click import Argument, Context, UsageError
 from click.shell_completion import CompletionItem
 
 from cmem_cmemc.completion import finalize_completion, get_completion_args
-from cmem_cmemc.context import CONTEXT
+from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.title_helper import TitleHelper
 
 
@@ -291,7 +291,7 @@ class ObjectList:
         if not isinstance(added_filter, Filter):
             raise TypeError("'filter_' parameter must be an instance OR a subclass of Filter")
         if added_filter.name in self.filters:
-            raise ValueError(f"Filter {added_filter.name} already exists")
+            raise UsageError(f"Filter {added_filter.name} already exists")
         self.filters[added_filter.name] = added_filter
 
     def remove_filter(self, filter_name: str) -> None:
@@ -361,7 +361,7 @@ class ObjectList:
                 incomplete=incomplete,
             )
 
-        CONTEXT.set_connection_from_params(ctx.find_root().params)
+        ApplicationContext.set_connection_from_params(ctx.find_root().params)
         # This will filter the object list with name/values filter from the command line,
         # up to the current filter which values are completed
         objects = self.apply_filters(ctx=ctx, filter_=previous_filter)

cmem_cmemc/parameter_types/path.py

@@ -8,6 +8,8 @@ import smart_open
 from click.core import Context, Parameter
 from smart_open import compression
 
+from cmem_cmemc.context import ApplicationContext
+
 
 class ClickSmartPath(click.Path):
     """Custom Click smart_path ParamType"""
@@ -61,6 +63,11 @@ class ClickSmartPath(click.Path):
         """Open the file and return the file handle."""
         if file_path == "-":
             return click.open_file(file_path, mode=mode)
+        if transport_params is None:
+            transport_params = {}
+        transport_params["timeout"] = os.getenv(
+            "CMEM_TRANSPORT_TIMEOUT", ApplicationContext.DEFAULT_EXTERNAL_HTTP_TIMEOUT
+        )
         return smart_open.open(  # type: ignore[no-any-return]
             file_path,
             mode,

cmem_cmemc/placeholder.py

@@ -0,0 +1,69 @@
+"""Class to retrieve query placeholder resources
+
+note: this is not the correct place for this class - it should be cmem_client or cmempy
+"""
+
+from cmem.cmempy.queries import SparqlQuery
+
+FETCH_PLACEHOLDERS = """
+PREFIX shui: <https://vocab.eccenca.com/shui/>
+PREFIX rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+
+SELECT ?key ?valueQuery
+FROM <{{graph}}>
+WHERE {
+  ?placeholder a shui:QueryPlaceholder ;
+    shui:QueryPlaceholder_key ?key ;
+    shui:QueryPlaceholder_valueQuery ?valueQueryR ;
+    shui:QueryPlaceholder_usedInQuery <{{query}}> .
+  ?valueQueryR shui:queryText ?valueQuery
+}
+"""
+
+
+class QueryPlaceholder:
+    """A query placeholder"""
+
+    key: str
+    value_query: SparqlQuery
+
+    def __init__(self, key: str, value_query: str):
+        self.key = key
+        self.value_query = SparqlQuery(value_query, query_type="SELECT")
+
+    def complete(self, incomplete: str = "") -> list[str] | list[tuple[str, str]]:  # noqa: ARG002
+        """Prepare a list of placeholder values"""
+        result = self.value_query.get_json_results()
+        projection_vars = result["head"]["vars"]
+        bindings = result["results"]["bindings"]
+        if "value" not in projection_vars:
+            return []
+
+        if "description" not in projection_vars:
+            values_without_description = []
+            for _ in bindings:
+                value = str(_["value"]["value"])
+                values_without_description.append(value)
+            return values_without_description
+
+        values_with_description = []
+        for _ in bindings:
+            value = str(_["value"]["value"])
+            description = str(_["description"]["value"])
+            values_with_description.append((value, description))
+        return values_with_description
+
+
+def get_placeholders_for_query(iri: str) -> dict[str, QueryPlaceholder]:
+    """Get a placeholder dict for a query"""
+    placeholders = {}
+    results = SparqlQuery(
+        FETCH_PLACEHOLDERS,
+        query_type="SELECT",
+    ).get_json_results(placeholder={"graph": "https://ns.eccenca.com/data/queries/", "query": iri})
+    for binding in results["results"]["bindings"]:
+        key = binding["key"]["value"]
+        value_query = binding["valueQuery"]["value"]
+        placeholders[key] = QueryPlaceholder(key=key, value_query=value_query)
+    return placeholders