cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc1__py3-none-any.whl

cmem_cmemc/commands/admin.py

@@ -1,5 +1,6 @@
 """admin commands for cmem command line interface."""
-from datetime import UTC, datetime
+
+from datetime import datetime, timezone
 
 import click
 import jwt
@@ -9,23 +10,34 @@ from cmem.cmempy.health import get_complete_status_info
 from dateutil.relativedelta import relativedelta
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.acl import acl
 from cmem_cmemc.commands.client import client
 from cmem_cmemc.commands.metrics import metrics
+from cmem_cmemc.commands.migration import migration
 from cmem_cmemc.commands.store import store
 from cmem_cmemc.commands.user import user
 from cmem_cmemc.commands.workspace import workspace
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.utils import struct_to_table
 
+WARNING_MIGRATION = (
+    "Your workspace configuration version does not match your DataPlatform version. "
+    "Please consider migrating your workspace configuration (admin store migrate command)."
+)
+WARNING_SHAPES = (
+    "Your ShapeCatalog version does not match your DataPlatform version. "
+    "Please consider updating your bootstrap data (admin store boostrap command)."
+)
+
 
 def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> None:
     """Check grace period of CMEM license."""
-    if "license" not in data["dp"]["info"]:
+    if "license" not in data["explore"]["info"]:
         # DP < 24.1 has no cmem license information here
         return
-    license_ = data["dp"]["info"]["license"]
+    license_ = data["explore"]["info"]["license"]
     in_grace_period: bool = license_.get("inGracePeriod", False)
     if in_grace_period:
         cmem_license_end = license_["validDate"]
@@ -37,14 +49,14 @@ def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> Non
 
 def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exit_1: str) -> None:
     """Check grace period of graphdb license."""
-    if "licenseExpiration" not in data["dp"]["info"]["store"]:
+    if "licenseExpiration" not in data["explore"]["info"]["store"]:
         # DP < 24.1 has no graph license information here
         return
-    expiration_date_str = data["dp"]["info"]["store"]["licenseExpiration"]
-    expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=UTC)
+    expiration_date_str = data["explore"]["info"]["store"]["licenseExpiration"]
+    expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=timezone.utc)
     grace_starts = expiration_date - relativedelta(months=months)
-    if grace_starts < datetime.now(tz=UTC):
-        graphdb_license_end = data["dp"]["info"]["store"]["licenseExpiration"]
+    if grace_starts < datetime.now(tz=timezone.utc):
+        graphdb_license_end = data["explore"]["info"]["store"]["licenseExpiration"]
         output = f"Your GraphDB license expires on {graphdb_license_end}."
         if exit_1 == "always":
             raise ValueError(output)
@@ -83,7 +95,7 @@ def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exi
     "--raw", is_flag=True, help="Outputs combined raw JSON output of the health/info endpoints."
 )
 @click.pass_obj
-def status_command(  # noqa: C901
+def status_command(  # noqa: C901, PLR0912
     app: ApplicationContext, key: str, exit_1: str, enforce_table: bool, raw: bool
 ) -> None:
     """Output health and version information.
@@ -105,20 +117,12 @@ def status_command(  # noqa: C901
     Example: cmemc config list | parallel --ctag cmemc -c {} admin status
     """
     _ = get_complete_status_info()
-    if "error" in _["di"]:
-        app.echo_debug(_["di"]["error"])
-    if "error" in _["dp"]:
-        app.echo_debug(_["dp"]["error"])
-    if "error" in _["dm"]:
-        app.echo_debug(_["dm"]["error"])
-    basic_status = (
-        _["dp"]["healthy"],
-        _["di"]["healthy"],
-        _["dm"]["healthy"],
-        _["shapes"]["healthy"],
-        _["store"]["healthy"],
-    )
-    if exit_1 in ("always", "error") and ("DOWN" in basic_status or "UNKNOWN" in basic_status):
+    if "error" in _["build"]:
+        app.echo_debug(_["build"]["error"])
+    if "error" in _["explore"]:
+        app.echo_debug(_["explore"]["error"])
+
+    if exit_1 in ("always", "error") and (_["overall"]["healthy"] != "UP"):
         raise ValueError(
             f"One or more major status flags are DOWN or UNKNOWN: {_!r}",
         )
@@ -135,22 +139,28 @@ def status_command(  # noqa: C901
         app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
         return
     app.check_versions()
-    if _["shapes"]["version"] not in (_["dp"]["version"], "UNKNOWN"):
-        output = (
-            "Your ShapeCatalog version does not match your DataPlatform "
-            "version. Please consider updating your bootstrap data."
-        )
+    _workspace_config = _["explore"]["info"].get("workspaceConfiguration", {})
+    if _workspace_config.get("workspacesToMigrate"):
         if exit_1 == "always":
-            raise ValueError(output)
-        app.echo_warning(output)
+            raise ValueError(WARNING_MIGRATION)
+        app.echo_warning(WARNING_MIGRATION)
+
+    if _["shapes"]["version"] not in (_["explore"]["version"], "UNKNOWN"):
+        if exit_1 == "always":
+            raise ValueError(WARNING_SHAPES)
+        app.echo_warning(WARNING_SHAPES)
+
     _check_cmem_license(app=app, data=_, exit_1=exit_1)
     _check_graphdb_license(app=app, data=_, months=1, exit_1=exit_1)
+    if _["store"]["type"] != "GRAPHDB":
+        store_version = _["store"]["type"] + "/" + _["store"]["version"]
+    else:
+        store_version = _["store"]["version"]
     table = [
-        ("DP", _["dp"]["version"], _["dp"]["healthy"]),
-        ("DI", _["di"]["version"], _["di"]["healthy"]),
-        ("DM", _["dm"]["version"], _["dm"]["healthy"]),
+        ("EXPLORE", _["explore"]["version"], _["explore"]["healthy"]),
+        ("BUILD", _["build"]["version"], _["build"]["healthy"]),
         ("SHAPES", _["shapes"]["version"], _["shapes"]["healthy"]),
-        (_["store"]["type"], _["store"]["version"], _["store"]["healthy"]),
+        ("STORE", store_version, _["store"]["healthy"]),
     ]
     app.echo_info_table(
         table,
@@ -224,3 +234,4 @@ admin.add_command(store)
 admin.add_command(user)
 admin.add_command(acl)
 admin.add_command(client)
+admin.add_command(migration)

cmem_cmemc/commands/client.py

@@ -10,7 +10,8 @@ from cmem.cmempy.keycloak.client import (
 )
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
 
 NO_CLIENT_ERROR = (

cmem_cmemc/commands/config.py

@@ -1,7 +1,9 @@
 """configuration commands for cmem command line interface."""
+
 import click
 
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import KNOWN_CONFIG_KEYS, ApplicationContext
 
 

cmem_cmemc/commands/dataset.py

@@ -1,7 +1,7 @@
 """dataset commands for cmem command line interface."""
+
 import json
 import re
-from pathlib import Path
 
 import click
 import requests.exceptions
@@ -23,9 +23,13 @@ from cmem.cmempy.workspace.projects.resources.resource import (
 from cmem.cmempy.workspace.search import list_items
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.resource import resource
+from cmem_cmemc.completion import get_dataset_file_mapping
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 from cmem_cmemc.utils import check_or_select_project, struct_to_table
 
 DATASET_FILTER_TYPES = sorted(["project", "regex", "tag", "type"])
@@ -137,7 +141,7 @@ def _post_file_resource(
     post_resource(
         project_id=project_id,
         dataset_id=dataset_id,
-        file_resource=click.open_file(local_file_name, "rb"),
+        file_resource=ClickSmartPath.open(local_file_name),
     )
     app.echo_success("done")
 
@@ -187,7 +191,7 @@ def _upload_file_resource(
     create_resource(
         project_name=project_id,
         resource_name=remote_file_name,
-        file_resource=click.open_file(local_file_name, "rb"),
+        file_resource=ClickSmartPath.open(local_file_name),
         replace=replace,
     )
     app.echo_success("done")
@@ -284,23 +288,15 @@ def _check_or_set_dataset_type(
 
     """
     source = Path(dataset_file).name if dataset_file else ""
-    target = parameter_dict["file"] if "file" in parameter_dict else ""
-    suggestions = (
-        (".ttl", "file"),
-        (".csv", "csv"),
-        (".xlsx", "excel"),
-        (".xml", "xml"),
-        (".json", "json"),
-        (".jsonl", "json"),
-        (".orc", "orc"),
-        (".zip", "multiCsv"),
-        (".yaml", "text"),
-        (".yml", "text"),
-    )
+    target = parameter_dict.get("file", "")
+    suggestions = [
+        (extension, info["type"]) for extension, info in get_dataset_file_mapping().items()
+    ]
     if not dataset_type:
         for check, type_ in suggestions:
             if source.endswith(check) or target.endswith(check):
                 dataset_type = type_
+                break
         if not dataset_type:
             raise ValueError("Missing parameter. Please specify a dataset " "type with '--type'.")
         app.echo_warning(
@@ -440,7 +436,13 @@ def list_command(
                 _["label"],
             ]
             table.append(row)
-        app.echo_info_table(table, headers=["Dataset ID", "Type", "Label"], sort_column=2)
+        app.echo_info_table(
+            table,
+            headers=["Dataset ID", "Type", "Label"],
+            sort_column=2,
+            empty_table_message="No datasets found. "
+            "Use the `dataset create` command to create a new dataset.",
+        )
 
 
 @click.command(cls=CmemcCommand, name="delete")
@@ -525,7 +527,9 @@ def delete_command(
 @click.command(cls=CmemcCommand, name="download")
 @click.argument("dataset_id", type=click.STRING, shell_complete=completion.dataset_ids)
 @click.argument(
-    "output_path", required=True, type=click.Path(allow_dash=True, dir_okay=False, writable=True)
+    "output_path",
+    required=True,
+    type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True),
 )
 @click.option(
     "--replace",
@@ -588,7 +592,7 @@ def download_command(
     "input_path",
     required=True,
     shell_complete=completion.dataset_files,
-    type=click.Path(allow_dash=True, dir_okay=False, writable=True),
+    type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True, remote_okay=True),
 )
 @click.pass_obj
 def upload_command(app: ApplicationContext, dataset_id: str, input_path: str) -> None:
@@ -642,7 +646,7 @@ def inspect_command(app: ApplicationContext, dataset_id: str, raw: bool) -> None
     "DATASET_FILE",
     required=False,
     shell_complete=completion.dataset_files,
-    type=click.Path(allow_dash=False, readable=True, exists=True),
+    type=ClickSmartPath(allow_dash=False, readable=True, exists=True, remote_okay=True),
 )
 @click.option(
     "--type",
@@ -725,9 +729,7 @@ def create_command(  # noqa: PLR0913
         return
 
     # transform the parameter list of tuple to a dictionary
-    parameter_dict = {}
-    for key, value in parameter:
-        parameter_dict[key] = value
+    parameter_dict = dict(parameter)
 
     dataset_type = _check_or_set_dataset_type(
         app=app,
@@ -762,6 +764,7 @@ def create_command(  # noqa: PLR0913
         # add file parameter for the project if needed
         if "file" not in parameter_dict:
             parameter_dict["file"] = Path(dataset_file).name
+
         _upload_file_resource(
             app=app,
             project_id=project_id,

cmem_cmemc/commands/graph.py

@@ -1,8 +1,10 @@
 """graph commands for cmem command line interface."""
+
+import gzip
 import hashlib
+import io
 import json
 import os
-from pathlib import Path
 from xml.dom import minidom  # nosec
 from xml.etree.ElementTree import (  # nosec
     Element,
@@ -22,9 +24,12 @@ from six.moves.urllib.parse import quote
 from treelib import Tree
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.validation import validation_group
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 from cmem_cmemc.utils import (
     convert_uri_to_filename,
     get_graphs,
@@ -77,11 +82,14 @@ def _get_graph_to_file(  # noqa: PLR0913
                 nl=False,
             )
     # create and write the .ttl content file
-    if overwrite is True:
-        triple_file = click.open_file(file_path, "wb")
-    else:
-        triple_file = click.open_file(file_path, "ab")
-    with graph_api.get_streamed(graph_iri, accept=mime_type) as response:
+    mode = "wb" if overwrite is True else "ab"
+
+    with (
+        gzip.open(file_path, mode)
+        if file_path.endswith(".gz")
+        else click.open_file(file_path, mode) as triple_file,
+        graph_api.get_streamed(graph_iri, accept=mime_type) as response,
+    ):
         response.raise_for_status()
         for chunk in response.iter_content(chunk_size=None):
             if chunk:
@@ -94,7 +102,9 @@ def _get_graph_to_file(  # noqa: PLR0913
         app.echo_success("done")
 
 
-def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -> dict:
+def _get_export_names(
+    app: ApplicationContext, iris: list[str], template: str, file_extension: str = "ttl"
+) -> dict:
     """Get a dictionary of generated file names based on a template.
 
     Args:
@@ -102,6 +112,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
         app: the context click application
         iris: list of graph iris
         template (str): the template string to use
+        file_extension(str): the file extension to use
 
     Returns:
     -------
@@ -120,7 +131,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
             hash=hashlib.sha256(iri.encode("utf-8")).hexdigest(),
             iriname=convert_uri_to_filename(iri),
         )
-        _name_created = Template(template).render(template_data) + ".ttl"
+        _name_created = f"{Template(template).render(template_data)}.{file_extension}"
         _names[iri] = _name_created
     if len(_names.values()) != len(set(_names.values())):
         raise ValueError(
@@ -459,7 +470,27 @@ def list_command(
             _["label"]["title"],
         ]
         table.append(row)
-    app.echo_info_table(table, headers=["Graph IRI", "Type", "Label"], sort_column=2)
+    app.echo_info_table(
+        table,
+        headers=["Graph IRI", "Type", "Label"],
+        sort_column=2,
+        empty_table_message="No graphs found. "
+        "Use the `graph import` command to import a graph from a file, or "
+        "use the `admin store bootstrap` command to import the default graphs.",
+    )
+
+
+def _validate_export_command_input_parameters(
+    output_dir: str, output_file: str, compress: str, create_catalog: bool
+) -> None:
+    """Validate export command input parameters combinations"""
+    if output_dir and create_catalog and compress:
+        raise click.UsageError(
+            "Cannot create a catalog file when using a compressed graph file."
+            " Please remove either the --create-catalog or --compress option."
+        )
+    if output_file == "- " and compress:
+        raise click.UsageError("Cannot output a binary file to terminal. Use --output-file option.")
 
 
 # pylint: disable=too-many-arguments,too-many-locals
@@ -480,12 +511,12 @@ def list_command(
 )
 @click.option(
     "--output-dir",
-    type=click.Path(writable=True, file_okay=False),
+    type=ClickSmartPath(writable=True, file_okay=False),
     help="Export to this directory.",
 )
 @click.option(
     "--output-file",
-    type=click.Path(writable=True, allow_dash=True, dir_okay=False),
+    type=ClickSmartPath(writable=True, allow_dash=True, dir_okay=False),
     default="-",
     show_default=True,
     shell_complete=completion.triple_files,
@@ -516,6 +547,11 @@ def list_command(
     type=click.Choice(["application/n-triples", "text/turtle"]),
     help="Define the requested mime type",
 )
+@click.option(
+    "--compress",
+    type=click.Choice(["gzip"]),
+    help="Compress the exported graph files.",
+)
 @click.argument(
     "iris",
     nargs=-1,
@@ -534,6 +570,7 @@ def export_command(  # noqa: PLR0913
     template: str,
     mime_type: str,
     iris: list[str],
+    compress: str,
 ) -> None:
     """Export graph(s) as NTriples to stdout (-), file or directory.
 
@@ -542,6 +579,7 @@ def export_command(  # noqa: PLR0913
     In case of directory export, .graph and .ttl files will be created
     for each graph.
     """
+    _validate_export_command_input_parameters(output_dir, output_file, compress, create_catalog)
     iris = _check_and_extend_exported_graphs(iris, all_, include_imports, get_graphs_as_dict())
 
     count: int = len(iris)
@@ -551,7 +589,8 @@ def export_command(  # noqa: PLR0913
         app.echo_debug("output is directory")
         # pre-calculate all filenames with the template,
         # in order to output errors on naming clashes as early as possible
-        _names = _get_export_names(app, iris, template)
+        _names = _get_export_names(app, iris, template, "ttl.gz" if compress else "ttl")
+        _graph_file_names = _get_export_names(app, iris, template, "ttl.graph")
         # create directory
         if not Path(output_dir).exists():
             app.echo_warning("Output directory does not exist: " + "will create it.")
@@ -560,7 +599,7 @@ def export_command(  # noqa: PLR0913
         for current, iri in enumerate(iris, start=1):
             # join with given output directory and normalize full path
             triple_file_name = os.path.normpath(Path(output_dir) / _names[iri])
-            graph_file_name = triple_file_name + ".graph"
+            graph_file_name = os.path.normpath(Path(output_dir) / _graph_file_names[iri])
             # output directory is created lazy
             Path(triple_file_name).parent.mkdir(parents=True, exist_ok=True)
             # create and write the .ttl.graph metadata file
@@ -574,6 +613,10 @@ def export_command(  # noqa: PLR0913
         return
     # no output directory set -> file export
     if output_file == "-":
+        if compress:
+            raise click.UsageError(
+                "Cannot output a binary file to terminal. Use --output-file option."
+            )
         # in case a file is stdout,
         # all triples from all graphs go in and other output is suppressed
         app.echo_debug("output is stdout")
@@ -581,6 +624,9 @@ def export_command(  # noqa: PLR0913
             _get_graph_to_file(iri, output_file, app, mime_type=mime_type)
     else:
         # in case a file is given, all triples from all graphs go in
+        if compress and not output_file.endswith(".gz"):
+            output_file = output_file + ".gz"
+
         app.echo_debug("output is file")
         for current, iri in enumerate(iris, start=1):
             _get_graph_to_file(
@@ -593,6 +639,28 @@ def export_command(  # noqa: PLR0913
             )
 
 
+def validate_input_path(input_path: str) -> None:
+    """Validate input path
+
+    This function checks the provided folder for any .ttl or .nt files
+    that have corresponding .gz files. If such files are found, it raises a ValueError.
+    """
+    files = os.listdir(input_path)
+
+    # Check for files with the given extensions (.ttl and .nt)
+    rdf_files = [f for f in files if f.endswith((".ttl", ".nt"))]
+
+    # Check for corresponding .gz files
+    gz_files = [f"{f}.gz" for f in rdf_files]
+    conflicting_files = [f for f in gz_files if f in files]
+
+    if conflicting_files:
+        raise ValueError(
+            f"The following RDF files (.ttl/.nt) have corresponding '.gz' files,"
+            f" which is not allowed: {', '.join(conflicting_files)}"
+        )
+
+
 @click.command(cls=CmemcCommand, name="import")
 @click.option(
     "--replace",
@@ -611,7 +679,7 @@ def export_command(  # noqa: PLR0913
     "input_path",
     required=True,
     shell_complete=completion.triple_files,
-    type=click.Path(allow_dash=False, readable=True),
+    type=ClickSmartPath(allow_dash=False, readable=True, remote_okay=True),
 )
 @click.argument("iri", type=click.STRING, required=False, shell_complete=completion.graph_uris)
 @click.pass_obj
@@ -647,6 +715,7 @@ def import_command(
         )
     graphs: list
     if Path(input_path).is_dir():
+        validate_input_path(input_path)
         if iri is None:
             # in case a directory is the source (and no IRI is given),
             # the graph/nt file structure is crawled
@@ -654,6 +723,7 @@ def import_command(
         else:
             # in case a directory is the source AND IRI is given
             graphs = [(file, iri) for file in Path(input_path).glob("*.ttl")]
+            graphs += [(file, iri) for file in Path(input_path).glob("*.ttl.gz")]
     elif Path(input_path).is_file():
         if iri is None:
             raise ValueError(
@@ -678,7 +748,21 @@ def import_command(
             continue
         # prevents re-replacing of graphs in a single run
         _replace = False if graph_iri in processed_graphs else replace
-        graph_api.post_streamed(graph_iri, triple_file, replace=_replace)
+        _buffer = io.BytesIO()
+        transport_prams = {}
+        if Path(str(triple_file)).schema in ["http", "https"]:
+            transport_prams["headers"] = {
+                "Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
+                " q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
+            }
+        with ClickSmartPath.open(triple_file, transport_params=transport_prams) as _:
+            _buffer.write(_.read())
+        _buffer.seek(0)
+        response = graph_api.post_streamed(graph_iri, _buffer, replace=_replace)
+        request_headers = response.request.headers
+        request_headers.pop("Authorization")
+        app.echo_debug(f"cmemc request headers: {request_headers}")
+        app.echo_debug(f"server response headers: {response.headers}")
         app.echo_success("replaced" if _replace else "added")
         # refresh access conditions in case of dropped AC graph
         if graph_iri == refresh.AUTHORIZATION_GRAPH_URI: