cmem-cmemc 25.1.1__tar.gz → 25.2.0__tar.gz

{cmem_cmemc-25.1.1 → cmem_cmemc-25.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cmem-cmemc
-Version: 25.1.1
+Version: 25.2.0
 Summary: Command line client for eccenca Corporate Memory
 License: Apache-2.0
 Author: eccenca
@@ -27,7 +27,7 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Utilities
 Requires-Dist: beautifulsoup4 (>=4.13.3,<5.0.0)
 Requires-Dist: certifi (>=2024.2.2)
-Requires-Dist: click (>=8.1.8,<9.0.0)
+Requires-Dist: click (>=8.1.8,<8.2.0)
 Requires-Dist: click-didyoumean (>=0.3.1,<0.4.0)
 Requires-Dist: click-help-colors (>=0.9.4,<0.10.0)
 Requires-Dist: cmem-cmempy (==25.1.0)

{cmem_cmemc-25.1.1 → cmem_cmemc-25.2.0}/cmem_cmemc/command_group.py

@@ -65,6 +65,7 @@ class CmemcGroup(HelpColorsGroup, DYMGroup):
                 "validation": self.color_for_command_groups,
                 "migrate": self.color_for_writing_commands,
                 "migrations": self.color_for_command_groups,
+                "imports": self.color_for_command_groups,
             },
         )
         super().__init__(*args, **kwargs)

{cmem_cmemc-25.1.1 → cmem_cmemc-25.2.0}/cmem_cmemc/commands/admin.py

@@ -24,11 +24,11 @@ from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.utils import struct_to_table
 
 WARNING_MIGRATION = (
-    "Your workspace configuration version does not match your DataPlatform version. "
+    "Your workspace configuration version does not match your Explore version. "
     "Please consider migrating your workspace configuration (admin store migrate command)."
 )
 WARNING_SHAPES = (
-    "Your ShapeCatalog version does not match your DataPlatform version. "
+    "Your ShapeCatalog version does not match your Explore version. "
     "Please consider updating your bootstrap data (admin store boostrap command)."
 )
 
@@ -108,7 +108,7 @@ def status_command(  # noqa: C901, PLR0912
     Additionally, this command informs you in one of these cases:
     (1) A warning, if the target version of your cmemc client is newer than the version
     of your backend.
-    (2) A warning, if the ShapeCatalog has a different version than your DataPlatform component.
+    (2) A warning, if the ShapeCatalog has a different version than your Explore component.
     (3) An error, if your Corporate Memory license is expired (grace period).
     (4) A warning, if your Graph DB license will expire in less than a month.
 

{cmem_cmemc-25.1.1 → cmem_cmemc-25.2.0}/cmem_cmemc/commands/config.py

@@ -119,8 +119,8 @@ def config() -> CmemcGroup:  # type: ignore[empty-body]
 
     \b
     SSL_VERIFY=False    - for ignoring certificate issues (not recommended)
-    DP_API_ENDPOINT=URL - to point to a non-standard DataPlatform location
-    DI_API_ENDPOINT=URL - to point to a non-standard DataIntegration location
+    DP_API_ENDPOINT=URL - to point to a non-standard Explore backend (DataPlatform) location
+    DI_API_ENDPOINT=URL - to point to a non-standard Build (DataIntegration) location
     OAUTH_TOKEN_URI=URL - to point to an external IdentityProvider location
     OAUTH_USER=username - only if OAUTH_GRANT_TYPE=password
     OAUTH_PASSWORD=password - only if OAUTH_GRANT_TYPE=password

{cmem_cmemc-25.1.1 → cmem_cmemc-25.2.0}/cmem_cmemc/commands/graph.py

@@ -14,41 +14,34 @@ from xml.etree.ElementTree import (  # nosec
 )
 
 import click
-from click import Argument, ClickException, UsageError
+from click import ClickException, Context, UsageError
 from cmem.cmempy.config import get_cmem_base_uri
 from cmem.cmempy.dp.authorization import refresh
 from cmem.cmempy.dp.proxy import graph as graph_api
-from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, get_graph_imports
+from cmem.cmempy.dp.proxy.graph import get_graph_imports
 from cmem.cmempy.dp.proxy.sparql import get as sparql_api
 from jinja2 import Template
 from six.moves.urllib.parse import quote
-from treelib import Tree
 
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.commands.graph_imports import graph_imports_list, imports_group
 from cmem_cmemc.commands.validation import validation_group
+from cmem_cmemc.constants import UNKNOWN_GRAPH_ERROR
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.parameter_types.path import ClickSmartPath
 from cmem_cmemc.smart_path import SmartPath
 from cmem_cmemc.utils import (
+    RdfGraphData,
     convert_uri_to_filename,
     get_graphs,
     get_graphs_as_dict,
     iri_to_qname,
     read_rdf_graph_files,
+    tuple_to_list,
 )
 
-UNKNOWN_GRAPH_ERROR = "The graph {} is not accessible or does not exist."
-
-
-def tuple_to_list(ctx: ApplicationContext, param: Argument, value: tuple) -> list:  # noqa: ARG001
-    """Get a list from a tuple
-
-    Used as callback to have mutable values
-    """
-    return list(value)
-
 
 def count_graph(graph_iri: str) -> int:
     """Count triples in a graph and return integer."""
@@ -142,63 +135,6 @@ def _get_export_names(
     return _names
 
 
-def _create_node_label(iri: str, graphs: dict) -> str:
-    """Create a label for a node in the tree."""
-    if iri not in graphs:
-        return "[missing: " + iri + "]"
-    title = graphs[iri]["label"]["title"]
-    return f"{title} -- {iri}"
-
-
-def _add_tree_nodes_recursive(tree: Tree, structure: dict, iri: str, graphs: dict) -> Tree:
-    """Add all child nodes of iri from structure to tree.
-
-    Call recursively until no child node can be used as parent anymore.
-
-    Args:
-    ----
-        tree: the graph where to add the nodes
-        structure: the result dict of get_graph_import_tree()
-        iri: The IRI of the parent
-        graphs: the result of get_graphs()
-
-    Returns:
-    -------
-        the new treelib.Tree object with the additional nodes
-
-    """
-    if not tree.contains(iri):
-        tree.create_node(tag=_create_node_label(iri, graphs), identifier=iri)
-    if iri not in structure:
-        return tree
-    for child in structure[iri]:
-        tree.create_node(tag=_create_node_label(child, graphs), identifier=child, parent=iri)
-    for child in structure[iri]:
-        if child in structure:
-            tree = _add_tree_nodes_recursive(tree, structure, child, graphs)
-    return tree
-
-
-def _add_ignored_nodes(tree: Tree, structure: dict) -> Tree:
-    """Add all child nodes as ignored nodes.
-
-    Args:
-    ----
-        tree: the graph where to add the nodes
-        structure: the result dict of get_graph_import_tree()
-
-    Returns:
-    -------
-        the new treelib.Tree object with the additional nodes
-
-    """
-    if len(structure.keys()) > 0:
-        for parent in structure:
-            for children in structure[parent]:
-                tree.create_node(tag="[ignored: " + children + "]", parent=parent)
-    return tree
-
-
 def _get_graphs_filtered(filter_name: str, filter_value: str) -> list[dict]:
     """Get graphs but filtered according to name and value."""
     # not filtered means all graphs
@@ -322,29 +258,7 @@ def _create_xml_catalog_file(app: ApplicationContext, names: dict, output_dir: s
     app.echo_success(f"XML catalog file created as {file_name}.")
 
 
-def _prepare_tree_output_id_only(iris: list[str], graphs: dict) -> str:
-    """Prepare a sorted, de-duplicated IRI list of graph imports."""
-    output_iris = []
-    for iri in iris:
-        # get response for one requested graph
-        api_response = get_graph_import_tree(iri)
-
-        # add all imported IRIs to the IRI list
-        # add the requested graph as well
-        output_iris.append(iri)
-        for top_graph in api_response["tree"]:
-            output_iris.append(top_graph)
-            for sub_graph in api_response["tree"][top_graph]:
-                output_iris.append(sub_graph)  # noqa: PERF402
-
-    # prepare a sorted, de-duplicated IRI list of existing graphs
-    # and create a line-by-line output of it
-    output_iris = sorted(set(output_iris), key=lambda x: x.lower())
-    filtered_iris = [iri for iri in output_iris if iri in graphs]
-    return "\n".join(filtered_iris[0:]) + "\n"
-
-
-@click.command(cls=CmemcCommand, name="tree")
+@click.command(cls=CmemcCommand, name="tree", hidden=True)
 @click.option("-a", "--all", "all_", is_flag=True, help="Show tree of all (readable) graphs.")
 @click.option("--raw", is_flag=True, help="Outputs raw JSON of the graph importTree API response.")
 @click.option(
@@ -362,64 +276,17 @@ def _prepare_tree_output_id_only(iris: list[str], graphs: dict) -> str:
     shell_complete=completion.graph_uris,
     callback=tuple_to_list,
 )
-@click.pass_obj
-def tree_command(
-    app: ApplicationContext, all_: bool, raw: bool, id_only: bool, iris: list[str]
-) -> None:
-    """Show graph tree(s) of the owl:imports hierarchy.
-
-    You can output one or more trees of the import hierarchy.
-
-    Imported graphs which do not exist are shown as `[missing: IRI]`.
-    Imported graphs which will result in an import cycle are shown as
-    `[ignored: IRI]`.
-    Each graph is shown with label and IRI.
-    """
-    graphs = get_graphs_as_dict()
-    if not iris and not all_:
-        raise UsageError(
-            "Either specify at least one graph IRI or use the "
-            "--all option to show the owl:imports tree of all graphs."
-        )
-    if all_:
-        iris = [str(_) for _ in graphs]
-
-    for iri in iris:
-        if iri not in graphs:
-            raise ClickException(UNKNOWN_GRAPH_ERROR.format(iri))
-
-    iris = sorted(iris, key=lambda x: graphs[x]["label"]["title"].lower())
-
-    if raw:
-        for iri in iris:
-            # direct output of the response for one requested graph
-            app.echo_info_json(get_graph_import_tree(iri))
-        return
-
-    if id_only:
-        app.echo_result(_prepare_tree_output_id_only(iris, graphs), nl=False)
-        return
-
-    # normal execution
-    output = ""
-    for iri in iris:
-        # get response for on requested graph
-        api_response = get_graph_import_tree(iri)
-
-        tree = _add_tree_nodes_recursive(Tree(), api_response["tree"], iri, graphs)
-        tree = _add_ignored_nodes(tree, api_response["ignored"])
-
-        # strip empty lines from the tree.show output
-        output += os.linesep.join(
-            [
-                line
-                for line in tree.show(key=lambda x: x.tag.lower(), stdout=False).splitlines()  # type: ignore[arg-type, return-value]
-                if line.strip()
-            ]
-        )
-        output += "\n"
-    # result output
-    app.echo_result(output, nl=False)
+@click.pass_context
+def tree_command(ctx: Context, all_: bool, raw: bool, id_only: bool, iris: list[str]) -> None:
+    """(Hidden) Deprecated: use 'graph imports tree' instead."""
+    app: ApplicationContext = ctx.obj
+    app.echo_warning(
+        "The 'tree' command is deprecated and will be removed with the next major release."
+        " Please use the 'graph imports tree' command instead."
+    )
+    imports_cmd = graph.commands["imports"]
+    tree_cmd = imports_cmd.commands["tree"]  # type: ignore[attr-defined]
+    ctx.invoke(tree_cmd, all_=all_, raw=raw, id_only=id_only, iris=iris)
 
 
 @click.command(cls=CmemcCommand, name="list")
@@ -494,6 +361,15 @@ def _validate_export_command_input_parameters(
         raise UsageError("Cannot output a binary file to terminal. Use --output-file option.")
 
 
+def _write_graph_imports(ctx: Context, filename: str, iri: str) -> None:
+    imports = graph_imports_list.apply_filters(ctx=ctx, filter_=[("to-graph", iri)])
+    if imports:
+        imports_file = click.open_file(filename, "w")
+        for _ in imports:
+            imports_file.write(_["from_graph"] + "\n")
+        imports_file.close()
+
+
 # pylint: disable=too-many-arguments,too-many-locals
 @click.command(cls=CmemcCommand, name="export")
 @click.option("-a", "--all", "all_", is_flag=True, help="Export all readable graphs.")
@@ -503,6 +379,12 @@ def _validate_export_command_input_parameters(
     help="Export selected graph(s) and all graphs which are imported from "
     "these selected graph(s).",
 )
+@click.option(
+    "--include-import-statements",
+    is_flag=True,
+    help="Save graph imports information from other graphs to the exported graphs "
+    "and write *.imports files.",
+)
 @click.option(
     "--create-catalog",
     is_flag=True,
@@ -560,11 +442,12 @@ def _validate_export_command_input_parameters(
     shell_complete=completion.graph_uris,
     callback=tuple_to_list,
 )
-@click.pass_obj
-def export_command(  # noqa: PLR0913
-    app: ApplicationContext,
+@click.pass_context
+def export_command(  # noqa: C901, PLR0913
+    ctx: Context,
     all_: bool,
     include_imports: bool,
+    include_import_statements: bool,
     create_catalog: bool,
     output_dir: str,
     output_file: str,
@@ -580,6 +463,7 @@ def export_command(  # noqa: PLR0913
     In case of directory export, .graph and .ttl files will be created
     for each graph.
     """
+    app: ApplicationContext = ctx.obj
     _validate_export_command_input_parameters(output_dir, output_file, compress, create_catalog)
     iris = _check_and_extend_exported_graphs(iris, all_, include_imports, get_graphs_as_dict())
 
@@ -595,6 +479,8 @@ def export_command(  # noqa: PLR0913
             app, iris, template, f"{extension}.gz" if compress else f"{extension}"
         )
         _graph_file_names = _get_export_names(app, iris, template, f"{extension}.graph")
+        _graph_imports_file_names = _get_export_names(app, iris, template, f"{extension}.imports")
+
         # create directory
         if not SmartPath(output_dir).exists():
             app.echo_warning("Output directory does not exist: " + "will create it.")
@@ -604,14 +490,21 @@ def export_command(  # noqa: PLR0913
             # join with given output directory and normalize full path
             triple_file_name = os.path.normpath(SmartPath(output_dir) / _names[iri])
             graph_file_name = os.path.normpath(SmartPath(output_dir) / _graph_file_names[iri])
+            imports_file_name = os.path.normpath(
+                SmartPath(output_dir) / _graph_imports_file_names[iri]
+            )
             # output directory is created lazy
             SmartPath(triple_file_name).parent.mkdir(parents=True, exist_ok=True)
             # create and write the .ttl.graph metadata file
             graph_file = click.open_file(graph_file_name, "w")
             graph_file.write(iri + "\n")
+            graph_file.close()
+            if include_import_statements:
+                _write_graph_imports(ctx=ctx, filename=imports_file_name, iri=iri)
             _get_graph_to_file(
                 iri, triple_file_name, app, numbers=(current, count), mime_type=mime_type
             )
+
         if create_catalog:
             _create_xml_catalog_file(app, _names, output_dir)
         return
@@ -706,6 +599,64 @@ def _get_buffer_and_content_type(
     return buffer, content_type
 
 
+def _create_graph_imports(ctx: Context, graphs: list[RdfGraphData]) -> None:
+    # Locate and invoke the 'create' subcommand under 'graph imports' command
+    app: ApplicationContext = ctx.obj
+    imports_cmd = graph.commands["imports"]
+    create_cmd = imports_cmd.commands["create"]  # type: ignore[attr-defined]
+    for _ in graphs:
+        if not _.graph_imports:
+            continue
+        for _import in _.graph_imports:
+            from_graph = _import
+            to_graph = _.graph_iri
+            imports = graph_imports_list.apply_filters(
+                ctx=ctx, filter_=[("from-graph", from_graph), ("to-graph", to_graph)]
+            )
+            if imports:
+                app.echo_info(
+                    f"Creating graph import from {from_graph} to {to_graph} ... ", nl=False
+                )
+                app.echo_warning("exists")
+            else:
+                ctx.invoke(create_cmd, from_graph=from_graph, to_graph=to_graph)
+
+
+def _process_input_directory(input_path: str, iri: str) -> list[RdfGraphData]:
+    if iri is None:
+        # in case a directory is the source (and no IRI is given),
+        # the graph/nt file structure is crawled
+        graphs = read_rdf_graph_files(input_path)
+    else:
+        # in case a directory is the source AND IRI is given
+        graphs = []
+        for _ in _get_graph_supported_formats():
+            extension = mimetypes.guess_extension(_)
+            graphs += [
+                RdfGraphData(str(file), iri, [])
+                for file in SmartPath(input_path).glob(f"*{extension}")
+            ]
+            graphs += [
+                RdfGraphData(str(file), iri, [])
+                for file in SmartPath(input_path).glob(f"*{extension}.gz")
+            ]
+    return graphs
+
+
+def _validate_graph_imports(graphs_to_import: list[RdfGraphData]) -> None:
+    graphs = {_["iri"] for _ in get_graphs()}
+    graphs.update({_.graph_iri for _ in graphs_to_import})
+    graph_imports: set[str] = set()
+    for graph_data in graphs_to_import:
+        for item in graph_data.graph_imports:
+            graph_imports.add(str(item))
+    if graph_imports - graphs:
+        raise click.UsageError(
+            f"The following graphs are not available, "
+            f"so you can not add import statements to them: {','.join(graph_imports - graphs)}"
+        )
+
+
 @click.command(cls=CmemcCommand, name="import")
 @click.option(
     "--replace",
@@ -720,6 +671,11 @@ def _get_buffer_and_content_type(
     "beginning of the process, so that you can still add multiple "
     "files to one single graph (if it does not exist).",
 )
+@click.option(
+    "--include-import-statements",
+    is_flag=True,
+    help="Use *.imports files to re-apply the graph imports of the imported graphs.",
+)
 @click.argument(
     "input_path",
     required=True,
@@ -727,12 +683,13 @@ def _get_buffer_and_content_type(
     type=ClickSmartPath(allow_dash=False, readable=True, remote_okay=True),
 )
 @click.argument("iri", type=click.STRING, required=False, shell_complete=completion.graph_uris)
-@click.pass_obj
-def import_command(
-    app: ApplicationContext,
+@click.pass_context
+def import_command(  # noqa: PLR0913
+    ctx: Context,
     input_path: str,
     replace: bool,
     skip_existing: bool,
+    include_import_statements: bool,
     iri: str,
 ) -> None:
     """Import graph(s) to the store.
@@ -752,35 +709,25 @@ def import_command(
 
     Note: Directories are scanned on the first level only (not recursively).
     """
+    app: ApplicationContext = ctx.obj
     if replace and skip_existing:
         raise UsageError(
             "The options --replace and --skip-existing are mutually "
             "exclusive, so please remove one of them."
         )
     # is an array of tuples like this [('path/to/triple.file', 'graph IRI')]
-    graphs: list[tuple[str, str]]
+    graphs: list[RdfGraphData]
     if SmartPath(input_path).is_dir():
         validate_input_path(input_path)
-        if iri is None:
-            # in case a directory is the source (and no IRI is given),
-            # the graph/nt file structure is crawled
-            graphs = read_rdf_graph_files(input_path)
-        else:
-            # in case a directory is the source AND IRI is given
-            graphs = []
-            for _ in _get_graph_supported_formats():
-                extension = mimetypes.guess_extension(_)
-                graphs += [(str(file), iri) for file in SmartPath(input_path).glob(f"*{extension}")]
-                graphs += [
-                    (str(file), iri) for file in SmartPath(input_path).glob(f"*{extension}.gz")
-                ]
-
+        graphs = _process_input_directory(input_path, iri)
+        if include_import_statements:
+            _validate_graph_imports(graphs)
     elif SmartPath(input_path).is_file():
         if iri is None:
             raise UsageError(
                 "Either specify an input file AND a graph IRI or an input directory ONLY."
             )
-        graphs = [(input_path, iri)]
+        graphs = [RdfGraphData(input_path, iri, [])]
     else:
         raise NotImplementedError(
             "Input from special files (socket, FIFO, device file) is not supported."
@@ -790,7 +737,9 @@ def import_command(
     processed_graphs: set = set()
     count: int = len(graphs)
     current: int = 1
-    for triple_file, graph_iri in graphs:
+    for _ in graphs:
+        triple_file = _.file_path
+        graph_iri = _.graph_iri
         app.echo_info(
             f"Import file {current}/{count}: " f"{graph_iri} from {triple_file} ... ", nl=False
         )
@@ -814,6 +763,9 @@ def import_command(
             app.echo_debug("Access conditions refreshed.")
         processed_graphs.add(graph_iri)
         current += 1
+    # create graph imports
+    if include_import_statements:
+        _create_graph_imports(ctx, graphs)
 
 
 @click.command(cls=CmemcCommand, name="delete")
@@ -824,6 +776,9 @@ def import_command(
     help="Delete selected graph(s) and all writeable graphs which are "
     "imported from these selected graph(s).",
 )
+@click.option(
+    "--include-import-statements", is_flag=True, help="Delete import reference of deleted graphs"
+)
 @click.argument(
     "iris",
     nargs=-1,
@@ -831,16 +786,25 @@ def import_command(
     shell_complete=completion.writable_graph_uris,
     callback=tuple_to_list,
 )
-@click.pass_obj
+@click.pass_context
 def delete_command(
-    app: ApplicationContext, all_: bool, include_imports: bool, iris: list[str]
+    ctx: Context,
+    all_: bool,
+    include_imports: bool,
+    include_import_statements: bool,
+    iris: list[str],
 ) -> None:
     """Delete graph(s) from the store."""
+    app: ApplicationContext = ctx.obj
     graphs = get_graphs_as_dict(writeable=True, readonly=False)
     iris = _check_and_extend_exported_graphs(iris, all_, include_imports, graphs)
-
+    imports_to_be_deleted = []
     count: int = len(iris)
     for current, iri in enumerate(iris, start=1):
+        imports_to_be_deleted += graph_imports_list.apply_filters(
+            ctx=ctx, filter_=[("to-graph", iri)]
+        )
+
         app.echo_info(f"Drop graph {current}/{count}: {iri} ... ", nl=False)
         graph_api.delete(iri)
         app.echo_success("done")
@@ -848,6 +812,13 @@ def delete_command(
         if iri == refresh.AUTHORIZATION_GRAPH_URI:
             refresh.get()
             app.echo_debug("Access conditions refreshed.")
+    if include_import_statements:
+        # Locate and invoke the 'delete' subcommand under 'graph imports' command
+        imports_cmd = graph.commands["imports"]
+        delete_cmd = imports_cmd.commands["delete"]  # type: ignore[attr-defined]
+        for _ in imports_to_be_deleted:
+            if _["from_graph"] not in iris:
+                ctx.invoke(delete_cmd, from_graph=_["from_graph"], to_graph=_["to_graph"])
 
 
 @click.command(cls=CmemcCommand, name="open")
@@ -914,3 +885,4 @@ graph.add_command(delete_command)
 graph.add_command(import_command)
 graph.add_command(open_command)
 graph.add_command(validation_group)
+graph.add_command(imports_group)