cmem-cmemc 24.2.0rc1__py3-none-any.whl → 24.3.0rc1__py3-none-any.whl

cmem_cmemc/commands/metrics.py

@@ -1,39 +1,155 @@
 """metrics commands for cmem command line interface."""
 
 import click
-from prometheus_client import Metric
+from click import Argument, Context
+from click.shell_completion import CompletionItem
+from cmem.cmempy.api import request
+from cmem.cmempy.config import get_cmem_base_uri, get_di_api_endpoint, get_dp_api_endpoint
+from prometheus_client.parser import text_string_to_metric_families
+from requests import HTTPError
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
-from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.context import CONTEXT, ApplicationContext
+from cmem_cmemc.object_list import (
+    DirectValuePropertyFilter,
+    ObjectList,
+    compare_regex,
+    compare_str_equality,
+)
 from cmem_cmemc.utils import (
     metric_get_labels,
-    metrics_get_dict,
-    metrics_get_family,
     struct_to_table,
 )
 
 
-def _filter_samples(family: Metric, label_filter: tuple[tuple[str, str], ...]) -> list:
+def get_all_metrics(ctx: click.Context) -> list[dict]:
+    """Get metrics data for object list"""
+    known_metrics_urls: list[tuple[str, str]] = [
+        ("explore", get_dp_api_endpoint() + "/actuator/prometheus"),
+        ("build", get_di_api_endpoint() + "/metrics"),
+        ("store", get_dp_api_endpoint() + "/actuator/proxy/graphdb/metrics/structures"),
+        ("store", get_dp_api_endpoint() + "/actuator/proxy/graphdb/metrics/infrastructure"),
+        ("store", get_dp_api_endpoint() + "/actuator/proxy/graphdb/metrics/cluster"),
+        ("store", get_dp_api_endpoint() + "/actuator/proxy/graphdb/metrics/repository/cmem"),
+    ]
+    app: ApplicationContext = ctx.obj
+    all_metrics: list[dict] = []
+    for job, url in known_metrics_urls:
+        try:
+            families = text_string_to_metric_families(request(url).text)
+        except HTTPError as error:
+            if app is not None:
+                # when in completion mode, obj is not set :-(
+                app.echo_debug(str(error))
+            continue
+        for family in families:
+            documentation = family.documentation
+            if documentation == "":
+                documentation = f"No documentation available for {job}:{family.name}"
+            new_metric = {
+                "id": f"{job}:{family.name}",
+                "job": job,
+                "name": family.name,
+                "type": family.type,
+                "documentation": documentation,
+                "labels": metric_get_labels(family),
+                "samples": family.samples,
+                # "object": family
+            }
+            if family.type not in ("unknown", "histogram"):
+                all_metrics.append(new_metric)
+    return all_metrics
+
+
+metrics_list = ObjectList(
+    name="metrics",
+    get_objects=get_all_metrics,
+    filters=[
+        DirectValuePropertyFilter(
+            name="job",
+            description="Filter metrics by job ID.",
+            property_key="job",
+        ),
+        DirectValuePropertyFilter(
+            name="name",
+            description="Filter metrics by regex matching the name.",
+            property_key="name",
+            compare=compare_regex,
+            fixed_completion=[],
+        ),
+        DirectValuePropertyFilter(
+            name="type",
+            description="Filter metrics by type.",
+            property_key="type",
+        ),
+        DirectValuePropertyFilter(
+            name="id",
+            description="Filter metrics by ID.",
+            property_key="id",
+            compare=compare_str_equality,
+        ),
+    ],
+)
+
+
+def _complete_metrics_id(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:  # noqa: ARG001
+    """Prepare a list of metric identifier."""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    candidates = [(_["id"], _["documentation"]) for _ in metrics_list.apply_filters(ctx=ctx)]
+    return completion.finalize_completion(candidates=candidates, incomplete=incomplete)
+
+
+def _complete_metric_label_filter(
+    ctx: Context,
+    param: Argument,  # noqa: ARG001
+    incomplete: str,
+) -> list[CompletionItem]:
+    """Prepare a list of label name or values"""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    args = completion.get_completion_args(incomplete)
+    incomplete = incomplete.lower()
+    options: list[str] = []
+    try:
+        metric_id = ctx.args[0]
+        metric = metrics_list.apply_filters(ctx=ctx, filter_=[("id", metric_id)])[0]
+    except IndexError:
+        # means: --filter is used before we have a metrics ID
+        return []
+    labels = metric["labels"]
+    if args[len(args) - 1] in "--filter":
+        # we are in the name position
+        options = list(labels.keys())
+    if args[len(args) - 2] in "--filter":
+        label_name = args[len(args) - 1]
+        # we are in the value position
+        options = labels[label_name]
+    return completion.finalize_completion(candidates=options, incomplete=incomplete)
+
+
+def _filter_samples(family: dict, label_filter: tuple[tuple[str, str], ...]) -> list:
     """Filter samples by labels."""
-    all_samples = family.samples
+    family_name = family.get("name")
+    all_samples: list[list] = family["samples"]
     if not label_filter:
         return all_samples
-    labels = metric_get_labels(family)
+    labels: dict[str, list[str]] = family["labels"]
     samples = []
     for sample in all_samples:
         matching_labels = 0
+        sample_labels = sample[1]
         for name, value in label_filter:
             if name not in labels:
                 raise ValueError(
-                    f"The metric '{family.name}' does " f"not have a label named '{name}'."
+                    f"The metric '{family_name}' does " f"not have a label named '{name}'."
                 )
             if value not in labels[name]:
                 raise ValueError(
-                    f"The metric '{family.name}' does "
+                    f"The metric '{family_name}' does "
                     f"not have a label '{name}' with the value '{value}'."
                 )
-            if name in sample.labels and sample.labels[name] == value:
+            if name in sample_labels and sample_labels[name] == value:
                 matching_labels += 1
         if matching_labels == len(label_filter):
             # all filter match
@@ -43,20 +159,12 @@ def _filter_samples(family: Metric, label_filter: tuple[tuple[str, str], ...]) -
 
 # pylint: disable-msg=too-many-arguments
 @click.command(cls=CmemcCommand, name="get")
-@click.argument("metric_id", required=True, type=click.STRING, shell_complete=completion.metric_ids)
-@click.option(
-    "--job",
-    "job_id",
-    type=click.Choice(["DP"]),
-    default="DP",
-    show_default=True,
-    help="The job from which the metrics data is fetched.",
-)
+@click.argument("metric_id", required=True, type=click.STRING, shell_complete=_complete_metrics_id)
 @click.option(
     "--filter",
     "label_filter",
     type=(str, str),
-    shell_complete=completion.metric_label_filter,
+    shell_complete=_complete_metric_label_filter,
     multiple=True,
     help="A set of label name/value pairs in order to filter the samples "
     "of the requested metric family. Each metric has a different set "
@@ -74,11 +182,10 @@ def _filter_samples(family: Metric, label_filter: tuple[tuple[str, str], ...]) -
     "even for single row tables.",
 )
 @click.option("--raw", is_flag=True, help="Outputs raw prometheus sample classes.")
-@click.pass_obj
-def get_command(  # noqa: PLR0913
-    app: ApplicationContext,
+@click.pass_context
+def get_command(
+    ctx: click.Context,
     metric_id: str,
-    job_id: str,
     label_filter: tuple[tuple[str, str], ...],
     raw: bool,
     enforce_table: bool,
@@ -90,32 +197,33 @@ def get_command(  # noqa: PLR0913
     command. A metric can contain multiple samples.
     These samples are distinguished by labels (name and value).
     """
-    app.echo_debug(f"Get metrics family '{metric_id}' for job '{job_id}'")
-    family = metrics_get_family(job_id, metric_id)
-    app.echo_debug(f"Filter family '{metric_id}' with filter '{label_filter}'")
-    samples = _filter_samples(family, label_filter)
+    app = ctx.obj
+    data = metrics_list.apply_filters(ctx=ctx, filter_=[("id", metric_id)])
+    if len(data) == 0:
+        raise click.UsageError(
+            f"No metric with ID '{metric_id}' found. "
+            "Use the `metrics list` command to list all metrics."
+        )
+    if len(data) > 1:
+        raise click.UsageError("Unknown Error - More than one metric with ID '{metric_id}' found.")
+    metric = data[0]
+
+    samples = _filter_samples(metric, label_filter)
+    if raw:
+        app.echo_info_json(samples)
+        return
+
     if len(samples) == 0:
         raise ValueError(
             "No data - the given label combination filtered out "
             f"all available samples of the metric {metric_id}."
         )
 
-    if raw:
-        for sample in samples:
-            app.echo_info(str(sample))
-        return
-
     if len(samples) == 1 and enforce_table is not True:
         app.echo_info(str(samples[0].value))
         return
 
-    if family.type == "histogram":
-        raise NotImplementedError(
-            "Visualization of histogram metrics is not supported yet.\n"
-            "Please use the 'metrics inspect' command to get the raw data."
-        )
-
-    label_dict = metric_get_labels(family, clean=True)
+    label_dict = metric["labels"]
     table = []
     for sample in samples:
         row = [sample.labels[key] for key in sorted(label_dict.keys())]
@@ -127,42 +235,39 @@ def get_command(  # noqa: PLR0913
 
 
 @click.command(cls=CmemcCommand, name="inspect")
-@click.argument("metric_id", required=True, type=click.STRING, shell_complete=completion.metric_ids)
-@click.option(
-    "--job",
-    "job_id",
-    type=click.Choice(["DP"]),
-    default="DP",
-    show_default=True,
-    help="The job from which the metrics data is fetched.",
-)
+@click.argument("metric_id", required=True, type=click.STRING, shell_complete=_complete_metrics_id)
 @click.option("--raw", is_flag=True, help="Outputs raw JSON of the table data.")
-@click.pass_obj
-def inspect_command(app: ApplicationContext, metric_id: str, job_id: str, raw: bool) -> None:
+@click.pass_context
+def inspect_command(ctx: Context, metric_id: str, raw: bool) -> None:
     """Inspect a metric.
 
     This command outputs the data of a metric.
     The first table includes basic metadata about the metric.
     The second table includes sample labels and values.
     """
-    family = metrics_get_family(job_id, metric_id)
+    app = ctx.obj
+    data = metrics_list.apply_filters(ctx=ctx, filter_=[("id", metric_id)])
+    if len(data) == 0:
+        raise click.UsageError(
+            f"No metric with ID '{metric_id}' found. "
+            "Use the `metrics list` command to list all metrics."
+        )
+    if len(data) > 1:
+        raise click.UsageError("Unknown Error - More than one metric with ID '{metric_id}' found.")
     if raw:
-        app.echo_info_json({"family": family.__dict__, "labels": metric_get_labels(family)})
+        app.echo_info_json(data)
         return
-    app.echo_info_table(struct_to_table(family.__dict__), headers=["Key", "Value"], sort_column=0)
-    app.echo_info_table(
-        struct_to_table(metric_get_labels(family)), headers=["Label", "Value"], sort_column=0
-    )
+    app.echo_info_table(struct_to_table(data), headers=["Key", "Value"], sort_column=0)
 
 
 @click.command(cls=CmemcCommand, name="list")
 @click.option(
-    "--job",
-    "job_id",
-    type=click.Choice(["DP"]),
-    default="DP",
-    show_default=True,
-    help="The job from which the metrics data is fetched.",
+    "--filter",
+    "filter_",
+    type=(str, str),
+    multiple=True,
+    help=metrics_list.get_filter_help_text(),
+    shell_complete=metrics_list.complete_values,
 )
 @click.option(
     "--id-only",
@@ -172,8 +277,10 @@ def inspect_command(app: ApplicationContext, metric_id: str, job_id: str, raw: b
 @click.option(
     "--raw", is_flag=True, help="Outputs (sorted) JSON dict, parsed from the metrics API output."
 )
-@click.pass_obj
-def list_command(app: ApplicationContext, job_id: str, id_only: bool, raw: bool) -> None:
+@click.pass_context
+def list_command(
+    ctx: click.Context, filter_: tuple[tuple[str, str]], id_only: bool, raw: bool
+) -> None:
     """List metrics for a specific job.
 
     For each metric, the output table shows the metric ID,
@@ -181,24 +288,33 @@ def list_command(app: ApplicationContext, job_id: str, id_only: bool, raw: bool)
     are describing the samples (L) and a count of how many samples are
     currently available for a metric (S).
     """
-    data = metrics_get_dict(job_id=job_id)
+    app = ctx.obj
+    data = metrics_list.apply_filters(ctx=ctx, filter_=filter_)
     if raw:
-        output = {}
-        for key, family in sorted(data.items()):
-            output[key] = family.__dict__
-        app.echo_info_json(output)
+        app.echo_info_json(data)
         return
     if id_only:
-        for key, _ in sorted(data.items()):
-            app.echo_info(key)
+        for _ in sorted([_.get("id") for _ in data]):
+            app.echo_info(_)
         return
 
-    table = []
-    for _, family in sorted(data.items()):
-        labels = metric_get_labels(family)
-        table.append([family.name, family.type, len(labels), len(family.samples)])
-
-    app.echo_info_table(table, headers=["ID", "Type", "L", "S"], sort_column=0)
+    table = [
+        [
+            _.get("id"),
+            _.get("type"),
+            len(_.get("labels")),
+            len(_.get("samples")),
+            _.get("documentation"),
+        ]
+        for _ in data
+    ]
+    app.echo_info_table(
+        table,
+        headers=["ID", "Type", "L", "S", "Documentation"],
+        sort_column=0,
+        caption=f"{len(table)} metrics families of {get_cmem_base_uri()}",
+        empty_table_message="No metrics families available.",
+    )
 
 
 @click.group(cls=CmemcGroup)

cmem_cmemc/commands/migration.py

@@ -0,0 +1,265 @@
+"""migrations command group"""
+
+from typing import TYPE_CHECKING
+
+import click
+from click import Argument, Context
+from click.shell_completion import CompletionItem
+
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.completion import check_option_in_params, finalize_completion
+from cmem_cmemc.context import CONTEXT
+from cmem_cmemc.migrations.access_conditions_243 import (
+    MoveAccessConditionsToNewGraph,
+    RenameAuthVocabularyResources,
+)
+from cmem_cmemc.migrations.bootstrap_data import MigrateBootstrapData
+from cmem_cmemc.migrations.shapes_widget_integrations_243 import (
+    ChartsOnNodeShapesToToWidgetIntegrations,
+    ChartsOnPropertyShapesToWidgetIntegrations,
+    WorkflowTriggerPropertyShapesToWidgetIntegrations,
+)
+from cmem_cmemc.migrations.workspace_configurations import MigrateWorkspaceConfiguration
+from cmem_cmemc.object_list import (
+    DirectListPropertyFilter,
+    DirectValuePropertyFilter,
+    ObjectList,
+    compare_regex,
+    transform_lower,
+)
+
+if TYPE_CHECKING:
+    from cmem_cmemc.context import ApplicationContext
+    from cmem_cmemc.migrations.abc import MigrationRecipe
+
+
+def get_migrations(ctx: click.Context) -> list[dict]:  # noqa: ARG001
+    """Get users for object list"""
+    data = [
+        {
+            "id": _.id,
+            "description": _.description,
+            "component": _.component,
+            "first_version": _.first_version,
+            "last_version": _.last_version,
+            "tags": _.tags,
+            "object": _,
+        }
+        for _ in [
+            MoveAccessConditionsToNewGraph(),
+            RenameAuthVocabularyResources(),
+            MigrateBootstrapData(),
+            MigrateWorkspaceConfiguration(),
+            ChartsOnNodeShapesToToWidgetIntegrations(),
+            ChartsOnPropertyShapesToWidgetIntegrations(),
+            WorkflowTriggerPropertyShapesToWidgetIntegrations(),
+        ]
+    ]
+    data.sort(key=lambda x: x["first_version"])
+    return data
+
+
+def complete_migration_ids(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
+    """Prepare a list of migration recipe IDs"""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    options = []
+    for _ in get_migrations(ctx=ctx):
+        id_ = _["id"]
+        description = _["description"]
+        if check_option_in_params(id_, ctx.params.get(param.name)):  # type: ignore[attr-defined, arg-type]
+            continue
+        options.append((id_, description))
+    return finalize_completion(candidates=options, incomplete=incomplete)
+
+
+migrations_list = ObjectList(
+    name="migration recipes",
+    get_objects=get_migrations,
+    filters=[
+        DirectValuePropertyFilter(
+            name="id",
+            description="Filter migrations by id.",
+            property_key="id",
+            transform=transform_lower,
+        ),
+        DirectValuePropertyFilter(
+            name="description",
+            description="Filter migrations by regex over description.",
+            property_key="description",
+            compare=compare_regex,
+            fixed_completion=[],
+        ),
+        DirectValuePropertyFilter(
+            name="first_version",
+            description="Filter migrations by first version which is target of this migration.",
+            property_key="first_version",
+            transform=transform_lower,
+        ),
+        DirectListPropertyFilter(
+            name="tag",
+            description="Filter migrations by tags",
+            property_key="tags",
+        ),
+    ],
+)
+
+
+@click.command(cls=CmemcCommand, name="list")
+@click.option(
+    "--filter",
+    "filter_",
+    type=(str, str),
+    multiple=True,
+    help=migrations_list.get_filter_help_text(),
+    shell_complete=migrations_list.complete_values,
+)
+@click.option(
+    "--id-only",
+    is_flag=True,
+    help="Lists only IDs. This is useful for piping the IDs into other commands.",
+)
+@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
+@click.pass_context
+def list_command(
+    ctx: click.Context, filter_: tuple[tuple[str, str]], id_only: bool, raw: bool
+) -> None:
+    """List migration recipies.
+
+    This command lists all available migration recipies
+    """
+    app: ApplicationContext = ctx.obj
+    data = migrations_list.apply_filters(ctx=ctx, filter_=filter_)
+    if raw:
+        # https://stackoverflow.com/questions/17665809/
+        app.echo_info_json([(_, _.pop("object"))[0] for _ in data])
+        return
+    if id_only:
+        for _ in sorted([_.get("id") for _ in data]):
+            app.echo_info(_)
+        return
+
+    table = [
+        [
+            _.get("id"),
+            _.get("description"),
+            ", ".join(sorted(_.get("tags"))),
+            f"{_.get('first_version')} ({_.get('component')})",
+        ]
+        for _ in data
+    ]
+    app.echo_info_table(
+        table,
+        headers=["ID", "Description", "Tags", "First Version"],
+        sort_column=3,
+        caption=f"{len(table)} migration(s)",
+        empty_table_message="No migrations available.",
+    )
+
+
+@click.command(cls=CmemcCommand, name="execute")
+@click.argument(
+    "migration_id", type=click.STRING, required=False, shell_complete=complete_migration_ids
+)
+@click.option(
+    "--filter",
+    "filter_",
+    type=(str, str),
+    multiple=True,
+    help=migrations_list.get_filter_help_text(),
+    shell_complete=migrations_list.complete_values,
+)
+@click.option(
+    "-a",
+    "--all",
+    "all_",
+    is_flag=True,
+    help="Execute all needed migrations.",
+)
+@click.option("--test-only", is_flag=True, help="Only test, do not execute migrations.")
+@click.option("--id-only", is_flag=True, help="Lists only recipe identifier. ")
+@click.pass_context
+def execute_command(  # noqa: PLR0913
+    ctx: click.Context,
+    migration_id: str,
+    filter_: tuple[tuple[str, str]],
+    all_: bool,
+    test_only: bool,
+    id_only: bool,
+) -> None:
+    """Execute needed migration recipes.
+
+    This command executes one or more migration recipes.
+    Each recipe has a check method to determine if a migration is needed.
+    In addition to that, the current component version needs to match the specified
+    first-last-version range of the recipe.
+
+    Recipes are executed ordered by first_version.
+
+    Here are some argument examples, in order to see how to use this command:
+    `execute --all --test-only` will list all needed migrations (but not execute them),
+    `execute --filter tag system` will apply all migrations which target system data,
+    `execute bootstrap-data` will apply bootstrap-data migration if needed.
+    """
+    app: ApplicationContext = ctx.obj
+    if not all_ and not migration_id and not filter_:
+        raise click.UsageError(
+            "You can execute a single recipe, some recipes based on filter, or all recipes. "
+            "See the documentation for more information."
+        )
+    data = migrations_list.apply_filters(ctx=ctx, filter_=filter_)
+    if migration_id:
+        data = migrations_list.apply_filters(ctx=ctx, filter_=[("id", migration_id)])
+        if not data:
+            raise click.UsageError(
+                f"Migration recipe '{migration_id}' not found. "
+                "Use the 'migration list' command to get available migration recipes."
+            )
+    applied_counter = 0
+    for _ in data:
+        recipe: MigrationRecipe = _.get("object")
+        if not recipe.version_matches():
+            app.echo_debug(f"Migration '{recipe.id}' does not match component version.")
+            continue
+        if not recipe.is_applicable():
+            app.echo_debug(f"Migration '{recipe.id}' is not applicable.")
+            continue
+        app.echo_debug(f"Migration '{recipe.id}' could be applied.")
+        applied_counter += 1
+
+        app.echo_info(recipe.id, condition=id_only)
+        app.echo_info(f"{recipe.description} ({recipe.id}) ... ", nl=False, condition=not id_only)
+        if test_only:
+            app.echo_warning("needed", condition=not id_only)
+            continue
+        recipe.apply()
+        app.echo_success("done", condition=not id_only)
+    app.echo_success("No migration needed.", condition=(not applied_counter and not id_only))
+
+
+@click.group(cls=CmemcGroup)
+def migration() -> CmemcGroup:  # type: ignore[empty-body]
+    """List and apply migration recipes.
+
+    With this command group, you can check your instance for needed
+    migration activities as well as apply them to your data.
+
+    Beside an ID and a description, migration recipes have the following
+    metadata: 'First Version' is the first Corporate Memory version,
+    where this recipe is maybe applicable. The recipe will never be applied
+    to a version below this version. 'Tags' is a classification of the recipe
+    with regard to the target data, it migrates.
+
+    The following tags are important:
+    'system' recipes target data structures
+    which are needed to run the most basic functionality properly. These recipes
+    can and should be applied after each version upgrade.
+    'user' recipes can change user and / or customizing data.
+    'acl' recipes migrate access condition data.
+    'shapes' recipes migrate shape data.
+    'config' recipes migrate configuration data.
+    """
+
+
+migration.add_command(list_command)
+migration.add_command(execute_command)