esgvoc 2.0.2__py3-none-any.whl

esgvoc/cli/drs.py

@@ -0,0 +1,267 @@
+import shlex
+import sys
+from typing import List, Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+import esgvoc.api as ev
+from esgvoc.apps.drs.generator import DrsGenerator
+from esgvoc.apps.drs.report import DrsGenerationReport, DrsValidationReport
+from esgvoc.apps.drs.validator import DrsValidator
+from esgvoc.core.exceptions import EsgvocValueError
+
+app = typer.Typer()
+console = Console()
+
+
+# Predefined list of projects and DRS types
+# projects = ["cmip5", "cmip6","cmip6plus", "cmip7"]
+projects = ev.get_all_projects()
+drs_types = ["filename", "directory", "dataset"]
+
+
+def display(table):
+    """
+    Function to display a rich table in the console.
+
+    :param table: The table to be displayed
+    """
+    console = Console(record=True, width=200)
+    console.print(table)
+
+
+@app.command()
+def drsvalid(
+    drs_entries: Optional[List[str]] = typer.Argument(
+        None, help="List of DRS validation inputs in the form <project> <drstype> <string>"
+    ),
+    file: Optional[typer.FileText] = typer.Option(
+        None,
+        "--file",
+        "-f",
+        help="File containing DRS validation inputs, one per line in the form <project> <drstype> <string>",
+    ),
+    verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed validation results"),
+    output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the DRS entries validation"),
+    rm_prefix: Optional[str] = typer.Option(
+        None, "-p", "--prefix", help="Remove given prefix from all checked directory"
+    ),
+    pedantic: Optional[bool] = typer.Option(
+        False,
+        "-e",
+        "--enforce",
+        help="Enable pedantic mode, enforcing strict compliance, mean that warnings are now errors.",
+    ),
+) -> List[DrsValidationReport]:
+    """
+    Validates DRS strings for a specific project and type.
+
+    Args:
+        drs_entries (Optional[List[str]]): A list of DRS validation inputs in the form <project> <drstype> <string>.
+        file (Optional[typer.FileText]): File containing DRS validation inputs, one per line.
+        verbose (bool): If true, prints detailed validation results.
+
+    Usage Examples:
+        # Validate multiple filenames for CMIP6
+        drsvalid cmip6 filename file1.nc file2.nc file3.nc
+
+        # Validate using a file
+        drsvalid --file drs_input.txt
+    """
+    current_project = None
+    current_drs_type = None
+    reports = []
+
+    entries = drs_entries or []
+
+    if not sys.stdin.isatty():  # Check if input is being piped via stdin
+        entries.extend(el for line in sys.stdin for el in shlex.split(line))
+
+    if file:
+        entries.extend(el for line in file for el in line.strip().split(" "))
+
+    i = 0
+    while i < len(entries):
+        if entries[i] in ["", " "]:
+            i += 1
+            continue
+
+        if entries[i] in projects:
+            current_project = entries[i]
+            i += 1
+            continue
+        if entries[i] in drs_types:
+            current_drs_type = entries[i]
+            i += 1
+            continue
+
+        if current_project is None:
+            raise typer.BadParameter(f"Invalid project: {entries[i]}")
+
+        if current_drs_type is None:
+            raise typer.BadParameter(f"Invalid drs_type: {entries[i]}")
+
+        string = entries[i]
+        i += 1
+        validator = DrsValidator(current_project, pedantic=pedantic)
+        report = None
+        match current_drs_type:
+            case "filename":
+                report = validator.validate_file_name(string)
+            case "directory":
+                if rm_prefix:
+                    prefix = rm_prefix + "/" if rm_prefix[-1] != "/" else ""
+                else:
+                    prefix = None
+                report = validator.validate_directory(string, prefix)
+            case "dataset":
+                report = validator.validate_dataset_id(string)
+            case _:
+                raise EsgvocValueError(f"unsupported drs type '{current_drs_type}'")
+        reports.append(report)
+
+    if verbose:
+        table = Table(title="Validation result")
+        table.add_column("entry", style="cyan")
+        table.add_column("project & drs_type", style="cyan")
+        table.add_column("warnings", style="magenta")
+        table.add_column("errors", style="red")
+        table.add_column("valid")
+
+        for report in reports:
+            entry = str(report.expression)
+            proj_and_type = str(report.project_id) + " " + report.type + " "
+            warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
+            errors = "\n".join(["⚠️ " + str(error) for error in report.errors])
+            valid = "✅ Valid" if report else "❌ Invalid"
+
+            table.add_row("-" * 4, "-" * 4, "-" * 4, "-" * 4, "-" * 4)
+            table.add_row(entry, proj_and_type, warnings, errors, valid)
+
+        console.print(table)
+    elif output:
+        with open(output, "w") as f:
+            for report in reports:
+                f.write(str(report) + "\n")
+        console.print(f"DRS validation entries saved to [green]{output}[/green]")
+
+    else:
+        for report in reports:
+            console.print(str(report))
+
+    return reports
+
+
+@app.command()
+def drsgen(
+    drs_entries: Optional[List[str]] = typer.Argument(
+        None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_terms>"
+    ),
+    file: Optional[typer.FileText] = typer.Option(
+        None,
+        "--file",
+        "-f",
+        help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_terms>",
+    ),
+    verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed generation results"),
+    output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the generated DRS entries"),
+) -> List[DrsGenerationReport]:
+    """
+    Generates DRS strings for a specific project and type based on input bag of terms.
+
+    Args:
+        drs_entries (Optional[List[str]]): A list of inputs in the form <project> <drstype> <bag_of_terms>.
+        file (Optional[typer.FileText]): File containing DRS generation inputs, one per line.
+        verbose (bool): If true, prints detailed generation results.
+        output (Optional[str]): File path to save the generated DRS entries.
+
+    Usage Examples:
+        # Generate multiple filenames for CMIP6
+        drsgen cmip6 filename var1=tas var2=pr
+
+        # Generate using a file
+        drsgen --file drs_input.txt
+    """
+    current_project = None
+    current_drs_type = None
+    generated_reports = []
+
+    entries = drs_entries or []
+
+    if not sys.stdin.isatty():  # Check if input is being piped via stdin
+        entries.extend(el for line in sys.stdin for el in shlex.split(line))
+
+    if file:
+        entries.extend(el for line in file for el in shlex.split(line))
+
+    i = 0
+    while i < len(entries):
+        if entries[i] in ["", " "]:
+            i += 1
+            continue
+        if entries[i] in projects:
+            current_project = entries[i]
+            i += 1
+            continue
+        if entries[i] in drs_types:
+            current_drs_type = entries[i]
+            i += 1
+            continue
+
+        if current_project is None:
+            raise typer.BadParameter(f"Invalid project: {entries[i]}")
+
+        if current_drs_type is None:
+            raise typer.BadParameter(f"Invalid drs_type: {entries[i]}")
+
+        bag_of_terms = entries[i]
+        bag_of_terms = set(entries[i].split(" "))
+        i += 1
+
+        generator = DrsGenerator(current_project)
+        report = None
+        match current_drs_type:
+            case "filename":
+                report = generator.generate_file_name_from_bag_of_terms(bag_of_terms)
+            case "directory":
+                report = generator.generate_directory_from_bag_of_terms(bag_of_terms)
+            case "dataset":
+                report = generator.generate_dataset_id_from_bag_of_terms(bag_of_terms)
+            case _:
+                raise EsgvocValueError(f"unsupported drs type '{current_drs_type}'")
+        generated_reports.append(report)
+
+    if verbose:
+        table = Table(title="Generation result")
+        table.add_column("deduced mapping entry", style="cyan")
+        table.add_column("warnings", style="magenta")
+        table.add_column("errors", style="red")
+        table.add_column("result", style="green", width=10)
+        for report in generated_reports:
+            entry = str(report.mapping_used)
+            warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
+            errors = "\n".join([f"🔍 {error}" for error in report.errors])
+            result = report.generated_drs_expression
+            table.add_row(entry, warnings, errors, result)
+            table.add_row("----", "----", "----", "----")
+            if table.columns[3].width is not None and len(result) > table.columns[3].width:
+                table.columns[3].width = len(result) + 1
+        console.print(table)
+
+    elif output:
+        with open(output, "w") as f:
+            for report in generated_reports:
+                f.write(str(report) + "\n")
+        console.print(f"Generated entries saved to [green]{output}[/green]")
+
+    else:
+        for report in generated_reports:
+            console.print(str(report))
+
+    return generated_reports
+
+
+if __name__ == "__main__":
+    app()

esgvoc/cli/find.py

@@ -0,0 +1,138 @@
+import re
+from typing import Any
+
+import typer
+from pydantic import BaseModel
+from requests import logging
+from rich.console import Console
+from rich.json import JSON
+from rich.table import Table
+
+from esgvoc.api.projects import (
+    find_collections_in_project,
+    find_items_in_project,
+    find_terms_in_all_projects,
+    find_terms_in_collection,
+    find_terms_in_project,
+    get_all_projects,
+)
+from esgvoc.api.universe import (
+    find_data_descriptors_in_universe,
+    find_items_in_universe,
+    find_terms_in_data_descriptor,
+    find_terms_in_universe,
+)
+
+app = typer.Typer()
+console = Console()
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def validate_key_format(key: str):
+    """
+    Validate if the key matches the XXXX:YYYY:ZZZZ format.
+    """
+    if not re.match(r"^[a-zA-Z0-9\/_-]*:[a-zA-Z0-9\/_-]*:[a-zA-Z0-9\/_.-]*$", key):
+        raise typer.BadParameter(f"Invalid key format: {key}. Must be XXXX:YYYY:ZZZZ.")
+    return key.split(":")
+
+
+def handle_universe(expression: str, data_descriptor_id: str | None, term_id: str | None, options=None):
+    _LOGGER.debug(f"Handling universe with data_descriptor_id={data_descriptor_id}, term_id={term_id}")
+
+    if data_descriptor_id:
+        return find_terms_in_data_descriptor(expression, data_descriptor_id)
+        # BaseModel|dict[str: BaseModel]|None:
+
+    else:
+        return find_terms_in_universe(expression)
+        # dict[str, dict]:
+
+
+def handle_project(expression: str, project_id: str, collection_id: str | None, term_id: str | None, options=None):
+    _LOGGER.debug(f"Handling project {project_id} with Y={collection_id}, Z={term_id}, options = {options}")
+
+    if project_id == "all":
+        return find_terms_in_all_projects(expression)
+
+    elif collection_id:
+        return find_terms_in_collection(expression, project_id, collection_id)
+        # dict[str, BaseModel]|None:
+
+    else:
+        res = find_terms_in_project(expression, project_id)
+        if res is None:
+            return None
+        else:
+            return res
+        # dict[str, dict]:
+
+
+def handle_unknown(x: str | None, y: str | None, z: str | None):
+    print(f"Something wrong in X,Y or Z : X={x}, Y={y}, Z={z}")
+
+
+def display(data: Any):
+    if isinstance(data, BaseModel):
+        # Pydantic Model
+        console.print(JSON.from_data(data.model_dump()))
+    elif isinstance(data, dict):
+        # Dictionary as JSON
+        console.print(data.keys())
+    elif isinstance(data, list):
+        # List as Table
+        table = Table(title="List")
+        table.add_column("Index")
+        table.add_column("Item")
+        for i, item in enumerate(data):
+            table.add_row(str(i), str(item))
+        console.print(table)
+    else:
+        # Fallback to simple print
+        console.print(data)
+
+
+@app.command()
+def find(expression: str, keys: list[str] = typer.Argument(..., help="List of keys in XXXX:YYYY:ZZZZ format")):
+    """
+    Retrieve a specific value from the database system.\n
+    This command allows you to fetch a value by specifying the universe/project, data_descriptor/collection,
+    and term in a structured format.\n
+    \n
+
+    Usage:\n
+        `find <expression> <project>:<collection>:<term>`\n
+    \n
+    Arguments:\n
+        <expression>\t The full text search expression.
+        <project>\tThe project id to query. like `cmip6plus`\n
+        <collection>\tThe collection id in the specified database.\n
+        <term>\t\tThe term id within the specified collection.\n
+    \n
+    Example:
+    \n
+    Notes:\n
+        - Ensure data exist in your system before using this command (use `esgvoc status` command to see whats available).\n
+        - Use a colon (`:`) to separate the parts of the argument.  \n
+        - if more than one argument is given i.e get X:Y:Z A:B:C the 2 results are appended. \n
+    \n
+    """
+    known_projects = get_all_projects()
+    _LOGGER.debug(f"Processed expression: {expression}")
+
+    # Validate and process each key
+    for key in keys:
+        validated_key = validate_key_format(key)
+        _LOGGER.debug(f"Processed key: {validated_key}")
+        where, what, who = validated_key
+        what = what if what != "" else None
+        who = who if who != "" else None
+        if where == "" or where == "universe":
+            res = handle_universe(expression, what, who)
+        elif where in known_projects:
+            res = handle_project(expression, where, what, who, None)
+        else:
+            res = handle_unknown(where, what, who)
+
+        display(res)

esgvoc/cli/get.py

@@ -0,0 +1,155 @@
+import logging
+import re
+from typing import Any, List, Optional
+
+import typer
+from pydantic import BaseModel
+from rich.console import Console
+from rich.json import JSON
+from rich.table import Table
+
+from esgvoc.api.projects import (
+    get_all_collections_in_project,
+    get_all_projects,
+    get_all_terms_in_collection,
+    get_term_in_collection,
+    get_term_in_project,
+)
+from esgvoc.api.universe import (
+    get_all_data_descriptors_in_universe,
+    get_all_terms_in_data_descriptor,
+    get_term_in_data_descriptor,
+    get_term_in_universe,
+)
+
+app = typer.Typer()
+console = Console()
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def validate_key_format(key: str):
+    """
+    Validate if the key matches the XXXX:YYYY:ZZZZ format.
+    """
+    if not re.match(r"^[a-zA-Z0-9\/_-]*:[a-zA-Z0-9\/_-]*:[a-zA-Z0-9\/_.-]*$", key):
+        raise typer.BadParameter(f"Invalid key format: {key}. Must be XXXX:YYYY:ZZZZ.")
+    return key.split(":")
+
+
+def handle_universe(data_descriptor_id: str | None, term_id: str | None, options=None):
+    _LOGGER.debug(f"Handling universe with data_descriptor_id={data_descriptor_id}, term_id={term_id}")
+    if data_descriptor_id and term_id:
+        return get_term_in_data_descriptor(data_descriptor_id, term_id, options)
+        # BaseModel|dict[str: BaseModel]|None:
+
+    elif term_id:
+        return get_term_in_universe(term_id, options)
+        # dict[str, BaseModel] | dict[str, dict[str, BaseModel]] | None:
+
+    elif data_descriptor_id:
+        return get_all_terms_in_data_descriptor(data_descriptor_id, options)
+        # dict[str, BaseModel]|None:
+
+    else:
+        return get_all_data_descriptors_in_universe()
+        # dict[str, dict]:
+
+
+def handle_project(project_id: str, collection_id: str | None, term_id: str | None, options=None):
+    _LOGGER.debug(f"Handling project {project_id} with Y={collection_id}, Z={term_id}, options = {options}")
+
+    if project_id and collection_id and term_id:
+        return get_term_in_collection(project_id, collection_id, term_id, options)
+        # BaseModel|dict[str: BaseModel]|None:
+
+    elif term_id:
+        return get_term_in_project(project_id, term_id, options)
+        # dict[str, BaseModel] | dict[str, dict[str, BaseModel]] | None:
+
+    elif collection_id:
+        return get_all_terms_in_collection(project_id, collection_id, options)
+        # dict[str, BaseModel]|None:
+
+    else:
+        res = get_all_collections_in_project(project_id)
+        if res is None:
+            return None
+        else:
+            return res
+        # dict[str, dict]:
+
+
+def handle_unknown(x: str | None, y: str | None, z: str | None):
+    print(f"Something wrong in X,Y or Z : X={x}, Y={y}, Z={z}")
+
+
+def display(data: Any):
+    if isinstance(data, BaseModel):
+        # Pydantic Model - use mode='json' to serialize datetime and other types
+        console.print(JSON.from_data(data.model_dump(mode="json")))
+    elif isinstance(data, dict):
+        # Dictionary as JSON
+        console.print(data.keys())
+    elif isinstance(data, list):
+        # List as Table
+        table = Table(title="List")
+        table.add_column("Index")
+        table.add_column("Item")
+        for i, item in enumerate(data):
+            table.add_row(str(i), str(item))
+        console.print(table)
+    else:
+        # Fallback to simple print
+        console.print(data)
+
+
+@app.command()
+def get(
+    keys: List[str] = typer.Argument(..., help="List of keys in XXXX:YYYY:ZZZZ format"),
+    select: Optional[List[str]] = typer.Option(None, "--select", help="keys selected for the result"),
+):
+    """
+    Retrieve a specific value from the database system.\n
+    This command allows you to fetch a value by specifying the universe/project, data_descriptor/collection,
+    and term in a structured format.\n
+    \n
+
+    Usage:\n
+        `get <project>:<collection>:<term>`\n
+    \n
+    Arguments:\n
+        <project>\tThe project id to query. like `cmip6plus`\n
+        <collection>\tThe collection id in the specified database.\n
+        <term>\t\tThe term id within the specified collection.\n
+    \n
+    Example:
+        To retrieve the value from the "cmip6plus" project, under the "institution_id" column, the term with the identifier "ipsl", you would use: \n
+            `get cmip6plus:institution_id:ipsl`\n
+        The default project is the universe CV : the argument would be like `universe:institution:ipsl` or `:institution:ipsl` \n
+        - to get list of available term from universe institution `:institution:` \n
+        \n
+    \n
+    Notes:\n
+        - Ensure data exist in your system before using this command (use `esgvoc status` command to see whats available).\n
+        - Use a colon (`:`) to separate the parts of the argument.  \n
+        - if more than one argument is given i.e get X:Y:Z A:B:C the 2 results are appended. \n
+    \n
+    """
+    known_projects = get_all_projects()
+
+    # Validate and process each key
+    for key in keys:
+        validated_key = validate_key_format(key)
+        _LOGGER.debug(f"Processed key: {validated_key}")
+        where, what, who = validated_key
+        what = what if what != "" else None
+        who = who if who != "" else None
+        if where == "" or where == "universe":
+            res = handle_universe(what, who, select)
+        elif where in known_projects:
+            res = handle_project(where, what, who, select)
+        else:
+            res = handle_unknown(where, what, who)
+
+        display(res)

esgvoc/cli/install.py

@@ -0,0 +1,41 @@
+import typer
+from esgvoc.core.service import current_state
+
+app = typer.Typer()
+
+@app.command()
+def install():
+    """Initialize default config and apply settings"""
+    try:
+        typer.echo("Initialized default configuration")
+
+        # Check if any components are in offline mode
+        offline_components = []
+        if current_state.universe.offline_mode:
+            offline_components.append("universe")
+        for project_name, project in current_state.projects.items():
+            if project.offline_mode:
+                offline_components.append(project_name)
+
+        if offline_components:
+            typer.echo(f"Note: The following components are in offline mode: {', '.join(offline_components)}")
+            typer.echo("Only local repositories and databases will be used.")
+
+        current_state.synchronize_all()
+
+        # Ensure versions are properly fetched after synchronization
+        current_state.fetch_versions()
+        current_state.get_state_summary()
+
+        # Display final status after installation
+        from rich.console import Console
+        console = Console()
+        typer.echo("\nInstallation completed. Final status:")
+        console.print(current_state.table())
+
+    except Exception as e:
+        typer.echo(f"Error during installation: {str(e)}", err=True)
+        raise typer.Exit(1)
+
+if __name__ == "__main__":
+    app()

esgvoc/cli/main.py

@@ -0,0 +1,60 @@
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from esgvoc.cli.clean import app as clean_app
+from esgvoc.cli.cmor import app as cmor_app
+from esgvoc.cli.config import app as config_app
+from esgvoc.cli.drs import app as drs_app
+from esgvoc.cli.find import app as find_app
+from esgvoc.cli.get import app as get_app
+from esgvoc.cli.install import app as install_app
+from esgvoc.cli.offline import app as offline_app
+from esgvoc.cli.status import app as status_app
+from esgvoc.cli.test_cv import app as test_cv_app
+from esgvoc.cli.valid import app as valid_app
+from esgvoc.core.service.configuration.setting import ServiceSettings
+
+app = typer.Typer()
+console = Console()
+
+# Register the subcommands
+app.add_typer(get_app)
+app.add_typer(status_app)
+app.add_typer(valid_app)
+app.add_typer(install_app)
+app.add_typer(drs_app)
+app.add_typer(config_app, name="config")
+app.add_typer(offline_app, name="offline")
+app.add_typer(clean_app, name="clean")
+app.add_typer(test_cv_app, name="test")
+app.add_typer(find_app)
+app.add_typer(cmor_app)
+
+# maybe remove during a future refactor
+
+
+@app.command()
+def list_projects():
+    """List all available projects with their default configurations."""
+    default_configs = ServiceSettings._get_default_project_configs()
+
+    table = Table(title="Available Projects")
+    table.add_column("Project Name", style="cyan")
+    table.add_column("Repository", style="green")
+    table.add_column("Default Branch", style="yellow")
+    table.add_column("Local Path", style="blue")
+
+    for project_name, config in default_configs.items():
+        table.add_row(project_name, config["github_repo"], config["branch"], config["local_path"])
+
+    console.print(table)
+    console.print(f"\n[blue]Total: {len(default_configs)} projects available[/blue]")
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()