esgvoc 0.1.2__py3-none-any.whl

esgvoc/cli/get.py

@@ -0,0 +1,142 @@
+
+from typing import Any
+from esgvoc.api.projects import find_terms_in_collection, find_terms_in_project, get_all_collections_in_project, get_all_projects, get_all_terms_in_collection
+from esgvoc.api.universe import find_terms_in_data_descriptor, find_terms_in_universe, get_all_data_descriptors_in_universe, get_all_terms_in_data_descriptor
+from pydantic import BaseModel
+from requests import logging
+from rich.table import Table
+import typer
+import re
+from rich.json import JSON
+from rich.console import Console
+
+app = typer.Typer()
+console = Console()
+
+_LOGGER = logging.getLogger(__name__)
+
+def validate_key_format(key: str):
+    """
+    Validate if the key matches the XXXX:YYYY:ZZZZ format.
+    """
+    if not re.match(r"^(\w*-?\w*)*:(\w*-?\w*)*:(\w*-?\w*)*$", key):
+        raise typer.BadParameter(f"Invalid key format: {key}. Must be XXXX:YYYY:ZZZZ.")
+    return key.split(":")
+
+
+def handle_universe(data_descriptor_id:str|None,term_id:str|None, options=None):
+    _LOGGER.debug(f"Handling universe with data_descriptor_id={data_descriptor_id}, term_id={term_id}") 
+
+    if data_descriptor_id and term_id:
+        return find_terms_in_data_descriptor(data_descriptor_id,term_id,options)
+        # BaseModel|dict[str: BaseModel]|None:
+
+    elif term_id:
+        return find_terms_in_universe(term_id,options)
+        # dict[str, BaseModel] | dict[str, dict[str, BaseModel]] | None:
+
+
+    elif data_descriptor_id:
+        return get_all_terms_in_data_descriptor(data_descriptor_id)
+        # dict[str, BaseModel]|None:
+
+    else:
+        return get_all_data_descriptors_in_universe()
+        # dict[str, dict]:
+
+def handle_project(project_id:str,collection_id:str|None,term_id:str|None,options=None):
+    _LOGGER.debug(f"Handling project {project_id} with Y={collection_id}, Z={term_id}, options = {options}")
+    
+    if project_id and collection_id and term_id:
+        return find_terms_in_collection(project_id,collection_id,term_id)
+        # BaseModel|dict[str: BaseModel]|None:
+
+    elif term_id:
+        return find_terms_in_project(project_id, term_id,options)
+        # dict[str, BaseModel] | dict[str, dict[str, BaseModel]] | None:
+
+
+    elif collection_id:
+        return get_all_terms_in_collection(project_id, collection_id)
+        # dict[str, BaseModel]|None:
+
+    else:
+        res = get_all_collections_in_project(project_id)
+        if res is None:
+            return None
+        else:
+            return res
+        # dict[str, dict]:
+
+
+def handle_unknown(x:str|None,y:str|None,z:str|None):
+    print(f"Something wrong in X,Y or Z : X={x}, Y={y}, Z={z}")
+
+
+def display(data:Any):
+
+    if isinstance(data, BaseModel):
+        # Pydantic Model
+        console.print(JSON.from_data(data.model_dump()))
+    elif isinstance(data, dict):
+        # Dictionary as JSON
+        console.print(data.keys())
+    elif isinstance(data, list):
+        # List as Table
+        table = Table(title="List")
+        table.add_column("Index")
+        table.add_column("Item")
+        for i, item in enumerate(data):
+            table.add_row(str(i), str(item))
+        console.print(table)
+    else:
+        # Fallback to simple print
+        console.print(data)
+
+@app.command()
+def get(keys: list[str] = typer.Argument(..., help="List of keys in XXXX:YYYY:ZZZZ format")):
+    """
+    Retrieve a specific value from the database system.
+    This command allows you to fetch a value by specifying the universe/project, data_descriptor/collection, 
+    and term in a structured format.
+
+    Usage:
+        `get <project>:<collection>:<term>`
+
+    Arguments:
+        <project>        The name of the project to query. like `cmip6plus`
+        <collection>     The name of the collection in the specified database.
+        <term>           The name or term within the specified collection.
+
+    Example:
+        To retrieve the value from the "cmip6plus" project, under the "institution_id" column, 
+        in the term with the identifier "ipsl", you would use:
+            `get cmip6plus:institution_id:ipsl`
+        The default project is the universe CV : the argument would be like `universe:institution:ipsl` or `:institution:ipsl`
+        - to get list of available term from universe institution `:institution:` 
+
+    Notes:
+        - Ensure data exist in your system before using this command (use status command to see whats available).
+        - Use a colon (`:`) to separate the parts of the argument.  
+        - if more than one argument is given i.e get X:Y:Z A:B:C the 2 results are appended. 
+
+    """ 
+    known_projects = get_all_projects()
+
+    # Validate and process each key
+    for key in keys:
+        validated_key = validate_key_format(key)
+        _LOGGER.debug(f"Processed key: {validated_key}")
+        where,what,who = validated_key
+        what = what if what!="" else None
+        who = who if who!="" else None
+        if where == "" or where=="universe": 
+            res = handle_universe(what,who)
+        elif where in known_projects:
+            res = handle_project(where,what,who,{})
+        else:
+            res = handle_unknown(where,what,who)
+        
+        display(res)
+
+

esgvoc/cli/install.py

@@ -0,0 +1,14 @@
+import typer
+from esgvoc.core.service import esg_voc
+
+app = typer.Typer()
+
+@app.command()
+def install():
+    """
+    Command to clone and build necessary db with the latest available version
+    
+    """
+    esg_voc.install()
+    
+    

esgvoc/cli/main.py

@@ -0,0 +1,22 @@
+
+import typer
+from esgvoc.cli.config import app as config_app
+from esgvoc.cli.get import app as get_app
+from esgvoc.cli.status import app as status_app
+from esgvoc.cli.valid import app as valid_app
+from esgvoc.cli.install import app as install_app
+    
+
+app = typer.Typer()
+
+# Register the subcommands
+app.add_typer(config_app)
+app.add_typer(get_app)
+app.add_typer(status_app)
+app.add_typer(valid_app)
+app.add_typer(install_app)
+
+def main():
+    app()
+if __name__ == "__main__":
+    main()

esgvoc/cli/status.py

@@ -0,0 +1,26 @@
+from esgvoc.core import service
+import typer
+from rich.console import Console
+
+app = typer.Typer()
+console = Console()
+
+
+def display(table):
+    console = Console(record=True,width=200)
+    console.print(table)
+
+
+
+@app.command()
+def status():
+    """
+    Command to display status 
+    i.e summary of version of usable ressources (between remote/cached)  
+    
+    """
+
+    service.state_service.get_state_summary()
+    display(service.state_service.table())
+
+    

esgvoc/cli/valid.py

@@ -0,0 +1,156 @@
+
+from typing import List
+from esgvoc.api.projects import (
+    valid_term, 
+    valid_term_in_collection, 
+    valid_term_in_project, 
+    valid_term_in_all_projects
+)
+from esgvoc.api import BasicValidationErrorVisitor
+from requests import logging
+from rich.table import Table
+from sqlmodel import except_
+import typer
+import re
+from rich.console import Console
+
+app = typer.Typer()
+console = Console()
+
+_LOGGER = logging.getLogger(__name__)
+
+@app.command()
+def valid(
+    strings_targets: List[str] = typer.Argument(
+        ..., 
+        help=(
+            "Pairs of strings to validate against a key in the form '<StringToValidate> <Project:Collection:Term>'.\n"
+            "Multiple pairs can be provided. The key '<Project:Collection:Term>' consists of three parts:\n"
+            "- 'Project' (optional)\n"
+            "- 'Collection' (optional)\n"
+            "- 'Term' (optional)\n"
+            "Only the ':' separators are mandatory. For example:\n"
+            "  - 'my_string ::'\n"
+            "  - 'my_string Project::'\n"
+            "  - 'my_string Project:Collection:'\n"
+            "  - 'my_string Project:Collection:Term'\n"
+            "The function validates based on the provided parts."
+        )
+    ),
+    verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed validation results")
+):
+    """
+    Validates one or more strings against specified Project:Collection:Term configurations.
+
+    Depending on the provided key structure, the function performs different validation operations:
+    - If all are None (e.g., "::"), validates the term across all projects (`valid_term_in_all_projects`).
+    - If Term is None (e.g., "Project:Collection:"), validates the term in the specified collection (`valid_term_in_collection`).
+    - If Term and Collection are None (e.g., "Project::"), validates the term in the specified project (`valid_term_in_project`).
+    - If all are specified (e.g., "Project:Collection:Term"), validates the term exactly (`valid_term`).
+
+    Parameters:
+        strings_targets (List[str]): A list of validation pairs, where each pair consists of:
+            - A string to validate.
+            - A key in the form '<Project:Collection:Term>'.
+    Usage :
+        Valid one: 
+        esgvocab valid IPSL cmip6plus:institution_id:ipsl
+        esgvocab valid IPSL cmip6plus:institution_id:
+        esgvocab valid IPSL cmip6plus::
+        esgvocab valid IPSL ::
+        
+        Unvalid one:
+        esgvocab valid IPSL_invalid cmip6plus:institution_id:ipsl
+        esgvocab valid IPSL cmip6plus:institution_id:isl <= term cant be found
+        esgvocab valid IPSL cmip6plus:institutin_id:ispl <= collection cant be found
+        esgvocab valid IPSL cmip6pls:institution_id:ispl <= project cant be found
+
+        Multiple validation for all known projects: 
+        esgvocab valid IPSL :: IPS :: 
+            result will be [True, False]
+        
+        esgvocab valid --verbose IPS :: IPSL :: 
+            result will be 
+            ┏━━━━━━━━┳━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+            ┃ String ┃ Key ┃ Result     ┃ Errors                      ┃
+            ┡━━━━━━━━╇━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+            │ IPS    │ ::  │ ❌ Invalid │ did not found matching term │
+            │ IPSL   │ ::  │ ✅ Valid   │ None                        │
+            └────────┴─────┴────────────┴─────────────────────────────┘
+    Returns:
+        List[bool]: Validation results for each pair in the input.
+    """
+    results = []
+    detailed_results = []
+
+    # Combine string and target into pairs
+    pairs = [strings_targets[i] + " " + strings_targets[i + 1] for i in range(0, len(strings_targets), 2)]
+
+    # Validate each string against each target
+    for validation in pairs:
+        match = re.match(r"(.+)\s+([^:]*):([^:]*):([^:]*)", validation)
+        if not match:
+            console.print(f"[red]Invalid input format: {validation}[/red]")
+            results.append(False)
+            detailed_results.append({"validation": validation, "errors": ["Invalid input format"]})
+            continue
+
+        string_to_validate, project, collection, term = match.groups()
+        exception_message= None
+        try:
+            # Perform the appropriate validation
+            if project and collection and term:
+                validation_result = valid_term(string_to_validate, project, collection, term)
+            elif project and collection:
+                validation_result = valid_term_in_collection(string_to_validate, project, collection)
+            elif project:
+                validation_result = valid_term_in_project(string_to_validate, project)
+            else:
+                validation_result = valid_term_in_all_projects(string_to_validate)
+
+        except Exception as e:
+            validation_result=False
+            exception_message = repr(e)
+        
+        # Handle validation result
+
+        if validation_result:
+            results.append(True)
+            detailed_results.append({"validation": validation, "errors": []})
+        else:
+            # Parse and collect errors for verbose mode
+            if validation_result == []:
+                detailed_results.append({"validation":validation, "errors":["did not found matching term"]})
+
+            results.append(False)
+            if project and collection and term and exception_message is None:
+                visitor = BasicValidationErrorVisitor()
+                errors = [error.accept(visitor) for error in validation_result.errors]
+                detailed_results.append({"validation": validation, "errors": errors})
+            if exception_message is not None:
+                detailed_results.append({"validation": validation, "errors": [exception_message]})
+
+
+    # Output results
+    if verbose:
+        table = Table(title="Validation Results")
+        table.add_column("String", style="cyan")
+        table.add_column("Key", style="magenta")
+        table.add_column("Result", style="green" if all(results) else "red")
+        table.add_column("Errors", style="red")
+
+        for detail in detailed_results:
+            validation = detail["validation"]
+            validation_parts = validation.split()
+            string = validation_parts[0]
+            key = validation_parts[1] if len(validation_parts) > 1 else "::"
+            result = "✅ Valid" if detail["errors"] == [] else "❌ Invalid"
+            print(detail)
+            errors = "\n".join(detail["errors"]) if detail["errors"] else "None"
+            table.add_row(string, key, result, errors)
+
+        console.print(table)
+    else:
+        console.print(results)
+
+    return results

esgvoc/core/constants.py

@@ -0,0 +1,13 @@
+DIRNAME_AND_FILENAME_SEPARATOR = "_"
+PROJECT_SPECS_FILENAME = "project_specs.json"
+PROJECT_ID_JSON_KEY = "project_id"
+CONTEXT_FILENAME = "000_context.jsonld"
+CONTEXT_JSON_KEY = "@context"
+TERM_ID_JSON_KEY = 'id'
+COMPOSITE_PARTS_JSON_KEY = 'parts'
+COMPOSITE_SEPARATOR_JSON_KEY = 'separator'
+PATTERN_JSON_KEY = 'regex'
+TERM_TYPE_JSON_KEY = 'type'
+DRS_SPECS_JSON_KEY = 'drs_name'
+SQLITE_FIRST_PK = 1
+DATA_DESCRIPTOR_JSON_KEY = "@base"

esgvoc/core/data_handler.py

@@ -0,0 +1,133 @@
+
+import os
+import json
+import logging
+from functools import cached_property
+from typing import Any, Optional, Dict
+import requests
+from pyld import jsonld
+from pydantic import BaseModel, model_validator, ConfigDict
+
+from esgvoc.api.data_descriptors import DATA_DESCRIPTOR_CLASS_MAPPING
+# Configure logging
+_LOGGER = logging.getLogger(__name__)
+
+mapping = DATA_DESCRIPTOR_CLASS_MAPPING
+
+def unified_document_loader(uri: str) -> Dict:
+    """Load a document from a local file or a remote URI."""
+    if uri.startswith(("http://", "https://")):
+        response = requests.get(uri, headers={"accept": "application/json"}, verify=False)
+        if response.status_code == 200:
+            return response.json()
+        else:
+            _LOGGER.error(f"Failed to fetch remote document: {response.status_code} - {response.text}")
+            return {}
+    else:
+        with open(uri, "r") as f:
+            return json.load(f)
+
+class JsonLdResource(BaseModel):
+    uri: str
+    local_path: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @model_validator(mode="before")
+    @classmethod
+    def set_local_path(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Set the local path to an absolute path if provided."""
+        local_path = values.get("local_path")
+        if local_path:
+            values["local_path"] = os.path.abspath(local_path) + "/"
+        jsonld.set_document_loader(lambda uri,options:{
+            "contextUrl": None,  # No special context URL
+            "documentUrl": uri,  # The document's actual URL
+            "document": unified_document_loader(uri),  # The parsed JSON-LD document
+        }) 
+        return values
+    
+    @cached_property
+    def json_dict(self) -> Dict:
+        """Fetch the original JSON data."""
+        _LOGGER.debug(f"Fetching JSON data from {self.uri}")
+        return unified_document_loader(self.uri)
+
+    @cached_property
+    def expanded(self) -> Any:
+        """Expand the JSON-LD data."""
+        _LOGGER.debug(f"Expanding JSON-LD data for {self.uri}")
+        return jsonld.expand(self.uri, options={"base": self.uri})
+    
+    @cached_property
+    def context(self) -> Dict:
+        """Fetch and return the JSON content of the '@context'."""
+        
+        context_data =JsonLdResource(uri="/".join(self.uri.split("/")[:-1])+"/"+self.json_dict["@context"]) 
+        # Works only in relative path declaration
+
+        context_value = context_data.json_dict
+        if isinstance(context_value, str):
+            # It's a URI, fetch it
+            _LOGGER.info(f"Fetching context from URI: {context_value}")
+            return unified_document_loader(context_value)
+        elif isinstance(context_value, dict):
+            # Embedded context
+            _LOGGER.info("Using embedded context.")
+            return context_value
+        else:
+            _LOGGER.warning("No valid '@context' found.")
+            return {}
+
+    @cached_property
+    def normalized(self) -> str:
+        """Normalize the JSON-LD data."""
+        _LOGGER.info(f"Normalizing JSON-LD data for {self.uri}")
+        return jsonld.normalize(
+            self.uri, options={"algorithm": "URDNA2015", "format": "application/n-quads"}
+        )
+
+    @cached_property
+    def python(self) -> Optional[Any]:
+        """Map the data to a Pydantic model based on URI."""
+        _LOGGER.info(f"Mapping data to a Pydantic model for {self.uri}")
+        model_key = self._extract_model_key(self.uri)
+        if model_key and model_key in mapping:
+            model = mapping[model_key]
+            return model(**self.json_dict)
+        _LOGGER.warning(f"No matching model found for key: {model_key}")
+        return None
+
+    def _extract_model_key(self, uri: str) -> Optional[str]:
+        """Extract a model key from the URI."""
+        parts = uri.strip("/").split("/")
+        if len(parts) >= 2:
+            return parts[-2]
+        return None
+
+    @property
+    def info(self) -> str:
+        """Return a detailed summary of the data."""
+        res = f"{'#' * 100}\n"
+        res += f"###   {self.uri.split('/')[-1]}   ###\n"
+        res += f"JSON Version:\n {json.dumps(self.json_dict, indent=2)}\n"
+        res += f"URI: {self.uri}\n"
+        res += f"JSON Version:\n {json.dumps(self.json_dict, indent=2)}\n"
+        res += f"Expanded Version:\n {json.dumps(self.expanded, indent=2)}\n"
+        res += f"Normalized Version:\n {self.normalized}\n"
+        res += f"Pydantic Model Instance:\n {self.python}\n"
+        return res
+
+
+if __name__ == "__main__":
+    ## For Universe 
+    #online
+    # d = Data(uri = "https://espri-mod.github.io/mip-cmor-tables/activity/cmip.json")
+    # print(d.info)
+    #offline
+    #print(Data(uri = ".cache/repos/mip-cmor-tables/activity/cmip.json").info)
+    ## for Project 
+    #d = Data(uri = "https://espri-mod.github.io/CMIP6Plus_CVs/activity_id/cmip.json")
+    #print(d.info)
+    #offline
+    print(JsonLdResource(uri = ".cache/repos/CMIP6Plus_CVs/activity_id/cmip.json").info)

esgvoc/core/db/__init__.py

@@ -0,0 +1,5 @@
+from esgvoc.core.db.connection import read_json_file
+from esgvoc.core.db.connection import DBConnection
+
+
+__all__ = ["DBConnection", "read_json_file"]

esgvoc/core/db/connection.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+import json
+from sqlalchemy import Engine
+from sqlmodel import Session, create_engine
+
+
+class DBConnection:
+    SQLITE_URL_PREFIX = 'sqlite://'
+    def __init__(self, db_file_path: Path, echo: bool = False) -> None:
+        self.engine = create_engine(f'{DBConnection.SQLITE_URL_PREFIX}/{db_file_path}', echo=echo)
+        self.name = db_file_path.stem
+        self.file_path = db_file_path.absolute()
+
+    def set_echo(self, echo: bool) -> None:
+        self.engine.echo = echo
+
+    def get_engine(self) -> Engine:
+        return self.engine
+
+    def create_session(self) -> Session:
+        return Session(self.engine)
+
+    def get_name(self) -> str|None:
+        return self.name
+
+    def get_file_path(self) -> Path:
+        return self.file_path
+
+
+def read_json_file(json_file_path: Path) -> dict:
+    return json.loads(json_file_path.read_text())

esgvoc/core/db/models/mixins.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+from sqlmodel import Field
+
+
+class TermKind(Enum):
+    PLAIN = "plain"
+    PATTERN = "pattern"
+    COMPOSITE = "composite"
+    MIXED = 'mixed'
+
+
+class PkMixin:
+    pk: int | None = Field(default=None, primary_key=True)
+
+
+class IdMixin:
+    id: str = Field(index=True)

esgvoc/core/db/models/project.py

@@ -0,0 +1,65 @@
+import logging
+from pathlib import Path
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.sqlite import JSON
+from sqlmodel import Column, Field, Relationship, SQLModel
+
+import esgvoc.core.db.connection as db
+from esgvoc.core.db.models.mixins import IdMixin, PkMixin, TermKind
+
+_LOGGER = logging.getLogger("project_db_creation")
+
+
+class Project(SQLModel, PkMixin, IdMixin, table=True):
+    __tablename__ = "projects"
+    specs: dict = Field(sa_column=sa.Column(JSON))
+    git_hash: str
+    collections: list["Collection"] = Relationship(back_populates="project")
+
+
+class Collection(SQLModel, PkMixin, IdMixin, table=True):
+    __tablename__ = "collections"
+    data_descriptor_id: str = Field(index=True)
+    context: dict = Field(sa_column=sa.Column(JSON))
+    project_pk: int | None = Field(default=None, foreign_key="projects.pk")
+    project: Project = Relationship(back_populates="collections")
+    terms: list["PTerm"] = Relationship(back_populates="collection")
+    term_kind: TermKind = Field(sa_column=Column(sa.Enum(TermKind)))
+
+
+class PTerm(SQLModel, PkMixin, IdMixin, table=True):
+    __tablename__ = "pterms"
+    specs: dict = Field(sa_column=sa.Column(JSON))
+    kind: TermKind = Field(sa_column=Column(sa.Enum(TermKind)))
+    collection_pk: int | None = Field(default=None, foreign_key="collections.pk")
+    collection: Collection = Relationship(back_populates="terms")
+
+
+def create_drs_name_index():
+    PTerm.__table_args__ = sa.Index(
+        "drs_name_index", PTerm.__table__.c.specs["drs_name"]
+    )
+
+
+def project_create_db(db_file_path: Path):
+    try:
+        connection = db.DBConnection(db_file_path)
+    except Exception as e:
+        msg = f'Unable to create SQlite file at {db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+    try:
+        tables_to_be_created = [SQLModel.metadata.tables['projects'],
+                                SQLModel.metadata.tables['collections'],
+                                SQLModel.metadata.tables['pterms']]
+        create_drs_name_index()
+        SQLModel.metadata.create_all(connection.get_engine(), tables=tables_to_be_created)
+    except Exception as e:
+        msg = f'Unable to create tables in SQLite database at {db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+
+
+if __name__ == "__main__":
+    pass