esgvoc 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

esgvoc/core/db/project_ingestion.py

@@ -1,20 +1,21 @@
 import logging
 from pathlib import Path
 
-import esgvoc.core.constants
-from esgvoc.core.data_handler import JsonLdResource
-from esgvoc.core.db.connection import DBConnection
-from esgvoc.core.service.data_merger import DataMerger
-from esgvoc.core.db.models.mixins import TermKind
 from pydantic import BaseModel
+from sqlalchemy import text
 
+import esgvoc.core.constants
 import esgvoc.core.db.connection as db
-from esgvoc.core.db.connection import read_json_file
-from esgvoc.core.db.models.project import Collection, Project, PTerm
-import  esgvoc.core.service as service
+import esgvoc.core.service as service
+from esgvoc.core.data_handler import JsonLdResource
+from esgvoc.core.db.connection import DBConnection, read_json_file
+from esgvoc.core.db.models.mixins import TermKind
+from esgvoc.core.db.models.project import PCollection, Project, PTerm
+from esgvoc.core.exceptions import EsgvocDbError
+from esgvoc.core.service.data_merger import DataMerger
 
+_LOGGER = logging.getLogger(__name__)
 
-_LOGGER = logging.getLogger("project_ingestion")
 
 def infer_term_kind(json_specs: dict) -> TermKind:
     if esgvoc.core.constants.PATTERN_JSON_KEY in json_specs:
@@ -25,67 +26,69 @@ def infer_term_kind(json_specs: dict) -> TermKind:
         return TermKind.PLAIN
 
 
-def ingest_metadata_project(connection:DBConnection,git_hash):
+def ingest_metadata_project(connection: DBConnection, git_hash):
     with connection.create_session() as session:
-        project = Project(id=str(connection.file_path.stem), git_hash=git_hash,specs={})
-        session.add(project)    
+        project = Project(id=str(connection.file_path.stem), git_hash=git_hash, specs={})
+        session.add(project)
         session.commit()
 
-###############################
+
 def get_data_descriptor_id_from_context(collection_context: dict) -> str:
-    data_descriptor_url = collection_context[esgvoc.core.constants.CONTEXT_JSON_KEY][esgvoc.core.constants.DATA_DESCRIPTOR_JSON_KEY]
+    data_descriptor_url = collection_context[esgvoc.core.constants.CONTEXT_JSON_KEY][
+        esgvoc.core.constants.DATA_DESCRIPTOR_JSON_KEY
+    ]  # noqa E211
     return Path(data_descriptor_url).name
 
 
-def instantiate_project_term(universe_term_json_specs: dict,
-                             project_term_json_specs_update: dict,
-                             pydantic_class: type[BaseModel]) -> dict:
+def instantiate_project_term(
+    universe_term_json_specs: dict, project_term_json_specs_update: dict, pydantic_class: type[BaseModel]
+) -> dict:
     term_from_universe = pydantic_class(**universe_term_json_specs)
-    updated_term = term_from_universe.model_copy(
-        update=project_term_json_specs_update, deep=True
-    )
+    updated_term = term_from_universe.model_copy(update=project_term_json_specs_update, deep=True)
     return updated_term.model_dump()
 
 
-def ingest_collection(collection_dir_path: Path,
-                      project: Project,
-                      project_db_session) -> None:
-
-
+def ingest_collection(collection_dir_path: Path, project: Project, project_db_session) -> None:
     collection_id = collection_dir_path.name
     collection_context_file_path = collection_dir_path.joinpath(esgvoc.core.constants.CONTEXT_FILENAME)
     try:
         collection_context = read_json_file(collection_context_file_path)
         data_descriptor_id = get_data_descriptor_id_from_context(collection_context)
     except Exception as e:
-        msg = f'Unable to read project context file {collection_context_file_path}. Abort.'
+        msg = f"unable to read project context file {collection_context_file_path}"
         _LOGGER.fatal(msg)
-        raise RuntimeError(msg) from e
+        raise EsgvocDbError(msg) from e
     # [KEEP]
-    collection = Collection(
+    collection = PCollection(
         id=collection_id,
         context=collection_context,
         project=project,
         data_descriptor_id=data_descriptor_id,
-        term_kind="") # we ll know it only when we ll add a term (hypothesis all term have the same kind in a collection
+        term_kind="",
+    )  # We ll know it only when we ll add a term
+    # (hypothesis all term have the same kind in a collection) # noqa E116
     term_kind_collection = None
 
     for term_file_path in collection_dir_path.iterdir():
         _LOGGER.debug(f"found term path : {term_file_path}")
-        if term_file_path.is_file() and term_file_path.suffix==".json": 
+        if term_file_path.is_file() and term_file_path.suffix == ".json":
             try:
-                json_specs = DataMerger(data=JsonLdResource(uri =str(term_file_path)),
-                                        # locally_available={"https://espri-mod.github.io/mip-cmor-tables":".cache/repos/WCRP-universe"}).merge_linked_json()[-1]
-                                        locally_available={"https://espri-mod.github.io/mip-cmor-tables":service.current_state.universe.local_path}).merge_linked_json()[-1]
-
+                locally_avail = {
+                    "https://espri-mod.github.io/mip-cmor-tables": service.current_state.universe.local_path
+                }
+                json_specs = DataMerger(
+                    data=JsonLdResource(uri=str(term_file_path)),
+                    # locally_available={"https://espri-mod.github.io/mip-cmor-tables":".cache/repos/WCRP-universe"}).merge_linked_json()[-1]
+                    locally_available=locally_avail,
+                ).merge_linked_json()[-1]
                 term_kind = infer_term_kind(json_specs)
                 term_id = json_specs["id"]
 
                 if term_kind_collection is None:
                     term_kind_collection = term_kind
-                
+
             except Exception as e:
-                _LOGGER.warning(f'Unable to read term {term_file_path}. Skip.\n{str(e)}')
+                _LOGGER.warning(f"Unable to read term {term_file_path}. Skip.\n{str(e)}")
                 continue
             try:
                 term = PTerm(
@@ -105,51 +108,62 @@ def ingest_collection(collection_dir_path: Path,
         collection.term_kind = term_kind_collection
     project_db_session.add(collection)
 
-def ingest_project(project_dir_path: Path,
-                   project_db_file_path: Path,
-                   git_hash : str
-                   ):
+
+def ingest_project(project_dir_path: Path, project_db_file_path: Path, git_hash: str):
     try:
         project_connection = db.DBConnection(project_db_file_path)
     except Exception as e:
-        msg = f'Unable to read project SQLite file at {project_db_file_path}. Abort.'
+        msg = f"unable to read project SQLite file at {project_db_file_path}"
         _LOGGER.fatal(msg)
-        raise RuntimeError(msg) from e
-        
+        raise EsgvocDbError(msg) from e
+
     with project_connection.create_session() as project_db_session:
+        project_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.PROJECT_SPECS_FILENAME)
         try:
-            project_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.PROJECT_SPECS_FILENAME)
             project_json_specs = read_json_file(project_specs_file_path)
             project_id = project_json_specs[esgvoc.core.constants.PROJECT_ID_JSON_KEY]
         except Exception as e:
-            msg = f'Unable to read project specs file  {project_specs_file_path}. Abort.'
+            msg = f"unable to read project specs file  {project_specs_file_path}"
             _LOGGER.fatal(msg)
-            raise RuntimeError(msg) from e
-        
-        project = Project(id=project_id, specs=project_json_specs,git_hash=git_hash)
+            raise EsgvocDbError(msg) from e
+
+        project = Project(id=project_id, specs=project_json_specs, git_hash=git_hash)
         project_db_session.add(project)
-        
 
         for collection_dir_path in project_dir_path.iterdir():
-            if collection_dir_path.is_dir() and (collection_dir_path / "000_context.jsonld").exists(): #TODO maybe put that in settings
+            # TODO maybe put that in settings
+            if collection_dir_path.is_dir() and (collection_dir_path / "000_context.jsonld").exists():
                 _LOGGER.debug(f"found collection dir : {collection_dir_path}")
                 try:
-                    ingest_collection(collection_dir_path,
-                                      project,
-                                      project_db_session)
+                    ingest_collection(collection_dir_path, project, project_db_session)
                 except Exception as e:
-                    msg = f'Unexpected error while ingesting collection {collection_dir_path}. Abort.'
+                    msg = f"unexpected error while ingesting collection {collection_dir_path}"
                     _LOGGER.fatal(msg)
-                    raise RuntimeError(msg) from e
+                    raise EsgvocDbError(msg) from e
         project_db_session.commit()
 
-
-
-
-
-
-
-
-
-
-
+        # Well, the following instructions are not data duplication. It is more building an index.
+        # Read: https://sqlite.org/fts5.html
+        try:
+            sql_query = (
+                "INSERT INTO pterms_fts5(pk, id, specs, kind, collection_pk) "  # noqa: S608
+                + "SELECT pk, id, specs, kind, collection_pk FROM pterms;"
+            )
+            project_db_session.exec(text(sql_query))  # type: ignore
+        except Exception as e:
+            msg = f"unable to insert rows into pterms_fts5 table for {project_db_file_path}"
+            _LOGGER.fatal(msg)
+            raise EsgvocDbError(msg) from e
+        project_db_session.commit()
+        try:
+            sql_query = (
+                "INSERT INTO pcollections_fts5(pk, id, data_descriptor_id, context, "  # noqa: S608
+                + "project_pk, term_kind) SELECT pk, id, data_descriptor_id, context, "
+                + "project_pk, term_kind FROM pcollections;"
+            )
+            project_db_session.exec(text(sql_query))  # type: ignore
+        except Exception as e:
+            msg = f"unable to insert rows into pcollections_fts5 table for {project_db_file_path}"
+            _LOGGER.fatal(msg)
+            raise EsgvocDbError(msg) from e
+        project_db_session.commit()

esgvoc/core/db/universe_ingestion.py

@@ -1,20 +1,22 @@
 import logging
 from pathlib import Path
 
-import esgvoc.core.constants
-from esgvoc.core.data_handler import JsonLdResource
-from esgvoc.core.service.data_merger import DataMerger
+from sqlalchemy import text
 from sqlmodel import Session, select
 
+import esgvoc.core.constants
 import esgvoc.core.db.connection as db
+import esgvoc.core.service as service
+from esgvoc.core.data_handler import JsonLdResource
 from esgvoc.core.db.connection import read_json_file
 from esgvoc.core.db.models.mixins import TermKind
-from esgvoc.core.db.models.universe import UDataDescriptor, UTerm, Universe
-from esgvoc.core.db.models.universe import universe_create_db
-import esgvoc.core.service as service
+from esgvoc.core.db.models.universe import UDataDescriptor, Universe, UTerm, universe_create_db
+from esgvoc.core.exceptions import EsgvocDbError
+from esgvoc.core.service.data_merger import DataMerger
 
 _LOGGER = logging.getLogger(__name__)
 
+
 def infer_term_kind(json_specs: dict) -> TermKind:
     if esgvoc.core.constants.PATTERN_JSON_KEY in json_specs:
         return TermKind.PATTERN
@@ -28,94 +30,123 @@ def ingest_universe(universe_repo_dir_path: Path, universe_db_file_path: Path) -
     try:
         connection = db.DBConnection(universe_db_file_path)
     except Exception as e:
-        msg = f'Unable to read universe SQLite file at {universe_db_file_path}. Abort.'
+        msg = f"Unable to read universe SQLite file at {universe_db_file_path}. Abort."
         _LOGGER.fatal(msg)
         raise IOError(msg) from e
 
-    for data_descriptor_dir_path in universe_repo_dir_path.iterdir(): 
-        if data_descriptor_dir_path.is_dir() and (data_descriptor_dir_path / "000_context.jsonld").exists(): # TODO maybe put that in setting
+    for data_descriptor_dir_path in universe_repo_dir_path.iterdir():
+        if (
+            data_descriptor_dir_path.is_dir() and (data_descriptor_dir_path / "000_context.jsonld").exists()
+        ):  # TODO may be put that in setting
             try:
                 ingest_data_descriptor(data_descriptor_dir_path, connection)
             except Exception as e:
-                msg = f'Unexpected error while processing data descriptor {data_descriptor_dir_path}. Abort.'
+                msg = f"unexpected error while processing data descriptor {data_descriptor_dir_path}"
                 _LOGGER.fatal(msg)
-                raise RuntimeError(msg) from e
-        
-def ingest_metadata_universe(connection,git_hash):
+                raise EsgvocDbError(msg) from e
+
     with connection.create_session() as session:
-        universe = Universe(git_hash=git_hash)
-        session.add(universe)    
+        # Well, the following instructions are not data duplication. It is more building an index.
+        # Read: https://sqlite.org/fts5.html
+        try:
+            sql_query = (
+                "INSERT INTO uterms_fts5(pk, id, specs, kind, data_descriptor_pk) "
+                + "SELECT pk, id, specs, kind, data_descriptor_pk FROM uterms;"
+            )  # noqa: S608
+            session.exec(text(sql_query))  # type: ignore
+        except Exception as e:
+            msg = f"unable to insert rows into uterms_fts5 table for {universe_db_file_path}"
+            _LOGGER.fatal(msg)
+            raise EsgvocDbError(msg) from e
+        session.commit()
+        try:
+            sql_query = (
+                "INSERT INTO udata_descriptors_fts5(pk, id, universe_pk, context, term_kind) "
+                + "SELECT pk, id, universe_pk, context, term_kind FROM udata_descriptors;"
+            )  # noqa: S608
+            session.exec(text(sql_query))  # type: ignore
+        except Exception as e:
+            msg = f"unable to insert rows into udata_descriptors_fts5 table for {universe_db_file_path}"
+            _LOGGER.fatal(msg)
+            raise EsgvocDbError(msg) from e
         session.commit()
 
-def ingest_data_descriptor(data_descriptor_path: Path,
-                           connection: db.DBConnection) -> None:
 
+def ingest_metadata_universe(connection, git_hash):
+    with connection.create_session() as session:
+        universe = Universe(git_hash=git_hash)
+        session.add(universe)
+        session.commit()
 
+
+def ingest_data_descriptor(data_descriptor_path: Path, connection: db.DBConnection) -> None:
     data_descriptor_id = data_descriptor_path.name
 
     context_file_path = data_descriptor_path.joinpath(esgvoc.core.constants.CONTEXT_FILENAME)
     try:
         context = read_json_file(context_file_path)
     except Exception as e:
-        msg = f'Unable to read the context file {context_file_path} of data descriptor \
-               {data_descriptor_id}. Skip.\n{str(e)}'
+        msg = f"Unable to read the context file {context_file_path} of data descriptor \
+               {data_descriptor_id}. Skip.\n{str(e)}"
         _LOGGER.warning(msg)
-        return        
+        return
 
     with connection.create_session() as session:
-        data_descriptor = UDataDescriptor(id=data_descriptor_id,
-                                         context=context,
-                                         term_kind="") # we ll know it only when we ll add a term (hypothesis all term have the same kind in a data_descriptor)
+        # We ll know it only when we ll add a term (hypothesis all term have the same kind in a data_descriptor)
+        data_descriptor = UDataDescriptor(id=data_descriptor_id, context=context, term_kind="")
         term_kind_dd = None
 
         _LOGGER.debug(f"add data_descriptor : {data_descriptor_id}")
         for term_file_path in data_descriptor_path.iterdir():
             _LOGGER.debug(f"found term path : {term_file_path}, {term_file_path.suffix}")
             if term_file_path.is_file() and term_file_path.suffix == ".json":
-                
                 try:
-                    json_specs=DataMerger(data=JsonLdResource(uri=str(term_file_path)),
-                                          locally_available={"https://espri-mod.github.io/mip-cmor-tables":service.current_state.universe.local_path}).merge_linked_json()[-1]
+                    locally_available = {
+                        "https://espri-mod.github.io/mip-cmor-tables": service.current_state.universe.local_path
+                    }
+
+                    json_specs = DataMerger(
+                        data=JsonLdResource(uri=str(term_file_path)), locally_available=locally_available
+                    ).merge_linked_json()[-1]
+
                     term_kind = infer_term_kind(json_specs)
                     term_id = json_specs["id"]
 
                     if term_kind_dd is None:
                         term_kind_dd = term_kind
-
                 except Exception as e:
-                    _LOGGER.warning(f'Unable to read term {term_file_path} for data descriptor {data_descriptor_path}. Skip.\n{str(e)}')
+                    _LOGGER.warning(
+                        f"Unable to read term {term_file_path} for data descriptor "
+                        + f"{data_descriptor_path}. Skip.\n{str(e)}"
+                    )
                     continue
                 if term_id and json_specs and data_descriptor and term_kind:
-                    _LOGGER.debug("adding {term_id}")
+                    _LOGGER.debug(f"adding {term_id}")
                     term = UTerm(
                         id=term_id,
                         specs=json_specs,
                         data_descriptor=data_descriptor,
                         kind=term_kind,
                     )
+
                     session.add(term)
         if term_kind_dd is not None:
             data_descriptor.term_kind = term_kind_dd
         session.add(data_descriptor)
         session.commit()
 
-def get_universe_term(data_descriptor_id: str,
-                     term_id: str,
-                     universe_db_session: Session) -> tuple[TermKind, dict]:
-    statement = (
-        select(UTerm)
-        .join(UDataDescriptor)
-        .where(UDataDescriptor.id == data_descriptor_id, UTerm.id == term_id)
-    )
+
+def get_universe_term(data_descriptor_id: str, term_id: str, universe_db_session: Session) -> tuple[TermKind, dict]:
+    statement = select(UTerm).join(UDataDescriptor).where(UDataDescriptor.id == data_descriptor_id, UTerm.id == term_id)
     results = universe_db_session.exec(statement)
     term = results.one()
     return term.kind, term.specs
 
 
 if __name__ == "__main__":
-    #ingest_universe(db.UNIVERSE_DIR_PATH, db.UNIVERSE_DB_FILE_PATH)
     import os
+
     root_dir = Path(str(os.getcwd())).parent.parent
     print(root_dir)
-    universe_create_db(root_dir /  Path(".cache/dbs/universe.sqlite"))
-    ingest_universe(root_dir / Path(".cache/repos/mip-cmor-tables"),root_dir /  Path(".cache/dbs/universe.sqlite"))
+    universe_create_db(root_dir / Path(".cache/dbs/universe.sqlite"))
+    ingest_universe(root_dir / Path(".cache/repos/mip-cmor-tables"), root_dir / Path(".cache/dbs/universe.sqlite"))

esgvoc/core/exceptions.py

@@ -0,0 +1,33 @@
+class EsgvocException(Exception):
+    """
+    Class base of all ESGVOC errors.
+    """
+    pass
+
+
+class EsgvocNotFoundError(EsgvocException):
+    """
+    Represents the not found errors.
+    """
+    pass
+
+
+class EsgvocValueError(EsgvocException):
+    """
+    Represents value errors.
+    """
+    pass
+
+
+class EsgvocDbError(EsgvocException):
+    """
+    Represents errors relative to data base management.
+    """
+    pass
+
+
+class EsgvocNotImplementedError(EsgvocException):
+    """
+    Represents not implemented errors.
+    """
+    pass

esgvoc/core/logging_handler.py

@@ -1,4 +1,26 @@
 import logging.config
-from pathlib import Path
 
-logging.config.fileConfig(f"{Path(__file__).parent}/logging.conf", disable_existing_loggers=False)
+LOGGING_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'esgvoc_formatter': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
+        },
+    },
+    'handlers': {
+        'esgvoc_stdout': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'esgvoc_formatter',
+        },
+    },
+    'loggers': {
+        'esgvoc': {
+            'handlers': ['esgvoc_stdout'],
+            'level': 'ERROR',
+            'propagate': False,
+        }
+    }
+}
+
+logging.config.dictConfig(LOGGING_CONFIG)