esgvoc 0.1.2__py3-none-any.whl

esgvoc/core/db/models/universe.py

@@ -0,0 +1,59 @@
+import logging
+from pathlib import Path
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.sqlite import JSON
+from sqlmodel import Column, Field, Relationship, SQLModel
+
+import esgvoc.core.db.connection as db
+from esgvoc.core.db.models.mixins import IdMixin, PkMixin, TermKind
+
+_LOGGER = logging.getLogger("universe_db_creation")
+
+
+class Universe(SQLModel, PkMixin, table=True):
+    __tablename__ = "universes"
+    git_hash: str
+    data_descriptors: list["DataDescriptor"] = Relationship(back_populates="universe")
+
+
+class DataDescriptor(SQLModel, PkMixin, IdMixin, table=True):
+    __tablename__ = "data_descriptors"
+    context: dict = Field(sa_column=sa.Column(JSON))
+    universe_pk: int | None = Field(default=None, foreign_key="universes.pk")
+    universe: Universe = Relationship(back_populates="data_descriptors")
+    terms: list["UTerm"] = Relationship(back_populates="data_descriptor")
+    term_kind: TermKind = Field(sa_column=Column(sa.Enum(TermKind)))
+
+
+class UTerm(SQLModel, PkMixin, IdMixin, table=True):
+    __tablename__ = "uterms"
+    specs: dict = Field(sa_column=sa.Column(JSON))
+    kind: TermKind = Field(sa_column=Column(sa.Enum(TermKind)))
+    data_descriptor_pk: int | None = Field(
+        default=None, foreign_key="data_descriptors.pk"
+    )
+    data_descriptor: DataDescriptor = Relationship(back_populates="terms")
+
+
+def universe_create_db(db_file_path: Path) -> None:
+    try:
+        connection = db.DBConnection(db_file_path)
+    except Exception as e:
+        msg = f'Unable to create SQLite file at {db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+    try:
+        # Avoid creating project tables.
+        tables_to_be_created = [SQLModel.metadata.tables['uterms'],
+                                SQLModel.metadata.tables['data_descriptors'],
+                                SQLModel.metadata.tables['universes']]
+        SQLModel.metadata.create_all(connection.get_engine(), tables=tables_to_be_created)
+    except Exception as e:
+        msg = f'Unable to create tables in SQLite database at {db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+
+
+if __name__ == "__main__":
+    pass

esgvoc/core/db/project_ingestion.py

@@ -0,0 +1,152 @@
+import logging
+from pathlib import Path
+
+import esgvoc.core.constants
+from esgvoc.core.data_handler import JsonLdResource
+from esgvoc.core.db.connection import DBConnection
+from esgvoc.core.service.data_merger import DataMerger
+from esgvoc.core.db.models.mixins import TermKind
+from pydantic import BaseModel
+
+import esgvoc.core.db.connection as db
+from esgvoc.core.db.connection import read_json_file
+from esgvoc.core.db.models.project import Collection, Project, PTerm
+
+
+_LOGGER = logging.getLogger("project_ingestion")
+
+def infer_term_kind(json_specs: dict) -> TermKind:
+    if esgvoc.core.constants.PATTERN_JSON_KEY in json_specs:
+        return TermKind.PATTERN
+    elif esgvoc.core.constants.COMPOSITE_PARTS_JSON_KEY in json_specs:
+        return TermKind.COMPOSITE
+    else:
+        return TermKind.PLAIN
+
+
+def ingest_metadata_project(connection:DBConnection,git_hash):
+    with connection.create_session() as session:
+        project = Project(id=str(connection.file_path.stem), git_hash=git_hash,specs={})
+        session.add(project)    
+        session.commit()
+
+###############################
+def get_data_descriptor_id_from_context(collection_context: dict) -> str:
+    data_descriptor_url = collection_context[esgvoc.core.constants.CONTEXT_JSON_KEY][esgvoc.core.constants.DATA_DESCRIPTOR_JSON_KEY]
+    return Path(data_descriptor_url).name
+
+
+def instantiate_project_term(universe_term_json_specs: dict,
+                             project_term_json_specs_update: dict,
+                             pydantic_class: type[BaseModel]) -> dict:
+    term_from_universe = pydantic_class(**universe_term_json_specs)
+    updated_term = term_from_universe.model_copy(
+        update=project_term_json_specs_update, deep=True
+    )
+    return updated_term.model_dump()
+
+
+def ingest_collection(collection_dir_path: Path,
+                      project: Project,
+                      project_db_session) -> None:
+
+
+    collection_id = collection_dir_path.name
+    collection_context_file_path = collection_dir_path.joinpath(esgvoc.core.constants.CONTEXT_FILENAME)
+    try:
+        collection_context = read_json_file(collection_context_file_path)
+        data_descriptor_id = get_data_descriptor_id_from_context(collection_context)
+    except Exception as e:
+        msg = f'Unable to read project context file {collection_context_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+    # [KEEP]
+    collection = Collection(
+        id=collection_id,
+        context=collection_context,
+        project=project,
+        data_descriptor_id=data_descriptor_id,
+        term_kind="") # we ll know it only when we ll add a term (hypothesis all term have the same kind in a collection
+    term_kind_collection = None
+
+    for term_file_path in collection_dir_path.iterdir():
+        _LOGGER.debug(f"found term path : {term_file_path}")
+        if term_file_path.is_file() and term_file_path.suffix==".json": 
+            try:
+                json_specs = DataMerger(data=JsonLdResource(uri =str(term_file_path)),
+                                        locally_available={"https://espri-mod.github.io/mip-cmor-tables":".cache/repos/mip-cmor-tables"}).merge_linked_json()[-1]
+                term_kind = infer_term_kind(json_specs)
+                term_id = json_specs["id"]
+
+                if term_kind_collection is None:
+                    term_kind_collection = term_kind
+                
+            except Exception as e:
+                _LOGGER.warning(f'Unable to read term {term_file_path}. Skip.\n{str(e)}')
+                continue
+            try:
+                term = PTerm(
+                    id=term_id,
+                    specs=json_specs,
+                    collection=collection,
+                    kind=term_kind,
+                )
+                project_db_session.add(term)
+            except Exception as e:
+                _LOGGER.error(
+                    f"fail to find term {term_id} in data descriptor {data_descriptor_id} "
+                    + f"for the collection {collection_id} of the project {project.id}. Skip {term_id}.\n{str(e)}"
+                )
+                continue
+    if term_kind_collection:
+        collection.term_kind = term_kind_collection
+    project_db_session.add(collection)
+
+def ingest_project(project_dir_path: Path,
+                   project_db_file_path: Path,
+                   git_hash : str
+                   ):
+    try:
+        project_connection = db.DBConnection(project_db_file_path)
+    except Exception as e:
+        msg = f'Unable to read project SQLite file at {project_db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise RuntimeError(msg) from e
+        
+    with project_connection.create_session() as project_db_session:
+        try:
+            project_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.PROJECT_SPECS_FILENAME)
+            project_json_specs = read_json_file(project_specs_file_path)
+            project_id = project_json_specs[esgvoc.core.constants.PROJECT_ID_JSON_KEY]
+        except Exception as e:
+            msg = f'Unable to read project specs file  {project_specs_file_path}. Abort.'
+            _LOGGER.fatal(msg)
+            raise RuntimeError(msg) from e
+        
+        project = Project(id=project_id, specs=project_json_specs,git_hash=git_hash)
+        project_db_session.add(project)
+        
+
+        for collection_dir_path in project_dir_path.iterdir():
+            if collection_dir_path.is_dir() and (collection_dir_path / "000_context.jsonld").exists(): #TODO maybe put that in settings
+                _LOGGER.debug(f"found collection dir : {collection_dir_path}")
+                try:
+                    ingest_collection(collection_dir_path,
+                                      project,
+                                      project_db_session)
+                except Exception as e:
+                    msg = f'Unexpected error while ingesting collection {collection_dir_path}. Abort.'
+                    _LOGGER.fatal(msg)
+                    raise RuntimeError(msg) from e
+        project_db_session.commit()
+
+
+
+
+
+
+
+
+
+
+

esgvoc/core/db/universe_ingestion.py

@@ -0,0 +1,120 @@
+import logging
+from pathlib import Path
+
+import esgvoc.core.constants
+from esgvoc.core.data_handler import JsonLdResource
+from esgvoc.core.service.data_merger import DataMerger
+from sqlmodel import Session, select
+
+import esgvoc.core.db.connection as db
+from esgvoc.core.db.connection import read_json_file
+from esgvoc.core.db.models.mixins import TermKind
+from esgvoc.core.db.models.universe import DataDescriptor, UTerm, Universe
+from esgvoc.core.db.models.universe import universe_create_db
+
+_LOGGER = logging.getLogger(__name__)
+
+def infer_term_kind(json_specs: dict) -> TermKind:
+    if esgvoc.core.constants.PATTERN_JSON_KEY in json_specs:
+        return TermKind.PATTERN
+    elif esgvoc.core.constants.COMPOSITE_PARTS_JSON_KEY in json_specs:
+        return TermKind.COMPOSITE
+    else:
+        return TermKind.PLAIN
+
+
+def ingest_universe(universe_repo_dir_path: Path, universe_db_file_path: Path) -> None:
+    try:
+        connection = db.DBConnection(universe_db_file_path)
+    except Exception as e:
+        msg = f'Unable to read universe SQLite file at {universe_db_file_path}. Abort.'
+        _LOGGER.fatal(msg)
+        raise IOError(msg) from e
+
+    for data_descriptor_dir_path in universe_repo_dir_path.iterdir(): 
+        if data_descriptor_dir_path.is_dir() and (data_descriptor_dir_path / "000_context.jsonld").exists(): # TODO maybe put that in setting  
+            try:
+                ingest_data_descriptor(data_descriptor_dir_path, connection)
+            except Exception as e:
+                msg = f'Unexpected error while processing data descriptor {data_descriptor_dir_path}. Abort.'
+                _LOGGER.fatal(msg)
+                raise RuntimeError(msg) from e
+        
+def ingest_metadata_universe(connection,git_hash):
+    with connection.create_session() as session:
+        universe = Universe(git_hash=git_hash)
+        session.add(universe)    
+        session.commit()
+
+def ingest_data_descriptor(data_descriptor_path: Path,
+                           connection: db.DBConnection) -> None:
+
+
+    data_descriptor_id = data_descriptor_path.name
+
+    context_file_path = data_descriptor_path.joinpath(esgvoc.core.constants.CONTEXT_FILENAME)
+    try:
+        context = read_json_file(context_file_path)
+    except Exception as e:
+        msg = f'Unable to read the context file {context_file_path} of data descriptor \
+               {data_descriptor_id}. Skip.\n{str(e)}'
+        _LOGGER.warning(msg)
+        return        
+
+
+    with connection.create_session() as session:
+        data_descriptor = DataDescriptor(id=data_descriptor_id,
+                                         context=context,
+                                         term_kind="") # we ll know it only when we ll add a term (hypothesis all term have the same kind in a data_descriptor)
+        term_kind_dd = None
+
+        _LOGGER.debug(f"add data_descriptor : {data_descriptor_id}")
+        for term_file_path in data_descriptor_path.iterdir():
+            _LOGGER.debug(f"found term path : {term_file_path}, {term_file_path.suffix}")
+            if term_file_path.is_file() and term_file_path.suffix == ".json":
+                try:
+                    json_specs=DataMerger(data=JsonLdResource(uri=str(term_file_path)),
+                                          locally_available={"https://espri-mod.github.io/mip-cmor-tables":".cache/repos/mip-cmor-tables"}).merge_linked_json()[-1]
+                    term_kind = infer_term_kind(json_specs)
+                    term_id = json_specs["id"]
+
+                    if term_kind_dd is None:
+                        term_kind_dd = term_kind
+
+                except Exception as e:
+                    _LOGGER.warning(f'Unable to read term {term_file_path} for data descriptor {data_descriptor_path}. Skip.\n{str(e)}')
+                    continue
+                if term_id and json_specs and data_descriptor and term_kind:
+                    _LOGGER.debug("adding {term_id}")
+                    term = UTerm(
+                        id=term_id,
+                        specs=json_specs,
+                        data_descriptor=data_descriptor,
+                        kind=term_kind,
+                    )
+                    session.add(term)
+        if term_kind_dd is not None:
+            data_descriptor.term_kind = term_kind_dd
+        session.add(data_descriptor)
+        session.commit()
+
+def get_universe_term(data_descriptor_id: str,
+                     term_id: str,
+                     universe_db_session: Session) -> tuple[TermKind, dict]:
+    statement = (
+        select(UTerm)
+        .join(DataDescriptor)
+        .where(DataDescriptor.id == data_descriptor_id, UTerm.id == term_id)
+    )
+    results = universe_db_session.exec(statement)
+    term = results.one()
+    return term.kind, term.specs
+
+
+if __name__ == "__main__":
+    #ingest_universe(db.UNIVERSE_DIR_PATH, db.UNIVERSE_DB_FILE_PATH)
+    import os
+    root_dir = Path(str(os.getcwd())).parent.parent
+    print(root_dir)
+    universe_create_db(root_dir /  Path(".cache/dbs/universe.sqlite"))
+    ingest_universe(root_dir / Path(".cache/repos/mip-cmor-tables"),root_dir /  Path(".cache/dbs/universe.sqlite"))

esgvoc/core/logging.conf

@@ -0,0 +1,21 @@
+[loggers]
+keys=root
+
+[logger_root]
+level=ERROR 
+handlers=stdout
+
+[handlers]
+keys=stdout
+
+[handler_stdout]
+class=StreamHandler
+level=NOTSET
+formatter=stdout
+args=(sys.stdout,)
+
+[formatters]
+keys=stdout
+
+[formatter_stdout]
+format=%(asctime)s [%(levelname)s] %(name)s: %(message)s

esgvoc/core/logging_handler.py

@@ -0,0 +1,4 @@
+import logging.config
+from pathlib import Path
+
+logging.config.fileConfig(f"{Path(__file__).parent}/logging.conf")

esgvoc/core/repo_fetcher.py

@@ -0,0 +1,259 @@
+import os
+import subprocess
+import requests
+from pydantic import BaseModel, ValidationError
+from typing import List, Optional
+from contextlib import contextmanager
+import logging
+import sys
+
+_LOGGER = logging.getLogger(__name__)
+
+@contextmanager
+def redirect_stdout_to_log(level=logging.INFO):
+    """
+    Redirect stdout to the global _LOGGER temporarily.
+    """
+    class StreamToLogger:
+        def __init__(self, log_level):
+            self.log_level = log_level
+
+        def write(self, message):
+            if message.strip():  # Avoid logging empty lines
+                _LOGGER.debug(self.log_level, message.strip())
+
+        def flush(self):
+            pass  # No-op for compatibility
+
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = StreamToLogger(level)
+    sys.stderr = StreamToLogger(level)
+    try:
+        yield
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+
+
+class GitHubRepository(BaseModel):
+    id: int
+    name: str
+    full_name: str
+    description: Optional[str]
+    html_url: str
+    stargazers_count: int
+    forks_count: int
+    language: Optional[str]
+    created_at: str
+    updated_at: str
+
+class GitHubBranch(BaseModel):
+    name: str
+    commit: dict
+    protected: bool
+
+
+class RepoFetcher:
+    """
+    DataFetcher is responsible for fetching data from external sources such as GitHub.
+    """
+
+    def __init__(self, base_url: str = "https://api.github.com",local_path: str = ".cache/repos"):
+        self.base_url = base_url
+        self.repo_dir = local_path
+
+    def fetch_repositories(self, user: str) -> List[GitHubRepository]:
+        """
+        Fetch repositories of a given GitHub user.
+        :param user: GitHub username
+        :return: List of GitHubRepository objects
+        """
+        url = f"{self.base_url}/users/{user}/repos"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch data: {response.status_code} - {response.text}")
+
+        try:
+            data = response.json()
+            return [GitHubRepository(**repo) for repo in data]
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+    def fetch_repository_details(self, owner: str, repo: str) -> GitHubRepository:
+        """
+        Fetch details of a specific repository.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :return: GitHubRepository object
+        """
+        url = f"{self.base_url}/repos/{owner}/{repo}"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch data: {response.status_code} - {response.text}")
+
+        try:
+            data = response.json()
+            return GitHubRepository(**data)
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+
+    def fetch_branch_details(self, owner: str, repo: str, branch: str) -> GitHubBranch:
+        """
+        Fetch details of a specific branch in a repository.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :param branch: Branch name
+        :return: GitHubBranch object
+        """
+        url = f"{self.base_url}/repos/{owner}/{repo}/branches/{branch}"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch branch data: {response.status_code} - {response.text}")
+
+        try:
+            return GitHubBranch(**response.json())
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+    def list_directory(self,owner, repo, branch='main'):
+        """
+        List directories in the root of a GitHub repository.
+
+        :param owner: GitHub username or organization name.
+        :param repo: Repository name.
+        :param branch: Branch name (default: 'main').
+        :return: List of directories in the repository.
+        """
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/?ref={branch}"
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad responses
+        contents = response.json()
+        directories = [item['name'] for item in contents if item['type'] == 'dir']
+        return directories
+
+    def list_files(self,owner, repo, directory, branch='main'):
+        """
+        List files in a specific directory of a GitHub repository.
+
+        :param owner: GitHub username or organization name.
+        :param repo: Repository name.
+        :param directory: Target directory path within the repo.
+        :param branch: Branch name (default: 'main').
+        :return: List of files in the specified directory.
+        """
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{directory}?ref={branch}"
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad responses
+        contents = response.json()
+        files = [item['name'] for item in contents if item['type'] == 'file']
+        return files
+    
+    def clone_repository(self, owner: str, repo: str, branch: Optional[str] = None):
+        """
+        Clone a GitHub repository to a target directory.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :param target_dir: The directory where the repository should be cloned.
+        :param branch: (Optional) The branch to clone. Clones the default branch if None.
+        """
+        repo_url = f"https://github.com/{owner}/{repo}.git"
+
+        command = ["git", "clone", repo_url, f"{self.repo_dir}/{repo}"]
+        if branch:
+            command.extend(["--branch", branch])
+        with redirect_stdout_to_log():
+
+            try:
+                subprocess.run(command, check=True)
+                _LOGGER.debug(f"Repository cloned successfully into {self.repo_dir}/{repo}")
+            except subprocess.CalledProcessError:
+                try:
+                    current_work_dir = os.getcwd()
+                    os.chdir(f"{self.repo_dir}/{repo}")
+                    command = ["git", "pull"]
+                    subprocess.run(command, check=True)
+                    os.chdir(current_work_dir)
+
+
+                except Exception as e:
+                    raise Exception(f"Failed to clone repository: {e}")
+
+    def get_github_version_with_api(self, owner: str, repo: str, branch: str ="main"):
+        """ Fetch the latest commit version (or any other versioning scheme) from GitHub. """
+        details = self.fetch_branch_details( owner, repo, branch)
+        return details.commit.get('sha')
+
+    def get_github_version(self, owner: str, repo: str, branch: str="main"):
+        """ Fetch the latest commit version (or any other versioning scheme) from GitHub. with command git fetch """
+        repo_url = f"https://github.com/{owner}/{repo}.git"
+        command = ["git", "ls-remote", repo_url, f"{self.repo_dir}/{repo}"]
+        if branch:
+            command.extend([branch])
+
+        # with redirect_stdout_to_log():
+        output=None
+        try:
+            result = subprocess.run(command,   capture_output=True,
+                                                text=True,
+                                                check=True)
+            # Parse the output to get the commit hash
+            output = result.stdout.strip()
+            _LOGGER.debug(f"Repository fetch successfully from {self.repo_dir}/{repo}")
+        except Exception as e:
+
+            _LOGGER.debug("error in with git fetch " +  repr(e))
+        if output is not None:
+            commit_hash = output.split()[0]
+            return commit_hash
+        return None
+
+        # return git_hash
+
+    def get_local_repo_version(self, repo_path: str, branch: Optional[str] = "main"):
+        """ Check the version of the local repository by fetching the latest commit hash. """
+        # repo_path = os.path.join(self.repo_dir, repo)
+        if os.path.exists(repo_path):
+            #print("EXIST")
+            command = ["git", "-C", repo_path]
+            if branch:
+                command.extend(["switch", branch])
+            # Ensure we are on the correct branch
+            with redirect_stdout_to_log():
+                subprocess.run(command,
+                                stdout=subprocess.PIPE,  # Capture stdout
+                                stderr=subprocess.PIPE,  # Capture stderr
+                                text=True)                # Decode output as text
+                # Get the latest commit hash (SHA) from the local repository
+                commit_hash = subprocess.check_output(["git", "-C", repo_path, "rev-parse", "HEAD"],
+                                                      stderr=subprocess.PIPE,
+                                                      text=True).strip()
+            return commit_hash
+        return None
+
+if __name__ == "__main__":
+    fetcher = RepoFetcher()
+    
+    # Fetch repositories for a user
+    #repos = fetcher.fetch_repositories("ESPRI-Mod")
+    #for repo in repos:
+    #    print(repo)
+
+    # Fetch a specific repository's details
+    #repo_details = fetcher.fetch_repository_details("ESPRI-Mod", "mip-cmor-tables")
+    #"print(repo_details)
+    #branch_details = fetcher.fetch_branch_details("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    #print(branch_details)
+    
+    fetcher.clone_repository("ESPRI-Mod","mip-cmor-tables", branch="uni_proj_ld")
+    
+    #a =fetcher.get_github_version("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    #print(a)
+    #a = fetcher.get_local_repo_version("mip-cmor-tables","uni_proj_ld")
+    #print(a)
+
+    fetcher.clone_repository("ESPRI-Mod","CMIP6Plus_CVs", branch="uni_proj_ld")

esgvoc/core/service/__init__.py

@@ -0,0 +1,8 @@
+from esgvoc.core.service.settings import ServiceSettings
+from esgvoc.core.service.state import StateService
+from pathlib import Path
+
+settings_path = Path(__file__).parent / "settings.toml"
+service_settings = ServiceSettings.load_from_file(str(settings_path))
+state_service = StateService(service_settings)
+