esgvoc 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

esgvoc/core/repo_fetcher.py

@@ -1,19 +1,23 @@
+import logging
 import os
 import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from typing import List, Optional
+
 import requests
 from pydantic import BaseModel, ValidationError
-from typing import List, Optional
-from contextlib import contextmanager
-import logging
-import sys
 
 _LOGGER = logging.getLogger(__name__)
 
+
 @contextmanager
 def redirect_stdout_to_log(level=logging.INFO):
     """
     Redirect stdout to the global _LOGGER temporarily.
     """
+
     class StreamToLogger:
         def __init__(self, log_level):
             self.log_level = log_level
@@ -48,6 +52,7 @@ class GitHubRepository(BaseModel):
     created_at: str
     updated_at: str
 
+
 class GitHubBranch(BaseModel):
     name: str
     commit: dict
@@ -59,7 +64,7 @@ class RepoFetcher:
     DataFetcher is responsible for fetching data from external sources such as GitHub.
     """
 
-    def __init__(self, base_url: str = "https://api.github.com",local_path: str = ".cache/repos"):
+    def __init__(self, base_url: str = "https://api.github.com", local_path: str = ".cache/repos"):
         self.base_url = base_url
         self.repo_dir = local_path
 
@@ -100,7 +105,6 @@ class RepoFetcher:
         except ValidationError as e:
             raise Exception(f"Data validation error: {e}")
 
-
     def fetch_branch_details(self, owner: str, repo: str, branch: str) -> GitHubBranch:
         """
         Fetch details of a specific branch in a repository.
@@ -120,7 +124,7 @@ class RepoFetcher:
         except ValidationError as e:
             raise Exception(f"Data validation error: {e}")
 
-    def list_directory(self,owner, repo, branch='main'):
+    def list_directory(self, owner, repo, branch="main"):
         """
         List directories in the root of a GitHub repository.
 
@@ -133,10 +137,10 @@ class RepoFetcher:
         response = requests.get(url)
         response.raise_for_status()  # Raise an error for bad responses
         contents = response.json()
-        directories = [item['name'] for item in contents if item['type'] == 'dir']
+        directories = [item["name"] for item in contents if item["type"] == "dir"]
         return directories
 
-    def list_files(self,owner, repo, directory, branch='main'):
+    def list_files(self, owner, repo, directory, branch="main"):
         """
         List files in a specific directory of a GitHub repository.
 
@@ -150,10 +154,10 @@ class RepoFetcher:
         response = requests.get(url)
         response.raise_for_status()  # Raise an error for bad responses
         contents = response.json()
-        files = [item['name'] for item in contents if item['type'] == 'file']
+        files = [item["name"] for item in contents if item["type"] == "file"]
         return files
-    
-    def clone_repository(self, owner: str, repo: str, branch: Optional[str] = None, local_path: str|None = None):
+
+    def clone_repository(self, owner: str, repo: str, branch: Optional[str] = None, local_path: str | None = None):
         """
         Clone a GitHub repository to a target directory.
         :param owner: Repository owner
@@ -162,52 +166,47 @@ class RepoFetcher:
         :param branch: (Optional) The branch to clone. Clones the default branch if None.
         """
         repo_url = f"https://github.com/{owner}/{repo}.git"
-        destination = local_path if local_path else f"{self.repo_dir}/{repo}" 
+        destination = local_path if local_path else f"{self.repo_dir}/{repo}"
 
         command = ["git", "clone", repo_url, destination]
         if branch:
             command.extend(["--branch", branch])
         with redirect_stdout_to_log():
-
             try:
-                subprocess.run(command, check=True)
-                _LOGGER.debug(f"Repository cloned successfully into {destination}")
-            except subprocess.CalledProcessError:
-                try:
+                if not Path(destination).exists():
+                    subprocess.run(command, check=True)
+                    _LOGGER.debug(f"Repository cloned successfully into {destination}")
+                else:
                     current_work_dir = os.getcwd()
-                    os.chdir(f"{self.repo_dir}/{repo}")
+                    os.chdir(f"{destination}")
                     command = ["git", "pull"]
                     subprocess.run(command, check=True)
                     os.chdir(current_work_dir)
 
+            except Exception as e:
+                raise Exception(f"Failed to clone repository: {e}")
 
-                except Exception as e:
-                    raise Exception(f"Failed to clone repository: {e}")
-
-    def get_github_version_with_api(self, owner: str, repo: str, branch: str ="main"):
-        """ Fetch the latest commit version (or any other versioning scheme) from GitHub. """
-        details = self.fetch_branch_details( owner, repo, branch)
-        return details.commit.get('sha')
+    def get_github_version_with_api(self, owner: str, repo: str, branch: str = "main"):
+        """Fetch the latest commit version (or any other versioning scheme) from GitHub."""
+        details = self.fetch_branch_details(owner, repo, branch)
+        return details.commit.get("sha")
 
-    def get_github_version(self, owner: str, repo: str, branch: str="main"):
-        """ Fetch the latest commit version (or any other versioning scheme) from GitHub. with command git fetch """
+    def get_github_version(self, owner: str, repo: str, branch: str = "main"):
+        """Fetch the latest commit version (or any other versioning scheme) from GitHub. with command git fetch"""
         repo_url = f"https://github.com/{owner}/{repo}.git"
         command = ["git", "ls-remote", repo_url, f"{self.repo_dir}/{repo}"]
         if branch:
             command.extend([branch])
 
         # with redirect_stdout_to_log():
-        output=None
+        output = None
         try:
-            result = subprocess.run(command,   capture_output=True,
-                                                text=True,
-                                                check=True)
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
             # Parse the output to get the commit hash
             output = result.stdout.strip()
             _LOGGER.debug(f"Repository fetch successfully from {self.repo_dir}/{repo}")
         except Exception as e:
-
-            _LOGGER.debug("error in with git fetch " +  repr(e))
+            _LOGGER.debug("error in with git fetch " + repr(e))
         if output is not None:
             commit_hash = output.split()[0]
             return commit_hash
@@ -216,45 +215,48 @@ class RepoFetcher:
         # return git_hash
 
     def get_local_repo_version(self, repo_path: str, branch: Optional[str] = "main"):
-        """ Check the version of the local repository by fetching the latest commit hash. """
+        """Check the version of the local repository by fetching the latest commit hash."""
         # repo_path = os.path.join(self.repo_dir, repo)
         if os.path.exists(repo_path):
-            #print("EXIST")
+            # print("EXIST")
             command = ["git", "-C", repo_path]
             if branch:
                 command.extend(["switch", branch])
             # Ensure we are on the correct branch
             with redirect_stdout_to_log():
-                subprocess.run(command,
-                                stdout=subprocess.PIPE,  # Capture stdout
-                                stderr=subprocess.PIPE,  # Capture stderr
-                                text=True)                # Decode output as text
+                subprocess.run(
+                    command,
+                    stdout=subprocess.PIPE,  # Capture stdout
+                    stderr=subprocess.PIPE,  # Capture stderr
+                    text=True,
+                )  # Decode output as text
                 # Get the latest commit hash (SHA) from the local repository
-                commit_hash = subprocess.check_output(["git", "-C", repo_path, "rev-parse", "HEAD"],
-                                                      stderr=subprocess.PIPE,
-                                                      text=True).strip()
+                commit_hash = subprocess.check_output(
+                    ["git", "-C", repo_path, "rev-parse", "HEAD"], stderr=subprocess.PIPE, text=True
+                ).strip()
             return commit_hash
         return None
 
+
 if __name__ == "__main__":
     fetcher = RepoFetcher()
-    
+
     # Fetch repositories for a user
-    #repos = fetcher.fetch_repositories("ESPRI-Mod")
-    #for repo in repos:
+    # repos = fetcher.fetch_repositories("ESPRI-Mod")
+    # for repo in repos:
     #    print(repo)
 
     # Fetch a specific repository's details
-    #repo_details = fetcher.fetch_repository_details("ESPRI-Mod", "mip-cmor-tables")
-    #"print(repo_details)
-    #branch_details = fetcher.fetch_branch_details("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
-    #print(branch_details)
-    
-    fetcher.clone_repository("ESPRI-Mod","mip-cmor-tables", branch="uni_proj_ld")
-    
-    #a =fetcher.get_github_version("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
-    #print(a)
-    #a = fetcher.get_local_repo_version("mip-cmor-tables","uni_proj_ld")
-    #print(a)
-
-    fetcher.clone_repository("ESPRI-Mod","CMIP6Plus_CVs", branch="uni_proj_ld")
+    # repo_details = fetcher.fetch_repository_details("ESPRI-Mod", "mip-cmor-tables")
+    # "print(repo_details)
+    # branch_details = fetcher.fetch_branch_details("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    # print(branch_details)
+
+    fetcher.clone_repository("ESPRI-Mod", "mip-cmor-tables", branch="uni_proj_ld")
+
+    # a =fetcher.get_github_version("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    # print(a)
+    # a = fetcher.get_local_repo_version("mip-cmor-tables","uni_proj_ld")
+    # print(a)
+
+    fetcher.clone_repository("ESPRI-Mod", "CMIP6Plus_CVs", branch="uni_proj_ld")

esgvoc/core/service/data_merger.py

@@ -1,7 +1,7 @@
-
+import logging
 from typing import Dict, List, Set
+
 from esgvoc.core.data_handler import JsonLdResource
-import logging
 
 logger = logging.getLogger(__name__)
 
@@ -9,16 +9,23 @@ logger = logging.getLogger(__name__)
 def merge_dicts(original: list, custom: list) -> dict:
     """Shallow merge: Overwrites original data with custom data."""
     b = original[0]
-    a = custom[0]    
+    a = custom[0]
     merged = {**{k: v for k, v in a.items() if k != "@id"}, **{k: v for k, v in b.items() if k != "@id"}}
     return merged
 
-def merge(uri:str)->Dict:
+
+def merge(uri: str) -> Dict:
     mdm = DataMerger(data=JsonLdResource(uri=uri))
     return mdm.merge_linked_json()[-1]
 
+
 class DataMerger:
-    def __init__(self, data: JsonLdResource, allowed_base_uris: Set[str]={"https://espri-mod.github.io/mip-cmor-tables"}, locally_available:dict = {}):
+    def __init__(
+        self,
+        data: JsonLdResource,
+        allowed_base_uris: Set[str] = {"https://espri-mod.github.io/mip-cmor-tables"},
+        locally_available: dict = {},
+    ):
         self.data = data
         self.allowed_base_uris = allowed_base_uris
         self.locally_available = locally_available
@@ -29,42 +36,49 @@ class DataMerger:
 
     def _get_next_id(self, data: dict) -> str | None:
         """Extract the next @id from the data if it is a valid customization reference."""
-        if isinstance(data,list):
+        if isinstance(data, list):
             data = data[0]
         if "@id" in data and self._should_resolve(data["@id"]):
-            return data["@id"] + ".json" 
+            return data["@id"] + ".json"
         return None
 
     def merge_linked_json(self) -> List[Dict]:
-        """Fetch and merge data recursively, returning a list of progressively merged Data json instances."""
-        result_list = [self.data.json_dict]  # Start with the original json object
-        visited = set(self.data.uri)  # Track visited URIs to prevent cycles
-        current_data = self.data
-        #print(current_data.expanded)
-
-        while True:
-            next_id = self._get_next_id(current_data.expanded[0])
-            if not next_id or next_id in visited or not self._should_resolve(next_id):
-                break
-            visited.add(next_id)
-
-            # Fetch and merge the next customization
-            #do we have it in local ? if so use it instead of remote 
-            for local_repo in self.locally_available.keys():
-                if next_id.startswith(local_repo):
-                    next_id = next_id.replace(local_repo,self.locally_available[local_repo])
-            next_data_instance = JsonLdResource(uri=next_id)
-            merged_json_data = merge_dicts([current_data.json_dict], [next_data_instance.json_dict])
-            next_data_instance.json_dict = merged_json_data
-
-            # Add the merged instance to the result list
-            result_list.append(merged_json_data)
-            current_data = next_data_instance
-
-        return result_list
+        try:
+            """Fetch and merge data recursively, returning a list of progressively merged Data json instances."""
+            result_list = [self.data.json_dict]  # Start with the original json object
+            visited = set(self.data.uri)  # Track visited URIs to prevent cycles
+            current_data = self.data
+            # print(current_data.expanded)
+            while True:
+                next_id = self._get_next_id(current_data.expanded[0])
+
+                if not next_id or next_id in visited or not self._should_resolve(next_id):
+                    break
+
+                visited.add(next_id)
+
+                # Fetch and merge the next customization
+                # do we have it in local ? if so use it instead of remote
+                for local_repo in self.locally_available.keys():
+                    if next_id.startswith(local_repo):
+                        next_id = next_id.replace(local_repo, self.locally_available[local_repo])
+
+                next_data_instance = JsonLdResource(uri=next_id)
+                merged_json_data = merge_dicts([current_data.json_dict], [next_data_instance.json_dict])
+                next_data_instance.json_dict = merged_json_data
+
+                # Add the merged instance to the result list
+                result_list.append(merged_json_data)
+                current_data = next_data_instance
+            return result_list
+        except Exception as e:
+            print(self.data)
+            print(e)
+
 
 if __name__ == "__main__":
     import warnings
+
     warnings.simplefilter("ignore")
 
     # test from institution_id ipsl exapnd and merge with institution ipsl
@@ -80,4 +94,3 @@ if __name__ == "__main__":
     # print(mdm.merge_linked_json())
     #
     #
-