esgvoc 2.0.2__py3-none-any.whl

esgvoc/core/repo_fetcher.py

@@ -0,0 +1,345 @@
+import logging
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from typing import List, Optional
+
+import requests
+from pydantic import BaseModel, ValidationError
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@contextmanager
+def redirect_stdout_to_log(level=logging.INFO):
+    """
+    Redirect stdout to the global _LOGGER temporarily.
+    """
+
+    class StreamToLogger:
+        def __init__(self, log_level):
+            self.log_level = log_level
+
+        def write(self, message):
+            if message.strip():  # Avoid logging empty lines
+                _LOGGER.debug(self.log_level, message.strip())
+
+        def flush(self):
+            pass  # No-op for compatibility
+
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = StreamToLogger(level)
+    sys.stderr = StreamToLogger(level)
+    try:
+        yield
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+
+
+class GitHubRepository(BaseModel):
+    id: int
+    name: str
+    full_name: str
+    description: Optional[str]
+    html_url: str
+    stargazers_count: int
+    forks_count: int
+    language: Optional[str]
+    created_at: str
+    updated_at: str
+
+
+class GitHubBranch(BaseModel):
+    name: str
+    commit: dict
+    protected: bool
+
+
+class RepoFetcher:
+    """
+    DataFetcher is responsible for fetching data from external sources such as GitHub.
+    """
+
+    def __init__(self, base_url: str = "https://api.github.com", local_path: str = ".cache/repos", offline_mode: bool = False):
+        self.base_url = base_url
+        self.repo_dir = local_path
+        self.offline_mode = offline_mode
+
+    def fetch_repositories(self, user: str) -> List[GitHubRepository]:
+        """
+        Fetch repositories of a given GitHub user.
+        :param user: GitHub username
+        :return: List of GitHubRepository objects
+        """
+        if self.offline_mode:
+            raise Exception("Cannot fetch repositories in offline mode")
+
+        url = f"{self.base_url}/users/{user}/repos"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch data: {response.status_code} - {response.text}")
+
+        try:
+            data = response.json()
+            return [GitHubRepository(**repo) for repo in data]
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+    def fetch_repository_details(self, owner: str, repo: str) -> GitHubRepository:
+        """
+        Fetch details of a specific repository.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :return: GitHubRepository object
+        """
+        if self.offline_mode:
+            raise Exception("Cannot fetch repository details in offline mode")
+
+        url = f"{self.base_url}/repos/{owner}/{repo}"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch data: {response.status_code} - {response.text}")
+
+        try:
+            data = response.json()
+            return GitHubRepository(**data)
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+    def fetch_branch_details(self, owner: str, repo: str, branch: str) -> GitHubBranch:
+        """
+        Fetch details of a specific branch in a repository.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :param branch: Branch name
+        :return: GitHubBranch object
+        """
+        if self.offline_mode:
+            raise Exception("Cannot fetch branch details in offline mode")
+
+        url = f"{self.base_url}/repos/{owner}/{repo}/branches/{branch}"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch branch data: {response.status_code} - {response.text}")
+
+        try:
+            return GitHubBranch(**response.json())
+        except ValidationError as e:
+            raise Exception(f"Data validation error: {e}")
+
+    def list_directory(self, owner, repo, branch="main"):
+        """
+        List directories in the root of a GitHub repository.
+
+        :param owner: GitHub username or organization name.
+        :param repo: Repository name.
+        :param branch: Branch name (default: 'main').
+        :return: List of directories in the repository.
+        """
+        if self.offline_mode:
+            raise Exception("Cannot list directories in offline mode")
+
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/?ref={branch}"
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad responses
+        contents = response.json()
+        directories = [item["name"] for item in contents if item["type"] == "dir"]
+        return directories
+
+    def list_files(self, owner, repo, directory, branch="main"):
+        """
+        List files in a specific directory of a GitHub repository.
+
+        :param owner: GitHub username or organization name.
+        :param repo: Repository name.
+        :param directory: Target directory path within the repo.
+        :param branch: Branch name (default: 'main').
+        :return: List of files in the specified directory.
+        """
+        if self.offline_mode:
+            raise Exception("Cannot list files in offline mode")
+
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{directory}?ref={branch}"
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad responses
+        contents = response.json()
+        files = [item["name"] for item in contents if item["type"] == "file"]
+        return files
+
+    def clone_repository(self, owner: str, repo: str, branch: Optional[str] = None, local_path: str | None = None, shallow: bool = True):
+        """
+        Clone a GitHub repository to a target directory.
+        :param owner: Repository owner
+        :param repo: Repository name
+        :param target_dir: The directory where the repository should be cloned.
+        :param branch: (Optional) The branch to clone. Clones the default branch if None.
+        :param shallow: (Optional) If True, performs a shallow clone with --depth 1. Default is True.
+        """
+        if self.offline_mode:
+            raise Exception("Cannot clone repository in offline mode")
+
+        repo_url = f"https://github.com/{owner}/{repo}.git"
+        destination = local_path if local_path else f"{self.repo_dir}/{repo}"
+
+        command = ["git", "clone", repo_url, destination]
+        if shallow:
+            command.extend(["--depth", "1"])
+        if branch:
+            command.extend(["--branch", branch])
+        with redirect_stdout_to_log():
+            try:
+                if not Path(destination).exists():
+                    subprocess.run(command, check=True)
+                    _LOGGER.debug(f"Repository cloned successfully into {destination}")
+                else:
+                    current_work_dir = os.getcwd()
+                    os.chdir(f"{destination}")
+
+                    # Clean up any conflicted state first
+                    try:
+                        subprocess.run(["git", "reset", "--hard"], capture_output=True, check=False)
+                        subprocess.run(["git", "clean", "-fd"], capture_output=True, check=False)
+                    except Exception:
+                        pass
+
+                    # Check if the requested branch exists locally
+                    try:
+                        result = subprocess.run(
+                            ["git", "rev-parse", "--verify", f"refs/heads/{branch}"],
+                            capture_output=True,
+                            check=False
+                        )
+                        branch_exists_locally = result.returncode == 0
+                    except Exception:
+                        branch_exists_locally = False
+
+                    if not branch_exists_locally and branch:
+                        # If branch doesn't exist locally, we need to fetch it
+                        # For shallow repos, we need to unshallow first or fetch the specific branch
+                        try:
+                            # Try to fetch the specific branch
+                            subprocess.run(["git", "fetch", "origin", f"{branch}:{branch}"], check=True)
+                            _LOGGER.debug(f"Fetched new branch {branch}")
+                        except subprocess.CalledProcessError:
+                            # If that fails, unshallow and try again
+                            subprocess.run(["git", "fetch", "--unshallow"], check=True)
+                            subprocess.run(["git", "fetch", "origin", f"{branch}:{branch}"], check=True)
+                            _LOGGER.debug(f"Unshallowed repo and fetched branch {branch}")
+
+                    # Switch to the requested branch if specified
+                    if branch:
+                        subprocess.run(["git", "checkout", branch], check=True)
+                        _LOGGER.debug(f"Switched to branch {branch}")
+
+                    # For shallow repos that switched branches, just ensure we have latest
+                    # to avoid merge conflicts from different commit histories
+                    if branch and not branch_exists_locally:
+                        # We already fetched the branch, no need for additional reset
+                        _LOGGER.debug(f"Switched to newly fetched branch {branch}")
+                    else:
+                        # Pull latest changes for normal updates
+                        try:
+                            subprocess.run(["git", "pull"], check=True)
+                        except subprocess.CalledProcessError:
+                            # If pull fails, try to fetch and reset
+                            subprocess.run(["git", "fetch"], check=True)
+                            # Check if remote tracking branch exists
+                            try:
+                                subprocess.run(["git", "rev-parse", f"origin/{branch}"], capture_output=True, check=True)
+                                subprocess.run(["git", "reset", "--hard", f"origin/{branch}"], check=True)
+                                _LOGGER.debug(f"Reset to origin/{branch} after pull failure")
+                            except subprocess.CalledProcessError:
+                                # Remote tracking branch doesn't exist, just continue
+                                _LOGGER.debug(f"No remote tracking branch for {branch}, continuing")
+                    os.chdir(current_work_dir)
+
+            except Exception as e:
+                raise Exception(f"Failed to clone repository: {e}")
+
+    def get_github_version_with_api(self, owner: str, repo: str, branch: str = "main"):
+        """Fetch the latest commit version (or any other versioning scheme) from GitHub."""
+        if self.offline_mode:
+            raise Exception("Cannot get GitHub version in offline mode")
+        details = self.fetch_branch_details(owner, repo, branch)
+        return details.commit.get("sha")
+
+    def get_github_version(self, owner: str, repo: str, branch: str = "main"):
+        """Fetch the latest commit version (or any other versioning scheme) from GitHub. with command git fetch"""
+        if self.offline_mode:
+            _LOGGER.debug("Cannot get GitHub version in offline mode")
+            return None
+
+        repo_url = f"https://github.com/{owner}/{repo}.git"
+        command = ["git", "ls-remote", repo_url, f"{self.repo_dir}/{repo}"]
+        if branch:
+            command.extend([branch])
+
+        # with redirect_stdout_to_log():
+        output = None
+        try:
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+            # Parse the output to get the commit hash
+            output = result.stdout.strip()
+            _LOGGER.debug(f"Repository fetch successfully from {self.repo_dir}/{repo}")
+        except Exception as e:
+            _LOGGER.debug("error in with git fetch " + repr(e))
+        if output is not None:
+            commit_hash = output.split()[0]
+            return commit_hash
+        return None
+
+        # return git_hash
+
+    def get_local_repo_version(self, repo_path: str, branch: Optional[str] = "main"):
+        """Check the version of the local repository by fetching the latest commit hash."""
+        # repo_path = os.path.join(self.repo_dir, repo)
+        if os.path.exists(repo_path):
+            # print("EXIST")
+            command = ["git", "-C", repo_path]
+            if branch:
+                command.extend(["switch", branch])
+            # Ensure we are on the correct branch
+            with redirect_stdout_to_log():
+                subprocess.run(
+                    command,
+                    stdout=subprocess.PIPE,  # Capture stdout
+                    stderr=subprocess.PIPE,  # Capture stderr
+                    text=True,
+                )  # Decode output as text
+                # Get the latest commit hash (SHA) from the local repository
+                commit_hash = subprocess.check_output(
+                    ["git", "-C", repo_path, "rev-parse", "HEAD"], stderr=subprocess.PIPE, text=True
+                ).strip()
+            return commit_hash
+        return None
+
+
+if __name__ == "__main__":
+    fetcher = RepoFetcher()
+
+    # Fetch repositories for a user
+    # repos = fetcher.fetch_repositories("ESPRI-Mod")
+    # for repo in repos:
+    #    print(repo)
+
+    # Fetch a specific repository's details
+    # repo_details = fetcher.fetch_repository_details("ESPRI-Mod", "mip-cmor-tables")
+    # "print(repo_details)
+    # branch_details = fetcher.fetch_branch_details("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    # print(branch_details)
+
+    fetcher.clone_repository("ESPRI-Mod", "mip-cmor-tables", branch="uni_proj_ld")
+
+    # a =fetcher.get_github_version("ESPRI-Mod", "mip-cmor-tables", "uni_proj_ld")
+    # print(a)
+    # a = fetcher.get_local_repo_version("mip-cmor-tables","uni_proj_ld")
+    # print(a)
+
+    fetcher.clone_repository("ESPRI-Mod", "CMIP6Plus_CVs", branch="uni_proj_ld")

esgvoc/core/service/__init__.py

@@ -0,0 +1,41 @@
+# from esgvoc.core.service.config_register import ConfigManager
+# from esgvoc.core.service.settings import ServiceSettings
+# from esgvoc.core.service.state import StateService
+#
+# config_manager = ConfigManager()
+# active_setting = config_manager.get_active_config()
+# active_setting["base_dir"] = str(config_manager.config_dir / config_manager.get_active_config_name())
+# service_settings = ServiceSettings.from_config(active_setting)
+# state_service = StateService(service_settings)
+
+
+from esgvoc.core.service.configuration.config_manager import ConfigManager
+from esgvoc.core.service.configuration.setting import ServiceSettings
+from esgvoc.core.service.state import StateService
+
+config_manager : ConfigManager | None = None
+current_state : StateService | None = None
+
+def get_config_manager():
+    global config_manager
+    if config_manager is None:
+
+        config_manager = ConfigManager(ServiceSettings, app_name="esgvoc", app_author="ipsl", default_settings=ServiceSettings._get_default_settings())
+        active_config_name= config_manager.get_active_config_name()
+        config_manager.data_config_dir = config_manager.data_dir / active_config_name
+        config_manager.data_config_dir.mkdir(parents=True, exist_ok=True)
+
+    return config_manager   
+
+
+def get_state():
+    global current_state
+    if config_manager is not None:
+        service_settings = config_manager.get_active_config()
+        current_state = StateService(service_settings)
+    return current_state
+
+# Singleton Access Function
+config_manager = get_config_manager()
+current_state = get_state()
+

esgvoc/core/service/configuration/config_manager.py

@@ -0,0 +1,196 @@
+import toml
+import logging
+from pathlib import Path
+from platformdirs import PlatformDirs
+from typing import Type, TypeVar, Generic, Protocol
+
+# Setup logging
+# Use WARNING level to see important messages (errors, warnings) but not debug/info spam
+logging.basicConfig(
+    level=logging.WARNING,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+# Explicitly set data_merger logger to WARNING since something else seems to change it to ERROR
+logging.getLogger("esgvoc.core.service.data_merger").setLevel(logging.WARNING)
+
+# Define a generic type for configuration
+T = TypeVar("T", bound="ConfigSchema")
+
+
+class ConfigSchema(Protocol):
+    """Protocol for application-specific configuration classes."""
+
+    @classmethod
+    def load_from_file(cls, file_path: str): ...
+
+    def save_to_file(self, file_path: str): ...
+
+
+class ConfigManager(Generic[T]):
+    def __init__(self, config_cls: Type[T], app_name: str, app_author: str, default_settings: dict | None = None):
+        """
+        Initialize the configuration manager.
+        - config_cls: A class that implements `ConfigSchema` (e.g., ServiceSettings).
+        - app_name: Name of the application (used for directory paths).
+        - app_author: Name of the author/organization (used for directory paths).
+        """
+        self.config_cls = config_cls
+        self.dirs = PlatformDirs(app_name, app_author)
+
+        # Define standard paths
+        self.config_dir = Path(self.dirs.user_config_path).expanduser().resolve()
+        self.data_dir = Path(self.dirs.user_data_path).expanduser().resolve()
+        self.data_config_dir = None  # depends on loaded settings
+
+        self.cache_dir = Path(self.dirs.user_cache_path).expanduser().resolve()
+
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        self.registry_path = self.config_dir / "config_registry.toml"
+        self.default_config_path = self.config_dir / "default_setting.toml"
+        self.default_settings = default_settings
+        self._init_registry()
+
+    def _init_registry(self):
+        """Initialize the registry file if it doesn't exist."""
+        if not self.registry_path.exists():
+            logger.info("Initializing configuration registry...")
+            registry = {"configs": {"default": str(self.default_config_path)}, "active": "default"}
+            self._save_toml(self.registry_path, registry)
+        # Ensure the default settings file exists and save it if necessary
+        if not self.default_config_path.exists():
+            if self.default_settings:
+                logger.info("Saving default settings...")
+                self._save_toml(self.default_config_path, self.default_settings)
+            else:
+                logger.warning("No default settings provided.")
+
+    def _load_toml(self, path: Path) -> dict:
+        """Load TOML data from a file."""
+        if not path.exists():
+            logger.error(f"Configuration file not found: {path}")
+            raise FileNotFoundError(f"Configuration file not found: {path}")
+        with open(path, "r") as f:
+            return toml.load(f)
+
+    def _save_toml(self, path: Path, data: dict) -> None:
+        """Save TOML data to a file."""
+        with open(path, "w") as f:
+            toml.dump(data, f)
+
+    def _get_active_config_path(self) -> Path:
+        """Retrieve the path of the active configuration file."""
+        registry = self._load_toml(self.registry_path)
+        active_config_name = registry["active"]
+        return Path(registry["configs"][active_config_name])
+
+    def get_config(self, config_name: str) -> T:
+        """Load the configuration as an instance of the given config schema."""
+        registry = self._load_toml(self.registry_path)
+        if config_name not in registry["configs"]:
+            logger.error(f"Config '{config_name}' not found in registry.")
+            raise ValueError(f"Config '{config_name}' not found in registry.")
+        config_path = registry["configs"][config_name]
+        return self.config_cls.load_from_file(str(config_path))
+
+    def get_active_config(self) -> T:
+        """Load the active configuration as an instance of the given config schema."""
+        active_config_path = self._get_active_config_path()
+        active_config_name = self.get_active_config_name()
+
+        settings = self.config_cls.load_from_file(str(active_config_path))
+        # Set the config name if the settings support it (duck typing)
+        if hasattr(settings, 'set_config_name'):
+            settings.set_config_name(active_config_name)
+        return settings
+
+    def get_active_config_name(self) -> str:
+        """Retrieve the config name from the registry"""
+        registry = self._load_toml(self.registry_path)
+        return registry["active"]
+
+    def save_config(self, config_data: dict, name: str | None = None) -> None:
+        """Save the modified configuration to the corresponding file and update the registry."""
+
+        if name:
+            # If a name is provided, save the configuration with that name
+            config_path = self.config_dir / f"{name}.toml"
+            self._save_toml(config_path, config_data)
+
+            # Update the registry with the new config name
+            registry = self._load_toml(self.registry_path)
+            registry["configs"][name] = str(config_path)
+            registry["active"] = name
+            self._save_toml(self.registry_path, registry)
+
+            logger.info(f"Saved configuration to {config_path} and updated registry.")
+        else:
+            # If no name is provided, give the user a default name, like "user_config"
+            default_name = "user_config"
+            config_path = self.config_dir / f"{default_name}.toml"
+
+            # Check if the user_config already exists, if so, warn them
+            if config_path.exists():
+                logger.warning(f"{default_name}.toml already exists. Overwriting with the new config.")
+
+            # Save the configuration with the default name
+            self._save_toml(config_path, config_data)
+
+            # Update the registry with the new config name
+            registry = self._load_toml(self.registry_path)
+            registry["configs"][default_name] = str(config_path)
+            registry["active"] = default_name
+            self._save_toml(self.registry_path, registry)
+
+            logger.info(f"Saved new configuration to {config_path} and updated registry.")
+
+    def save_active_config(self, config: T):
+        """Save the current configuration to the active file."""
+        active_config_path = self._get_active_config_path()
+        config.save_to_file(str(active_config_path))
+
+    def switch_config(self, config_name: str):
+        """Switch to a different configuration."""
+        registry = self._load_toml(self.registry_path)
+        if config_name not in registry["configs"]:
+            logger.error(f"Config '{config_name}' not found in registry.")
+            raise ValueError(f"Config '{config_name}' not found in registry.")
+        registry["active"] = config_name
+
+        self._save_toml(self.registry_path, registry)
+        logger.info(f"Switched to configuration: {config_name}")
+
+    def list_configs(self) -> dict:
+        """Return a list of available configurations."""
+        return self._load_toml(self.registry_path)["configs"]
+
+    def add_config(self, config_name: str, config_data: dict):
+        """Add a new configuration."""
+        registry = self._load_toml(self.registry_path)
+        if config_name in registry["configs"]:
+            raise ValueError(f"Config '{config_name}' already exists.")
+        config_path = self.config_dir / f"{config_name}.toml"
+        self._save_toml(config_path, config_data)
+        registry["configs"][config_name] = str(config_path)
+        self._save_toml(self.registry_path, registry)
+
+    def remove_config(self, config_name: str):
+        """Remove a configuration."""
+        registry = self._load_toml(self.registry_path)
+        if config_name == "default":
+            raise ValueError("Cannot remove the default configuration.")
+        if config_name not in registry["configs"]:
+            raise ValueError(f"Config '{config_name}' not found.")
+        del registry["configs"][config_name]
+        config_path = self.config_dir / f"{config_name}.toml"
+        config_path.unlink()
+
+        self._save_toml(self.registry_path, registry)
+        logger.info(f"Removed configuration: {config_name}")
+        if registry["active"] not in registry["configs"]:
+            self.switch_config("default")
+            logger.info("active configuration doesnot exist anymore : Switch to default configuration")