esgvoc 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

esgvoc/core/service/configuration/setting.py

@@ -1,7 +1,34 @@
 from typing import ClassVar, Dict, Optional
+from pathlib import Path
 
 import toml
 from pydantic import BaseModel, Field
+from platformdirs import PlatformDirs
+
+
+def resolve_path_to_absolute(relative_path: Optional[str], config_name: Optional[str] = None) -> Optional[str]:
+    """
+    Convert a relative path to an absolute path without modifying the original.
+    This is used for internal path resolution only.
+    """
+    if relative_path is None:
+        return None
+
+    path_obj = Path(relative_path)
+
+    if path_obj.is_absolute():
+        return str(path_obj.resolve())
+
+    # Handle dot-relative paths (./... or ../..) relative to current working directory
+    if relative_path.startswith('.'):
+        return str((Path.cwd() / relative_path).resolve())
+
+    # Handle plain relative paths using PlatformDirs with config name (default behavior)
+    dirs = PlatformDirs("esgvoc", "ipsl")
+    base_path = Path(dirs.user_data_path).expanduser().resolve()
+    if config_name:
+        base_path = base_path / config_name
+    return str(base_path / relative_path)
 
 
 class ProjectSettings(BaseModel):
@@ -10,6 +37,20 @@ class ProjectSettings(BaseModel):
     branch: Optional[str] = "main"
     local_path: Optional[str] = None
     db_path: Optional[str] = None
+    offline_mode: bool = False
+    _config_name: Optional[str] = None
+
+    def set_config_name(self, config_name: str):
+        """Set the config name for path resolution."""
+        self._config_name = config_name
+
+    def get_absolute_local_path(self) -> Optional[str]:
+        """Get the absolute local path without modifying the stored value."""
+        return resolve_path_to_absolute(self.local_path, self._config_name)
+
+    def get_absolute_db_path(self) -> Optional[str]:
+        """Get the absolute db path without modifying the stored value."""
+        return resolve_path_to_absolute(self.db_path, self._config_name)
 
 
 class UniverseSettings(BaseModel):
@@ -17,71 +58,118 @@ class UniverseSettings(BaseModel):
     branch: Optional[str] = None
     local_path: Optional[str] = None
     db_path: Optional[str] = None
+    offline_mode: bool = False
+    _config_name: Optional[str] = None
+
+    def set_config_name(self, config_name: str):
+        """Set the config name for path resolution."""
+        self._config_name = config_name
+
+    def get_absolute_local_path(self) -> Optional[str]:
+        """Get the absolute local path without modifying the stored value."""
+        return resolve_path_to_absolute(self.local_path, self._config_name)
+
+    def get_absolute_db_path(self) -> Optional[str]:
+        """Get the absolute db path without modifying the stored value."""
+        return resolve_path_to_absolute(self.db_path, self._config_name)
 
 
 class ServiceSettings(BaseModel):
     universe: UniverseSettings
     projects: Dict[str, ProjectSettings] = Field(default_factory=dict)
 
-    # 🔹 Centralized default project configurations
-    DEFAULT_PROJECT_CONFIGS: ClassVar[Dict[str, dict]] = {
-        "cmip6": {
-            "project_name": "cmip6",
-            "github_repo": "https://github.com/WCRP-CMIP/CMIP6_CVs",
-            "branch": "esgvoc",
-            "local_path": "repos/CMIP6_CVs",
-            "db_path": "dbs/cmip6.sqlite",
-        },
-        "cmip6plus": {
-            "project_name": "cmip6plus",
-            "github_repo": "https://github.com/WCRP-CMIP/CMIP6Plus_CVs",
-            "branch": "esgvoc",
-            "local_path": "repos/CMIP6Plus_CVs",
-            "db_path": "dbs/cmip6plus.sqlite",
-        },
-        "input4mip": {
-            "project_name": "input4mip",
-            "github_repo": "https://github.com/PCMDI/input4MIPs_CVs",
-            "branch": "esgvoc",
-            "local_path": "repos/Input4MIP_CVs",
-            "db_path": "dbs/input4mips.sqlite",
-        },
-        "obs4ref": {
-            "project_name": "obs4ref",
-            "github_repo": "https://github.com/Climate-REF/Obs4REF_CVs",
-            "branch": "main",
-            "local_path": "repos/obs4REF_CVs",
-            "db_path": "dbs/obs4ref.sqlite",
-        },
-        "cordex-cmip6": {
-            "project_name": "cordex-cmip6",
-            "github_repo": "https://github.com/WCRP-CORDEX/cordex-cmip6-cv",
-            "branch": "esgvoc",
-            "local_path": "repos/cordex-cmip6-cv",
-            "db_path": "dbs/cordex-cmip6.sqlite",
-        },
-        "cmip7": {
-            "project_name": "cmip7",
-            "github_repo": "https://github.com/WCRP-CMIP/CMIP7-CVs",
-            "branch": "esgvoc",
-            "local_path": "repos/CMIP7-CVs",
-            "db_path": "dbs/cmip7.sqlite",
-        },
-    }
+    def set_config_name(self, config_name: str):
+        """Set the config name for all settings components."""
+        self.universe.set_config_name(config_name)
+        for project_settings in self.projects.values():
+            project_settings.set_config_name(config_name)
 
-    # 🔹 Default settings - only includes cmip6 and cmip6plus by default
-    DEFAULT_SETTINGS: ClassVar[dict] = {
-        "universe": {
-            "github_repo": "https://github.com/WCRP-CMIP/WCRP-universe",
-            "branch": "esgvoc",
-            "local_path": "repos/WCRP-universe",
-            "db_path": "dbs/universe.sqlite",
-        },
-        "projects": [
-            DEFAULT_PROJECT_CONFIGS["cmip6"],
-            DEFAULT_PROJECT_CONFIGS["cmip6plus"],
-        ],
-    }
+    @staticmethod
+    def _get_default_base_path() -> Path:
+        """Get the default base path for data storage using PlatformDirs."""
+        dirs = PlatformDirs("esgvoc", "ipsl")
+        return Path(dirs.user_data_path).expanduser().resolve()
+
+    @classmethod
+    def _get_default_project_configs(cls) -> Dict[str, dict]:
+        """Generate default project configurations with relative paths."""
+        return {
+            "cmip6": {
+                "project_name": "cmip6",
+                "github_repo": "https://github.com/WCRP-CMIP/CMIP6_CVs",
+                "branch": "esgvoc",
+                "local_path": "repos/CMIP6_CVs",
+                "db_path": "dbs/cmip6.sqlite",
+                "offline_mode": False,
+            },
+            "cmip6plus": {
+                "project_name": "cmip6plus",
+                "github_repo": "https://github.com/WCRP-CMIP/CMIP6Plus_CVs",
+                "branch": "esgvoc",
+                "local_path": "repos/CMIP6Plus_CVs",
+                "db_path": "dbs/cmip6plus.sqlite",
+                "offline_mode": False,
+            },
+            "input4mip": {
+                "project_name": "input4mip",
+                "github_repo": "https://github.com/PCMDI/input4MIPs_CVs",
+                "branch": "esgvoc",
+                "local_path": "repos/Input4MIP_CVs",
+                "db_path": "dbs/input4mips.sqlite",
+                "offline_mode": False,
+            },
+            "obs4ref": {
+                "project_name": "obs4ref",
+                "github_repo": "https://github.com/Climate-REF/Obs4REF_CVs",
+                "branch": "main",
+                "local_path": "repos/obs4REF_CVs",
+                "db_path": "dbs/obs4ref.sqlite",
+                "offline_mode": False,
+            },
+            "cordex-cmip6": {
+                "project_name": "cordex-cmip6",
+                "github_repo": "https://github.com/WCRP-CORDEX/cordex-cmip6-cv",
+                "branch": "esgvoc",
+                "local_path": "repos/cordex-cmip6-cv",
+                "db_path": "dbs/cordex-cmip6.sqlite",
+                "offline_mode": False,
+            },
+            "cmip7": {
+                "project_name": "cmip7",
+                "github_repo": "https://github.com/WCRP-CMIP/CMIP7-CVs",
+                "branch": "esgvoc",
+                "local_path": "repos/CMIP7-CVs",
+                "db_path": "dbs/cmip7.sqlite",
+                "offline_mode": False,
+            },
+        }
+
+    @classmethod
+    def _get_default_settings(cls) -> dict:
+        """Generate default settings with relative paths."""
+        project_configs = cls._get_default_project_configs()
+        return {
+            "universe": {
+                "github_repo": "https://github.com/WCRP-CMIP/WCRP-universe",
+                "branch": "esgvoc",
+                "local_path": "repos/WCRP-universe",
+                "db_path": "dbs/universe.sqlite",
+                "offline_mode": False,
+            },
+            "projects": [
+                project_configs["cmip6"],
+                project_configs["cmip6plus"],
+            ],
+        }
+
+    # 🔹 Properties that provide access to the dynamic configurations
+    @property
+    def DEFAULT_PROJECT_CONFIGS(self) -> Dict[str, dict]:
+        return self._get_default_project_configs()
+
+    @property
+    def DEFAULT_SETTINGS(self) -> dict:
+        return self._get_default_settings()
 
     @classmethod
     def load_from_file(cls, file_path: str) -> "ServiceSettings":
@@ -89,7 +177,7 @@ class ServiceSettings(BaseModel):
         try:
             data = toml.load(file_path)
         except FileNotFoundError:
-            data = cls.DEFAULT_SETTINGS.copy()  # Use defaults if the file is missing
+            data = cls._get_default_settings().copy()  # Use defaults if the file is missing
 
         projects = {p["project_name"]: ProjectSettings(**p) for p in data.pop("projects", [])}
         return cls(universe=UniverseSettings(**data["universe"]), projects=projects)
@@ -97,7 +185,7 @@ class ServiceSettings(BaseModel):
     @classmethod
     def load_default(cls) -> "ServiceSettings":
         """Load default settings."""
-        return cls.load_from_dict(cls.DEFAULT_SETTINGS)
+        return cls.load_from_dict(cls._get_default_settings())
 
     @classmethod
     def load_from_dict(cls, config_data: dict) -> "ServiceSettings":
@@ -136,12 +224,13 @@ class ServiceSettings(BaseModel):
         if project_name in self.projects:
             return False  # Project already exists
 
-        if project_name not in self.DEFAULT_PROJECT_CONFIGS:
+        default_configs = self._get_default_project_configs()
+        if project_name not in default_configs:
             raise ValueError(
-                f"Unknown project '{project_name}'. Available defaults: {list(self.DEFAULT_PROJECT_CONFIGS.keys())}"
+                f"Unknown project '{project_name}'. Available defaults: {list(default_configs.keys())}"
             )
 
-        config = self.DEFAULT_PROJECT_CONFIGS[project_name].copy()
+        config = default_configs[project_name].copy()
         self.projects[project_name] = ProjectSettings(**config)
         return True
 
@@ -193,6 +282,11 @@ class ServiceSettings(BaseModel):
         if project_name not in self.projects:
             return False
 
+        # Handle boolean conversion for offline_mode if present
+        if 'offline_mode' in kwargs:
+            if isinstance(kwargs['offline_mode'], str):
+                kwargs['offline_mode'] = kwargs['offline_mode'].lower() in ("true", "1", "yes", "on")
+
         # Get current config and update with new values
         current_config = self.projects[project_name].model_dump()
         current_config.update(kwargs)
@@ -203,7 +297,7 @@ class ServiceSettings(BaseModel):
 
     def get_available_default_projects(self) -> list[str]:
         """Return list of available default project names."""
-        return list(self.DEFAULT_PROJECT_CONFIGS.keys())
+        return list(self._get_default_project_configs().keys())
 
     def has_project(self, project_name: str) -> bool:
         """Check if a project exists in the current configuration."""
@@ -218,11 +312,13 @@ class ServiceSettings(BaseModel):
 def main():
     # Create default settings (only cmip6 and cmip6plus)
     settings = ServiceSettings.load_default()
-    print(f"Default projects: {list(settings.projects.keys())}")  # ['cmip6', 'cmip6plus']
+    # ['cmip6', 'cmip6plus']
+    print(f"Default projects: {list(settings.projects.keys())}")
 
     # See what other projects are available to add
     available = settings.get_available_default_projects()
-    print(f"Available default projects: {available}")  # ['cmip6', 'cmip6plus', 'input4mip', 'obs4mip']
+    # ['cmip6', 'cmip6plus', 'input4mip', 'obs4mip']
+    print(f"Available default projects: {available}")
 
     # Add optional projects when needed
     added_input4mip = settings.add_project_from_default("input4mip")

esgvoc/core/service/data_merger.py

@@ -46,7 +46,8 @@ class DataMerger:
         try:
             """Fetch and merge data recursively, returning a list of progressively merged Data json instances."""
             result_list = [self.data.json_dict]  # Start with the original json object
-            visited = set(self.data.uri)  # Track visited URIs to prevent cycles
+            # Track visited URIs to prevent cycles
+            visited = set(self.data.uri)
             current_data = self.data
             # print(current_data.expanded)
             while True:

esgvoc/core/service/esg_voc.py

@@ -1,10 +1,9 @@
-
 import logging
 import os
 from rich.logging import RichHandler
 from rich.console import Console
 import shutil
-import esgvoc.core.service as service 
+import esgvoc.core.service as service
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -13,67 +12,66 @@ _LOGGER.addHandler(rich_handler)
 
 
 def reset_init_repo():
-    service_settings = service.service_settings 
+    service_settings = service.service_settings
     if (service_settings.universe.local_path) and os.path.exists(service_settings.universe.local_path):
         shutil.rmtree(service_settings.universe.local_path)
 
-    for _, proj in service_settings.projects.items():    
+    for _, proj in service_settings.projects.items():
         if (proj.local_path) and os.path.exists(proj.local_path):
             shutil.rmtree(proj.local_path)
     service.state_service.get_state_summary()
 
+
 def reset_init_db():
-    service_settings = service.service_settings 
+    service_settings = service.service_settings
     if (service_settings.universe.db_path) and os.path.exists(service_settings.universe.db_path):
         os.remove(service_settings.universe.db_path)
-    for _, proj in service_settings.projects.items():    
+    for _, proj in service_settings.projects.items():
         if (proj.db_path) and os.path.exists(proj.db_path):
             os.remove(proj.db_path)
     service.state_service.get_state_summary()
 
+
 def reset_init_all():
     reset_init_db()
     reset_init_repo()
 
+
 def display(table):
-    console = Console(record=True,width=200)
+    console = Console(record=True, width=200)
     console.print(table)
 
+
 def install():
     service.state_service.synchronize_all()
 
 
-
-
-
-
 if __name__ == "__main__":
 
-    def Nothing(): # IT WORKS
+    def Nothing():  # IT WORKS
         reset_init_all()
         display(service.state_service.table())
         service.state_service.universe.sync()
         display(service.state_service.table())
-        for _,proj in service.state_service.projects.items():
+        for _, proj in service.state_service.projects.items():
             proj.sync()
         display(service.state_service.table())
-   
-    def OnlyLocal(): #IT ALSO WORKS
+
+    def OnlyLocal():  # IT ALSO WORKS
         reset_init_db()
         service.state_service.universe.github_access = False
-        for _,proj in service.state_service.projects.items():
+        for _, proj in service.state_service.projects.items():
             proj.github_access = False
         display(service.state_service.table())
 
         service.state_service.universe.sync()
         display(service.state_service.table())
-        for _,proj in service.state_service.projects.items():
+        for _, proj in service.state_service.projects.items():
             proj.sync()
         display(service.state_service.table())
 
     # TODO Some other test to do to be complete:
-    # Change the settings ... for now .. let say nobody change the settings ! 
+    # Change the settings ... for now .. let say nobody change the settings !
 
     OnlyLocal()
     # service.state_service.synchronize_all()
-    

esgvoc/core/service/state.py

@@ -11,16 +11,14 @@ from esgvoc.core.db.connection import DBConnection
 from esgvoc.core.db.models.project import Project
 from esgvoc.core.db.models.universe import Universe
 from esgvoc.core.repo_fetcher import RepoFetcher
-from esgvoc.core.service.configuration.setting import (ProjectSettings,
-                                                       ServiceSettings,
-                                                       UniverseSettings)
+from esgvoc.core.service.configuration.setting import ProjectSettings, ServiceSettings, UniverseSettings
 
 logger = logging.getLogger(__name__)
 
 
 class BaseState:
     def __init__(
-        self, github_repo: str, branch: str = "main", local_path: Optional[str] = None, db_path: Optional[str] = None
+        self, github_repo: str, branch: str = "main", local_path: Optional[str] = None, db_path: Optional[str] = None, offline_mode: bool = False
     ):
         from esgvoc.core.service import config_manager
 
@@ -28,26 +26,24 @@ class BaseState:
 
         self.github_repo: str = github_repo
         self.branch: str = branch
-        self.github_access: bool = True  # False if we dont have internet and some other cases
+        self.offline_mode: bool = offline_mode
+        # False if we dont have internet and some other cases
+        # In offline mode, disable github access from the start
+        self.github_access: bool = not offline_mode
         self.github_version: str | None = None
 
-        self.local_path: str | None = self._get_absolute_path(str(self.base_dir), local_path)
+        self.local_path: str | None = local_path
         self.local_access: bool = True  # False if we dont have cloned the remote repo yet
         self.local_version: str | None = None
 
-        self.db_path: str | None = self._get_absolute_path(str(self.base_dir), db_path)
+        self.db_path: str | None = db_path
         self.db_access: bool = True  # False if we cant access the db for some reason
         self.db_version: str | None = None
 
-        self.rf = RepoFetcher(local_path=str(self.base_dir))
+        self.rf = RepoFetcher(local_path=str(self.base_dir), offline_mode=offline_mode)
         self.db_connection: DBConnection | None = None
         self.db_sqlmodel: Universe | Project | None = None
 
-    def _get_absolute_path(self, base_dir: str, path: str | None) -> str | None:
-        if base_dir != "" and path is not None:
-            return base_dir + "/" + path
-        if base_dir == "":
-            return path
 
     def fetch_version_local(self):
         if self.local_path:
@@ -60,11 +56,16 @@ class BaseState:
                 self.local_access = False
 
     def fetch_version_remote(self):
+        if self.offline_mode:
+            logger.debug("Skipping remote version fetch due to offline mode")
+            self.github_access = False
+            return
+
         if self.github_repo:
             owner = None
             repo = None
             try:
-                owner, repo = self.github_repo.lstrip("https://github.com/").split("/")
+                owner, repo = self.github_repo.removeprefix("https://github.com/").split("/")
                 self.github_version = self.rf.get_github_version(owner, repo, self.branch)
                 self.github_access = True
                 logger.debug(f"Latest GitHub commit: {self.github_version}")
@@ -128,8 +129,19 @@ class BaseState:
             else False,
         }
 
-    def clone_remote(self):
-        owner, repo = self.github_repo.lstrip("https://github.com/").split("/")
+    def clone_remote(self, force_clean=False):
+        if self.offline_mode:
+            logger.warning("Cannot clone remote repository in offline mode")
+            return
+
+        # If force_clean is True or if local repo exists and we're handling divergence,
+        # remove the existing local repository to ensure clean state
+        if force_clean and self.local_path and os.path.exists(self.local_path):
+            print(f"Removing existing local repository: {self.local_path}")
+            import shutil
+            shutil.rmtree(self.local_path)
+
+        owner, repo = self.github_repo.removeprefix("https://github.com/").split("/")
         # TODO add destination "local_path" in clone_repo, done in a wierd way Improve that:
         self.rf.clone_repository(owner, repo, self.branch, self.local_path)
         self.fetch_version_local()
@@ -138,8 +150,7 @@ class BaseState:
         from esgvoc.core.db.models.project import project_create_db
         from esgvoc.core.db.models.universe import universe_create_db
         from esgvoc.core.db.project_ingestion import ingest_project
-        from esgvoc.core.db.universe_ingestion import (
-            ingest_metadata_universe, ingest_universe)
+        from esgvoc.core.db.universe_ingestion import ingest_metadata_universe, ingest_universe
 
         if self.db_path:
             if os.path.exists(self.db_path):
@@ -168,6 +179,26 @@ class BaseState:
     def sync(self):
         summary = self.check_sync_status()
         updated = False
+
+        if self.offline_mode:
+            print("Running in offline mode - only using local repositories and databases")
+            if self.local_access:
+                if not summary["local_db_sync"] and summary["local_db_sync"] is not None:
+                    self.build_db()
+                    updated = True
+                else:
+                    print("Cache db is uptodate from local repository")
+            elif not self.db_access:  # it can happen if the db is created but not filled
+                if self.local_path and os.path.exists(self.local_path):
+                    self.build_db()
+                    updated = True
+                else:
+                    print(f"No local repository found at {self.local_path} - cannot sync in offline mode")
+            else:
+                print("Nothing to sync in offline mode - local repository and database are up to date")
+            return updated
+
+        # Online sync logic with offline-to-online transition detection
         if (
             self.github_access
             and summary["github_db_sync"] is None
@@ -183,7 +214,11 @@ class BaseState:
                 self.build_db()
                 updated = True
             elif not summary["github_local_sync"]:
-                self.clone_remote()
+                # Critical fix: when local and remote diverge in online mode,
+                # prioritize remote truth by completely removing local repo and re-cloning
+                print(f"Local and remote repositories have diverged (local: {summary['local'][:8] if summary['local'] else 'N/A'}, remote: {summary['github'][:8] if summary['github'] else 'N/A'})")
+                print("Prioritizing remote repository truth - removing local repository and re-cloning from GitHub...")
+                self.clone_remote(force_clean=True)
                 self.build_db()
                 updated = True
             else:  # can be simply build in root and clone if neccessary
@@ -206,7 +241,10 @@ class BaseState:
 
 class StateUniverse(BaseState):
     def __init__(self, settings: UniverseSettings):
-        super().__init__(**settings.model_dump())
+        params = settings.model_dump()
+        params['local_path'] = settings.get_absolute_local_path()
+        params['db_path'] = settings.get_absolute_db_path()
+        super().__init__(**params)
         self.db_sqlmodel = Universe
 
 
@@ -214,6 +252,8 @@ class StateProject(BaseState):
     def __init__(self, settings: ProjectSettings):
         mdict = settings.model_dump()
         self.project_name = mdict.pop("project_name")
+        mdict['local_path'] = settings.get_absolute_local_path()
+        mdict['db_path'] = settings.get_absolute_db_path()
         super().__init__(**mdict)
         self.db_sqlmodel = Project
 
@@ -241,19 +281,17 @@ class StateService:
 
     def synchronize_all(self):
         print("sync universe")
+        if self.universe.offline_mode:
+            print("Universe is in offline mode")
         universe_updated = self.universe.sync()
         print("sync projects")
-        for project in self.projects.values():
+        for project_name, project in self.projects.items():
+            if project.offline_mode:
+                print(f"Project {project_name} is in offline mode")
             project_updated = project.sync()
             if universe_updated and not project_updated:
                 project.build_db()
         self.connect_db()
-        
-        # Display state table after synchronization
-        table = self.table()
-        from rich.console import Console
-        console = Console()
-        console.print(table)
 
     def table(self):
         table = Table(show_header=False, show_lines=True)

{esgvoc-1.1.1.dist-info → esgvoc-1.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: esgvoc
-Version: 1.1.1
+Version: 1.1.3
 Summary: python library and CLI to interact with WCRP CVs
 Project-URL: Repository, https://github.com/ESGF/esgf-vocab
 Author-email: Sébastien Gardoll <sebastien@gardoll.fr>, Guillaume Levavasseur <guillaume.levavasseur@ipsl.fr>, Laurent Troussellier <laurent.troussellier@ipsl.fr>
@@ -11,6 +11,7 @@ Requires-Dist: jinja2>=3.1.6
 Requires-Dist: platformdirs>=4.3.6
 Requires-Dist: pydantic>=2.9.2
 Requires-Dist: pyld>=2.0.4
+Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: requests>=2.32.3
 Requires-Dist: sqlalchemy>=2.0.36
 Requires-Dist: sqlmodel>=0.0.22