esgvoc 2.0.2__py3-none-any.whl

esgvoc/core/service/esg_voc.py

@@ -0,0 +1,77 @@
+import logging
+import os
+from rich.logging import RichHandler
+from rich.console import Console
+import shutil
+import esgvoc.core.service as service
+
+_LOGGER = logging.getLogger(__name__)
+
+rich_handler = RichHandler(rich_tracebacks=True)
+_LOGGER.addHandler(rich_handler)
+
+
+def reset_init_repo():
+    service_settings = service.service_settings
+    if (service_settings.universe.local_path) and os.path.exists(service_settings.universe.local_path):
+        shutil.rmtree(service_settings.universe.local_path)
+
+    for _, proj in service_settings.projects.items():
+        if (proj.local_path) and os.path.exists(proj.local_path):
+            shutil.rmtree(proj.local_path)
+    service.state_service.get_state_summary()
+
+
+def reset_init_db():
+    service_settings = service.service_settings
+    if (service_settings.universe.db_path) and os.path.exists(service_settings.universe.db_path):
+        os.remove(service_settings.universe.db_path)
+    for _, proj in service_settings.projects.items():
+        if (proj.db_path) and os.path.exists(proj.db_path):
+            os.remove(proj.db_path)
+    service.state_service.get_state_summary()
+
+
+def reset_init_all():
+    reset_init_db()
+    reset_init_repo()
+
+
+def display(table):
+    console = Console(record=True, width=200)
+    console.print(table)
+
+
+def install():
+    service.state_service.synchronize_all()
+
+
+if __name__ == "__main__":
+
+    def Nothing():  # IT WORKS
+        reset_init_all()
+        display(service.state_service.table())
+        service.state_service.universe.sync()
+        display(service.state_service.table())
+        for _, proj in service.state_service.projects.items():
+            proj.sync()
+        display(service.state_service.table())
+
+    def OnlyLocal():  # IT ALSO WORKS
+        reset_init_db()
+        service.state_service.universe.github_access = False
+        for _, proj in service.state_service.projects.items():
+            proj.github_access = False
+        display(service.state_service.table())
+
+        service.state_service.universe.sync()
+        display(service.state_service.table())
+        for _, proj in service.state_service.projects.items():
+            proj.sync()
+        display(service.state_service.table())
+
+    # TODO Some other test to do to be complete:
+    # Change the settings ... for now .. let say nobody change the settings !
+
+    OnlyLocal()
+    # service.state_service.synchronize_all()

esgvoc/core/service/resolver_config.py

@@ -0,0 +1,56 @@
+"""Configuration for JSON-LD reference resolution behavior."""
+
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class ResolverConfig:
+    """
+    Configuration for controlling JSON-LD ID reference resolution behavior.
+
+    This class provides fine-grained control over how the DataMerger resolves
+    nested @id references, including depth limits, string filtering, and
+    file resolution strategies.
+    """
+
+    # Recursion control
+    max_depth: int = 5
+    """Maximum recursion depth when resolving nested references"""
+
+    # String filtering for primitive resolution
+    max_string_length: int = 100
+    """Maximum length for strings to be considered as ID references"""
+
+    exclude_patterns: List[str] = field(default_factory=lambda: [" ", ".", "http", "/", "@"])
+    """Patterns that disqualify a string from being resolved as an ID reference"""
+
+    # File resolution strategies
+    fallback_dirs: List[str] = field(default_factory=lambda: ["horizontal_grid", "vertical_grid", "grid"])
+    """Alternative directories to search when a term file is not found"""
+
+    min_path_parts: int = 3
+    """Minimum number of path components required for alternate directory search"""
+
+    # Network and I/O
+    verify_ssl: bool = True
+    """Whether to verify SSL certificates when fetching remote resources"""
+
+    enable_caching: bool = True
+    """Whether to cache fetched terms to improve performance"""
+
+    cache_size: int = 128
+    """Maximum number of terms to cache (when caching is enabled)"""
+
+    # Logging and debugging
+    log_depth_warnings: bool = True
+    """Whether to log warnings when max_depth is exceeded"""
+
+    def __post_init__(self):
+        """Validate configuration values."""
+        if self.max_depth < 1:
+            raise ValueError("max_depth must be at least 1")
+        if self.max_string_length < 1:
+            raise ValueError("max_string_length must be at least 1")
+        if self.cache_size < 1:
+            raise ValueError("cache_size must be at least 1")

esgvoc/core/service/state.py

@@ -0,0 +1,324 @@
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+from rich.table import Table
+from sqlalchemy.exc import NoResultFound
+from sqlmodel import select
+
+from esgvoc.core.db.connection import DBConnection
+from esgvoc.core.db.models.project import Project
+from esgvoc.core.db.models.universe import Universe
+from esgvoc.core.repo_fetcher import RepoFetcher
+from esgvoc.core.service.configuration.setting import ProjectSettings, ServiceSettings, UniverseSettings
+
+logger = logging.getLogger(__name__)
+
+
+class BaseState:
+    def __init__(
+        self, github_repo: str, branch: str = "main", local_path: Optional[str] = None, db_path: Optional[str] = None, offline_mode: bool = False
+    ):
+        from esgvoc.core.service import config_manager
+
+        self.base_dir = config_manager.data_config_dir  # needed for repofetcher
+
+        self.github_repo: str = github_repo
+        self.branch: str = branch
+        self.offline_mode: bool = offline_mode
+        # False if we dont have internet and some other cases
+        # In offline mode, disable github access from the start
+        self.github_access: bool = not offline_mode
+        self.github_version: str | None = None
+
+        self.local_path: str | None = local_path
+        self.local_access: bool = True  # False if we dont have cloned the remote repo yet
+        self.local_version: str | None = None
+
+        self.db_path: str | None = db_path
+        self.db_access: bool = True  # False if we cant access the db for some reason
+        self.db_version: str | None = None
+
+        self.rf = RepoFetcher(local_path=str(self.base_dir), offline_mode=offline_mode)
+        self.db_connection: DBConnection | None = None
+        self.db_sqlmodel: Universe | Project | None = None
+
+
+    def fetch_version_local(self):
+        if self.local_path:
+            try:
+                self.local_version = self.rf.get_local_repo_version(self.local_path, self.branch)
+                logger.debug(f"Local repo commit: {self.local_version}")
+                self.local_access = True
+            except Exception as e:
+                logger.exception(f"Failed to fetch local repo version: {e}")
+                self.local_access = False
+
+    def fetch_version_remote(self):
+        if self.offline_mode:
+            logger.debug("Skipping remote version fetch due to offline mode")
+            self.github_access = False
+            return
+
+        if self.github_repo:
+            owner = None
+            repo = None
+            try:
+                owner, repo = self.github_repo.removeprefix("https://github.com/").split("/")
+                self.github_version = self.rf.get_github_version(owner, repo, self.branch)
+                self.github_access = True
+                logger.debug(f"Latest GitHub commit: {self.github_version}")
+            except IndexError as e:
+                self.github_access = False
+            except Exception as e:
+                logger.exception(
+                    f"Failed to fetch GitHub version: {e} ,for {self.github_repo},owner : {owner}, repo : {repo},branch : {self.branch}"
+                )
+                self.github_access = False
+
+            if self.github_version is None:
+                self.github_access = False
+
+    def connect_db(self):
+        if self.db_path:
+            if not os.path.exists(self.db_path):
+                self.db_access = False
+            else:
+                self.db_connection = DBConnection(db_file_path=Path(self.db_path))
+
+    def fetch_version_db(self):
+        if self.db_path:
+            if not os.path.exists(self.db_path):
+                self.db_version = None
+                self.db_access = False
+            else:
+                try:
+                    with self.db_connection.create_session() as session:
+                        self.db_version = session.exec(select(self.db_sqlmodel.git_hash)).one()
+                        self.db_access = True
+                except NoResultFound:
+                    logger.debug(f"Unable to find git_hash in {self.db_path}")
+                except Exception as e:
+                    logger.debug(f"Unable to find git_has in {self.db_path} cause {e}")
+
+        else:
+            self.db_version = None
+            self.db_access = False
+
+    def fetch_versions(self):
+        if self.github_access:
+            self.fetch_version_remote()
+        self.fetch_version_local()
+        self.fetch_version_db()
+
+    def check_sync_status(self):
+        self.fetch_versions()
+        return {
+            "github": self.github_version if self.github_version else None,
+            "local": self.local_version if self.local_version else None,
+            "db": self.db_version if self.db_version else None,
+            "github_local_sync": self.github_version == self.local_version
+            if self.github_access and self.github_version and self.local_version
+            else False,
+            "local_db_sync": self.local_version == self.db_version
+            if self.local_access and self.local_version
+            else False,
+            "github_db_sync": self.github_version == self.db_version
+            if self.github_access and self.github_version
+            else False,
+        }
+
+    def clone_remote(self, force_clean=False):
+        if self.offline_mode:
+            logger.warning("Cannot clone remote repository in offline mode")
+            return
+
+        # If force_clean is True or if local repo exists and we're handling divergence,
+        # remove the existing local repository to ensure clean state
+        if force_clean and self.local_path and os.path.exists(self.local_path):
+            print(f"Removing existing local repository: {self.local_path}")
+            import shutil
+            shutil.rmtree(self.local_path)
+
+        owner, repo = self.github_repo.removeprefix("https://github.com/").split("/")
+        # TODO add destination "local_path" in clone_repo, done in a wierd way Improve that:
+        self.rf.clone_repository(owner, repo, self.branch, self.local_path)
+        self.fetch_version_local()
+
+    def build_db(self):
+        from esgvoc.core.db.models.project import project_create_db
+        from esgvoc.core.db.models.universe import universe_create_db
+        from esgvoc.core.db.project_ingestion import ingest_project
+        from esgvoc.core.db.universe_ingestion import ingest_metadata_universe, ingest_universe
+
+        if self.db_path:
+            if os.path.exists(self.db_path):
+                os.remove(self.db_path)
+            else:
+                os.makedirs(Path(self.db_path).parent, exist_ok=True)
+
+            if self.db_sqlmodel == Universe:  # Ugly
+                print("Building Universe DB from ", self.local_path)
+                universe_create_db(Path(self.db_path))
+                self.db_connection = DBConnection(db_file_path=Path(self.db_path))
+
+                ingest_metadata_universe(self.db_connection, self.local_version)
+                print("Filling Universe DB")
+                if self.local_path:
+                    ingest_universe(Path(self.local_path), Path(self.db_path))
+
+            elif self.db_sqlmodel == Project:
+                print("Building Project DB from ", self.local_path)
+                project_create_db(Path(self.db_path))
+                print("Filling project DB")
+                if self.local_path and self.local_version:
+                    ingest_project(Path(self.local_path), Path(self.db_path), self.local_version)
+        self.fetch_version_db()
+
+    def sync(self):
+        summary = self.check_sync_status()
+        updated = False
+
+        if self.offline_mode:
+            print("Running in offline mode - only using local repositories and databases")
+            if self.local_access:
+                if not summary["local_db_sync"] and summary["local_db_sync"] is not None:
+                    self.build_db()
+                    updated = True
+                else:
+                    print("Cache db is uptodate from local repository")
+            elif not self.db_access:  # it can happen if the db is created but not filled
+                if self.local_path and os.path.exists(self.local_path):
+                    self.build_db()
+                    updated = True
+                else:
+                    print(f"No local repository found at {self.local_path} - cannot sync in offline mode")
+            else:
+                print("Nothing to sync in offline mode - local repository and database are up to date")
+            return updated
+
+        # Online sync logic with offline-to-online transition detection
+        if (
+            self.github_access
+            and summary["github_db_sync"] is None
+            and summary["local_db_sync"] is None
+            and summary["github_local_sync"] is None
+        ):
+            self.clone_remote()
+            self.build_db()
+            updated = True
+        elif self.github_access and not summary["github_db_sync"]:
+            if not summary["local_db_sync"] and summary["local_db_sync"] is not None:
+                self.clone_remote()
+                self.build_db()
+                updated = True
+            elif not summary["github_local_sync"]:
+                # Critical fix: when local and remote diverge in online mode,
+                # prioritize remote truth by completely removing local repo and re-cloning
+                print(f"Local and remote repositories have diverged (local: {summary['local'][:8] if summary['local'] else 'N/A'}, remote: {summary['github'][:8] if summary['github'] else 'N/A'})")
+                print("Prioritizing remote repository truth - removing local repository and re-cloning from GitHub...")
+                self.clone_remote(force_clean=True)
+                self.build_db()
+                updated = True
+            else:  # can be simply build in root and clone if neccessary
+                self.build_db()
+                updated = True
+        elif self.local_access:
+            if not summary["local_db_sync"] and summary["local_db_sync"] is not None:
+                self.build_db()
+                updated = True
+            else:
+                print("Cache db is uptodate from local repository")
+        elif not self.db_access:  # it can happen if the db is created but not filled
+            self.build_db()
+            updated = True
+        else:
+            print("Nothing to install, everything up to date")
+            print("Try 'esgvoc status' for more details")
+        return updated
+
+
+class StateUniverse(BaseState):
+    def __init__(self, settings: UniverseSettings):
+        params = settings.model_dump()
+        params['local_path'] = settings.get_absolute_local_path()
+        params['db_path'] = settings.get_absolute_db_path()
+        super().__init__(**params)
+        self.db_sqlmodel = Universe
+
+
+class StateProject(BaseState):
+    def __init__(self, settings: ProjectSettings):
+        mdict = settings.model_dump()
+        self.project_name = mdict.pop("project_name")
+        mdict['local_path'] = settings.get_absolute_local_path()
+        mdict['db_path'] = settings.get_absolute_db_path()
+        super().__init__(**mdict)
+        self.db_sqlmodel = Project
+
+
+class StateService:
+    def __init__(self, service_settings: ServiceSettings):
+        self.universe = StateUniverse(service_settings.universe)
+        self.projects = {name: StateProject(proj) for name, proj in service_settings.projects.items()}
+        self.connect_db()
+
+    def get_state_summary(self):
+        universe_status = self.universe.check_sync_status()
+        project_statuses = {name: proj.check_sync_status() for name, proj in self.projects.items()}
+        return {"universe": universe_status, "projects": project_statuses}
+
+    def fetch_versions(self):
+        self.universe.fetch_versions()
+        for _, proj_state in self.projects.items():
+            proj_state.fetch_versions()
+
+    def connect_db(self):
+        self.universe.connect_db()
+        for _, proj_state in self.projects.items():
+            proj_state.connect_db()
+
+    def synchronize_all(self):
+        print("sync universe")
+        if self.universe.offline_mode:
+            print("Universe is in offline mode")
+        universe_updated = self.universe.sync()
+        print("sync projects")
+        for project_name, project in self.projects.items():
+            if project.offline_mode:
+                print(f"Project {project_name} is in offline mode")
+            project_updated = project.sync()
+            if universe_updated and not project_updated:
+                project.build_db()
+        self.connect_db()
+
+    def table(self):
+        table = Table(show_header=False, show_lines=True)
+        table.add_row("", "Remote github repo", "Local repository", "Cache Database")
+        table.add_row("Universe path", self.universe.github_repo, self.universe.local_path, self.universe.db_path)
+        table.add_row("Version", self.universe.github_version, self.universe.local_version, self.universe.db_version)
+        for proj_name, proj in self.projects.items():
+            # table.add_row("","Remote github repo","Local repository","Cache Database")
+            table.add_row(f"{proj_name} path", proj.github_repo, proj.local_path, proj.db_path)
+            table.add_row("Version", proj.github_version, proj.local_version, proj.db_version)
+        return table
+
+
+if __name__ == "__main__":
+    # Load settings from file
+    service_settings = ServiceSettings.load_from_file("src/esgvoc/core/service/settings.toml")
+
+    # Initialize StateService
+    state_service = StateService(service_settings)
+    state_service.get_state_summary()
+
+    # Synchronize all
+    state_service.synchronize_all()
+
+    # pprint(state_service.universe.github_version)
+    # pprint(state_service.universe.local_version)
+    # pprint(state_service.universe.db_version)
+
+    # Check for differences
+    # pprint(state_service.find_version_differences())

esgvoc/core/service/string_heuristics.py

@@ -0,0 +1,98 @@
+"""Heuristics for determining if strings should be resolved as ID references."""
+
+from typing import List
+
+
+class StringHeuristics:
+    """
+    Determine if a string value should be resolved as an ID reference.
+
+    Uses configurable heuristics to distinguish between:
+    - ID references (e.g., "hadgem3_gc31_atmosphere") - should resolve
+    - Literal strings (e.g., "A long description...") - should not resolve
+    - URLs (e.g., "https://doi.org/...") - should not resolve
+    """
+
+    def __init__(self, max_length: int = 100, exclude_patterns: List[str] | None = None):
+        """
+        Initialize string heuristics.
+
+        Args:
+            max_length: Maximum length for strings to be considered as ID references.
+                       Longer strings are assumed to be content, not references.
+            exclude_patterns: Patterns that disqualify a string from being an ID reference.
+                            Defaults to [" ", ".", "http", "/", "@"] which filter out
+                            descriptions, URLs, DOIs, paths, and emails.
+        """
+        self.max_length = max_length
+        self.exclude_patterns = exclude_patterns or [" ", ".", "http", "/", "@"]
+
+    def is_resolvable(self, value: str) -> bool:
+        """
+        Check if a string looks like an ID reference that should be resolved.
+
+        Args:
+            value: The string to evaluate
+
+        Returns:
+            True if the string appears to be an ID reference, False otherwise
+
+        Example:
+            >>> heuristics = StringHeuristics()
+            >>> heuristics.is_resolvable("hadgem3_gc31_atmosphere")
+            True
+            >>> heuristics.is_resolvable("This is a long description text")
+            False
+            >>> heuristics.is_resolvable("https://doi.org/10.5194/gmd")
+            False
+        """
+        # Check length
+        if len(value) > self.max_length:
+            return False
+
+        # Check for exclude patterns
+        for pattern in self.exclude_patterns:
+            if pattern in value:
+                return False
+
+        return True
+
+    def should_skip_literal(self, expanded_data: dict) -> bool:
+        """
+        Check if the expanded data indicates this is a literal value (not a reference).
+
+        In JSON-LD, literal values are marked with @value in expanded form.
+
+        Args:
+            expanded_data: The expanded JSON-LD data
+
+        Returns:
+            True if this is a literal value that should not be resolved
+
+        Example:
+            >>> heuristics = StringHeuristics()
+            >>> heuristics.should_skip_literal({"@value": "some text"})
+            True
+            >>> heuristics.should_skip_literal({"@id": "some_term"})
+            False
+        """
+        return isinstance(expanded_data, dict) and "@value" in expanded_data
+
+    def has_id_in_expanded(self, expanded_data: dict) -> bool:
+        """
+        Check if the expanded data contains an @id, indicating it's a reference.
+
+        Args:
+            expanded_data: The expanded JSON-LD data
+
+        Returns:
+            True if the expanded data has an @id field
+
+        Example:
+            >>> heuristics = StringHeuristics()
+            >>> heuristics.has_id_in_expanded({"@id": "https://example.com/term"})
+            True
+            >>> heuristics.has_id_in_expanded({"@value": "literal"})
+            False
+        """
+        return isinstance(expanded_data, dict) and "@id" in expanded_data

esgvoc/core/service/term_cache.py

@@ -0,0 +1,108 @@
+"""Caching for resolved JSON-LD terms to improve performance."""
+
+import json
+import logging
+from functools import lru_cache
+from pathlib import Path
+from typing import Dict
+
+logger = logging.getLogger(__name__)
+
+
+class TermCache:
+    """
+    LRU cache for JSON-LD terms fetched from files or remote sources.
+
+    Caching reduces redundant file I/O and network calls when the same
+    terms are referenced multiple times during resolution.
+    """
+
+    def __init__(self, max_size: int = 128, enabled: bool = True):
+        """
+        Initialize the term cache.
+
+        Args:
+            max_size: Maximum number of terms to cache
+            enabled: Whether caching is enabled (can be disabled for debugging)
+        """
+        self.max_size = max_size
+        self.enabled = enabled
+        self._cache: Dict[str, dict] = {}
+        self._hits = 0
+        self._misses = 0
+
+    def get(self, uri: str) -> dict | None:
+        """
+        Retrieve a cached term by URI.
+
+        Args:
+            uri: The URI key for the cached term
+
+        Returns:
+            The cached term data, or None if not in cache
+        """
+        if not self.enabled:
+            return None
+
+        if uri in self._cache:
+            self._hits += 1
+            logger.debug(f"Cache hit for {uri}")
+            return self._cache[uri]
+
+        self._misses += 1
+        return None
+
+    def put(self, uri: str, data: dict) -> None:
+        """
+        Store a term in the cache.
+
+        Args:
+            uri: The URI key for the term
+            data: The term data to cache
+        """
+        if not self.enabled:
+            return
+
+        # Simple LRU: if cache is full, remove the oldest entry
+        if len(self._cache) >= self.max_size:
+            # Remove first item (oldest in insertion order for Python 3.7+)
+            oldest_key = next(iter(self._cache))
+            del self._cache[oldest_key]
+            logger.debug(f"Cache eviction: {oldest_key}")
+
+        self._cache[uri] = data
+        logger.debug(f"Cached {uri}")
+
+    def clear(self) -> None:
+        """Clear all cached terms."""
+        self._cache.clear()
+        self._hits = 0
+        self._misses = 0
+        logger.debug("Cache cleared")
+
+    def get_stats(self) -> Dict[str, int]:
+        """
+        Get cache statistics.
+
+        Returns:
+            Dictionary with cache hits, misses, size, and hit rate
+        """
+        total_requests = self._hits + self._misses
+        hit_rate = (self._hits / total_requests * 100) if total_requests > 0 else 0
+
+        return {
+            "hits": self._hits,
+            "misses": self._misses,
+            "size": len(self._cache),
+            "max_size": self.max_size,
+            "hit_rate_percent": round(hit_rate, 2),
+        }
+
+    def __repr__(self) -> str:
+        """String representation showing cache stats."""
+        stats = self.get_stats()
+        return (
+            f"TermCache(size={stats['size']}/{stats['max_size']}, "
+            f"hits={stats['hits']}, misses={stats['misses']}, "
+            f"hit_rate={stats['hit_rate_percent']}%)"
+        )