esgvoc 1.0.1__py3-none-any.whl → 1.1.1__py3-none-any.whl

esgvoc/apps/test_cv/example_usage.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+Example usage of the CV Testing Application
+
+This script demonstrates how to use the CVTester class programmatically.
+"""
+
+from pathlib import Path
+from rich.console import Console
+
+from .cv_tester import CVTester
+
+console = Console()
+
+
+def example_test_default_project():
+    """Example: Test a default project with its standard configuration"""
+    console.print("[bold blue]Example 1: Testing default obs4mip project[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Test with default obs4mip configuration
+        success = tester.run_complete_test(
+            project_name="obs4mip",
+            repo_path=".",  # Assuming we're in the CV repository
+        )
+
+        if success:
+            console.print("[green]✅ Default project test completed successfully[/green]")
+        else:
+            console.print("[red]❌ Default project test failed[/red]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_test_custom_branch():
+    """Example: Test a project with custom branch"""
+    console.print("[bold blue]Example 2: Testing obs4mip with custom branch[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Test with custom branch
+        success = tester.run_complete_test(
+            project_name="obs4mip",
+            branch="test-branch",  # Custom branch
+            repo_path=".",
+        )
+
+        if success:
+            console.print("[green]✅ Custom branch test completed successfully[/green]")
+        else:
+            console.print("[red]❌ Custom branch test failed[/red]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_test_universe_branch_override():
+    """Example: Test with custom universe branch"""
+    console.print("[bold blue]Example 2b: Testing input4mip with custom universe branch[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Test with custom universe branch
+        success = tester.run_complete_test(
+            project_name="input4mip",
+            branch="esgvoc",  # Project branch
+            universe_branch="esgvoc_dev",  # Custom universe branch
+            repo_path=".",
+        )
+
+        if success:
+            console.print("[green]✅ Universe branch override test completed successfully[/green]")
+        else:
+            console.print("[red]❌ Universe branch override test failed[/red]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_test_custom_repo():
+    """Example: Test with completely custom repository"""
+    console.print("[bold blue]Example 3: Testing custom repository[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Test with custom repo and branch
+        success = tester.run_complete_test(
+            project_name="obs4mip",  # Use obs4mip project structure
+            repo_url="https://github.com/my-org/my-custom-cvs",
+            branch="main",
+            repo_path=".",
+        )
+
+        if success:
+            console.print("[green]✅ Custom repository test completed successfully[/green]")
+        else:
+            console.print("[red]❌ Custom repository test failed[/red]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_test_custom_repo_and_universe():
+    """Example: Test with custom repository and custom universe branch"""
+    console.print("[bold blue]Example 3b: Testing custom repository with custom universe[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Test with custom repo, project branch, and universe branch
+        success = tester.run_complete_test(
+            project_name="obs4mip",  # Use obs4mip project structure
+            repo_url="https://github.com/my-org/my-custom-cvs",
+            branch="main",  # Project branch
+            universe_branch="development",  # Custom universe branch
+            repo_path=".",
+        )
+
+        if success:
+            console.print("[green]✅ Custom repository and universe test completed successfully[/green]")
+        else:
+            console.print("[red]❌ Custom repository and universe test failed[/red]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_individual_tests():
+    """Example: Run individual test components"""
+    console.print("[bold blue]Example 4: Running individual test components[/bold blue]")
+
+    tester = CVTester()
+
+    try:
+        # Step 1: Configure for testing with universe branch override
+        console.print("Step 1: Configuring...")
+        if not tester.configure_for_testing(
+            project_name="obs4mip",
+            branch="main",  # Custom project branch
+            universe_branch="esgvoc_dev"  # Custom universe branch
+        ):
+            console.print("[red]Configuration failed[/red]")
+            return
+
+        # Step 2: Sync CVs
+        console.print("Step 2: Synchronizing CVs...")
+        if not tester.synchronize_cvs():
+            console.print("[red]Synchronization failed[/red]")
+            return
+
+        # Step 3: Test repository structure only
+        console.print("Step 3: Testing repository structure...")
+        if not tester.test_repository_structure("."):
+            console.print("[red]Structure test failed[/red]")
+            return
+
+        # Step 4: Test API access only
+        console.print("Step 4: Testing API access...")
+        if not tester.test_esgvoc_api_access("obs4mip", "."):
+            console.print("[red]API test failed[/red]")
+            return
+
+        console.print("[green]✅ All individual tests completed successfully[/green]")
+
+    finally:
+        tester.cleanup()
+
+
+def example_list_available_projects():
+    """Example: List all available projects for testing"""
+    console.print("[bold blue]Example 5: Listing available projects[/bold blue]")
+
+    tester = CVTester()
+    projects = tester.get_available_projects()
+
+    console.print(f"Available projects ({len(projects)}):")
+    for project in projects:
+        console.print(f"  • {project}")
+
+
+def main():
+    """Run all examples"""
+    console.print("[bold green]CV Testing Application Examples[/bold green]\n")
+
+    # List available projects
+    example_list_available_projects()
+    console.print()
+
+    # Note: The following examples would need actual CV repositories to work
+    console.print("[yellow]Note: The following examples require actual CV repositories[/yellow]\n")
+
+    # Show example configurations (without actually running them)
+    console.print("[bold blue]Example configurations:[/bold blue]")
+    console.print("1. Test default project: CVTester().run_complete_test('obs4mip')")
+    console.print("2. Test custom branch: CVTester().run_complete_test('obs4mip', branch='test-branch')")
+    console.print("3. Test custom universe: CVTester().run_complete_test('input4mip', branch='esgvoc', universe_branch='esgvoc_dev')")
+    console.print(
+        "4. Test custom repo: CVTester().run_complete_test('obs4mip', repo_url='https://github.com/...', branch='main')"
+    )
+    console.print(
+        "5. Test custom repo + universe: CVTester().run_complete_test('obs4mip', repo_url='https://github.com/...', branch='main', universe_branch='dev')"
+    )
+    console.print(
+        "6. Individual tests: configure_for_testing(universe_branch='dev') -> synchronize_cvs() -> test_repository_structure() -> test_esgvoc_api_access()"
+    )
+
+
+if __name__ == "__main__":
+    main()
+

esgvoc/apps/vr/__init__.py

@@ -0,0 +1,12 @@
+"""
+Variable Registry (VR) App - A simplified tool for creating nested structures from branded variables.
+
+This module provides:
+- VRApp: Main application class for querying and structuring branded variables
+- create_nested_structure: Generic function for creating nested structures
+- variable_registry_structure: Function for creating the standard variable registry format
+"""
+
+from .vr_app import VRApp, create_nested_structure, variable_registry_structure
+
+__all__ = ["VRApp", "create_nested_structure", "variable_registry_structure"]

esgvoc/apps/vr/build_variable_registry.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""
+Script to build the variable registry structure from branded variables.
+This creates the nested JSON structure organized by CF Standard Name and Variable Root Name.
+"""
+
+import json
+from vr_app import VRApp
+
+
+def build_variable_registry():
+    """
+    Build the complete variable registry structure.
+
+    This creates a comprehensive JSON structure with all branded variables
+    organized by CF Standard Name and Variable Root Name.
+    """
+    print("Building Variable Registry...")
+    print("=" * 40)
+
+    with VRApp() as vr_app:
+        # Get all branded variables
+        print("Fetching all branded variables from the universe...")
+        all_terms = vr_app.get_all_branded_variables()
+        print(f"Found {len(all_terms)} total terms")
+
+        # Get statistics
+        stats = vr_app.get_statistics(all_terms)
+        print(f"\nStatistics:")
+        print(f"  Total terms: {stats['total_terms']}")
+        print(f"  Unique CF Standard Names: {stats['unique_cf_standard_names']}")
+        print(f"  Unique Variable Root Names: {stats['unique_variable_root_names']}")
+        print(f"  Unique Realms: {stats['unique_realms']}")
+
+        # Create complete variable registry
+        print("\nCreating complete variable registry...")
+        registry_all = vr_app.create_variable_registry()
+
+        # Create atmospheric variables registry
+        print("Creating atmospheric variables registry...")
+        registry_atmos = vr_app.create_variable_registry(filters={"realm": "atmos"})
+
+        # Export structures
+        print("\nExporting registry structures...")
+        vr_app.export_to_json(registry_all, "variable_registry_complete.json", indent=2)
+        vr_app.export_to_json(registry_atmos, "variable_registry_atmos.json", indent=2)
+
+        print("\n" + "=" * 50)
+        print("VARIABLE REGISTRY BUILD COMPLETED!")
+        print("=" * 50)
+        print("\nGenerated files:")
+        print("  - variable_registry_complete.json (all terms)")
+        print("  - variable_registry_atmos.json (atmospheric terms only)")
+
+        # Show sample structure
+        print(f"\nRegistry contains {len(registry_all.get('standard_name', {}))} CF Standard Names")
+        sample_names = list(registry_all.get("standard_name", {}).keys())[:5]
+        print(f"Sample CF Standard Names: {sample_names}")
+
+        return registry_all
+
+
+if __name__ == "__main__":
+    try:
+        build_variable_registry()
+    except Exception as e:
+        print(f"Error building variable registry: {e}")
+        import traceback
+
+        traceback.print_exc()
+

esgvoc/apps/vr/example_usage.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""
+Simple example showing how to use the VR App.
+This demonstrates basic usage patterns for the Variable Registry application.
+"""
+
+from vr_app import VRApp
+
+
+def main():
+    """
+    Example usage of the VR App showing common operations.
+    """
+    print("VR App Usage Example")
+    print("=" * 20)
+
+    # Initialize the app using context manager for proper cleanup
+    with VRApp() as vr_app:
+        # 1. Get basic statistics
+        print("\n1. Getting statistics...")
+        stats = vr_app.get_statistics()
+        print(f"   Total terms: {stats['total_terms']}")
+        print(f"   Unique CF Standard Names: {stats['unique_cf_standard_names']}")
+        print(f"   Unique Variable Root Names: {stats['unique_variable_root_names']}")
+
+        # 2. Create variable registry for atmospheric variables
+        print("\n2. Creating variable registry for atmospheric variables...")
+        atmos_registry = vr_app.create_variable_registry(filters={"realm": "atmos"})
+        vr_app.export_to_json(atmos_registry, "example_atmos_registry.json")
+        print("   Exported to: example_atmos_registry.json")
+
+        # 3. Create custom nested structure
+        print("\n3. Creating custom structure grouped by realm and CF standard name...")
+        custom_structure = vr_app.create_custom_nested_structure(
+            group_by_keys=["realm", "cf_standard_name"], metadata_config={0: ["cf_units"], 1: ["cf_sn_status"]}
+        )
+        vr_app.export_to_json(custom_structure, "example_custom_structure.json")
+        print("   Exported to: example_custom_structure.json")
+
+        # 4. Filter by specific variables
+        print("\n4. Filtering specific variables...")
+        filtered_terms = vr_app.get_branded_variables_subset({"cf_standard_name": "air_temperature", "realm": "atmos"})
+        print(f"   Found {len(filtered_terms)} air temperature terms in atmosphere")
+
+        # Show some sample branding suffixes
+        if filtered_terms:
+            print("   Sample branding suffixes:")
+            for term in filtered_terms[:3]:
+                print(f"     - {term.branding_suffix_name}")
+
+    print("\n" + "=" * 40)
+    print("Example completed successfully!")
+    print("Generated files:")
+    print("  - example_atmos_registry.json")
+    print("  - example_custom_structure.json")
+
+
+if __name__ == "__main__":
+    main()
+

esgvoc/apps/vr/vr_app.py

@@ -0,0 +1,333 @@
+import json
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Union
+
+from esgvoc import api
+from esgvoc.api import search
+from esgvoc.api.data_descriptors.data_descriptor import DataDescriptor
+from esgvoc.api.data_descriptors.known_branded_variable import KnownBrandedVariable
+
+
+def create_nested_structure(
+    terms: List[KnownBrandedVariable], group_by_keys: List[str], metadata_config: Optional[Dict[str, List[str]]] = None
+) -> Dict[str, Any]:
+    """
+    Create a nested structure from a list of terms using ordered grouping keys.
+
+    Args:
+        terms: List of KnownBrandedVariable terms
+        group_by_keys: Ordered list of field names to group by
+        metadata_config: Optional dict mapping group levels to metadata field names
+                        Format: {level_index: [field_names]}
+
+    Returns:
+        Nested dictionary structure
+    """
+    if not terms or not group_by_keys:
+        return {}
+
+    metadata_config = metadata_config or {}
+
+    def _build_nested_dict(
+        current_terms: List[KnownBrandedVariable], remaining_keys: List[str], level: int
+    ) -> Dict[str, Any]:
+        if not remaining_keys:
+            return [term.model_dump() for term in current_terms]
+
+        current_key = remaining_keys[0]
+        remaining_keys = remaining_keys[1:]
+
+        grouped = defaultdict(list)
+        metadata_by_group = {}
+
+        for term in current_terms:
+            group_value = getattr(term, current_key, None)
+            if group_value is not None:
+                grouped[group_value].append(term)
+
+                if level in metadata_config and group_value not in metadata_by_group:
+                    metadata_by_group[group_value] = {}
+                    for meta_field in metadata_config[level]:
+                        metadata_by_group[group_value][meta_field] = getattr(term, meta_field, None)
+
+        result = {}
+        for group_value, group_terms in grouped.items():
+            if level in metadata_config:
+                result[group_value] = metadata_by_group[group_value].copy()
+
+                if remaining_keys:
+                    nested_result = _build_nested_dict(group_terms, remaining_keys.copy(), level + 1)
+                    result[group_value].update(nested_result)
+                else:
+                    result[group_value]["items"] = [term.model_dump() for term in group_terms]
+            else:
+                result[group_value] = _build_nested_dict(group_terms, remaining_keys.copy(), level + 1)
+
+        return result
+
+    return _build_nested_dict(terms, group_by_keys.copy(), 0)
+
+
+def variable_registry_structure(terms: List[KnownBrandedVariable]) -> Dict[str, Any]:
+    """
+    Create the variable registry structure with CF Standard Name and VariableRootName grouping.
+
+    Args:
+        terms: List of KnownBrandedVariable terms
+
+    Returns:
+        Nested dictionary with the variable registry structure format
+    """
+    metadata_config = {
+        0: ["cf_units", "cf_sn_status"],  # CF Standard Name level
+        1: ["var_def_qualifier"],  # Variable Root Name level
+    }
+
+    group_by_keys = ["cf_standard_name", "variable_root_name"]
+
+    nested_data = create_nested_structure(terms, group_by_keys, metadata_config)
+
+    def _transform_to_registry_format(data: Dict[str, Any]) -> Dict[str, Any]:
+        result = {"standard_name": {}}
+
+        for std_name, std_data in data.items():
+            if isinstance(std_data, dict):
+                result["standard_name"][std_name] = {
+                    "units": std_data.get("cf_units", ""),
+                    "sn_status": std_data.get("cf_sn_status", ""),
+                    "variable_root_name": {},
+                }
+
+                for var_name, var_data in std_data.items():
+                    if var_name in ["cf_units", "cf_sn_status"]:
+                        continue
+
+                    if isinstance(var_data, dict):
+                        result["standard_name"][std_name]["variable_root_name"][var_name] = {
+                            "var_def_qualifier": var_data.get("var_def_qualifier", ""),
+                            "branding_suffix": {},
+                        }
+
+                        for suffix_name, suffix_data in var_data.items():
+                            if suffix_name == "var_def_qualifier":
+                                continue
+
+                            if isinstance(suffix_data, list):
+                                for term_data in suffix_data:
+                                    if isinstance(term_data, dict):
+                                        suffix_key = term_data.get("branding_suffix_name", "")
+                                        if suffix_key:
+                                            result["standard_name"][std_name]["variable_root_name"][var_name][
+                                                "branding_suffix"
+                                            ][suffix_key] = {
+                                                "brand_description": term_data.get("description", ""),
+                                                "bn_status": term_data.get("bn_status", ""),
+                                                "dimensions": term_data.get("dimensions", []),
+                                                "cell_methods": term_data.get("cell_methods", ""),
+                                                "cell_measures": term_data.get("cell_measures", ""),
+                                                "history": term_data.get("history", ""),
+                                                "temporal_label": term_data.get("temporal_label", ""),
+                                                "vertical_label": term_data.get("vertical_label", ""),
+                                                "horizontal_label": term_data.get("horizontal_label", ""),
+                                                "area_label": term_data.get("area_label", ""),
+                                                "realm": term_data.get("realm", ""),
+                                            }
+
+        return result
+
+    return _transform_to_registry_format(nested_data)
+
+
+class VRApp:
+    """
+    Variable Restructuring (VR) App for creating nested structures from branded variables.
+
+    This app allows querying known_branded_variable terms from the universe and
+    transforming them into nested JSON structures with customizable grouping.
+    """
+
+    def __init__(self):
+        self.universe_session = search.get_universe_session()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.universe_session:
+            self.universe_session.close()
+
+    def get_all_branded_variables(self) -> List[DataDescriptor]:
+        """
+        Get all known_branded_variable terms from the universe.
+
+        Returns:
+            List of KnownBrandedVariable terms
+        """
+        try:
+            terms = api.get_all_terms_in_data_descriptor("known_branded_variable")
+            return terms
+        except Exception as e:
+            print(f"Error fetching branded variables: {e}")
+            return []
+
+    def get_branded_variables_subset(self, filters: Dict[str, Any]) -> List[KnownBrandedVariable]:
+        """
+        Get a subset of known_branded_variable terms based on filters.
+
+        Args:
+            filters: Dictionary of field names and values to filter by
+
+        Returns:
+            List of filtered KnownBrandedVariable terms
+        """
+        all_terms = self.get_all_branded_variables()
+        filtered_terms = []
+
+        for term in all_terms:
+            match = True
+            for field, value in filters.items():
+                term_value = getattr(term, field, None)
+                if isinstance(value, list):
+                    if term_value not in value:
+                        match = False
+                        break
+                elif term_value != value:
+                    match = False
+                    break
+
+            if match:
+                filtered_terms.append(term)
+
+        return filtered_terms
+
+    def create_custom_nested_structure(
+        self,
+        terms: Optional[List[KnownBrandedVariable]] = None,
+        group_by_keys: List[str] = None,
+        metadata_config: Optional[Dict[str, List[str]]] = None,
+        filters: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Create a custom nested structure.
+
+        Args:
+            terms: Optional list of terms. If None, fetches all terms
+            group_by_keys: List of field names to group by
+            metadata_config: Optional metadata configuration
+            filters: Optional filters to apply when fetching terms
+
+        Returns:
+            Nested dictionary structure
+        """
+        if terms is None:
+            if filters:
+                terms = self.get_branded_variables_subset(filters)
+            else:
+                terms = self.get_all_branded_variables()
+
+        if not group_by_keys:
+            group_by_keys = ["cf_standard_name", "variable_root_name"]
+
+        return create_nested_structure(terms, group_by_keys, metadata_config)
+
+    def create_variable_registry(
+        self, terms: Optional[List[KnownBrandedVariable]] = None, filters: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """
+        Create the variable registry structure with CF Standard Name and VariableRootName grouping.
+
+        Args:
+            terms: Optional list of terms. If None, fetches all terms
+            filters: Optional filters to apply when fetching terms
+
+        Returns:
+            Nested dictionary with the variable registry structure format
+        """
+        if terms is None:
+            if filters:
+                terms = self.get_branded_variables_subset(filters)
+            else:
+                terms = self.get_all_branded_variables()
+
+        return variable_registry_structure(terms)
+
+    def export_to_json(self, structure: Dict[str, Any], filename: str, indent: int = 2) -> None:
+        """
+        Export a nested structure to a JSON file.
+
+        Args:
+            structure: The nested dictionary structure to export
+            filename: Output filename
+            indent: JSON indentation level
+        """
+        try:
+            with open(filename, "w", encoding="utf-8") as f:
+                json.dump(structure, f, indent=indent, ensure_ascii=False)
+            print(f"Structure exported to {filename}")
+        except Exception as e:
+            print(f"Error exporting to JSON: {e}")
+
+    def get_statistics(self, terms: Optional[List[KnownBrandedVariable]] = None) -> Dict[str, Any]:
+        """
+        Get statistics about the branded variables.
+
+        Args:
+            terms: Optional list of terms. If None, fetches all terms
+
+        Returns:
+            Dictionary with statistics
+        """
+        if terms is None:
+            terms = self.get_all_branded_variables()
+
+        stats = {
+            "total_terms": len(terms),
+            "unique_cf_standard_names": len(set(term.cf_standard_name for term in terms)),
+            "unique_variable_root_names": len(set(term.variable_root_name for term in terms)),
+            "unique_realms": len(set(term.realm for term in terms)),
+            "status_distribution": {},
+            "realm_distribution": {},
+        }
+
+        # Status distribution
+        for term in terms:
+            status = term.bn_status
+            stats["status_distribution"][status] = stats["status_distribution"].get(status, 0) + 1
+
+        # Realm distribution
+        for term in terms:
+            realm = term.realm
+            stats["realm_distribution"][realm] = stats["realm_distribution"].get(realm, 0) + 1
+
+        return stats
+
+
+def main():
+    """
+    Example usage of the VR App.
+    """
+    with VRApp() as vr_app:
+        # Get statistics
+        stats = vr_app.get_statistics()
+        print(f"Total terms: {stats['total_terms']}")
+        print(f"Unique CF Standard Names: {stats['unique_cf_standard_names']}")
+        print(f"Unique Variable Root Names: {stats['unique_variable_root_names']}")
+
+        # Create variable registry for a subset
+        filters = {"realm": "atmos"}
+        registry_struct = vr_app.create_variable_registry(filters=filters)
+
+        # Export to JSON
+        vr_app.export_to_json(registry_struct, "variable_registry_atmos.json")
+
+        # Create custom structure
+        custom_struct = vr_app.create_custom_nested_structure(
+            group_by_keys=["realm", "cf_standard_name"],
+            metadata_config={0: ["bn_status"], 1: ["cf_units", "cf_sn_status"]},
+        )
+
+        vr_app.export_to_json(custom_struct, "custom_structure.json")
+
+
+if __name__ == "__main__":
+    main()