esgvoc 1.0.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

esgvoc/cli/drs.py

@@ -16,12 +16,12 @@ app = typer.Typer()
 console = Console()
 
 
-
 # Predefined list of projects and DRS types
 # projects = ["cmip5", "cmip6","cmip6plus", "cmip7"]
 projects = ev.get_all_projects()
 drs_types = ["filename", "directory", "dataset"]
 
+
 def display(table):
     """
     Function to display a rich table in the console.
@@ -34,15 +34,26 @@ def display(table):
 
 @app.command()
 def drsvalid(
-    drs_entries: Optional[List[str]] = typer.Argument(None, help="List of DRS validation inputs in the form <project> <drstype> <string>"),
-    file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS validation inputs, one per line in the form <project> <drstype> <string>"),
+    drs_entries: Optional[List[str]] = typer.Argument(
+        None, help="List of DRS validation inputs in the form <project> <drstype> <string>"
+    ),
+    file: Optional[typer.FileText] = typer.Option(
+        None,
+        "--file",
+        "-f",
+        help="File containing DRS validation inputs, one per line in the form <project> <drstype> <string>",
+    ),
     verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed validation results"),
     output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the DRS entries validation"),
-    rm_prefix: Optional[str] = typer.Option(None,"-p","--prefix", help="Remove given prefix from all checked directory"),
-    pedantic: Optional[bool] = typer.Option(False,"-e","--enforce", help="Enable pedantic mode, enforcing strict compliance, mean that warnings are now errors.")
-
-
-
+    rm_prefix: Optional[str] = typer.Option(
+        None, "-p", "--prefix", help="Remove given prefix from all checked directory"
+    ),
+    pedantic: Optional[bool] = typer.Option(
+        False,
+        "-e",
+        "--enforce",
+        help="Enable pedantic mode, enforcing strict compliance, mean that warnings are now errors.",
+    ),
 ) -> List[DrsValidationReport]:
     """
     Validates DRS strings for a specific project and type.
@@ -68,14 +79,13 @@ def drsvalid(
     if not sys.stdin.isatty():  # Check if input is being piped via stdin
         entries.extend(el for line in sys.stdin for el in shlex.split(line))
 
-
     if file:
         entries.extend(el for line in file for el in line.strip().split(" "))
 
     i = 0
     while i < len(entries):
-        if entries[i] in [""," "]:
-            i+=1
+        if entries[i] in ["", " "]:
+            i += 1
             continue
 
         if entries[i] in projects:
@@ -102,9 +112,9 @@ def drsvalid(
                 report = validator.validate_file_name(string)
             case "directory":
                 if rm_prefix:
-                    prefix = rm_prefix+"/" if rm_prefix[-1]!="/" else ""
+                    prefix = rm_prefix + "/" if rm_prefix[-1] != "/" else ""
                 else:
-                    prefix=None
+                    prefix = None
                 report = validator.validate_directory(string, prefix)
             case "dataset":
                 report = validator.validate_dataset_id(string)
@@ -127,8 +137,8 @@ def drsvalid(
             errors = "\n".join(["⚠️ " + str(error) for error in report.errors])
             valid = "✅ Valid" if report else "❌ Invalid"
 
-            table.add_row("-"*4,"-"*4,"-"*4,"-"*4,"-"*4)
-            table.add_row(entry,proj_and_type, warnings, errors, valid)
+            table.add_row("-" * 4, "-" * 4, "-" * 4, "-" * 4, "-" * 4)
+            table.add_row(entry, proj_and_type, warnings, errors, valid)
 
         console.print(table)
     elif output:
@@ -137,7 +147,6 @@ def drsvalid(
                 f.write(str(report) + "\n")
         console.print(f"DRS validation entries saved to [green]{output}[/green]")
 
-
     else:
         for report in reports:
             console.print(str(report))
@@ -147,8 +156,15 @@ def drsvalid(
 
 @app.command()
 def drsgen(
-    drs_entries: Optional[List[str]] = typer.Argument(None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_terms>"),
-    file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_terms>"),
+    drs_entries: Optional[List[str]] = typer.Argument(
+        None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_terms>"
+    ),
+    file: Optional[typer.FileText] = typer.Option(
+        None,
+        "--file",
+        "-f",
+        help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_terms>",
+    ),
     verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed generation results"),
     output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the generated DRS entries"),
 ) -> List[DrsGenerationReport]:
@@ -182,8 +198,8 @@ def drsgen(
 
     i = 0
     while i < len(entries):
-        if entries[i] in [""," "]:
-            i+=1
+        if entries[i] in ["", " "]:
+            i += 1
             continue
         if entries[i] in projects:
             current_project = entries[i]
@@ -231,7 +247,7 @@ def drsgen(
             table.add_row(entry, warnings, errors, result)
             table.add_row("----", "----", "----", "----")
             if table.columns[3].width is not None and len(result) > table.columns[3].width:
-                table.columns[3].width = len(result)+1
+                table.columns[3].width = len(result) + 1
         console.print(table)
 
     elif output:
@@ -245,5 +261,7 @@ def drsgen(
             console.print(str(report))
 
     return generated_reports
+
+
 if __name__ == "__main__":
     app()

esgvoc/cli/main.py

@@ -6,6 +6,7 @@ from esgvoc.cli.find import app as find_app
 from esgvoc.cli.get import app as get_app
 from esgvoc.cli.install import app as install_app
 from esgvoc.cli.status import app as status_app
+from esgvoc.cli.test_cv import app as test_cv_app
 from esgvoc.cli.valid import app as valid_app
 
 app = typer.Typer()
@@ -17,6 +18,7 @@ app.add_typer(valid_app)
 app.add_typer(install_app)
 app.add_typer(drs_app)
 app.add_typer(config_app, name="config")
+app.add_typer(test_cv_app, name="test")
 app.add_typer(find_app)
 
 

esgvoc/cli/test_cv.py

@@ -0,0 +1,257 @@
+"""
+Test CV CLI commands
+
+Provides commands for testing project CVs and Universe CVs integrated with esgvoc CLI.
+"""
+
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from esgvoc.apps.test_cv.cv_tester import CVTester
+from esgvoc.core.service.configuration.setting import ServiceSettings
+
+app = typer.Typer()
+console = Console()
+
+
+@app.command()
+def list_projects():
+    """List all available CV projects that can be tested."""
+    tester = CVTester()
+    projects = tester.get_available_projects()
+
+    table = Table(title="Available CV Projects for Testing")
+    table.add_column("Project Name", style="cyan")
+    table.add_column("Repository", style="green")
+    table.add_column("Default Branch", style="yellow")
+    table.add_column("Local Path", style="blue")
+
+    for project_name in projects:
+        config = ServiceSettings.DEFAULT_PROJECT_CONFIGS[project_name]
+        table.add_row(project_name, config["github_repo"], config["branch"], config["local_path"])
+
+    console.print(table)
+    console.print(f"\n[blue]Total: {len(projects)} projects available for testing[/blue]")
+
+
+@app.command()
+def configure(
+    project: str = typer.Argument(..., help="Project name to configure for testing"),
+    repo_url: Optional[str] = typer.Option(None, "--repo", "-r", help="Custom repository URL"),
+    branch: Optional[str] = typer.Option(None, "--branch", "-b", help="Custom branch to test"),
+    universe_branch: Optional[str] = typer.Option(None, "--universe-branch", "-u", help="Custom universe branch"),
+    sync: bool = typer.Option(True, "--sync/--no-sync", help="Synchronize CVs after configuration"),
+):
+    """
+    Configure esgvoc with a specific project for testing.
+
+    Examples:
+        esgvoc test configure obs4mip
+        esgvoc test configure cmip6 --branch my-test-branch
+        esgvoc test configure cmip6 --universe-branch my-universe-branch
+        esgvoc test configure custom --repo https://github.com/me/my-cvs --branch main --universe-branch dev
+    """
+    tester = CVTester()
+
+    try:
+        # Configure
+        if not tester.configure_for_testing(project, repo_url, branch, None, universe_branch):
+            raise typer.Exit(1)
+
+        # Optionally synchronize
+        if sync:
+            if not tester.synchronize_cvs():
+                raise typer.Exit(1)
+
+        console.print(f"[green]✅ Successfully configured project '{project}' for testing[/green]")
+        if not sync:
+            console.print("[yellow]Note: CVs not synchronized. Run 'esgvoc test sync' to download.[/yellow]")
+
+    except Exception as e:
+        console.print(f"[red]❌ Configuration failed: {e}[/red]")
+        raise typer.Exit(1)
+
+
+@app.command()
+def sync():
+    """Synchronize/download CVs for the currently configured project."""
+    tester = CVTester()
+
+    try:
+        if not tester.synchronize_cvs():
+            raise typer.Exit(1)
+        console.print("[green]✅ CVs synchronized successfully[/green]")
+    except Exception as e:
+        console.print(f"[red]❌ Synchronization failed: {e}[/red]")
+        raise typer.Exit(1)
+
+
+@app.command()
+def structure(
+    path: str = typer.Argument(".", help="Path to CV repository to validate"),
+):
+    """
+    Test CV repository structure and file format compliance.
+
+    Validates:
+    - Collection directory structure
+    - JSONLD context files
+    - Element JSON files
+    - project_specs.json references
+
+    Examples:
+        esgvoc test structure .
+        esgvoc test structure /path/to/cv/repo
+    """
+    tester = CVTester()
+
+    try:
+        if not tester.test_repository_structure(path):
+            raise typer.Exit(1)
+        console.print("[green]✅ Repository structure validation passed[/green]")
+    except Exception as e:
+        console.print(f"[red]❌ Structure validation failed: {e}[/red]")
+        raise typer.Exit(1)
+
+
+@app.command()
+def api(
+    project: str = typer.Argument(..., help="Project name to test API access for"),
+    path: str = typer.Argument(".", help="Path to CV repository"),
+    debug_terms: bool = typer.Option(True, "--debug-terms/--no-debug-terms", help="Show detailed debugging info for missing terms"),
+):
+    """
+    Test esgvoc API access for all repository collections and elements.
+
+    Validates:
+    - Project is accessible via esgvoc API
+    - All repository collections are queryable
+    - All repository elements are accessible
+    - API functions work correctly
+
+    Examples:
+        esgvoc test api obs4mip .
+        esgvoc test api cmip6 /path/to/cmip6/repo
+    """
+    tester = CVTester(debug_missing_terms=debug_terms)
+
+    try:
+        if not tester.test_esgvoc_api_access(project, path):
+            raise typer.Exit(1)
+        console.print("[green]✅ ESGVoc API access validation passed[/green]")
+    except Exception as e:
+        console.print(f"[red]❌ API validation failed: {e}[/red]")
+        raise typer.Exit(1)
+
+
+@app.command()
+def run(
+    project: str = typer.Argument(..., help="Project name to test"),
+    path: Optional[str] = typer.Argument(None, help="Path to CV repository (auto-detected if not provided)"),
+    repo_url: Optional[str] = typer.Option(None, "--repo", "-r", help="Custom repository URL"),
+    branch: Optional[str] = typer.Option(None, "--branch", "-b", help="Custom branch to test"),
+    universe_branch: Optional[str] = typer.Option(None, "--universe-branch", "-u", help="Custom universe branch"),
+    debug_terms: bool = typer.Option(True, "--debug-terms/--no-debug-terms", help="Show detailed debugging info for missing terms"),
+):
+    """
+    Run complete CV test suite: configure, sync, structure, and API tests.
+
+    This is the comprehensive test that runs all validation steps:
+    1. Configure esgvoc with the specified project
+    2. Synchronize/download CVs
+    3. Validate repository structure
+    4. Test esgvoc API access
+
+    Examples:
+        esgvoc test run obs4mip
+        esgvoc test run cmip6 --branch my-test-branch
+        esgvoc test run cmip6 --universe-branch my-universe-branch
+        esgvoc test run cmip6 /path/to/custom/repo --branch my-test-branch --universe-branch dev
+        esgvoc test run custom --repo https://github.com/me/cvs --branch main --universe-branch main
+    """
+    tester = CVTester(debug_missing_terms=debug_terms)
+
+    try:
+        success = tester.run_complete_test(project, repo_url, branch, path, None, universe_branch)
+        if success:
+            console.print(f"[bold green]🎉 All tests passed for project '{project}'![/bold green]")
+        else:
+            # The detailed failure information is already printed by cv_tester
+            raise typer.Exit(1)
+    except Exception as e:
+        console.print(f"[red]❌ Test suite failed: {e}[/red]")
+        raise typer.Exit(1)
+    finally:
+        tester.cleanup()
+
+
+@app.command()
+def env(
+    command: str = typer.Argument(..., help="Environment mode command: 'configure' or 'test'"),
+    project: Optional[str] = typer.Option(None, "--project", "-p", help="Project name (auto-detected if not provided)"),
+    repo_url: Optional[str] = typer.Option(
+        None, "--repo-url", help="Repository URL (from REPO_URL env var if not provided)"
+    ),
+    branch: Optional[str] = typer.Option(None, "--branch", help="Branch (from TEST_BRANCH env var if not provided)"),
+    universe_branch: Optional[str] = typer.Option(None, "--universe-branch", help="Universe branch (from UNIVERSE_BRANCH env var if not provided)"),
+    debug_terms: bool = typer.Option(True, "--debug-terms/--no-debug-terms", help="Show detailed debugging info for missing terms"),
+):
+    """
+    Environment variable mode for CI/CD integration and automated testing.
+
+    Reads configuration from environment variables:
+    - REPO_URL: Repository URL to test
+    - TEST_BRANCH: Branch to test
+    - PROJECT_NAME: Project name (auto-detected if not set)
+    - UNIVERSE_BRANCH: Universe branch to test (optional)
+    - ESGVOC_LIBRARY_BRANCH: ESGVoc library branch (informational)
+
+    Examples:
+        # Set environment and run
+        export REPO_URL=https://github.com/me/obs4MIPs_CVs
+        export TEST_BRANCH=test-branch
+        export UNIVERSE_BRANCH=my-universe-branch
+        esgvoc test env configure
+        esgvoc test env test
+
+        # Or use options
+        esgvoc test env configure --project obs4mip --repo-url https://github.com/me/repo --branch main --universe-branch dev
+    """
+    import os
+
+    # Get config from environment or options
+    final_repo_url = repo_url or os.environ.get("REPO_URL")
+    final_branch = branch or os.environ.get("TEST_BRANCH")
+    final_universe_branch = universe_branch or os.environ.get("UNIVERSE_BRANCH")
+    final_project = project or os.environ.get("PROJECT_NAME")
+
+    # Auto-detect project if not provided
+    if not final_project:
+        from esgvoc.apps.test_cv.cv_tester import detect_project_name
+
+        final_project = detect_project_name()
+
+    if command == "configure":
+        if not final_repo_url or not final_branch:
+            console.print("[red]❌ REPO_URL and TEST_BRANCH are required for env configure[/red]")
+            console.print("Set environment variables or use --repo-url and --branch options")
+            raise typer.Exit(1)
+
+        # Use configure command
+        configure(final_project, final_repo_url, final_branch, final_universe_branch, sync=True)
+
+    elif command == "test":
+        # Use run command
+        run(final_project, None, final_repo_url, final_branch, final_universe_branch, debug_terms)
+
+    else:
+        console.print(f"[red]❌ Invalid env command '{command}'. Use 'configure' or 'test'[/red]")
+        raise typer.Exit(1)
+
+
+if __name__ == "__main__":
+    app()
+

esgvoc/core/constants.py

@@ -1,13 +1,16 @@
 DIRNAME_AND_FILENAME_SEPARATOR = "_"
-PROJECT_SPECS_FILENAME = "project_specs.json"
+PROJECT_SPECS_FILENAME = "project_specs.yaml"
+DRS_SPECS_FILENAME = "drs_specs.yaml"
+CATALOG_SPECS_FILENAME = "catalog_specs.yaml"
+ATTRIBUTES_SPECS_FILENAME = "attr_specs.yaml"
 PROJECT_ID_JSON_KEY = "project_id"
 CONTEXT_FILENAME = "000_context.jsonld"
 CONTEXT_JSON_KEY = "@context"
-TERM_ID_JSON_KEY = 'id'
-COMPOSITE_PARTS_JSON_KEY = 'parts'
-COMPOSITE_SEPARATOR_JSON_KEY = 'separator'
-PATTERN_JSON_KEY = 'regex'
-TERM_TYPE_JSON_KEY = 'type'
-DRS_SPECS_JSON_KEY = 'drs_name'
+TERM_ID_JSON_KEY = "id"
+COMPOSITE_PARTS_JSON_KEY = "parts"
+COMPOSITE_SEPARATOR_JSON_KEY = "separator"
+PATTERN_JSON_KEY = "regex"
+TERM_TYPE_JSON_KEY = "type"
+DRS_SPECS_JSON_KEY = "drs_name"
 SQLITE_FIRST_PK = 1
 DATA_DESCRIPTOR_JSON_KEY = "@base"

esgvoc/core/data_handler.py

@@ -1,4 +1,3 @@
-
 import os
 import json
 import logging
@@ -9,11 +8,13 @@ from pyld import jsonld
 from pydantic import BaseModel, model_validator, ConfigDict
 
 from esgvoc.api.data_descriptors import DATA_DESCRIPTOR_CLASS_MAPPING
+
 # Configure logging
 _LOGGER = logging.getLogger(__name__)
 
 mapping = DATA_DESCRIPTOR_CLASS_MAPPING
 
+
 def unified_document_loader(uri: str) -> Dict:
     """Load a document from a local file or a remote URI."""
     if uri.startswith(("http://", "https://")):
@@ -27,6 +28,7 @@ def unified_document_loader(uri: str) -> Dict:
         with open(uri, "r") as f:
             return json.load(f)
 
+
 class JsonLdResource(BaseModel):
     uri: str
     local_path: Optional[str] = None
@@ -40,13 +42,15 @@ class JsonLdResource(BaseModel):
         local_path = values.get("local_path")
         if local_path:
             values["local_path"] = os.path.abspath(local_path) + "/"
-        jsonld.set_document_loader(lambda uri,options:{
-            "contextUrl": None,  # No special context URL
-            "documentUrl": uri,  # The document's actual URL
-            "document": unified_document_loader(uri),  # The parsed JSON-LD document
-        }) 
+        jsonld.set_document_loader(
+            lambda uri, options: {
+                "contextUrl": None,  # No special context URL
+                "documentUrl": uri,  # The document's actual URL
+                "document": unified_document_loader(uri),  # The parsed JSON-LD document
+            }
+        )
         return values
-    
+
     @cached_property
     def json_dict(self) -> Dict:
         """Fetch the original JSON data."""
@@ -58,12 +62,12 @@ class JsonLdResource(BaseModel):
         """Expand the JSON-LD data."""
         _LOGGER.debug(f"Expanding JSON-LD data for {self.uri}")
         return jsonld.expand(self.uri, options={"base": self.uri})
-    
+
     @cached_property
     def context(self) -> Dict:
         """Fetch and return the JSON content of the '@context'."""
-        
-        context_data =JsonLdResource(uri="/".join(self.uri.split("/")[:-1])+"/"+self.json_dict["@context"]) 
+
+        context_data = JsonLdResource(uri="/".join(self.uri.split("/")[:-1]) + "/" + self.json_dict["@context"])
         # Works only in relative path declaration
 
         context_value = context_data.json_dict
@@ -83,9 +87,7 @@ class JsonLdResource(BaseModel):
     def normalized(self) -> str:
         """Normalize the JSON-LD data."""
         _LOGGER.info(f"Normalizing JSON-LD data for {self.uri}")
-        return jsonld.normalize(
-            self.uri, options={"algorithm": "URDNA2015", "format": "application/n-quads"}
-        )
+        return jsonld.normalize(self.uri, options={"algorithm": "URDNA2015", "format": "application/n-quads"})
 
     @cached_property
     def python(self) -> Optional[Any]:
@@ -120,14 +122,14 @@ class JsonLdResource(BaseModel):
 
 
 if __name__ == "__main__":
-    ## For Universe 
-    #online
+    ## For Universe
+    # online
     # d = Data(uri = "https://espri-mod.github.io/mip-cmor-tables/activity/cmip.json")
     # print(d.info)
-    #offline
-    #print(Data(uri = ".cache/repos/mip-cmor-tables/activity/cmip.json").info)
-    ## for Project 
-    #d = Data(uri = "https://espri-mod.github.io/CMIP6Plus_CVs/activity_id/cmip.json")
-    #print(d.info)
-    #offline
-    print(JsonLdResource(uri = ".cache/repos/CMIP6Plus_CVs/activity_id/cmip.json").info)
+    # offline
+    # print(Data(uri = ".cache/repos/mip-cmor-tables/activity/cmip.json").info)
+    ## for Project
+    # d = Data(uri = "https://espri-mod.github.io/CMIP6Plus_CVs/activity_id/cmip.json")
+    # print(d.info)
+    # offline
+    print(JsonLdResource(uri=".cache/repos/CMIP6Plus_CVs/activity_id/cmip.json").info)

esgvoc/core/db/connection.py

@@ -1,6 +1,7 @@
 import json
 from pathlib import Path
 
+import yaml
 from sqlalchemy import Engine
 from sqlmodel import Session, create_engine
 
@@ -31,3 +32,9 @@ class DBConnection:
 
 def read_json_file(json_file_path: Path) -> dict:
     return json.loads(json_file_path.read_text())
+
+
+def read_yaml_file(yaml_file_path: Path) -> dict:
+    with open(yaml_file_path, 'r') as file:
+        result = yaml.safe_load(file)
+    return result

esgvoc/core/db/project_ingestion.py

@@ -8,7 +8,7 @@ import esgvoc.core.constants
 import esgvoc.core.db.connection as db
 import esgvoc.core.service as service
 from esgvoc.core.data_handler import JsonLdResource
-from esgvoc.core.db.connection import DBConnection, read_json_file
+from esgvoc.core.db.connection import DBConnection, read_json_file, read_yaml_file
 from esgvoc.core.db.models.mixins import TermKind
 from esgvoc.core.db.models.project import PCollection, Project, PTerm
 from esgvoc.core.exceptions import EsgvocDbError
@@ -88,7 +88,10 @@ def ingest_collection(collection_dir_path: Path, project: Project, project_db_se
                     term_kind_collection = term_kind
 
             except Exception as e:
-                _LOGGER.warning(f"Unable to read term {term_file_path}. Skip.\n{str(e)}")
+                _LOGGER.warning(
+                    f"Unable to read term file {term_file_path} in collection '{collection_id}' "
+                    + f"of project '{project.id}'. Skip.\n{str(e)}"
+                )
                 continue
             try:
                 term = PTerm(
@@ -99,14 +102,28 @@ def ingest_collection(collection_dir_path: Path, project: Project, project_db_se
                 )
                 project_db_session.add(term)
             except Exception as e:
+                # Enhanced error reporting for term validation failures
                 _LOGGER.error(
-                    f"fail to find term {term_id} in data descriptor {data_descriptor_id} "
-                    + f"for the collection {collection_id} of the project {project.id}. Skip {term_id}.\n{str(e)}"
+                    f"Failed to create term '{term_id}' in collection '{collection_id}' "
+                    + f"of project '{project.id}' from file '{term_file_path}': {str(e)}"
                 )
                 continue
     if term_kind_collection:
         collection.term_kind = term_kind_collection
-    project_db_session.add(collection)
+    else:
+        # If we couldn't determine a term kind, use PLAIN as default and log warning
+        _LOGGER.warning(
+            f"No term kind determined for collection '{collection_id}' in project '{project.id}'. "
+            + "Using PLAIN as default. This might indicate empty collection or processing errors."
+        )
+        collection.term_kind = TermKind.PLAIN
+
+    try:
+        project_db_session.add(collection)
+    except Exception as e:
+        error_context = f"Failed to add collection '{collection_id}' to project '{project.id}'"
+        _LOGGER.error(f"{error_context}: {str(e)}")
+        raise EsgvocDbError(f"{error_context}: {str(e)}") from e
 
 
 def ingest_project(project_dir_path: Path, project_db_file_path: Path, git_hash: str):
@@ -119,15 +136,23 @@ def ingest_project(project_dir_path: Path, project_db_file_path: Path, git_hash:
 
     with project_connection.create_session() as project_db_session:
         project_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.PROJECT_SPECS_FILENAME)
+        drs_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.DRS_SPECS_FILENAME)
+        catalog_specs_file_path = project_dir_path.joinpath(esgvoc.core.constants.CATALOG_SPECS_FILENAME)
         try:
-            project_json_specs = read_json_file(project_specs_file_path)
-            project_id = project_json_specs[esgvoc.core.constants.PROJECT_ID_JSON_KEY]
+            raw_project_specs = read_yaml_file(project_specs_file_path)
+            project_id = raw_project_specs[esgvoc.core.constants.PROJECT_ID_JSON_KEY]
+            raw_drs_specs = read_yaml_file(drs_specs_file_path)
+            project_specs = raw_project_specs
+            project_specs['drs_specs'] = raw_drs_specs
+            if catalog_specs_file_path.exists():
+                raw_catalog_specs = read_yaml_file(catalog_specs_file_path)
+                project_specs['catalog_specs'] = raw_catalog_specs
         except Exception as e:
-            msg = f"unable to read project specs file  {project_specs_file_path}"
+            msg = f"unable to read specs files in {project_dir_path}"
             _LOGGER.fatal(msg)
             raise EsgvocDbError(msg) from e
 
-        project = Project(id=project_id, specs=project_json_specs, git_hash=git_hash)
+        project = Project(id=project_id, specs=project_specs, git_hash=git_hash)
         project_db_session.add(project)
 
         for collection_dir_path in project_dir_path.iterdir():

esgvoc/core/db/universe_ingestion.py

@@ -15,6 +15,7 @@ from esgvoc.core.exceptions import EsgvocDbError
 from esgvoc.core.service.data_merger import DataMerger
 
 _LOGGER = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
 
 def infer_term_kind(json_specs: dict) -> TermKind:
@@ -81,7 +82,6 @@ def ingest_metadata_universe(connection, git_hash):
 
 def ingest_data_descriptor(data_descriptor_path: Path, connection: db.DBConnection) -> None:
     data_descriptor_id = data_descriptor_path.name
-
     context_file_path = data_descriptor_path.joinpath(esgvoc.core.constants.CONTEXT_FILENAME)
     try:
         context = read_json_file(context_file_path)
@@ -108,7 +108,6 @@ def ingest_data_descriptor(data_descriptor_path: Path, connection: db.DBConnecti
                     json_specs = DataMerger(
                         data=JsonLdResource(uri=str(term_file_path)), locally_available=locally_available
                     ).merge_linked_json()[-1]
-
                     term_kind = infer_term_kind(json_specs)
                     term_id = json_specs["id"]