esgvoc 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

esgvoc/apps/drs/validator.py

@@ -2,17 +2,33 @@ from typing import cast
 
 import esgvoc.api.projects as projects
 import esgvoc.apps.drs.constants as constants
-from esgvoc.api import APIException
-from esgvoc.api.project_specs import (DrsCollection, DrsConstant, DrsPart,
-                                      DrsPartKind, DrsSpecification, DrsType,
-                                      ProjectSpecs)
-from esgvoc.apps.drs.report import (BlankTerm, ComplianceIssue, DrsIssue,
-                                    DrsValidationReport, ExtraChar,
-                                    ExtraSeparator, ExtraTerm,
-                                    FileNameExtensionIssue, InvalidTerm,
-                                    MissingTerm, ParsingIssue, Space,
-                                    Unparsable, ValidationError,
-                                    ValidationWarning)
+from esgvoc.api.project_specs import (
+    DrsCollection,
+    DrsConstant,
+    DrsPart,
+    DrsPartKind,
+    DrsSpecification,
+    DrsType,
+    ProjectSpecs,
+)
+from esgvoc.apps.drs.report import (
+    BlankTerm,
+    ComplianceIssue,
+    DrsIssue,
+    DrsValidationReport,
+    ExtraChar,
+    ExtraSeparator,
+    ExtraTerm,
+    FileNameExtensionIssue,
+    InvalidTerm,
+    MissingTerm,
+    ParsingIssue,
+    Space,
+    Unparsable,
+    ValidationError,
+    ValidationWarning,
+)
+from esgvoc.core.exceptions import EsgvocDbError, EsgvocNotFoundError
 
 
 class DrsApplication:
@@ -25,9 +41,9 @@ class DrsApplication:
         """The project id."""
         self.pedantic: bool = pedantic
         """Same as the option of GCC: turn warnings into errors. Default False."""
-        project_specs: ProjectSpecs|None = projects.find_project(project_id)
+        project_specs: ProjectSpecs | None = projects.get_project(project_id)
         if not project_specs:
-            raise APIException(f'unable to find project {project_id}')
+            raise EsgvocNotFoundError(f"unable to find project '{project_id}'")
         for specs in project_specs.drs_specs:
             match specs.type:
                 case DrsType.DIRECTORY:
@@ -40,7 +56,7 @@ class DrsApplication:
                     self.dataset_id_specs: DrsSpecification = specs
                     """The DRS dataset id specs of the project."""
                 case _:
-                    raise RuntimeError(f'unsupported DRS specs type {specs.type}')
+                    raise EsgvocDbError(f"unsupported DRS specs type '{specs.type}'")
 
     def _get_full_file_name_extension(self) -> str:
         """
@@ -55,8 +71,8 @@ class DrsApplication:
             full_extension = specs.properties[constants.FILE_NAME_EXTENSION_SEPARATOR_KEY] + \
                              specs.properties[constants.FILE_NAME_EXTENSION_KEY]
         else:
-            raise RuntimeError('missing properties in the DRS file name specifications of the ' +
-                               f'project {self.project_id}')
+            raise EsgvocDbError('missing properties in the DRS file name specifications of the ' +
+                                f"project '{self.project_id}'")
         return full_extension
 
 
@@ -66,7 +82,7 @@ class DrsValidator(DrsApplication):
     """
 
     def validate_directory(self, drs_expression: str,
-                           prefix: str|None = None) -> DrsValidationReport:
+                           prefix: str | None = None) -> DrsValidationReport:
         """
         Validate a DRS directory expression.
 
@@ -112,7 +128,7 @@ class DrsValidator(DrsApplication):
                                          [issue], [])
         return result
 
-    def validate(self, drs_expression: str, drs_type: DrsType|str) -> DrsValidationReport:
+    def validate(self, drs_expression: str, drs_type: DrsType | str) -> DrsValidationReport:
         """
         Validate a DRS expression.
 
@@ -131,14 +147,14 @@ class DrsValidator(DrsApplication):
             case DrsType.DATASET_ID:
                 return self.validate_dataset_id(drs_expression=drs_expression)
             case _:
-                raise RuntimeError(f'unsupported drs type {drs_type}')
+                raise EsgvocDbError(f"unsupported drs type '{drs_type}'")
 
     def _parse(self,
                drs_expression: str,
                separator: str,
-               drs_type: DrsType) -> tuple[list[str]|None,  # terms
+               drs_type: DrsType) -> tuple[list[str] | None,  # terms
                                            list[DrsIssue],  # Errors
-                                           list[DrsIssue]]: # Warnings
+                                           list[DrsIssue]]:  # Warnings
         errors: list[DrsIssue] = list()
         warnings: list[DrsIssue] = list()
         cursor_offset = 0
@@ -160,7 +176,7 @@ class DrsValidator(DrsApplication):
         terms = drs_expression.split(separator)
         if len(terms) < 2:
             errors.append(Unparsable(expected_drs_type=drs_type))
-            return None, errors, warnings # Early exit
+            return None, errors, warnings  # Early exit
         max_term_index = len(terms)
         cursor_position = initial_cursor_position = len(drs_expression) + 1
         has_white_term = False
@@ -178,7 +194,10 @@ class DrsValidator(DrsApplication):
             column = cursor_position+cursor_offset
             if (drs_type == DrsType.DIRECTORY) and (not has_white_term):
                 issue = ExtraSeparator(column=column)
-                warnings.append(issue)
+                if self.pedantic:
+                    errors.append(issue)
+                else:
+                    warnings.append(issue)
             else:
                 issue = ExtraChar(column=column)
                 errors.append(issue)
@@ -188,7 +207,7 @@ class DrsValidator(DrsApplication):
             if not term:
                 column = cursor_position + cursor_offset
                 issue = ExtraSeparator(column=column)
-                if (drs_type != DrsType.DIRECTORY) or self.pedantic or (index == 0):
+                if self.pedantic or drs_type != DrsType.DIRECTORY or index == 0:
                     errors.append(issue)
                 else:
                     warnings.append(issue)
@@ -200,10 +219,10 @@ class DrsValidator(DrsApplication):
                 del terms[index]
             cursor_position -= len_term + 1
 
-        # Mypy doesn't understand that ParsingIssues are DrsIssues...
-        sorted_errors = DrsValidator._sort_parser_issues(errors) # type: ignore
-        sorted_warnings = DrsValidator._sort_parser_issues(warnings) # type: ignore
-        return terms, sorted_errors, sorted_warnings # type: ignore
+        # Mypy doesn't understand that ParsingIssues are DrsIssues...
+        sorted_errors = DrsValidator._sort_parser_issues(errors)  # type: ignore
+        sorted_warnings = DrsValidator._sort_parser_issues(warnings)  # type: ignore
+        return terms, sorted_errors, sorted_warnings  # type: ignore
 
     @staticmethod
     def _sort_parser_issues(issues: list[ParsingIssue]) -> list[ParsingIssue]:
@@ -213,13 +232,9 @@ class DrsValidator(DrsApplication):
         match part.kind:
             case DrsPartKind.COLLECTION:
                 casted_part: DrsCollection = cast(DrsCollection, part)
-                try:
-                    matching_terms = projects.valid_term_in_collection(term,
-                                                                       self.project_id,
-                                                                       casted_part.collection_id)
-                except Exception as e:
-                    msg = f'problem while validating term: {e}.Abort.'
-                    raise APIException(msg) from e
+                matching_terms = projects.valid_term_in_collection(term,
+                                                                   self.project_id,
+                                                                   casted_part.collection_id)
                 if len(matching_terms) > 0:
                     return True
                 else:
@@ -228,7 +243,7 @@ class DrsValidator(DrsApplication):
                 part_casted: DrsConstant = cast(DrsConstant, part)
                 return part_casted.value != term
             case _:
-                raise RuntimeError(f'unsupported DRS specs part type {part.kind}')
+                raise EsgvocDbError(f"unsupported DRS specs part type '{part.kind}'")
 
     def _create_report(self,
                        type: DrsType,
@@ -245,7 +260,7 @@ class DrsValidator(DrsApplication):
                   specs: DrsSpecification) -> DrsValidationReport:
         terms, errors, warnings = self._parse(drs_expression, specs.separator, specs.type)
         if not terms:
-            return self._create_report(specs.type, drs_expression, errors, warnings) # Early exit.
+            return self._create_report(specs.type, drs_expression, errors, warnings)  # Early exit.
         term_index = 0
         term_max_index = len(terms)
         part_index = 0
@@ -259,27 +274,27 @@ class DrsValidator(DrsApplication):
                 part_index += 1
                 matching_code_mapping[part.__str__()] = 0
             elif part.kind == DrsPartKind.CONSTANT or \
-                 cast(DrsCollection, part).is_required:
+                 cast(DrsCollection, part).is_required:  # noqa E127
                 issue: ComplianceIssue = InvalidTerm(term=term,
-                                                      term_position=term_index+1,
-                                                      collection_id_or_constant_value=str(part))
+                                                     term_position=term_index+1,
+                                                     collection_id_or_constant_value=str(part))
                 errors.append(issue)
                 matching_code_mapping[part.__str__()] = 1
                 term_index += 1
                 part_index += 1
-            else: # The part is not required so try to match the term with the next part.
+            else:  # The part is not required so try to match the term with the next part.
                 part_index += 1
                 matching_code_mapping[part.__str__()] = -1
             if term_index == term_max_index:
                 break
         # Cases:
         # - All terms and collections have been processed.
-        # - Not enough term to process all collections.
+        # - Not enough term to process all collections.
         # - Extra terms left whereas all collections have been processed:
         #   + The last collections are required => report extra terms.
         #   + The last collections are not required and these terms were not validated by them.
         #     => Should report error even if the collections are not required.
-        if part_index < part_max_index: # Missing terms.
+        if part_index < part_max_index:  # Missing terms.
             for index in range(part_index, part_max_index):
                 part = specs.parts[index]
                 issue = MissingTerm(collection_id=str(part), collection_position=index+1)
@@ -288,43 +303,17 @@ class DrsValidator(DrsApplication):
                     errors.append(issue)
                 else:
                     warnings.append(issue)
-        elif term_index < term_max_index: # Extra terms.
+        elif term_index < term_max_index:  # Extra terms.
             part_index -= term_max_index - term_index
             for index in range(term_index, term_max_index):
                 term = terms[index]
                 part = specs.parts[part_index]
                 if part.kind != DrsPartKind.CONSTANT           and \
                    (not cast(DrsCollection, part).is_required) and \
-                    matching_code_mapping[part.__str__()] < 0:
+                    matching_code_mapping[part.__str__()] < 0: # noqa E125
                     issue = ExtraTerm(term=term, term_position=index, collection_id=str(part))
                 else:
                     issue = ExtraTerm(term=term, term_position=index, collection_id=None)
                 errors.append(issue)
                 part_index += 1
         return self._create_report(specs.type, drs_expression, errors, warnings)
-
-
-if __name__ == "__main__":
-    project_id = 'cmip6plus'
-    validator = DrsValidator(project_id)
-    drs_expressions = [
-".CMIP6Plus.CMIP.IPSL.  .MIROC6.amip..r2i2p1f2.ACmon.od550aer. ..gn",
-]
-    import time
-    for drs_expression in drs_expressions:
-        start_time = time.perf_counter_ns()
-        report = validator.validate_dataset_id(drs_expression)
-        stop_time = time.perf_counter_ns()
-        print(f'elapsed time: {(stop_time-start_time)/1000000} ms')
-        if report.nb_errors > 0:
-            print(f'error(s): {report.nb_errors}')
-            for error in report.errors:
-                print(error)
-        else:
-            print('error(s): 0')
-        if report.nb_warnings > 0:
-            print(f'warning(s): {report.nb_warnings}')
-            for warning in report.warnings:
-                print(warning)
-        else:
-            print('warning(s): 0')

esgvoc/apps/jsg/cmip6_template.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stac-extensions.github.io/cmip6/v1.0.0/schema.json#",
+  "title": "CMIP6 Extension",
+  "description": "STAC CMIP6 Extension for STAC Items and STAC Collection Summaries.",
+  "type": "object",
+  "required": [
+    "stac_extensions"
+  ],
+  "properties": {
+    "stac_extensions": {
+      "type": "array",
+      "contains": {
+        "const": "https://stac-extensions.github.io/cmip6/v1.0.0/schema.json"
+      }
+    }
+  },
+  "oneOf": [
+    {
+      "$comment": "This is the schema for STAC Items.",
+      "type": "object",
+      "required": [
+        "type",
+        "properties"
+      ],
+      "properties": {
+        "type": {
+          "const": "Feature"
+        },
+        "properties": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/require_any"
+            },
+            {
+              "$ref": "#/definitions/fields"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "$comment": "This is the schema for STAC Collections, or more specifically only Collection Summaries in this case. By default, only checks the existence of the properties, but not the schema of the summaries.",
+      "type": "object",
+      "required": [
+        "type",
+        "summaries"
+      ],
+      "properties": {
+        "type": {
+          "const": "Collection"
+        },
+        "summaries": {
+          "$ref": "#/definitions/require_any"
+        }
+      }
+    }
+  ],
+  "definitions": {
+    "require_any": {
+      "$comment": "Please list all fields here so that we can force the existence of one of them in other parts of the schemas."
+    },
+    "fields": {
+      "$comment": " Don't require fields here, do that above in the corresponding schema.",
+      "type": "object",
+      "properties": {
+      },
+      "patternProperties": {
+        "^(?!cmip6:)": {}
+      },
+      "additionalProperties": false
+    }
+  }
+}

esgvoc/apps/jsg/cmip6plus_template.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stac-extensions.github.io/cmip6plus/v1.0.0/schema.json#",
+  "title": "CMIP6Plus Extension",
+  "description": "STAC CMIP6Plus Extension for STAC Items and STAC Collection Summaries.",
+  "type": "object",
+  "required": [
+    "stac_extensions"
+  ],
+  "properties": {
+    "stac_extensions": {
+      "type": "array",
+      "contains": {
+        "const": "https://stac-extensions.github.io/cmip6plus/v1.0.0/schema.json"
+      }
+    }
+  },
+  "oneOf": [
+    {
+      "$comment": "This is the schema for STAC Items.",
+      "type": "object",
+      "required": [
+        "type",
+        "properties"
+      ],
+      "properties": {
+        "type": {
+          "const": "Feature"
+        },
+        "properties": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/require_any"
+            },
+            {
+              "$ref": "#/definitions/fields"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "$comment": "This is the schema for STAC Collections, or more specifically only Collection Summaries in this case. By default, only checks the existence of the properties, but not the schema of the summaries.",
+      "type": "object",
+      "required": [
+        "type",
+        "summaries"
+      ],
+      "properties": {
+        "type": {
+          "const": "Collection"
+        },
+        "summaries": {
+          "$ref": "#/definitions/require_any"
+        }
+      }
+    }
+  ],
+  "definitions": {
+    "require_any": {
+      "$comment": "Please list all fields here so that we can force the existence of one of them in other parts of the schemas."
+    },
+    "fields": {
+      "$comment": " Don't require fields here, do that above in the corresponding schema.",
+      "type": "object",
+      "properties": {
+      },
+      "patternProperties": {
+        "^(?!cmip6plus:)": {}
+      },
+      "additionalProperties": false
+    }
+  }
+}

esgvoc/apps/jsg/json_schema_generator.py

@@ -0,0 +1,185 @@
+import contextlib
+import json
+from pathlib import Path
+from typing import Iterable
+
+from sqlmodel import Session
+
+from esgvoc.api import projects, search
+from esgvoc.api.project_specs import (
+    GlobalAttributeSpecBase,
+    GlobalAttributeSpecSpecific,
+    GlobalAttributeVisitor,
+)
+from esgvoc.core.constants import DRS_SPECS_JSON_KEY, PATTERN_JSON_KEY
+from esgvoc.core.db.models.project import PCollection, TermKind
+from esgvoc.core.exceptions import EsgvocNotFoundError, EsgvocNotImplementedError
+
+KEY_SEPARATOR = ':'
+JSON_SCHEMA_TEMPLATE_DIR_PATH = Path(__file__).parent
+JSON_SCHEMA_TEMPLATE_FILE_NAME_TEMPLATE = '{project_id}_template.json'
+JSON_INDENTATION = 2
+
+
+def _process_plain(collection: PCollection, selected_field: str) -> list[str]:
+    result: list[str] = list()
+    for term in collection.terms:
+        if selected_field in term.specs:
+            value = term.specs[selected_field]
+            result.append(value)
+        else:
+            raise EsgvocNotFoundError(f'missing key {selected_field} for term {term.id} in ' +
+                                      f'collection {collection.id}')
+    return result
+
+
+def _process_composite(collection: PCollection, universe_session: Session,
+                       project_session: Session) -> str:
+    result = ""
+    for term in collection.terms:
+        _, parts = projects._get_composite_term_separator_parts(term)
+        for part in parts:
+            resolved_term = projects._resolve_term(part, universe_session, project_session)
+            if resolved_term.kind == TermKind.PATTERN:
+                result += resolved_term.specs[PATTERN_JSON_KEY]
+            else:
+                raise EsgvocNotImplementedError(f'{term.kind} term is not supported yet')
+    # Patterns terms are meant to be validated individually.
+    # So their regex are defined as a whole (begins by a ^, ends by a $).
+    # As the pattern is a concatenation of plain or regex, multiple ^ and $ can exist.
+    # The later, must be removed.
+    result = result.replace('^', '').replace('$', '')
+    result = f'^{result}$'
+    return result
+
+
+def _process_pattern(collection: PCollection) -> str:
+    # The generation of the value of the field pattern for the collections with more than one term
+    # is not specified yet.
+    if len(collection.terms) == 1:
+        term = collection.terms[0]
+        return term.specs[PATTERN_JSON_KEY]
+    else:
+        msg = f"unsupported collection of term pattern with more than one term for '{collection.id}'"
+        raise EsgvocNotImplementedError(msg)
+
+
+def _generate_attribute_key(project_id: str, attribute_name) -> str:
+    return f'{project_id}{KEY_SEPARATOR}{attribute_name}'
+
+
+class JsonPropertiesVisitor(GlobalAttributeVisitor, contextlib.AbstractContextManager):
+    def __init__(self, project_id: str) -> None:
+        self.project_id = project_id
+        # Project session can't be None here.
+        self.universe_session: Session = search.get_universe_session()
+        self.project_session: Session = projects._get_project_session_with_exception(project_id)
+        self.collections: dict[str, PCollection] = dict()
+        for collection in projects._get_all_collections_in_project(self.project_session):
+            self.collections[collection.id] = collection
+
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        self.project_session.close()
+        self.universe_session.close()
+        if exception_type is not None:
+            raise exception_value
+        return True
+
+    def _generate_attribute_property(self, attribute_name: str, source_collection: str,
+                                     selected_field: str) -> tuple[str, str | list[str]]:
+        property_value: str | list[str]
+        property_key: str
+        if source_collection not in self.collections:
+            raise EsgvocNotFoundError(f"collection '{source_collection}' referenced by attribute " +
+                                      f"{attribute_name} is not found")
+        collection = self.collections[source_collection]
+        match collection.term_kind:
+            case TermKind.PLAIN:
+                property_value = _process_plain(collection=collection,
+                                                selected_field=selected_field)
+                property_key = 'enum'
+            case TermKind.COMPOSITE:
+                property_value = _process_composite(collection=collection,
+                                                    universe_session=self.universe_session,
+                                                    project_session=self.project_session)
+                property_key = 'pattern'
+            case TermKind.PATTERN:
+                property_value = _process_pattern(collection)
+                property_key = 'pattern'
+            case _:
+                msg = f"unsupported term kind '{collection.term_kind}' " + \
+                      f"for global attribute {attribute_name}"
+                raise EsgvocNotImplementedError(msg)
+        return property_key, property_value
+
+    def visit_base_attribute(self, attribute_name: str, attribute: GlobalAttributeSpecBase) \
+            -> tuple[str, dict[str, str | list[str]]]:
+        attribute_key = _generate_attribute_key(self.project_id, attribute_name)
+        attribute_properties: dict[str, str | list[str]] = dict()
+        attribute_properties['type'] = attribute.value_type.value
+        property_key, property_value = self._generate_attribute_property(attribute_name,
+                                                                         attribute.source_collection,
+                                                                         DRS_SPECS_JSON_KEY)
+        attribute_properties[property_key] = property_value
+        return attribute_key, attribute_properties
+
+    def visit_specific_attribute(self, attribute_name: str, attribute: GlobalAttributeSpecSpecific) \
+            -> tuple[str, dict[str, str | list[str]]]:
+        attribute_key = _generate_attribute_key(self.project_id, attribute_name)
+        attribute_properties: dict[str, str | list[str]] = dict()
+        attribute_properties['type'] = attribute.value_type.value
+        property_key, property_value = self._generate_attribute_property(attribute_name,
+                                                                         attribute.source_collection,
+                                                                         attribute.specific_key)
+        attribute_properties[property_key] = property_value
+        return attribute_key, attribute_properties
+
+
+def _inject_global_attributes(json_root: dict, project_id: str, attribute_names: Iterable[str]) -> None:
+    attribute_properties = list()
+    for attribute_name in attribute_names:
+        attribute_key = _generate_attribute_key(project_id, attribute_name)
+        attribute_properties.append({"required": [attribute_key]})
+    json_root['definitions']['require_any']['anyOf'] = attribute_properties
+
+
+def _inject_properties(json_root: dict, properties: list[tuple]) -> None:
+    for property in properties:
+        json_root['definitions']['fields']['properties'][property[0]] = property[1]
+
+
+def generate_json_schema(project_id: str) -> str:
+    """
+    Generate json schema for the given project.
+
+    :param project_id: The id of the given project.
+    :type project_id: str
+    :returns: The content of a json schema
+    :rtype: str
+    :raises EsgvocNotFoundError: On missing information
+    :raises EsgvocNotImplementedError: On unexpected operations
+    """
+    file_name = JSON_SCHEMA_TEMPLATE_FILE_NAME_TEMPLATE.format(project_id=project_id)
+    template_file_path = JSON_SCHEMA_TEMPLATE_DIR_PATH.joinpath(file_name)
+    if template_file_path.exists():
+        project_specs = projects.get_project(project_id)
+        if project_specs:
+            if project_specs.global_attributes_specs:
+                with open(file=template_file_path, mode='r') as file, \
+                     JsonPropertiesVisitor(project_id) as visitor:
+                    file_content = file.read()
+                    root = json.loads(file_content)
+                    properties: list[tuple[str, dict[str, str | list[str]]]] = list()
+                    for attribute_name, attribute in project_specs.global_attributes_specs.items():
+                        attribute_key, attribute_properties = attribute.accept(attribute_name, visitor)
+                        properties.append((attribute_key, attribute_properties))
+                _inject_properties(root, properties)
+                _inject_global_attributes(root, project_id, project_specs.global_attributes_specs.keys())
+                return json.dumps(root, indent=JSON_INDENTATION)
+            else:
+                raise EsgvocNotFoundError(f"global attributes for the project '{project_id}' " +
+                                          "are not provided")
+        else:
+            raise EsgvocNotFoundError(f"project '{project_id}' is not found")
+    else:
+        raise EsgvocNotFoundError(f"template for project '{project_id}' is not found")