esgvoc 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

esgvoc/apps/drs/validator.py

@@ -1,26 +1,34 @@
 from typing import cast
-from esgvoc.api.project_specs import (ProjectSpecs,
-                               DrsType,
-                               DrsPart,
-                               DrsSpecification,
-                               DrsPartKind,
-                               DrsCollection,
-                               DrsConstant)
+
 import esgvoc.api.projects as projects
 import esgvoc.apps.drs.constants as constants
-from esgvoc.apps.drs.report import (DrsValidationReport,
-                                    DrsIssue,
-                                    ParserIssue,
-                                    ValidationIssue,
-                                    Space,
-                                    Unparsable,
-                                    ExtraSeparator,
-                                    ExtraChar,
-                                    BlankToken,
-                                    InvalidToken,
-                                    ExtraToken,
-                                    MissingToken,
-                                    FileNameExtensionIssue)
+from esgvoc.api.project_specs import (
+    DrsCollection,
+    DrsConstant,
+    DrsPart,
+    DrsPartKind,
+    DrsSpecification,
+    DrsType,
+    ProjectSpecs,
+)
+from esgvoc.apps.drs.report import (
+    BlankTerm,
+    ComplianceIssue,
+    DrsIssue,
+    DrsValidationReport,
+    ExtraChar,
+    ExtraSeparator,
+    ExtraTerm,
+    FileNameExtensionIssue,
+    InvalidTerm,
+    MissingTerm,
+    ParsingIssue,
+    Space,
+    Unparsable,
+    ValidationError,
+    ValidationWarning,
+)
+from esgvoc.core.exceptions import EsgvocDbError, EsgvocNotFoundError
 
 
 class DrsApplication:
@@ -33,7 +41,9 @@ class DrsApplication:
         """The project id."""
         self.pedantic: bool = pedantic
         """Same as the option of GCC: turn warnings into errors. Default False."""
-        project_specs: ProjectSpecs = projects.get_project_specs(project_id)
+        project_specs: ProjectSpecs | None = projects.get_project(project_id)
+        if not project_specs:
+            raise EsgvocNotFoundError(f"unable to find project '{project_id}'")
         for specs in project_specs.drs_specs:
             match specs.type:
                 case DrsType.DIRECTORY:
@@ -46,7 +56,7 @@ class DrsApplication:
                     self.dataset_id_specs: DrsSpecification = specs
                     """The DRS dataset id specs of the project."""
                 case _:
-                    raise ValueError(f'unsupported DRS specs type {specs.type}')
+                    raise EsgvocDbError(f"unsupported DRS specs type '{specs.type}'")
 
     def _get_full_file_name_extension(self) -> str:
         """
@@ -61,38 +71,33 @@ class DrsApplication:
             full_extension = specs.properties[constants.FILE_NAME_EXTENSION_SEPARATOR_KEY] + \
                              specs.properties[constants.FILE_NAME_EXTENSION_KEY]
         else:
-            raise ValueError('missing properties in the DRS file name specifications of the ' +
-                             f'project {self.project_id}')
+            raise EsgvocDbError('missing properties in the DRS file name specifications of the ' +
+                                f"project '{self.project_id}'")
         return full_extension
-    
-    def _get_specs(self, drs_type: DrsType|str) -> DrsSpecification:
-        match drs_type:
-            case DrsType.DIRECTORY:
-                specs = self.directory_specs
-            case DrsType.FILE_NAME:
-                specs = self.file_name_specs
-            case DrsType.DATASET_ID:
-                specs = self.dataset_id_specs
-            case _:
-                raise ValueError(f'unsupported DRS type {drs_type}')
-        return specs
-    
+
+
 class DrsValidator(DrsApplication):
     """
     Valid a DRS directory, dataset id and file name expression against a project.
     """
-   
-    def validate_directory(self, drs_expression: str) -> DrsValidationReport:
+
+    def validate_directory(self, drs_expression: str,
+                           prefix: str | None = None) -> DrsValidationReport:
         """
         Validate a DRS directory expression.
 
         :param drs_expression: A DRS directory expression.
         :type drs_expression: str
+        :param prefix: A directory prefix to be removed from the directory expression.
+        :type prefix: str|None
         :returns: A validation report.
         :rtype: DrsValidationReport
         """
+        if prefix:
+            # Remove prefix if present. Always returns a copy.
+            drs_expression = drs_expression.removeprefix(prefix)
         return self._validate(drs_expression, self.directory_specs)
-    
+
     def validate_dataset_id(self, drs_expression: str) -> DrsValidationReport:
         """
         Validate a DRS dataset id expression.
@@ -123,7 +128,7 @@ class DrsValidator(DrsApplication):
                                          [issue], [])
         return result
 
-    def validate(self, drs_expression: str, drs_type: DrsType|str) -> DrsValidationReport:
+    def validate(self, drs_expression: str, drs_type: DrsType | str) -> DrsValidationReport:
         """
         Validate a DRS expression.
 
@@ -134,15 +139,22 @@ class DrsValidator(DrsApplication):
         :returns: A validation report.
         :rtype: DrsValidationReport
         """
-        specs = self._get_specs(drs_type)
-        return self._validate(drs_expression, specs)
-    
+        match drs_type:
+            case DrsType.DIRECTORY:
+                return self.validate_directory(drs_expression=drs_expression)
+            case DrsType.FILE_NAME:
+                return self.validate_file_name(drs_expression=drs_expression)
+            case DrsType.DATASET_ID:
+                return self.validate_dataset_id(drs_expression=drs_expression)
+            case _:
+                raise EsgvocDbError(f"unsupported drs type '{drs_type}'")
+
     def _parse(self,
                drs_expression: str,
                separator: str,
-               drs_type: DrsType) -> tuple[list[str]|None,  # Tokens
+               drs_type: DrsType) -> tuple[list[str] | None,  # terms
                                            list[DrsIssue],  # Errors
-                                           list[DrsIssue]]: # Warnings
+                                           list[DrsIssue]]:  # Warnings
         errors: list[DrsIssue] = list()
         warnings: list[DrsIssue] = list()
         cursor_offset = 0
@@ -150,7 +162,7 @@ class DrsValidator(DrsApplication):
         start_with_space = drs_expression[0].isspace()
         end_with_space = drs_expression[-1].isspace()
         if start_with_space or end_with_space:
-            issue: ParserIssue = Space()
+            issue: ParsingIssue = Space()
             if self.pedantic:
                 errors.append(issue)
             else:
@@ -161,78 +173,77 @@ class DrsValidator(DrsApplication):
                 cursor_offset = previous_len - len(drs_expression)
             if end_with_space:
                 drs_expression = drs_expression.rstrip()
-        tokens = drs_expression.split(separator)
-        if len(tokens) < 2:
+        terms = drs_expression.split(separator)
+        if len(terms) < 2:
             errors.append(Unparsable(expected_drs_type=drs_type))
-            return None, errors, warnings # Early exit
-        max_token_index = len(tokens)
+            return None, errors, warnings  # Early exit
+        max_term_index = len(terms)
         cursor_position = initial_cursor_position = len(drs_expression) + 1
-        has_white_token = False
-        for index in range(max_token_index-1, -1, -1):
-            token = tokens[index]
-            if (is_white_token := token.isspace()) or (not token):
-                has_white_token = has_white_token or is_white_token
-                cursor_position -= len(token) + 1
-                del tokens[index]
+        has_white_term = False
+        for index in range(max_term_index-1, -1, -1):
+            term = terms[index]
+            if (is_white_term := term.isspace()) or (not term):
+                has_white_term = has_white_term or is_white_term
+                cursor_position -= len(term) + 1
+                del terms[index]
                 continue
             else:
                 break
         if cursor_position != initial_cursor_position:
-            max_token_index = len(tokens)
+            max_term_index = len(terms)
             column = cursor_position+cursor_offset
-            if (drs_type == DrsType.DIRECTORY) and (not has_white_token):
+            if (drs_type == DrsType.DIRECTORY) and (not has_white_term):
                 issue = ExtraSeparator(column=column)
-                warnings.append(issue)
+                if self.pedantic:
+                    errors.append(issue)
+                else:
+                    warnings.append(issue)
             else:
                 issue = ExtraChar(column=column)
                 errors.append(issue)
-        for index in range(max_token_index-1, -1, -1):
-            token = tokens[index]
-            len_token = len(token)
-            if not token:
+        for index in range(max_term_index-1, -1, -1):
+            term = terms[index]
+            len_term = len(term)
+            if not term:
                 column = cursor_position + cursor_offset
                 issue = ExtraSeparator(column=column)
-                if (drs_type != DrsType.DIRECTORY) or self.pedantic or (index == 0):
+                if self.pedantic or drs_type != DrsType.DIRECTORY or index == 0:
                     errors.append(issue)
                 else:
                     warnings.append(issue)
-                del tokens[index]
-            if token.isspace():
-                column = cursor_position + cursor_offset - len_token
-                issue = BlankToken(column=column)
+                del terms[index]
+            if term.isspace():
+                column = cursor_position + cursor_offset - len_term
+                issue = BlankTerm(column=column)
                 errors.append(issue)
-                del tokens[index]
-            cursor_position -= len_token + 1
-        
-        # Mypy doesn't understand that ParserIssues are DrsIssues...
-        sorted_errors = DrsValidator._sort_parser_issues(errors) # type: ignore
-        sorted_warnings = DrsValidator._sort_parser_issues(warnings) # type: ignore
-        return tokens, sorted_errors, sorted_warnings # type: ignore
-    
+                del terms[index]
+            cursor_position -= len_term + 1
+
+        # Mypy doesn't understand that ParsingIssues are DrsIssues...
+        sorted_errors = DrsValidator._sort_parser_issues(errors)  # type: ignore
+        sorted_warnings = DrsValidator._sort_parser_issues(warnings)  # type: ignore
+        return terms, sorted_errors, sorted_warnings  # type: ignore
+
     @staticmethod
-    def _sort_parser_issues(issues: list[ParserIssue]) -> list[ParserIssue]:
+    def _sort_parser_issues(issues: list[ParsingIssue]) -> list[ParsingIssue]:
         return sorted(issues, key=lambda issue: issue.column if issue.column else 0)
 
-    def _validate_token(self, token: str, part: DrsPart) -> bool:
+    def _validate_term(self, term: str, part: DrsPart) -> bool:
         match part.kind:
             case DrsPartKind.COLLECTION:
                 casted_part: DrsCollection = cast(DrsCollection, part)
-                try:
-                    matching_terms = projects.valid_term_in_collection(token,
-                                                                       self.project_id,
-                                                                       casted_part.collection_id)
-                except Exception as e:
-                    msg = f'problem while validating token: {e}.Abort.'
-                    raise ValueError(msg) from e
+                matching_terms = projects.valid_term_in_collection(term,
+                                                                   self.project_id,
+                                                                   casted_part.collection_id)
                 if len(matching_terms) > 0:
                     return True
                 else:
                     return False
             case DrsPartKind.CONSTANT:
                 part_casted: DrsConstant = cast(DrsConstant, part)
-                return part_casted.value != token
+                return part_casted.value != term
             case _:
-                raise ValueError(f'unsupported DRS specs part type {part.kind}')
+                raise EsgvocDbError(f"unsupported DRS specs part type '{part.kind}'")
 
     def _create_report(self,
                        type: DrsType,
@@ -240,93 +251,69 @@ class DrsValidator(DrsApplication):
                        errors: list[DrsIssue],
                        warnings: list[DrsIssue]) -> DrsValidationReport:
         return DrsValidationReport(project_id=self.project_id, type=type,
-                                   expression=drs_expression, errors=errors, warnings=warnings)
+                                   expression=drs_expression,
+                                   errors=cast(list[ValidationError], errors),
+                                   warnings=cast(list[ValidationWarning], warnings))
 
     def _validate(self,
                   drs_expression: str,
                   specs: DrsSpecification) -> DrsValidationReport:
-        tokens, errors, warnings = self._parse(drs_expression, specs.separator, specs.type)
-        if not tokens:
-            return self._create_report(specs.type, drs_expression, errors, warnings) # Early exit.
-        token_index = 0
-        token_max_index = len(tokens)
+        terms, errors, warnings = self._parse(drs_expression, specs.separator, specs.type)
+        if not terms:
+            return self._create_report(specs.type, drs_expression, errors, warnings)  # Early exit.
+        term_index = 0
+        term_max_index = len(terms)
         part_index = 0
         part_max_index = len(specs.parts)
         matching_code_mapping = dict()
         while part_index < part_max_index:
-            token = tokens[token_index]
+            term = terms[term_index]
             part = specs.parts[part_index]
-            if self._validate_token(token, part):
-                token_index += 1
+            if self._validate_term(term, part):
+                term_index += 1
                 part_index += 1
                 matching_code_mapping[part.__str__()] = 0
             elif part.kind == DrsPartKind.CONSTANT or \
-                 cast(DrsCollection, part).is_required:
-                issue: ValidationIssue = InvalidToken(token=token,
-                                                      token_position=token_index+1,
-                                                      collection_id_or_constant_value=str(part))
+                 cast(DrsCollection, part).is_required:  # noqa E127
+                issue: ComplianceIssue = InvalidTerm(term=term,
+                                                     term_position=term_index+1,
+                                                     collection_id_or_constant_value=str(part))
                 errors.append(issue)
                 matching_code_mapping[part.__str__()] = 1
-                token_index += 1
+                term_index += 1
                 part_index += 1
-            else: # The part is not required so try to match the token with the next part.
+            else:  # The part is not required so try to match the term with the next part.
                 part_index += 1
                 matching_code_mapping[part.__str__()] = -1
-            if token_index == token_max_index:
+            if term_index == term_max_index:
                 break
         # Cases:
-        # - All tokens and collections have been processed.
-        # - Not enough token to process all collections.
-        # - Extra tokens left whereas all collections have been processed:
-        #   + The last collections are required => report extra tokens.
-        #   + The last collections are not required and these tokens were not validated by them.
+        # - All terms and collections have been processed.
+        # - Not enough term to process all collections.
+        # - Extra terms left whereas all collections have been processed:
+        #   + The last collections are required => report extra terms.
+        #   + The last collections are not required and these terms were not validated by them.
         #     => Should report error even if the collections are not required.
-        if part_index < part_max_index: # Missing tokens.
+        if part_index < part_max_index:  # Missing terms.
             for index in range(part_index, part_max_index):
                 part = specs.parts[index]
-                issue = MissingToken(collection_id=str(part), collection_position=index+1)
+                issue = MissingTerm(collection_id=str(part), collection_position=index+1)
                 if part.kind == DrsPartKind.CONSTANT or \
                    cast(DrsCollection, part).is_required:
                     errors.append(issue)
                 else:
                     warnings.append(issue)
-        elif token_index < token_max_index: # Extra tokens.
-            part_index -= token_max_index - token_index
-            for index in range(token_index, token_max_index):
-                token = tokens[index]
+        elif term_index < term_max_index:  # Extra terms.
+            part_index -= term_max_index - term_index
+            for index in range(term_index, term_max_index):
+                term = terms[index]
                 part = specs.parts[part_index]
                 if part.kind != DrsPartKind.CONSTANT           and \
                    (not cast(DrsCollection, part).is_required) and \
-                    matching_code_mapping[part.__str__()] < 0:
-                    issue = ExtraToken(token=token, token_position=index, collection_id=str(part))
+                    matching_code_mapping[part.__str__()] < 0: # noqa E125
+                    issue = ExtraTerm(term=term, term_position=index, collection_id=str(part))
                 else:
-                    issue = ExtraToken(token=token, token_position=index, collection_id=None)
+                    issue = ExtraTerm(term=term, term_position=index, collection_id=None)
                 errors.append(issue)
                 part_index += 1
         return self._create_report(specs.type, drs_expression, errors, warnings)
-
-
-if __name__ == "__main__":
-    project_id = 'cmip6plus'
-    validator = DrsValidator(project_id)
-    drs_expressions = [
-".CMIP6Plus.CMIP.IPSL.  .MIROC6.amip..r2i2p1f2.ACmon.od550aer. ..gn",
-]
-    import time
-    for drs_expression in drs_expressions:
-        start_time = time.perf_counter_ns()
-        report = validator.validate_dataset_id(drs_expression)
-        stop_time = time.perf_counter_ns()
-        print(f'elapsed time: {(stop_time-start_time)/1000000} ms')
-        if report.nb_errors > 0:
-            print(f'error(s): {report.nb_errors}')
-            for error in report.errors:
-                print(error)
-        else:
-            print('error(s): 0')
-        if report.nb_warnings > 0:
-            print(f'warning(s): {report.nb_warnings}')
-            for warning in report.warnings:
-                print(warning)
-        else:
-            print('warning(s): 0')

esgvoc/cli/drs.py

@@ -1,13 +1,16 @@
-from esgvoc.apps.drs.generator import DrsGenerator
-from esgvoc.apps.drs.report import DrsValidationReport, DrsGeneratorReport
-from esgvoc.apps.drs.validator import DrsValidator
+import shlex
 import sys
+from typing import List, Optional
+
 import typer
 from rich.console import Console
 from rich.table import Table
-from typing import List, Optional
+
 import esgvoc.api as ev
-import shlex
+from esgvoc.apps.drs.generator import DrsGenerator
+from esgvoc.apps.drs.report import DrsGenerationReport, DrsValidationReport
+from esgvoc.apps.drs.validator import DrsValidator
+from esgvoc.core.exceptions import EsgvocValueError
 
 app = typer.Typer()
 console = Console()
@@ -35,6 +38,10 @@ def drsvalid(
     file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS validation inputs, one per line in the form <project> <drstype> <string>"),
     verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed validation results"),
     output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the DRS entries validation"),
+    rm_prefix: Optional[str] = typer.Option(None,"-p","--prefix", help="Remove given prefix from all checked directory"),
+    pedantic: Optional[bool] = typer.Option(False,"-e","--enforce", help="Enable pedantic mode, enforcing strict compliance, mean that warnings are now errors.")
+
+
 
 ) -> List[DrsValidationReport]:
     """
@@ -88,17 +95,21 @@ def drsvalid(
 
         string = entries[i]
         i += 1
-        validator = DrsValidator(current_project)
+        validator = DrsValidator(current_project, pedantic=pedantic)
         report = None
         match current_drs_type:
             case "filename":
                 report = validator.validate_file_name(string)
             case "directory":
-                report = validator.validate_directory(string)
+                if rm_prefix:
+                    prefix = rm_prefix+"/" if rm_prefix[-1]!="/" else ""
+                else:
+                    prefix=None
+                report = validator.validate_directory(string, prefix)
             case "dataset":
                 report = validator.validate_dataset_id(string)
             case _:
-                raise RuntimeError("drstype is not known")
+                raise EsgvocValueError(f"unsupported drs type '{current_drs_type}'")
         reports.append(report)
 
     if verbose:
@@ -111,7 +122,7 @@ def drsvalid(
 
         for report in reports:
             entry = str(report.expression)
-            proj_and_type = str(report.project_id) + " " + report.type + " "  
+            proj_and_type = str(report.project_id) + " " + report.type + " "
             warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
             errors = "\n".join(["⚠️ " + str(error) for error in report.errors])
             valid = "✅ Valid" if report else "❌ Invalid"
@@ -136,16 +147,16 @@ def drsvalid(
 
 @app.command()
 def drsgen(
-    drs_entries: Optional[List[str]] = typer.Argument(None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_tokens>"),
-    file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_tokens>"),
+    drs_entries: Optional[List[str]] = typer.Argument(None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_terms>"),
+    file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_terms>"),
     verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed generation results"),
     output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the generated DRS entries"),
-) -> List[DrsGeneratorReport]:
+) -> List[DrsGenerationReport]:
     """
-    Generates DRS strings for a specific project and type based on input bag of tokens.
+    Generates DRS strings for a specific project and type based on input bag of terms.
 
     Args:
-        drs_entries (Optional[List[str]]): A list of inputs in the form <project> <drstype> <bag_of_tokens>.
+        drs_entries (Optional[List[str]]): A list of inputs in the form <project> <drstype> <bag_of_terms>.
         file (Optional[typer.FileText]): File containing DRS generation inputs, one per line.
         verbose (bool): If true, prints detailed generation results.
         output (Optional[str]): File path to save the generated DRS entries.
@@ -189,21 +200,21 @@ def drsgen(
         if current_drs_type is None:
             raise typer.BadParameter(f"Invalid drs_type: {entries[i]}")
 
-        bag_of_tokens = entries[i]
-        bag_of_tokens = set(entries[i].split(" "))
+        bag_of_terms = entries[i]
+        bag_of_terms = set(entries[i].split(" "))
         i += 1
 
         generator = DrsGenerator(current_project)
         report = None
         match current_drs_type:
             case "filename":
-                report = generator.generate_file_name_from_bag_of_tokens(bag_of_tokens)
+                report = generator.generate_file_name_from_bag_of_terms(bag_of_terms)
             case "directory":
-                report = generator.generate_directory_from_bag_of_tokens(bag_of_tokens)
+                report = generator.generate_directory_from_bag_of_terms(bag_of_terms)
             case "dataset":
-                report = generator.generate_dataset_id_from_bag_of_tokens(bag_of_tokens)
+                report = generator.generate_dataset_id_from_bag_of_terms(bag_of_terms)
             case _:
-                raise RuntimeError("drstype is not known")
+                raise EsgvocValueError(f"unsupported drs type '{current_drs_type}'")
         generated_reports.append(report)
 
     if verbose:
@@ -216,7 +227,7 @@ def drsgen(
             entry = str(report.mapping_used)
             warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
             errors = "\n".join([f"🔍 {error}" for error in report.errors])
-            result = report.computed_drs_expression
+            result = report.generated_drs_expression
             table.add_row(entry, warnings, errors, result)
             table.add_row("----", "----", "----", "----")
             if table.columns[3].width is not None and len(result) > table.columns[3].width: