bam-masterdata 0.4.0__tar.gz → 0.4.2__tar.gz

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "python.defaultInterpreterPath": "~/Miniconda3/python.exe",
+  "python.defaultInterpreterPath": ".venv/bin/python",
   "python.terminal.activateEnvInCurrentTerminal": true,
   "editor.rulers": [90],
   "editor.renderWhitespace": "all",

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bam-masterdata
-Version: 0.4.0
+Version: 0.4.2
 Summary: The BAM Data Store utility tools and masterdata models wrapped in a Python package with a front-end web interface.
 Author-email: BAM Data Store <datastore@bam.de>
 Maintainer-email: "Jose M. Pizarro" <jose.pizarro-blanco@bam.de>

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.0'
-__version_tuple__ = version_tuple = (0, 4, 0)
+__version__ = version = '0.4.2'
+__version_tuple__ = version_tuple = (0, 4, 2)

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata/checker/checker.py

@@ -61,7 +61,7 @@ class MasterdataChecker:
             mode in ["self", "validate", "compare", "all", "individual"]
             and self.current_model is None
         ):
-            self.logger.info("Current model is missing. Loading now...")
+            self.logger.info("Current model is missing. Loading it from local files.")
             self.load_current_model()
 
         if (

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata/checker/masterdata_validator.py

@@ -35,7 +35,7 @@ class MasterdataValidator:
          Returns:
             dict: Validation results.
         """
-        self.logger.info("Starting validation process...", mode=mode)
+        self.logger.debug("Starting validation process...", mode=mode)
 
         # Reset validation results before running checks
         self.validation_results = {
@@ -45,28 +45,28 @@ class MasterdataValidator:
         }
 
         if mode in ["self", "all", "validate"]:
-            self.logger.info("Validating current model...")
+            self.logger.debug("Validating current model...")
             self._validate_model(self.current_model)
             self._extract_log_messages(
                 self.current_model, self.validation_results["current_model"]
             )
 
         if mode in ["incoming", "all", "validate"]:
-            self.logger.info("Validating new entities...")
+            self.logger.debug("Validating new entities...")
             self._validate_model(self.new_entities)
             self._extract_log_messages(
                 self.new_entities, self.validation_results["incoming_model"]
             )
 
         if mode in ["compare", "all"]:
-            self.logger.info("Comparing new entities with current model...")
+            self.logger.debug("Comparing new entities with current model...")
             self._compare_with_current_model(mode=mode)
             self._extract_log_messages(
                 self.new_entities, self.validation_results["comparisons"]
             )
 
         if mode == "individual":
-            self.logger.info(
+            self.logger.debug(
                 "Validating new entities and comparing them with current model..."
             )
             self.validation_results = {
@@ -318,7 +318,7 @@ class MasterdataValidator:
         """
         Compare new entities against the current model using validation rules.
         """
-        self.logger.info("Starting comparison with the current model...")
+        self.logger.debug("Starting comparison with the current model...")
 
         new_entity = False
 

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata/cli/cli.py

@@ -11,7 +11,7 @@ from decouple import config as environ
 from openpyxl import Workbook
 from rdflib import Graph
 
-from bam_masterdata.checker import MasterdataChecker, no_validation_errors
+from bam_masterdata.checker import MasterdataChecker
 from bam_masterdata.cli.entities_to_excel import entities_to_excel
 from bam_masterdata.cli.entities_to_rdf import entities_to_rdf
 from bam_masterdata.cli.fill_masterdata import MasterdataCodeGenerator
@@ -357,41 +357,22 @@ def export_to_rdf(force_delete, python_path, export_dir):
     )
 
 
-@cli.command(
-    name="checker",
-    help="Checks the files specified in the tag `--file-path` with respect to the ones specified in `--datamodel-path`.",
-)
-@click.option(
-    "--file-path",
-    "file_path",  # alias
-    type=click.Path(exists=True),
-    required=True,
-    help="""The path to the file or directory containing Python modules or the Excel file to be checked.""",
-)
-@click.option(
-    "--mode",
-    "mode",  # alias
-    type=click.Choice(
-        ["self", "incoming", "validate", "compare", "all", "individual"],
-        case_sensitive=False,
-    ),
-    default="all",
-    help="""Specify the mode for the checker. Options are:
-    "self" -> Validate only the current data model.
-    "incoming" -> Validate only the new entity structure.
-    "validate" -> Validate both the current model and new entities.
-    "compare" -> Compare new entities against the current model.
-    "all" -> Run all validations and comparison. (Default).
-    "individual" -> Run individual repositories validations.""",
-)
-@click.option(
-    "--datamodel-path",
-    "datamodel_path",  # alias
-    type=click.Path(exists=True, dir_okay=True),
-    default=DATAMODEL_DIR,
-    help="""Path to the directory containing the Python modules defining the datamodel (defaults to './bam_masterdata/datamodel/').""",
-)
-def checker(file_path, mode, datamodel_path):
+def run_checker(file_path: str, mode: str = "all", datamodel_path: str = DATAMODEL_DIR):
+    """
+
+    Run the masterdata checker on the specified file path and mode.
+
+    Args:
+        file_path (str): The path to the file or directory containing Python modules or the Excel file to be checked.
+        mode (str, optional): The mode for the checker. Defaults to "all". Options are:
+            "self" -> Validate only the current data model.
+            "incoming" -> Validate only the new entity structure.
+            "validate" -> Validate both the current model and new entities.
+            "compare" -> Compare new entities against the current model.
+            "all" -> Run all validations and comparison. (Default).
+            "individual" -> Run individual repositories validations.
+        datamodel_path (str, optional): Path to the directory containing the Python modules defining the datamodel. Defaults to DATAMODEL_DIR.
+    """
     # Instantiate the checker class and run validation
     checker = MasterdataChecker()
 
@@ -449,13 +430,43 @@ def checker(file_path, mode, datamodel_path):
                     f"There are problems in the individual repositories when comparing them with respect to bam-masterdata for entity {entity} that need to be solved"
                 )
 
-    # Check if no problems were found
-    if no_validation_errors(validation_results):
-        click.echo("No problems found in the datamodel and incoming model.")
-    else:
-        raise click.ClickException(
-            "Problems found in the datamodel. Please, check the logs."
-        )
+
+@cli.command(
+    name="checker",
+    help="Checks the files specified in the tag `--file-path` with respect to the ones specified in `--datamodel-path`.",
+)
+@click.option(
+    "--file-path",
+    "file_path",  # alias
+    type=click.Path(exists=True),
+    required=True,
+    help="""The path to the file or directory containing Python modules or the Excel file to be checked.""",
+)
+@click.option(
+    "--mode",
+    "mode",  # alias
+    type=click.Choice(
+        ["self", "incoming", "validate", "compare", "all", "individual"],
+        case_sensitive=False,
+    ),
+    default="all",
+    help="""Specify the mode for the checker. Options are:
+    "self" -> Validate only the current data model.
+    "incoming" -> Validate only the new entity structure.
+    "validate" -> Validate both the current model and new entities.
+    "compare" -> Compare new entities against the current model.
+    "all" -> Run all validations and comparison. (Default).
+    "individual" -> Run individual repositories validations.""",
+)
+@click.option(
+    "--datamodel-path",
+    "datamodel_path",  # alias
+    type=click.Path(exists=True, dir_okay=True),
+    default=DATAMODEL_DIR,
+    help="""Path to the directory containing the Python modules defining the datamodel (defaults to './bam_masterdata/datamodel/').""",
+)
+def checker(file_path, mode, datamodel_path):
+    run_checker(file_path=file_path, mode=mode, datamodel_path=datamodel_path)
 
 
 @cli.command(

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata/excel/excel_to_entities.py

@@ -1,4 +1,3 @@
-import json
 import os
 import re
 from typing import TYPE_CHECKING, Any, Optional, Union
@@ -12,6 +11,7 @@ import openpyxl
 
 from bam_masterdata.logger import logger
 from bam_masterdata.metadata.definitions import DataType
+from bam_masterdata.utils import VALIDATION_RULES_DIR
 
 
 class MasterdataExcelExtractor:
@@ -29,7 +29,7 @@ class MasterdataExcelExtractor:
         if not MasterdataExcelExtractor.VALIDATION_RULES:
             self.VALIDATION_RULES = load_validation_rules(
                 self.logger,
-                file_path="./bam_masterdata/checker/validation_rules/excel_validation_rules.json",
+                os.path.join(VALIDATION_RULES_DIR, "excel_validation_rules.json"),
             )
 
     def index_to_excel_column(self, index: int) -> str:
@@ -414,13 +414,21 @@ class MasterdataExcelExtractor:
             "Vocabulary code",
             "Metadata",
             "Dynamic script",
-            "Unique",
-            "Internal assignment",
+            # ! these are not used
+            # "Unique",
+            # "Internal assignment",
         ]
 
         # Determine the header row index
         header_index = start_index_row + 3
-        row_headers = [cell.value for cell in sheet[header_index]]
+        row_headers = [(cell.value, cell.coordinate) for cell in sheet[header_index]]
+        # And store how many properties are for the entity
+        n_properties = last_non_empty_row - header_index
+        if n_properties < 0:
+            self.logger.error(
+                f"No properties found for the entity in sheet {sheet.title} starting at row {start_index_row}."
+            )
+            return property_dict
 
         # Initialize a dictionary to store extracted columns
         extracted_columns: dict[str, list] = {term: [] for term in expected_terms}
@@ -428,8 +436,8 @@ class MasterdataExcelExtractor:
             extracted_columns["row_location"] = []
 
         # Extract columns for each expected term
-        for term in expected_terms:
-            if term not in row_headers:
+        for i, (term, coordinate) in enumerate(row_headers):
+            if term not in expected_terms:
                 log_func = (
                     self.logger.warning
                     if term
@@ -439,48 +447,55 @@ class MasterdataExcelExtractor:
                         "Section",
                         "Metadata",
                         "Dynamic script",
-                        "Unique",
-                        "Internal assignment",
                         "Vocabulary code",
+                        # ! these are not used
+                        # "Unique",
+                        # "Internal assignment",
                     )
                     else self.logger.error
                 )
                 log_func(f"'{term}' not found in the properties headers.", term=term)
                 continue
 
-            # Get column index and Excel letter
-            term_index = row_headers.index(term) + 1
-            term_letter = self.index_to_excel_column(term_index)
+            # Excel column letter from the coordinate
+            term_letter = coordinate[0]
 
             # Extract values from the column
-            for i in range(header_index + 1, last_non_empty_row + 1):
-                cell = sheet[term_letter + str(i)]
+            for cell_property in sheet[term_letter][header_index:last_non_empty_row]:
                 extracted_columns[term].append(
-                    self.process_term(term, cell.value, cell.coordinate, sheet.title)
+                    self.process_term(
+                        term, cell_property.value, cell_property.coordinate, sheet.title
+                    )
                 )
                 if self.row_cell_info:
-                    extracted_columns["row_location"].append(term_letter + str(i))
-            # for cell in sheet[term_letter][header_index:last_non_empty_row]:
-            #     extracted_columns[term].append(
-            #         self.process_term(term, cell.value, cell.coordinate, sheet.title)
-            #     )
+                    extracted_columns["row_location"].append(cell_property.coordinate)
 
         # Combine extracted values into a dictionary
-        for i in range(len(extracted_columns["Code"])):
-            property_dict[extracted_columns["Code"][i]] = {
-                "permId": extracted_columns["Code"][i],
-                "code": extracted_columns["Code"][i],
-                "description": extracted_columns["Description"][i],
-                "section": extracted_columns["Section"][i],
-                "mandatory": extracted_columns["Mandatory"][i],
-                "show_in_edit_views": extracted_columns["Show in edit views"][i],
-                "label": extracted_columns["Property label"][i],
-                "dataType": extracted_columns["Data type"][i],
-                "vocabularyCode": extracted_columns["Vocabulary code"][i],
-            }
+        for i in range(n_properties):
+            code = extracted_columns.get("Code", [])
+            if not code:
+                self.logger.error(
+                    f"'Code' not found in the properties headers for sheet {sheet.title}."
+                )
+                return property_dict
+            code = code[i]
+            property_dict[code] = {"permId": code, "code": code}
+            for key, pybis_val in {
+                "Description": "description",
+                "Section": "section",
+                "Mandatory": "mandatory",
+                "Show in edit views": "show_in_edit_views",
+                "Property label": "label",
+                "Data type": "dataType",
+                "Vocabulary code": "vocabularyCode",
+            }.items():
+                data_column = extracted_columns.get(key, [])
+                if not data_column:
+                    continue
+                property_dict[code][pybis_val] = data_column[i]
             if self.row_cell_info:
-                property_dict[extracted_columns["Code"][i]]["row_location"] = (
-                    extracted_columns["row_location"][i]
+                property_dict[code]["row_location"] = (
+                    extracted_columns.get("row_location")[i],
                 )
             # Only add optional fields if they exist in extracted_columns
             optional_fields = [

bam_masterdata-0.4.2/bam_masterdata/validation_rules/all_validation_rules.json

@@ -0,0 +1,76 @@
+{
+  "code": {
+    "pattern": "^\\$?[A-Za-z0-9_.]+$"
+  },
+  "description": {
+    "pattern": ".*",
+    "is_description": true
+  },
+  "validation_script": {
+    "pattern": "^[A-Za-z0-9_\\.]+$",
+    "allow_empty": true
+  },
+  "auto_generated_codes": {
+    "is_bool": true
+  },
+  "main_dataset_pattern": {
+    "pattern": ".*",
+    "allow_empty": true
+  },
+  "main_dataset_path": {
+    "pattern": ".*",
+    "allow_empty": true
+  },
+  "property_label": {
+    "pattern": ".*"
+  },
+  "data_type": {
+    "is_data": true
+  },
+  "vocabulary_code": {
+    "pattern": "^\\$?[A-Za-z0-9_.]+$",
+    "allow_empty": true
+  },
+  "object_code": {
+    "pattern": "^\\$?[A-Za-z0-9_.]+$",
+    "allow_empty": true
+  },
+  "metadata": {
+    "pattern": ".*",
+    "allow_empty": true
+  },
+  "url_template": {
+    "pattern": "https?://(?:www\\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+",
+    "is_url": true,
+    "allow_empty": true
+  },
+  "dynamic_script": {
+    "pattern": "^[A-Za-z0-9_\\.]+$",
+    "allow_empty": true
+  },
+  "mandatory": {
+    "is_bool": true
+  },
+  "show_in_edit_views": {
+    "is_bool": true
+  },
+  "section": {
+    "pattern": ".*"
+  },
+  "unique": {
+    "is_bool": true,
+    "allow_empty": true
+  },
+  "internal_assignment": {
+    "pattern": ".*",
+    "allow_empty": true
+  },
+  "generated_code_prefix": {
+    "pattern": ".*",
+    "extra_validation": "is_reduced_version"
+  },
+  "official": {
+    "is_bool": true,
+    "allow_empty": true
+  }
+}

bam_masterdata-0.4.2/bam_masterdata/validation_rules/validation_rules.json

@@ -0,0 +1,109 @@
+{
+  "defs_validation": {
+    "code": {
+      "pattern": "^\\$?[A-Za-z0-9_.-]+$"
+    },
+    "description": {
+      "pattern": ".*",
+      "allow_empty": true,
+      "is_description": true
+    },
+    "validation_script": {
+      "pattern": "^[A-Za-z0-9_\\.]+$",
+      "allow_empty": true
+    },
+    "generated_code_prefix": {
+      "pattern": ".*",
+      "extra_validation": "is_reduced_version"
+    },
+    "main_dataset_pattern": {
+      "pattern": ".*",
+      "allow_empty": true
+    },
+    "main_dataset_path": {
+      "pattern": ".*",
+      "allow_empty": true
+    },
+      "auto_generated_codes": {
+      "is_bool": true
+    },
+    "url_template": {
+      "pattern": "https?://(?:www\\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+",
+      "is_url": true,
+      "allow_empty": true
+    }
+  },
+  "properties_validation": {
+    "code": {
+      "pattern": "^\\$?[A-Za-z0-9_.-]+$"
+    },
+    "description": {
+      "pattern": ".*",
+      "allow_empty": true,
+      "is_description": true
+    },
+    "property_label": {
+      "pattern": ".*"
+    },
+    "data_type": {
+      "is_data": true
+    },
+    "vocabulary_code": {
+      "pattern": "^\\$?[A-Za-z0-9_.]+$",
+      "allow_empty": true
+    },
+    "object_code": {
+      "pattern": "^\\$?[A-Za-z0-9_.]+$",
+      "allow_empty": true
+    },
+    "metadata": {
+      "pattern": ".*",
+      "allow_empty": true
+    },
+    "dynamic_script": {
+      "pattern": "^[A-Za-z0-9_\\.]+$",
+      "allow_empty": true
+    },
+    "mandatory": {
+      "is_bool": true
+    },
+    "show_in_edit_views": {
+      "is_bool": true
+    },
+    "section": {
+      "pattern": "^([A-Z][a-z]*|[A-Z]+)( ([A-Z][a-z]*|[A-Z]+))*$",
+      "is_section": true,
+      "allow_empty": true
+    },
+    "unique": {
+      "is_bool": true,
+      "allow_empty": true
+    },
+    "internal_assignment": {
+      "pattern": ".*",
+      "allow_empty": true
+    }
+  },
+  "terms_validation": {
+    "code": {
+      "pattern": "^\\$?[A-Za-z0-9_.-]+$"
+    },
+    "description": {
+      "pattern": ".*",
+      "is_description": true,
+      "allow_empty": true
+    },
+    "url_template": {
+      "pattern": "https?://(?:www\\.)?[a-zA-Z0-9-._~:/?#@!$&'()*+,;=%]+",
+      "is_url": true,
+      "allow_empty": true
+    },
+    "label": {
+      "pattern": ".*"
+    },
+    "official": {
+      "is_bool": true,
+      "allow_empty": true
+    }
+  }
+}

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bam-masterdata
-Version: 0.4.0
+Version: 0.4.2
 Summary: The BAM Data Store utility tools and masterdata models wrapped in a Python package with a front-end web interface.
 Author-email: BAM Data Store <datastore@bam.de>
 Maintainer-email: "Jose M. Pizarro" <jose.pizarro-blanco@bam.de>

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/bam_masterdata.egg-info/SOURCES.txt

@@ -50,6 +50,7 @@ bam_masterdata/openbis/login.py
 bam_masterdata/utils/__init__.py
 bam_masterdata/utils/paths.py
 bam_masterdata/utils/utils.py
+bam_masterdata/validation_rules/all_validation_rules.json
 bam_masterdata/validation_rules/excel_validation_rules.json
 bam_masterdata/validation_rules/validation_rules.json
 docs/index.md

{bam_masterdata-0.4.0 → bam_masterdata-0.4.2}/tests/excel/test_excel_to_entities.py

@@ -711,8 +711,6 @@ def test_process_entity(
                 "Vocabulary code",
                 "Metadata",
                 "Dynamic script",
-                "Unique",
-                "Internal assignment",
             ],
             [
                 [
@@ -726,8 +724,6 @@ def test_process_entity(
                     "VOCAB_ABC",
                     "Meta info",
                     "Script example",
-                    "True",
-                    "Internal logic",
                 ],
                 [
                     "PROP_002",
@@ -740,8 +736,6 @@ def test_process_entity(
                     "VOCAB_DEF",
                     None,  # Metadata not provided
                     None,  # Dynamic script not provided
-                    None,  # Unique
-                    None,  # Internal assignment not provided
                 ],
             ],
             6,
@@ -758,8 +752,6 @@ def test_process_entity(
                     "vocabularyCode": "VOCAB_ABC",
                     "metadata": "Meta info",
                     "dynamic_script": "Script example",
-                    "unique": "True",
-                    "internal_assignment": "Internal logic",
                 },
                 "PROP_002": {
                     "permId": "PROP_002",
@@ -773,101 +765,11 @@ def test_process_entity(
                     "vocabularyCode": "VOCAB_DEF",
                     "metadata": None,
                     "dynamic_script": None,
-                    "unique": None,
-                    "internal_assignment": None,
                 },
             },
             None,
             None,
         ),
-        # Missing Header: "Data type"
-        (
-            [
-                "Code",
-                "Description",
-                "Mandatory",
-                "Show in edit views",
-                "Section",
-                "Property label",
-                "Vocabulary code",
-                "Metadata",
-                "Dynamic script",
-                "Unique",
-                "Internal assignment",
-            ],  # Missing "Data type"
-            [
-                [
-                    "PROP_001",
-                    "Sample description",
-                    "True",
-                    "False",
-                    "General",
-                    "Property Label",
-                    "VOCAB_ABC",
-                    "Meta info",
-                    "Script example",
-                    "True",
-                    "Internal logic",
-                ]
-            ],
-            4,
-            {},
-            "'Data type' not found in the properties headers.",
-            "error",
-        ),
-        # Invalid Data Type
-        (
-            [
-                "Code",
-                "Description",
-                "Mandatory",
-                "Show in edit views",
-                "Section",
-                "Property label",
-                "Data type",
-                "Vocabulary code",
-                "Metadata",
-                "Dynamic script",
-                "Unique",
-                "Internal assignment",
-            ],
-            [
-                [
-                    "PROP_003",
-                    "Description",
-                    "True",
-                    "False",
-                    "General",
-                    "Property Label",
-                    "INVALID_TYPE",
-                    "VOCAB_ABC",
-                    "Meta info",
-                    "Script example",
-                    "True",
-                    "Internal logic",
-                ]
-            ],
-            5,
-            {
-                "PROP_003": {
-                    "permId": "PROP_003",
-                    "code": "PROP_003",
-                    "description": "Description",
-                    "section": "General",
-                    "mandatory": True,
-                    "show_in_edit_views": False,
-                    "label": "Property Label",
-                    "dataType": "INVALID_TYPE",
-                    "vocabularyCode": "VOCAB_ABC",
-                    "metadata": "Meta info",
-                    "dynamic_script": "Script example",
-                    "unique": "True",
-                    "internal_assignment": "Internal logic",
-                }
-            },
-            f"Invalid data type value found in the Data type column at position G5 in TestSheet.The Data Type should be one of the following: {[dt.value for dt in DataType]}",
-            "error",
-        ),
         # Missing Code (Should be ignored)
         (
             [
@@ -880,8 +782,6 @@ def test_process_entity(
                 "Vocabulary code",
                 "Metadata",
                 "Dynamic script",
-                "Unique",
-                "Internal assignment",
             ],  # Missing "Code"
             [
                 [
@@ -894,13 +794,11 @@ def test_process_entity(
                     "VOCAB_ABC",
                     "Meta info",
                     "Script example",
-                    "True",
-                    "Internal logic",
                 ]
             ],
-            4,
+            5,
             {},
-            "'Code' not found in the properties headers.",
+            "'Code' not found in the properties headers for sheet TestSheet.",
             "error",
         ),
     ],