bam-masterdata 0.3.1__tar.gz → 0.4.0__tar.gz

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/.github/workflows/actions.yml

@@ -34,10 +34,10 @@ jobs:
         uv pip install -e '.[dev]'
     - name: mypy
       run: |
-        python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional --exclude dependencies bam_masterdata tests
+        python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional bam_masterdata tests
     - name: Test with pytest
       run: |
-        python -m pytest -sv --ignore=bam_masterdata/dependencies tests
+        python -m pytest -sv --ignore=tests/cli --ignore=tests/metadata/test_entities_dict.py tests
   build-and-install:
     runs-on: ubuntu-latest
     steps:
@@ -62,11 +62,11 @@ jobs:
         - uses: actions/checkout@v4
         - uses: chartboost/ruff-action@v1
           with:
-            args: "check . --exclude bam_masterdata/dependencies"
+            args: "check ."
   ruff-formatting:
       runs-on: ubuntu-latest
       steps:
         - uses: actions/checkout@v4
         - uses: chartboost/ruff-action@v1
           with:
-            args: "format . --check --verbose --exclude bam_masterdata/dependencies"
+            args: "format . --check --verbose"

bam_masterdata-0.4.0/AUTHORS

@@ -0,0 +1,23 @@
+# This is the list of BAM Masterdata's significant contributors.
+#
+# This does not necessarily list everyone who has contributed code,
+# especially since many employees of one corporation may be contributing to the
+# definition of object types, property types, and vocabularies in the Masterdata
+# datamodel.
+# To see the full list of code contributors, see the revision history in
+# source control.
+
+- Angela Ariza de Schellenberger
+- Ingo Bressler
+- Rukeia El-Athman
+- Çağtay Fabry
+- Tobias Fritsch
+- Ralf Herrmann
+- Zoltán Konthur
+- Julius Kruse
+- Pavlina Kruzikova
+- Tarakeshwar Lakshmipathy
+- Julien Lecompagnon
+- Jan Lisec
+- Mathias Röllig
+- Bastian Rühle

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/LICENSE

@@ -1,4 +1,4 @@
-MIT License
+MIT License - MODIFIED
 
 Copyright (c) 2025 Bundesanstalt für Materialforschung und -prüfung
 
@@ -19,3 +19,7 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+**MODIFICATION**: the datamodel files are excluded from this license agreement and
+we reserve all rights on them for developments. Appropriate recognition has to be
+done if you use parts of this datamodel in your own modelling.

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: bam-masterdata
-Version: 0.3.1
+Version: 0.4.0
 Summary: The BAM Data Store utility tools and masterdata models wrapped in a Python package with a front-end web interface.
 Author-email: BAM Data Store <datastore@bam.de>
 Maintainer-email: "Jose M. Pizarro" <jose.pizarro-blanco@bam.de>
-License: MIT License
+License: MIT License - MODIFIED
         
         Copyright (c) 2025 Bundesanstalt für Materialforschung und -prüfung
         
@@ -26,6 +26,9 @@ License: MIT License
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
         
+        **MODIFICATION**: the datamodel files are excluded from this license agreement and
+        we reserve all rights on them for developments. Appropriate recognition has to be
+        done if you use parts of this datamodel in your own modelling.
 Project-URL: Homepage, https://github.com/BAMresearch/bam-masterdata
 Project-URL: Bug Tracker, https://github.com/BAMresearch/bam-masterdata/issues
 Classifier: Natural Language :: English
@@ -39,6 +42,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+License-File: AUTHORS
 Requires-Dist: pybis~=1.37.1rc4
 Requires-Dist: openpyxl
 Requires-Dist: click
@@ -207,11 +211,31 @@ INFO    -  [14:07:47] Watching paths for changes: 'docs', 'mkdocs.yml'
 INFO    -  [14:07:47] Serving on http://127.0.0.1:8000/
 ```
 
-Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are inmediately reflected when the files are saved (the local web will automatically refresh).
+Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are immediately reflected when the files are saved (the local web will automatically refresh).
 
 ## Main contributors
 
+The main code developers are:
+
 | Name                | E-mail                                                       |
 | ------------------- | ------------------------------------------------------------ |
 | Carlos Madariaga    | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de)       |
 | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |
+
+
+The main datamodel developers are:
+
+- Angela Ariza de Schellenberger
+- Ingo Bressler
+- Rukeia El-Athman
+- Çağtay Fabry
+- Tobias Fritsch
+- Ralf Herrmann
+- Zoltán Konthur
+- Julius Kruse
+- Pavlina Kruzikova
+- Tarakeshwar Lakshmipathy
+- Julien Lecompagnon
+- Jan Lisec
+- Mathias Röllig
+- Bastian Rühle

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/README.md

@@ -146,11 +146,31 @@ INFO    -  [14:07:47] Watching paths for changes: 'docs', 'mkdocs.yml'
 INFO    -  [14:07:47] Serving on http://127.0.0.1:8000/
 ```
 
-Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are inmediately reflected when the files are saved (the local web will automatically refresh).
+Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are immediately reflected when the files are saved (the local web will automatically refresh).
 
 ## Main contributors
 
+The main code developers are:
+
 | Name                | E-mail                                                       |
 | ------------------- | ------------------------------------------------------------ |
 | Carlos Madariaga    | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de)       |
 | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |
+
+
+The main datamodel developers are:
+
+- Angela Ariza de Schellenberger
+- Ingo Bressler
+- Rukeia El-Athman
+- Çağtay Fabry
+- Tobias Fritsch
+- Ralf Herrmann
+- Zoltán Konthur
+- Julius Kruse
+- Pavlina Kruzikova
+- Tarakeshwar Lakshmipathy
+- Julien Lecompagnon
+- Jan Lisec
+- Mathias Röllig
+- Bastian Rühle

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.1'
-__version_tuple__ = version_tuple = (0, 3, 1)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/checker/checker.py

@@ -58,23 +58,23 @@ class MasterdataChecker:
 
         # Load required models based on the selected mode
         if (
-            mode in ["self", "validate", "compare", "all"]
+            mode in ["self", "validate", "compare", "all", "individual"]
             and self.current_model is None
         ):
             self.logger.info("Current model is missing. Loading now...")
             self.load_current_model()
 
         if (
-            mode in ["incoming", "validate", "compare", "all"]
+            mode in ["incoming", "validate", "compare", "all", "individual"]
             and self.new_entities is None
         ):
             raise ValueError(
-                "New entities must be loaded before validation in 'incoming', 'validate', 'compare', or 'all' modes."
+                "New entities must be loaded before validation in 'incoming', 'validate', 'individual', 'compare', or 'all' modes."
             )
 
         # Load the validation rules
         if (
-            mode in ["self", "incoming", "validate", "all"]
+            mode in ["self", "incoming", "validate", "all", "individual"]
             and self.validation_rules == {}
         ):
             self.validation_rules = load_validation_rules(self.logger)

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/checker/masterdata_validator.py

@@ -67,7 +67,7 @@ class MasterdataValidator:
 
         if mode == "individual":
             self.logger.info(
-                "Validating new DSST entities and comparing them with current model..."
+                "Validating new entities and comparing them with current model..."
             )
             self.validation_results = {
                 "incoming_model": {},
@@ -427,6 +427,11 @@ class MasterdataValidator:
                         is_terms=True,
                     )
 
+        if not self.validation_results.get("comparisons"):
+            logger.info(
+                "No critical conflicts found between new entities compared to the current model."
+            )
+
         return self.validation_results
 
     def _compare_assigned_properties(
@@ -524,7 +529,7 @@ class MasterdataValidator:
                 )
                 other_entity_props = {prop["code"] for prop in other_entity_properties}
 
-                if incoming_prop_codes == other_entity_props:
+                if (incoming_prop_codes == other_entity_props) and incoming_prop_codes:
                     log_message = (
                         "Entity will not be imported in openBIS. "
                         f"The entity {entity_code} at row {incoming_entity['defs'].get('row_location')} has the same properties defined as {other_entity_code}. "

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/checker/source_loader.py

@@ -64,18 +64,34 @@ class SourceLoader:
             transformed_data[entity_type] = {}
 
             for entity_name, entity_data in entities.items():
-                transformed_entity = {
-                    "properties": [],  # Now placed before "defs"
-                    "defs": {  # Metadata moved to the end
-                        "code": entity_data.get("code"),
-                        "description": entity_data.get("description", ""),
-                        "id": format_json_id(entity_name),  # PascalCase for entity ID
-                        "row_location": entity_data.get("row_location"),
-                        "validation_script": entity_data.get("validationPlugin")
-                        or None,  # Convert "" to None
-                        "iri": entity_data.get("iri") or None,  # Convert "" to None
-                    },
-                }
+                if entity_type == "vocabulary_types":
+                    transformed_entity = {
+                        "terms": [],  # Now placed before "defs"
+                        "defs": {  # Metadata moved to the end
+                            "code": entity_data.get("code"),
+                            "description": entity_data.get("description", ""),
+                            "id": format_json_id(
+                                entity_name
+                            ),  # PascalCase for entity ID
+                            "row_location": entity_data.get("row_location"),
+                            "url_template": entity_data.get("url_template") or None,
+                        },
+                    }
+                else:
+                    transformed_entity = {
+                        "properties": [],  # Now placed before "defs"
+                        "defs": {  # Metadata moved to the end
+                            "code": entity_data.get("code"),
+                            "description": entity_data.get("description", ""),
+                            "id": format_json_id(
+                                entity_name
+                            ),  # PascalCase for entity ID
+                            "row_location": entity_data.get("row_location"),
+                            "validation_script": entity_data.get("validationPlugin")
+                            or None,  # Convert "" to None
+                            "iri": entity_data.get("iri") or None,  # Convert "" to None
+                        },
+                    }
 
                 # Handle additional fields specific to dataset_types
                 if entity_type == "dataset_types":
@@ -123,6 +139,22 @@ class SourceLoader:
                         }
                         transformed_entity["properties"].append(transformed_property)
 
+                if "terms" in entity_data:
+                    for term_name, term_data in entity_data["terms"].items():
+                        transformed_term = {
+                            "code": term_data.get("code"),
+                            "description": term_data.get("description", ""),
+                            "id": format_json_id(
+                                term_name
+                            ),  # Now correctly formatted to PascalCase
+                            "row_location": term_data.get("row_location"),
+                            "url_template": term_data.get("url_template")
+                            or None,  # Convert "" to None
+                            "label": term_data.get("label"),
+                            "official": term_data.get("official"),
+                        }
+                        transformed_entity["terms"].append(transformed_term)
+
                 transformed_data[entity_type][entity_name] = transformed_entity
 
         return transformed_data

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/cli/cli.py

@@ -1,6 +1,8 @@
+import glob
+import inspect
 import json
 import os
-import subprocess
+import shutil
 import time
 from pathlib import Path
 
@@ -15,6 +17,7 @@ from bam_masterdata.cli.entities_to_rdf import entities_to_rdf
 from bam_masterdata.cli.fill_masterdata import MasterdataCodeGenerator
 from bam_masterdata.logger import logger
 from bam_masterdata.metadata.entities_dict import EntitiesDict
+from bam_masterdata.openbis.login import ologin
 from bam_masterdata.utils import (
     DATAMODEL_DIR,
     delete_and_create_dir,
@@ -410,7 +413,6 @@ def checker(file_path, mode, datamodel_path):
                 click.echo(
                     f"There are problems in the current model for entity {entity} that need to be solved"
                 )
-                click.echo(f"Errors: {errors}")
 
     # Check if there are problems with the incoming model
     if mode in ["incoming", "all", "validate"] and validation_results.get(
@@ -421,7 +423,6 @@ def checker(file_path, mode, datamodel_path):
                 click.echo(
                     f"There are problems in the incoming model in {file_path} for entity {entity} that need to be solved"
                 )
-                click.echo(f"Errors: {errors}")
 
     # Check if there are comparison problems
     if mode in ["compare", "all"] and validation_results.get("comparisons", {}):
@@ -430,7 +431,6 @@ def checker(file_path, mode, datamodel_path):
                 click.echo(
                     f"There are problems when checking the incoming model in {file_path} against the current model {datamodel_path} for entity {entity} that need to be solved"
                 )
-                click.echo(f"Errors: {errors}")
 
     # Check if there are individual repository problems
     if (
@@ -443,17 +443,131 @@ def checker(file_path, mode, datamodel_path):
                 click.echo(
                     f"There are problems in the individual repositories when validating them for entity {entity} that need to be solved"
                 )
-                click.echo(f"Errors: {errors}")
         for entity, errors in validation_results.get("comparisons", {}).items():
             if errors:
                 click.echo(
                     f"There are problems in the individual repositories when comparing them with respect to bam-masterdata for entity {entity} that need to be solved"
                 )
-                click.echo(f"Errors: {errors}")
 
     # Check if no problems were found
     if no_validation_errors(validation_results):
         click.echo("No problems found in the datamodel and incoming model.")
+    else:
+        raise click.ClickException(
+            "Problems found in the datamodel. Please, check the logs."
+        )
+
+
+@cli.command(
+    name="push_to_openbis",
+    help="Uploads to openBIS the entities contained in the file specified in the tag `--file-path` after passing correctly all the checks from the `checker`.",
+)
+@click.option(
+    "--file-path",
+    "file_path",  # alias
+    type=click.Path(exists=True),
+    required=True,
+    help="""The path to the file or directory containing Python modules or the Excel file to be checked.""",
+)
+@click.option(
+    "--datamodel-path",
+    "datamodel_path",  # alias
+    type=click.Path(exists=True, dir_okay=True),
+    default=DATAMODEL_DIR,
+    help="""Path to the directory containing the Python modules defining the datamodel (defaults to './bam_masterdata/datamodel/').""",
+)
+def push_to_openbis(file_path, datamodel_path):
+    # Check if the path is a single .py file OR a directory containing .py files
+    if file_path.endswith(".py") or (
+        os.path.isdir(file_path) and any(glob.glob(os.path.join(file_path, "*.py")))
+    ):
+        source_type = "python"
+    elif file_path.endswith(".xlsx"):
+        source_type = "excel"
+    else:
+        source_type = None
+        logger.warning(f"Unsupported source type for path: {file_path}")
+
+    # Handle source_type
+    if source_type == "python":
+        # Copy all .py files to the tmp directory
+        if os.path.isdir(file_path):
+            tmp_dir = file_path
+        else:
+            tmp_dir = os.path.dirname(file_path)
+        logger.info(f"Copied Python files to {tmp_dir}")
+
+    elif source_type == "excel":
+        tmp_dir = "./bam_masterdata/tmp"
+        os.makedirs(tmp_dir, exist_ok=True)
+
+        generator = MasterdataCodeGenerator(path=file_path, row_cell_info=True)
+
+        for module_name in ["collection", "dataset", "object", "vocabulary"]:
+            output_file = Path(os.path.join(tmp_dir, f"{module_name}_types.py"))
+
+            # Get the method from `MasterdataCodeGenerator`
+            code = getattr(generator, f"generate_{module_name}_types")().rstrip()
+
+            if code != "":
+                output_file.write_text(code + "\n", encoding="utf-8")
+                click.echo(f"Generated {module_name} types in {output_file}\n")
+            else:
+                click.echo(f"Skipping {module_name}_types.py (empty entity data)")
+        logger.info(f"Processed Excel file and exported to {tmp_dir}")
+
+    # Instantiate the checker class and run validation
+    checker = MasterdataChecker()
+
+    # Load current model from datamodel path
+    checker.load_current_model(datamodel_dir=datamodel_path)
+
+    # Load new entities from the specified file path (could be a Python file, directory, or Excel)
+    checker.load_new_entities(source=file_path)
+
+    # Run the checker in the specified mode
+    validation_results = checker.check(mode="individual")
+
+    if not validation_results.get("incoming_model"):
+        logger.info("No problems found in the new entities definition.")
+
+    # If there are no problems, push to openBIS
+    if all(
+        isinstance(value, dict) and all(not sub_value for sub_value in value.values())
+        for value in validation_results.values()
+    ):
+        click.echo("No problems found in the datamodel and incoming model.")
+        # Push to openBIS
+
+        url = environ("OPENBIS_URL")
+        openbis = ologin(url=url)
+        click.echo(f"Using the openBIS instance: {url}\n")
+
+        # Push each entity type
+        for module_path in listdir_py_modules(tmp_dir):
+            module = import_module(module_path=module_path)
+            for _, obj in inspect.getmembers(module, inspect.isclass):
+                if hasattr(obj, "defs") and callable(getattr(obj, "to_openbis")):
+                    obj_instance = obj()
+                    obj_instance.to_openbis(openbis=openbis, logger=logger)
+
+    else:
+        logger.error(
+            f"The checking of the new entities definition located in {file_path} did not pass. Please, check your entity definitions."
+        )
+        click.echo(
+            f"There are problems in the incoming model located in {file_path} that need to be solved"
+        )
+        return
+
+    if source_type == "excel":
+        # Clean up the tmp directory
+        shutil.rmtree(tmp_dir)
+        click.echo(f"Temporary files in {tmp_dir} have been deleted.")
+
+    logger.info(
+        f"Push to openBIS of the new entities located in {file_path} completed."
+    )
 
 
 if __name__ == "__main__":

{bam_masterdata-0.3.1 → bam_masterdata-0.4.0}/bam_masterdata/cli/fill_masterdata.py

@@ -18,6 +18,7 @@ class MasterdataCodeGenerator:
         self.row_cell_info = kwargs.get("row_cell_info", False)
         # * This part takes some time due to the loading of all entities from Openbis
         if url:
+            self.generator_type = "openbis"
             self.properties = OpenbisEntities(url=url).get_property_dict()
             self.collections = OpenbisEntities(url=url).get_collection_dict()
             self.datasets = OpenbisEntities(url=url).get_dataset_dict()
@@ -28,9 +29,11 @@ class MasterdataCodeGenerator:
                 f"Loaded OpenBIS entities in `MasterdataCodeGenerator` initialization {elapsed_time:.2f} seconds\n"
             )
         else:
+            self.generator_type = "excel"
             entities_dict = MasterdataExcelExtractor(
                 excel_path=path, row_cell_info=self.row_cell_info
             ).excel_to_entities()
+            self.properties = entities_dict.get("property_types", {})
             self.collections = entities_dict.get("collection_types", {})
             self.datasets = entities_dict.get("dataset_types", {})
             self.objects = entities_dict.get("object_types", {})
@@ -76,7 +79,7 @@ class MasterdataCodeGenerator:
 
         return parent_code, parent_class, class_name
 
-    def get_property_object_code(self, prop_code: str) -> str:
+    def get_property_object_code(self, prop_data: dict) -> str:
         """
         Get the object code (or vocabulary code) used for reference for the assigned property with `prop_code`.
 
@@ -86,18 +89,21 @@ class MasterdataCodeGenerator:
         Returns:
             str: The object/vocabulary code used for reference for the assigned property.
         """
-        data = self.properties.get(prop_code, {})
-        if not data:
+        if not prop_data:
             return ""
 
-        object_code = data.get("sampleType", "")
+        # TODO check excel extractor to add sampleType column
+        object_code = prop_data.get("sampleType", "")
         if object_code:
             return object_code
-        vocabulary_code = data.get("vocabulary", "")
-        if vocabulary_code:
-            return vocabulary_code
 
-        return ""
+        # TODO fix this patch and avoid using generator type
+        vocabulary_code = ""
+        if self.generator_type == "openbis":
+            vocabulary_code = prop_data.get("vocabulary", "")
+        elif self.generator_type == "excel":
+            vocabulary_code = prop_data.get("vocabularyType", "")
+        return vocabulary_code
 
     def add_properties(
         self, entities: dict, parent_code: str, data: dict, lines: list
@@ -115,14 +121,13 @@ class MasterdataCodeGenerator:
             data (dict): The data information for the entity as obtained from openBIS.
             lines (list): A list of strings to be printed to the Python module.
         """
-        parent_properties = entities.get(parent_code, {}).get("properties", {}).keys()
+        parent_properties_code = (
+            entities.get(parent_code, {}).get("properties", {}).keys()
+        )
         for prop_code, prop_data in data.get("properties", {}).items():
             # Skip "UNKNOWN" properties
-            if prop_code == "UNKNOWN":
-                continue
-
             # We check if the property is inherited from the parent class
-            if prop_code in parent_properties:
+            if prop_code == "UNKNOWN" or prop_code in parent_properties_code:
                 continue
 
             prop_name = prop_code.lstrip("$").replace(".", "_").lower()
@@ -134,11 +139,11 @@ class MasterdataCodeGenerator:
                 data_type = "OBJECT"
             lines.append(f'        data_type="{data_type}",')
             if data_type == "OBJECT":
-                object_code = self.get_property_object_code(prop_code=prop_code)
+                object_code = self.get_property_object_code(prop_data=prop_data)
                 if object_code:
                     lines.append(f'        object_code="{object_code}",')
             elif data_type == "CONTROLLEDVOCABULARY":
-                vocabulary_code = self.get_property_object_code(prop_code=prop_code)
+                vocabulary_code = self.get_property_object_code(prop_data=prop_data)
                 if vocabulary_code:
                     lines.append(f'        vocabulary_code="{vocabulary_code}",')