bam-masterdata 0.6.2__tar.gz → 0.7.0__tar.gz

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/.github/workflows/mkdocs-deploy.yml

@@ -22,8 +22,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install .[docu]
-      - name: Deploy docs
-        uses: mhausenblas/mkdocs-deploy-gh-pages@master
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --force
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CONFIG_FILE: mkdocs.yml
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/.gitignore

@@ -176,4 +176,7 @@ pytest-coverage.txt
 # artifacts
 artifacts
 # and tmp files
-tmp/
+tmp/
+
+# testing python file
+bam_masterdata/test.py

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bam-masterdata
-Version: 0.6.2
+Version: 0.7.0
 Summary: The BAM Data Store utility tools and masterdata models wrapped in a Python package with a front-end web interface.
 Author-email: BAM Data Store <datastore@bam.de>
 Maintainer-email: "Jose M. Pizarro" <jose.pizarro-blanco@bam.de>
@@ -59,6 +59,7 @@ Requires-Dist: pytest-cov; extra == "dev"
 Provides-Extra: docu
 Requires-Dist: mkdocs-material; extra == "docu"
 Requires-Dist: mkdocs-click; extra == "docu"
+Requires-Dist: mkdocstrings; extra == "docu"
 Requires-Dist: mkdocstrings-python; extra == "docu"
 Dynamic: license-file
 
@@ -189,4 +190,5 @@ The main code developers are:
 | Name                | E-mail                                                       |
 | ------------------- | ------------------------------------------------------------ |
 | Carlos Madariaga    | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de)       |
+| Lucas Zimmermann    | [lucas.zimmermann@bam.de](mailto:lucas.zimmermann@bam.de)       |
 | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/README.md

@@ -125,4 +125,5 @@ The main code developers are:
 | Name                | E-mail                                                       |
 | ------------------- | ------------------------------------------------------------ |
 | Carlos Madariaga    | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de)       |
+| Lucas Zimmermann    | [lucas.zimmermann@bam.de](mailto:lucas.zimmermann@bam.de)       |
 | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.2'
-__version_tuple__ = version_tuple = (0, 6, 2)
+__version__ = version = '0.7.0'
+__version_tuple__ = version_tuple = (0, 7, 0)
+
+__commit_id__ = commit_id = 'g08075a92e'

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/checker/masterdata_validator.py

@@ -13,7 +13,7 @@ class MasterdataValidator:
         Args:
             new_entities (dict): The incoming datamodel.
             current_model (dict): The existing datamodel.
-            rules_path (str): Path to the validation rules JSON file.
+            validation_rules (dict): The validation rules to apply.
         """
         self.new_entities = new_entities
         self.current_model = current_model

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/checker/source_loader.py

@@ -46,7 +46,7 @@ class SourceLoader:
         else:
             raise NotImplementedError(f"Source type {self.source_type} not supported.")
 
-    def entities_to_json(self):
+    def entities_to_json(self) -> dict:
         """
         Transforms the dictionary of entities returned by the Excel extractor into a dictionary in JSON format for later check.
 
@@ -158,9 +158,3 @@ class SourceLoader:
                 transformed_data[entity_type][entity_name] = transformed_entity
 
         return transformed_data
-
-        # # Save the transformed JSON
-        # with open(output_json_path, "w", encoding="utf-8") as json_file:
-        #     json.dump(transformed_data, json_file, indent=2, ensure_ascii=False)
-
-        # print(f"Transformed JSON saved at: {output_json_path}")

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/cli/cli.py

@@ -9,18 +9,16 @@ from pathlib import Path
 import click
 from decouple import config as environ
 from openpyxl import Workbook
-from pybis import Openbis
 from rdflib import Graph
 
 from bam_masterdata.checker import MasterdataChecker
 from bam_masterdata.cli.entities_to_excel import entities_to_excel
 from bam_masterdata.cli.entities_to_rdf import entities_to_rdf
 from bam_masterdata.cli.fill_masterdata import MasterdataCodeGenerator
+from bam_masterdata.cli.run_parser import run_parser
 from bam_masterdata.logger import logger
-from bam_masterdata.metadata.entities import CollectionType, PropertyTypeAssignment
 from bam_masterdata.metadata.entities_dict import EntitiesDict
 from bam_masterdata.openbis.login import ologin
-from bam_masterdata.parsing import AbstractParser
 from bam_masterdata.utils import (
     DATAMODEL_DIR,
     delete_and_create_dir,
@@ -585,119 +583,6 @@ def push_to_openbis(file_path, datamodel_path):
     )
 
 
-def run_parser(
-    openbis: Openbis = None,
-    files_parser: dict[AbstractParser, list[str]] = {},
-    project_name: str = "PROJECT",
-    collection_name: str = "COLLECTION",
-) -> None:
-    """
-    Run the parsers on the specified files and collect the results.
-    Args:
-    login with save_token=True dont forget!!
-        files_parser (dict): A dictionary where keys are parser instances and values are lists of file paths to be parsed.
-        space (str): The space in openBIS where the entities will be stored.
-        project (str): The project in openBIS where the entities will be stored.
-        collection (str): The collection in openBIS where the entities will be stored.
-    """
-    # Ensure the space, project, and collection are set
-    if not (project_name or collection_name):
-        logger.error(
-            "The name of the project and collection must be specified for the parser to run."
-        )
-        return
-    # Ensure the files_parser is not empty
-    if not files_parser:
-        logger.error(
-            "No files or parsers to parse. Please provide valid file paths or contact an Admin to add missing parser."
-        )
-        return
-
-    # Specify the space and project for the data
-    space = openbis.get_space(openbis.username)
-    if space is None:
-        logger.error(
-            f"Space {openbis.username} does not exist in openBIS. Please create it first."
-        )
-        return
-    if project_name.upper() in [p.code for p in space.get_projects()]:
-        project = space.get_project(project_name)
-    else:
-        project = space.new_project(
-            code=project_name,
-            description="New project created via automated parsing with `bam_masterdata`.",
-        )
-    project.save()
-
-    # Create a new pybis `COLLECTION` to store the generated objects
-    if collection_name.upper() in [c.code for c in project.get_collections()]:
-        collection_openbis = space.get_collection(
-            f"/{openbis.username}/{project_name}/{collection_name}".upper()
-        )
-    else:
-        collection_openbis = openbis.new_collection(
-            code=collection_name,
-            type="COLLECTION",
-            project=project,
-        )
-    collection_openbis.save()
-
-    # Create a bam_masterdata CollectionType instance for storing parsed results
-    collection = CollectionType()
-    # Iterate over each parser and its associated files and store them in `collection`
-    for parser, files in files_parser.items():
-        parser.parse(files, collection, logger=logger)
-
-    # Map the objects added to CollectionType to objects in openBIS using pyBIS
-    openbis_id_map = {}
-    for object_id, object_instance in collection.attached_objects.items():
-        # Map PropertyTypeAssignment to pybis props dictionary
-        obj_props = {}
-        for key in object_instance._properties.keys():
-            value = getattr(object_instance, key, None)
-            if value is None or isinstance(value, PropertyTypeAssignment):
-                continue
-            obj_props[object_instance._property_metadata[key].code.lower()] = value
-
-        object_openbis = openbis.new_object(
-            type=object_instance.defs.code,
-            space=space,
-            project=project,
-            collection=collection_openbis,
-            props=obj_props,
-        )
-        object_openbis.save()
-
-        # save local and openbis IDs to map parent-child relationships
-        openbis_id_map[object_id] = object_openbis.identifier
-        click.echo(
-            f"Object {obj_props.get('$name')} stored in openBIS collection {collection_name}."
-        )
-    for _, files in files_parser.items():
-        # Upload the file to openBIS
-        dataset = openbis.new_dataset(
-            type="RAW_DATA",
-            files=files,
-            collection=collection_openbis,
-        )
-        dataset.save()
-        click.echo(f"Files uploaded to openBIS collection {collection_name}.")
-
-    # Map parent-child relationships
-    for parent_id, child_id in collection.relationships.values():
-        if parent_id in openbis_id_map and child_id in openbis_id_map:
-            parent_openbis_id = openbis_id_map[parent_id]
-            child_openbis_id = openbis_id_map[child_id]
-
-            child_openbis = openbis.get_object(child_openbis_id)
-            child_openbis.add_parents(parent_openbis_id)
-            child_openbis.save()
-
-            click.echo(
-                f"Linked child {child_openbis_id} to parent {parent_openbis_id} in collection {collection_name}."
-            )
-
-
 @cli.command(
     name="parser",
     help="Parses a list of files using the specified parsers and stores the results in openBIS.",
@@ -712,8 +597,8 @@ def run_parser(
     help="Parser name and file path tuple: 'ExampleParser file1.txt'",
 )
 @click.option(
-    "--project_name",
-    "project-name",  # alias
+    "--project-name",
+    "project_name",  # alias
     type=str,
     required=True,
     help="OpenBIS project name",
@@ -722,10 +607,17 @@ def run_parser(
     "--collection-name",
     "collection_name",  # alias
     type=str,
-    required=True,
+    required=False,
     help="OpenBIS collection name",
 )
-def parser(files_parser, project_name, collection_name):
+@click.option(
+    "--space-name",
+    "space_name",  # alias
+    type=str,
+    required=False,
+    help="OpenBIS space name",
+)
+def parser(files_parser, project_name, collection_name, space_name):
     parser_map = {}  # TODO load from configuration from yaml file
     parse_file_dict = {}
     for parser_key, filepath in files_parser:
@@ -739,9 +631,11 @@ def parser(files_parser, project_name, collection_name):
         parse_file_dict[parser_cls].append(filepath)
 
     run_parser(
-        files_parser=parse_file_dict,
+        openbis=ologin(url=environ("OPENBIS_URL")),
+        space_name=space_name,
         project_name=project_name,
         collection_name=collection_name,
+        files_parser=parse_file_dict,
     )
 
 

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/cli/excel_to_entities.py

@@ -644,11 +644,8 @@ class MasterdataExcelExtractor:
         """
         Extracts entities from an Excel file and returns them as a dictionary.
 
-        Args:
-            output_directory: The directory to store the output files.
-
         Returns:
-            A dictionary where each key is a normalized sheet name and the value is a dictionary
+            dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
             containing the extracted entities. Returns an empty dictionary if all sheets are empty.
         """
         sheets_dict: dict[str, dict[str, Any]] = {}

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/cli/fill_masterdata.py

@@ -58,7 +58,6 @@ class MasterdataCodeGenerator:
             class_names (dict): A dictionary with the class names of the entities.
             default (str): The default parent class if the parent class does not exist.
             lines (list): A list of strings to be printed to the Python module.
-            logger (BoundLoggerLazyProxy): The logger to log messages.
         Returns:
             tuple: The parent code, parent class, and class name of the entity.
         """
@@ -84,7 +83,7 @@ class MasterdataCodeGenerator:
         Get the object code (or vocabulary code) used for reference for the assigned property with `prop_code`.
 
         Args:
-            prop_code (str): The code of the assigned property.
+            prop_data (dict): The data information for the property as obtained from openBIS.
 
         Returns:
             str: The object/vocabulary code used for reference for the assigned property.

bam_masterdata-0.7.0/bam_masterdata/cli/run_parser.py

@@ -0,0 +1,180 @@
+from pybis import Openbis
+
+from bam_masterdata.logger import logger
+from bam_masterdata.metadata.entities import CollectionType, PropertyTypeAssignment
+from bam_masterdata.parsing import AbstractParser
+
+
+def run_parser(
+    openbis: Openbis | None = None,
+    space_name: str = "",
+    project_name: str = "PROJECT",
+    collection_name: str = "",
+    files_parser: dict[AbstractParser, list[str]] = {},
+) -> None:
+    """
+    Run the parsers on the specified files and collect the results.
+    login with save_token=True don't forget!!
+
+    Args:
+        openbis (Openbis): An instance of the Openbis class from pyBIS, already logged in.
+        space_name (str): The space in openBIS where the entities will be stored.
+        project_name (str): The project in openBIS where the entities will be stored.
+        collection_name (str): The collection in openBIS where the entities will be stored.
+        files_parser (dict): A dictionary where keys are parser instances and values are lists of file paths to be parsed. E.g., {MasterdataParserExample(): ["path/to/file.json", "path/to/another_file.json"]}
+    """
+    # Ensure openbis is provided
+    if openbis is None:
+        logger.error("An instance of Openbis must be provided for the parser to run.")
+        return
+    # Ensure the space, project, and collection are set
+    if not project_name:
+        logger.error("The Project name must be specified for the parser to run.")
+        return
+    # Ensure the files_parser is not empty
+    if not files_parser:
+        logger.error(
+            "No files or parsers to parse. Please provide valid file paths or contact an Admin to add missing parser."
+        )
+        return
+
+    # Specify the space
+    try:
+        space = openbis.get_space(space_name)
+    except Exception:
+        space = None
+    # If space is not found, use the user space
+    if space is None:
+        # user name as default space
+        for s in openbis.get_spaces():
+            if s.code.endswith(openbis.username.upper()):
+                space = s
+                logger.warning(
+                    f"Space {space_name} does not exist in openBIS. "
+                    f"Loading space for {openbis.username}."
+                )
+                break
+        # no space found
+        if space is None:
+            logger.error(
+                f"No usable Space for {openbis.username} in openBIS. Please create it first or notify an Admin."
+            )
+            return
+
+    # Get project if `project_name` already exists under the space or create a new one if it does not
+    if project_name.upper() in [p.code for p in space.get_projects()]:
+        project = space.get_project(project_name)
+    else:
+        logger.info("Replacing project code with uppercase and underscores.")
+        project = space.new_project(
+            code=project_name.replace(" ", "_").upper(),
+            description="New project created via automated parsing with `bam_masterdata`.",
+        )
+    project.save()
+
+    # Create a new pybis `COLLECTION` to store the generated objects
+    if not collection_name:
+        logger.info(
+            "No Collection name specified. Attaching objects directly to Project."
+        )
+        collection_openbis = project
+    else:
+        if collection_name.upper() in [c.code for c in project.get_collections()]:
+            collection_openbis = space.get_collection(
+                f"/{space_name}/{project_name}/{collection_name}".upper()
+            )
+        else:
+            logger.info("Replacing collection code with uppercase and underscores.")
+            collection_openbis = openbis.new_collection(
+                code=collection_name.replace(" ", "_").upper(),
+                type="DEFAULT_EXPERIMENT",
+                project=project,
+            )
+        collection_openbis.save()
+
+    # Create a bam_masterdata CollectionType instance for storing parsed results
+    collection = CollectionType()
+    # Iterate over each parser and its associated files and store them in `collection`
+    for parser, files in files_parser.items():
+        parser.parse(files, collection, logger=logger)
+
+    # Map the objects added to CollectionType to objects in openBIS using pyBIS
+    openbis_id_map = {}
+    for object_id, object_instance in collection.attached_objects.items():
+        # Map PropertyTypeAssignment to pybis props dictionary
+        obj_props = {}
+        for key in object_instance._properties.keys():
+            value = getattr(object_instance, key, None)
+            if value is None or isinstance(value, PropertyTypeAssignment):
+                continue
+            obj_props[object_instance._property_metadata[key].code.lower()] = value
+
+        # Check if object already exists in openBIS, and if so, notify and get for updating properties
+        if not object_instance.code:
+            if not collection_name:
+                object_openbis = openbis.new_object(
+                    type=object_instance.defs.code,
+                    space=space,
+                    project=project,
+                    props=obj_props,
+                )
+            else:
+                object_openbis = openbis.new_object(
+                    type=object_instance.defs.code,
+                    space=space,
+                    project=project,
+                    collection=collection_openbis,
+                    props=obj_props,
+                )
+            object_openbis.save()
+        else:
+            identifier = (
+                f"/{space_name}/{project_name}/{object_instance.code}"
+                if not collection_name
+                else f"/{space_name}/{project_name}/{collection_name}/{object_instance.code}"
+            )
+            object_openbis = space.get_object(identifier)
+            object_openbis.set_props(obj_props)
+            object_openbis.save()
+            logger.info(
+                f"Object {identifier} already exists in openBIS, updating properties."
+            )
+
+        # save local and openbis IDs to map parent-child relationships
+        openbis_id_map[object_id] = object_openbis.identifier
+
+    # Storing files as datasets in openBIS
+    for files in files_parser.values():
+        try:
+            if not collection_name:
+                # ! This won't work on a project -> datasets only attached to collections in pyBIS
+                dataset = openbis.new_dataset(
+                    type="RAW_DATA",
+                    files=files,
+                    project=project,
+                )
+            else:
+                dataset = openbis.new_dataset(
+                    type="RAW_DATA",
+                    files=files,
+                    collection=collection_openbis,
+                )
+            dataset.save()
+        except Exception as e:
+            logger.warning(f"Error uploading files {files} to openBIS: {e}")
+            continue
+        logger.info(f"Files uploaded to openBIS collection {collection_name}.")
+
+    # Map parent-child relationships
+    for parent_id, child_id in collection.relationships.values():
+        if parent_id in openbis_id_map and child_id in openbis_id_map:
+            parent_openbis_id = openbis_id_map[parent_id]
+            child_openbis_id = openbis_id_map[child_id]
+
+            child_openbis = openbis.get_object(child_openbis_id)
+            child_openbis.add_parents(parent_openbis_id)
+            child_openbis.save()
+
+            logger.info(
+                f"Linked child {child_openbis_id} to parent {parent_openbis_id} in collection {collection_name}."
+            )

{bam_masterdata-0.6.2 → bam_masterdata-0.7.0}/bam_masterdata/excel/excel_to_entities.py

@@ -645,11 +645,8 @@ class MasterdataExcelExtractor:
         """
         Extracts entities from an Excel file and returns them as a dictionary.
 
-        Args:
-            output_directory: The directory to store the output files.
-
         Returns:
-            A dictionary where each key is a normalized sheet name and the value is a dictionary
+            dict[str, dict[str, Any]]: A dictionary where each key is a normalized sheet name and the value is a dictionary
             containing the extracted entities. Returns an empty dictionary if all sheets are empty.
         """
         sheets_dict: dict[str, dict[str, Any]] = {}