mgtx-benchling-wrapper 0.1.0__py3-none-any.whl

mgtx_benchling_wrapper/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.2"

mgtx_benchling_wrapper/client/benchling_client.py

@@ -0,0 +1,67 @@
+from typing import Callable
+
+from benchling_sdk.benchling import Benchling
+from benchling_api_client.benchling_client import BenchlingApiClient
+from benchling_sdk.helpers.retry_helpers import RetryStrategy
+
+
+# -----------------------------
+# Configuration defaults
+# -----------------------------
+
+DEFAULT_TIMEOUT_SECONDS = 180
+MAX_RETRIES = 10
+BACKOFF_FACTOR = 2.0
+
+# -----------------------------
+# Client decorators
+# -----------------------------
+
+def with_default_timeout(client: BenchlingApiClient) -> BenchlingApiClient:
+    """
+    Apply default timeout to all Benchling API calls.
+    """
+    return client.with_timeout(DEFAULT_TIMEOUT_SECONDS)
+
+
+# -----------------------------
+# Factory
+# -----------------------------
+
+def with_backoff_timeout():
+    return RetryStrategy(
+        max_tries=MAX_RETRIES,
+        backoff_factor=BACKOFF_FACTOR
+    )
+
+def create_benchling_client(
+    *,
+    url: str,
+    auth,
+    client_decorator: Callable[[BenchlingApiClient], BenchlingApiClient] = with_default_timeout,
+) -> Benchling:
+    """
+    Factory for creating a Benchling SDK client.
+
+    Args:
+        url (str):
+            Benchling tenant base URL (e.g. https://acme.benchling.com)
+
+        auth:
+            Authentication method compatible with Benchling SDK
+            (e.g. ClientCredentialsOAuth2)
+
+        client_decorator (callable):
+            Optional decorator to customize the underlying API client
+            (timeouts, retries, logging, etc.)
+
+    Returns:
+        Benchling:
+            Configured Benchling SDK client
+    """
+    return Benchling(
+        url=url,
+        auth_method=auth,
+        client_decorator=client_decorator,
+        retry_strategy=with_backoff_timeout(),
+    )

mgtx_benchling_wrapper/context/benchling_context.py

@@ -0,0 +1,56 @@
+from functools import cache
+from typing import Optional
+
+from benchling_sdk.benchling import Benchling
+from benchling_sdk.apps.framework import App
+from benchling_sdk.auth.client_credentials_oauth2 import ClientCredentialsOAuth2
+from benchling_sdk.models.webhooks.v0 import WebhookEnvelopeV0
+
+from mgtx_benchling_wrapper.client.benchling_client import create_benchling_client
+
+
+class BenchlingContext:
+    """
+    Unified authentication + tenant context.
+    """
+
+    def __init__(
+        self,
+        *,
+        client_id: str,
+        client_secret: str,
+        base_url: str,
+        token_url: Optional[str] = None,
+        app_id: Optional[str] = None,
+    ):
+        self._client_id = client_id
+        self._client_secret = client_secret
+        self._base_url = base_url
+        self._token_url = token_url
+        self._app_id = app_id
+
+    @cache
+    def _auth(self):
+        return ClientCredentialsOAuth2(self._client_id, self._client_secret, self._token_url)
+
+    @cache
+    def benchling(self) -> Benchling:
+        return create_benchling_client(
+            url=self._base_url,
+            auth=self._auth(),
+        )
+
+    @cache
+    def app(self) -> App:
+        if self._app_id is None:
+            raise RuntimeError("App requested but no app_id provided")
+        return App(self._app_id, self.benchling())
+
+    @classmethod
+    def from_webhook(cls, webhook: WebhookEnvelopeV0, *, client_id, client_secret):
+        return cls(
+            client_id=client_id,
+            client_secret=client_secret,
+            base_url=webhook.base_url,
+            app_id=webhook.app.id,
+        )

mgtx_benchling_wrapper/utils/chunking.py

@@ -0,0 +1,6 @@
+def chunk_list(lst: list, chunk_n: int):
+    """
+    Chunking the list into chunks of chunk_n elements.
+    """
+    for i in range(0, len(lst), chunk_n):
+        yield lst[i:i + chunk_n]

mgtx_benchling_wrapper/utils/compare_dataframe_cols.py

@@ -0,0 +1,19 @@
+import pandas as pd
+
+def compare_dataframe_columns(
+        dataframe: pd.DataFrame,
+        new_dataframe: pd.DataFrame,
+        column_name: str
+) -> list[str]:
+
+    list_missing_values = []
+    org_column = list(dataframe[column_name])
+    new_column = list(new_dataframe[column_name])
+
+    for value in org_column:
+        if value in new_column:
+            if value not in list_missing_values:
+                if value is not None:
+                    list_missing_values.append(value)
+
+    return list_missing_values

mgtx_benchling_wrapper/utils/create_blob_payload.py

@@ -0,0 +1,38 @@
+import base64
+import hashlib
+import os.path
+
+def create_image_blob_payload(
+        image_path: str,
+        mime_type="image/png",
+        blob_type="VISUALIZATION"):
+    """
+    Create a payload for uploading an image blob through the API.
+    """
+    try:
+        with open(image_path, "rb") as image_file:
+            image_data = image_file.read()
+    except FileNotFoundError:
+        raise Exception(f"Blob file not found: {image_path}")
+
+    #get name of the image
+    _,tail = os.path.split(image_path)
+
+    name = tail.split(".")[0]
+
+    # Encode the binary data to base64
+    data64 = base64.b64encode(image_data).decode('utf-8')
+
+    # Calculate the MD5 hash of the image data
+    md5_hash = hashlib.md5(image_data).hexdigest()
+
+    # Create the payload
+    payload = {
+        "data64": data64,
+        "md5": md5_hash,
+        "mimeType": mime_type,
+        "name": name,
+        "type": blob_type
+    }
+
+    return payload, name

mgtx_benchling_wrapper/utils/datetime_parser.py

@@ -0,0 +1,43 @@
+import warnings
+import pandas as pd
+import re
+import datetime
+
+def _parse_datetime(value):
+    if pd.isna(value):
+        return None
+
+    if isinstance(value, (pd.Timestamp, datetime.date)):
+        return pd.Timestamp(value)
+
+    if _is_ambiguous_date(value):
+        warnings.warn(
+            f"Ambiguous date format detected '{value}'. "
+            f"Assuming day-first format (DD/MM/YYYY).",
+            UserWarning
+        )
+        return pd.to_datetime(value, errors="raise", dayfirst=True, format='mixed')
+
+    return pd.to_datetime(
+        value,
+        errors="raise",
+        dayfirst=True
+    )
+
+
+def _is_ambiguous_date(value: str) -> bool:
+    """
+    Detect ambiguous numeric date formats like 01/02/2024
+    where both day and month <= 12.
+    """
+    if not isinstance(value, str):
+        return False
+
+    pattern = r"^\d{1,2}[/-]\d{1,2}[/-]\d{4}$"
+    if not re.match(pattern, value.strip()):
+        return False
+
+    parts = re.split(r"[/-]", value)
+    day, month, _ = map(int, parts)
+
+    return day <= 12 and month <= 12

mgtx_benchling_wrapper/utils/deprecated.py

@@ -0,0 +1,78 @@
+import functools
+import inspect
+import warnings
+
+string_types = (type(b''), type(u''))
+
+
+def deprecated(reason):
+    """
+    This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+    """
+
+    if isinstance(reason, string_types):
+
+        # The @deprecated is used with a 'reason'.
+        #
+        # .. code-block:: python
+        #
+        #    @deprecated("please, use another function")
+        #    def old_function(x, y):
+        #      pass
+
+        def decorator(func1):
+
+            if inspect.isclass(func1):
+                fmt1 = "Call to deprecated class {name} ({reason})."
+            else:
+                fmt1 = "Call to deprecated function {name} ({reason})."
+
+            @functools.wraps(func1)
+            def new_func1(*args, **kwargs):
+                warnings.simplefilter('always', DeprecationWarning)
+                warnings.warn(
+                    fmt1.format(name=func1.__name__, reason=reason),
+                    category=DeprecationWarning,
+                    stacklevel=2
+                )
+                warnings.simplefilter('default', DeprecationWarning)
+                return func1(*args, **kwargs)
+
+            return new_func1
+
+        return decorator
+
+    elif inspect.isclass(reason) or inspect.isfunction(reason):
+
+        # The @deprecated is used without any 'reason'.
+        #
+        # .. code-block:: python
+        #
+        #    @deprecated
+        #    def old_function(x, y):
+        #      pass
+
+        func2 = reason
+
+        if inspect.isclass(func2):
+            fmt2 = "Call to deprecated class {name}."
+        else:
+            fmt2 = "Call to deprecated function {name}."
+
+        @functools.wraps(func2)
+        def new_func2(*args, **kwargs):
+            warnings.simplefilter('always', DeprecationWarning)
+            warnings.warn(
+                fmt2.format(name=func2.__name__),
+                category=DeprecationWarning,
+                stacklevel=2
+            )
+            warnings.simplefilter('default', DeprecationWarning)
+            return func2(*args, **kwargs)
+
+        return new_func2
+
+    else:
+        raise TypeError(repr(type(reason)))

mgtx_benchling_wrapper/utils/fields.py

@@ -0,0 +1,15 @@
+def create_fields_dict(
+        list_field_names: list,
+        list_field_values: list
+) -> dict:
+    """ Creates a dictionary with the format {field_name:'value':{field_value}}
+    Args:
+        list_field_names (list): list containing the field names (usually the column names of dataframe)
+        list_field_values (list): values of the fields (usually a row in a dataframe)
+    Return:
+        final_dict (dict): a dictionary of the format to input in fields.
+    """
+    list_word_value = ['value'] * len(list_field_names)
+    final_dict = {u: {v: w} for (u, v, w) in zip(list_field_names, list_word_value, list_field_values)}
+
+    return final_dict

mgtx_benchling_wrapper/utils/list_unique_values_col.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+def list_unique_values_in_column(
+        dataframe: pd.DataFrame,
+        column_name: str,
+        drop_na: bool = False
+) -> list[str]:
+    if drop_na:
+        return dataframe[column_name].dropna().unique().tolist()
+    else:
+        return dataframe[column_name].unique().tolist()

mgtx_benchling_wrapper/utils/logger.py

@@ -0,0 +1,38 @@
+import logging
+
+def get_logger(name: str, file_log_level: str | int, console_log_level: str | int, console_filter: str | None = None) -> logging.Logger:
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+
+    # Factory function for creating level filters
+    def level_filter(level_name: str):
+        def filter_func(record):
+            return record.levelname == level_name
+        return filter_func
+
+    # Create formatters
+    console_format = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | line:%(lineno)d | %(message)s')
+    file_format = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(message)s')
+
+
+    # initiating a console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel((console_log_level))
+    console_handler.setFormatter(console_format)
+    # Add filter if specified
+    if console_filter:
+        console_handler.addFilter(level_filter(console_filter))
+
+    # initiating a file handler
+    file_handler = logging.FileHandler(
+        filename='app.log',
+        mode='a',
+        encoding='utf-8')
+    file_handler.setLevel((file_log_level))
+    file_handler.setFormatter(file_format)
+
+    # add the handlers to the logger
+    logger.addHandler(console_handler)
+    logger.addHandler(file_handler)
+
+    return logger

mgtx_benchling_wrapper/utils/substitute_df_col_dict.py

@@ -0,0 +1,10 @@
+import pandas as pd
+
+def substitute_with_dict(
+        dataframe: pd.DataFrame,
+        column_name: str,
+        dict_orig_to_api: dict
+) -> pd.DataFrame:
+    dataframe[column_name] = dataframe[column_name].replace(dict_orig_to_api)
+
+    return dataframe

mgtx_benchling_wrapper/utils/validation_summary.py

@@ -0,0 +1,23 @@
+from mgtx_benchling_wrapper.workflows.models.types import ValidationResult
+
+def get_validation_summary(result: ValidationResult) -> str:
+    """Generate human-readable validation summary."""
+    if result.is_valid:
+        summary = "✓ Validation passed"
+        if result.warnings:
+            summary += f" with {len(result.warnings)} warning(s)"
+        return summary
+
+    summary = f"✗ Validation failed with {len(result.errors)} error(s):\n"
+
+    for i, error in enumerate(result.errors, 1):
+        summary += f"\n{i}. {error.message}"
+        if hasattr(error, 'context') and error.context:
+            summary += f"\n   Context: {error.context}"
+
+    if result.warnings:
+        summary += f"\n\nWarnings ({len(result.warnings)}):\n"
+        for warning in result.warnings:
+            summary += f"  - {warning}\n"
+
+    return summary

mgtx_benchling_wrapper/workflows/assays_result_ingestion.py

@@ -0,0 +1,256 @@
+from mgtx_benchling_wrapper.wrapper.facade import BenchlingWrapperFacade
+from mgtx_benchling_wrapper.utils.logger import get_logger
+from mgtx_benchling_wrapper.workflows.models.types import BlobType
+from .transformation.container_transformer import ContainersTransformer
+from .transformation.link_transformer import LinkTransformer
+from .transformation.blob_transformer import BlobTransformer
+from .transformation.dropdown_transformer import DropdownTransformer
+from .transformation.datetime_converter import DateTimeConverter
+from .transformation.exceptions import TransformationError
+from .handlers.result_archiver import ResultArchiver
+from .handlers.blob_handler import BlobHandler
+from .handlers.result_ingestion import ResultIngestion
+from .handlers.exceptions import HandlerError
+from .handlers.schema_handler import SchemaHandler
+from .validation.api_variable_validation import ApiParametersValidation
+from .validation.dataframe_validator import DataFrameValidator
+from .validation.input_param_validator import InputParamValidator
+from .validation.exceptions import ValidationError
+import pandas as pd
+from typing import Optional
+
+logger = get_logger(__name__,
+                    file_log_level='DEBUG',
+                    console_log_level='DEBUG',)
+
+class AssayResultIngestionWorkflow:
+    """ """
+    def __init__(self, wrapper: BenchlingWrapperFacade):
+        self._wrapper = wrapper
+        self._api_param_validator : ApiParametersValidation
+        self._schema_handler = SchemaHandler
+        self._input_param_validator : Optional[InputParamValidator] = None
+        self._df_validator : Optional[DataFrameValidator] = None
+        self._container_transformer : Optional[ContainersTransformer] = None
+        self._link_transformer : Optional[LinkTransformer] = None
+        self._blob_transformer : Optional[BlobTransformer] = None
+        self._dropdown_transformer : Optional[DropdownTransformer] = None
+        self._datetime_converter : Optional[DateTimeConverter] = None
+        self._result_archiver : Optional[ResultArchiver] = None
+        self._blob_handler: Optional[BlobHandler] = None
+        self._result_ingestion: Optional[ResultIngestion] = None
+
+    def assay_results_ingestion(
+            self,
+            list_dataframes: list[pd.DataFrame],
+            schema_id: str,
+            project_id: str,
+            unique_identifiers: list[str],
+            dict_variable_to_entity_type: dict[str, list[str]] = None,
+            archive: bool = False,
+            compare_on: list[str] = None,
+            entry_name: str = None,
+            commit_in_transaction: bool = True,
+            blob_type: BlobType = "image/png"
+    ) -> (list[list[str]]):
+        """
+        A function to ingest assay results on Benchling following the logic at MeiraGTx DSC.
+
+        Ingests a lists of results dataframes.
+
+        Data parser requires datetime formats to be in the form of DD/MM/YYYY formats.
+        Empty values for non-required columns to be set to None.
+
+        Parameters
+        ----------
+        list_dataframes: list[pd.DataFrame] :
+            A list of dataframes to ingest. Columns need to be named with the warehouse name of the target assay
+            results schema.
+            
+        schema_id: str :
+            assay results schema api id.
+            
+        project_id: str :
+            project api id.
+
+        unique_identifiers: list[str] :
+            a list of unique identifiers of the assay results schema. To be provided as the warehouse name.
+
+        dict_variable_to_entity_type: dict[str:list[str]] :
+             (Default value = None)
+             To be specified for assay result schema variables set to AnyEntity. Allows to restrict the entity or
+             container types to be ingested to an AnyEntity or AnyInventory variable.
+             The entity type should refer the front end name of the entity schema or container schema.
+
+        archive: bool :
+             (Default value = False)
+             Defines whether pre-existing results should be archived.
+
+        compare_on: list[str] :
+             (Default value = None)
+             A list of variables in the assay results schema to build a dataframe of pre-existing results on Benchling.
+             Variable names need to be provided as the warehouse name.
+
+             When archive is False and compare_on is None : Pre-existing results are searched for unique_identifiers.
+             If pre-existing results are found for an identifier, new results are not ingested for that identifier.
+
+             When archive is False and compare_on is provided : Pre-existing results retrieved based on
+             unique_identifiers and compare_on list, and a pre-existing results dataframe is created. Any rows matching
+             pre-existing values will not be ingested.
+
+             When archive is True and compare_on is None : Pre-existing results are retrieved based on variables in
+             unique_identifiers. Retrieved data is archived and new results are ingested.
+
+             When archive is True, compare_on cannot be provided.
+
+        entry_name: str :
+             (Default value = None)
+             Name of the entry to visualized ingested data. Can't be provided in commit_in_transaction is True
+
+        commit_in_transaction: bool :
+             (Default value = True)
+            When commit_in_transaction is True : Data is ingested to the back-end. entry_name needs to be set to None.
+            When commit_in_transaction is False : Data is ingested to a table in an entry mentioned in entry_name.
+            entry_name needs to be provided.
+
+        blob_type: BlobType :
+             (Default value = "image/png")
+              Defines the expected blob type. Allowed values are "image/png" and "image/jpeg".
+
+        Returns
+        -------
+        list_missing_variables_per_df: a list of lists. Each list will contain missing entities or containers not found
+        on Benchling.
+
+        """
+        #validate api inputs
+        self._api_param_validator = ApiParametersValidation(wrapper=self._wrapper)
+
+        try:
+            api_param_validation, table_id = self._api_param_validator.validation(
+                schema_id=schema_id,
+                project_id=project_id,
+                entry_name=entry_name,
+                commit_in_transaction = commit_in_transaction,
+
+            )
+            logger.info(f"Api input parameters passed validation.")
+        except ValidationError as e:
+            logger.error(f"Api Input validation failed: {e.message}", extra=e.context)
+            raise
+
+        #create schema after validation
+        self._schema_handler = SchemaHandler(wrapper=self._wrapper)
+
+        schema = self._schema_handler._build_schema_definition(schema_id, 'assay_results_schema')
+
+        #validation of input parameters
+        self._input_param_validator = InputParamValidator(schema=schema, wrapper=self._wrapper)
+        try:
+            input_param_result = self._input_param_validator.validate(
+                unique_identifiers = unique_identifiers,
+                compare_on = compare_on,
+                archive = archive,
+                blob_type = blob_type,
+                dict_variable_to_entity_type = dict_variable_to_entity_type)
+            logger.info(f"Input parameters passed validation.")
+        except ValidationError as e:
+            logger.error(f"Input validation failed: {e.message}", extra=e.context)
+            raise
+
+        list_missing_variables_per_df = []
+        for dataframe in list_dataframes:
+            # create dataframe validator
+            self._df_validator = DataFrameValidator(schema)
+
+            # validate dataframe Before continuing
+            try:
+                df_validation_result = self._df_validator.validate(dataframe, strict=True)
+                logger.info(f"Schema validation passed for {len(dataframe)} rows")
+            except ValidationError as e:
+                logger.error(f"Schema validation failed: {e.message}", extra=e.context)
+                raise
+
+            # create container_id column if needed
+            self._container_transformer = ContainersTransformer(schema, wrapper=self._wrapper)
+            try:
+                dataframe = self._container_transformer.transform(dataframe, dict_variable_to_entity_type)
+            except TransformationError as e:
+                logger.error(f"Container column transformation failed: {e.message}", extra=e.context)
+                raise
+
+            # create dataframe transformer
+            self._link_transformer = LinkTransformer(schema, wrapper=self._wrapper)
+
+            # transform dataframe
+            dataframe, list_missing_variables = self._link_transformer.transform(dataframe, dict_variable_to_entity_type)
+            list_missing_variables_per_df.append(list_missing_variables)
+
+            if len(dataframe) == 0: #go to the next dataframe if all entities are missing
+                logger.warning(f"No entities were found on benchling: {list_missing_variables}.")
+                continue
+
+            # create blobs transformer
+            self._blob_transformer = BlobTransformer(schema, wrapper=self._wrapper)
+
+            # transform blobs in dataframe if they exist
+            try:
+                dataframe, dict_name_to_payload = self._blob_transformer.transform(dataframe, blob_type=blob_type)
+            except TransformationError as e:
+                logger.error(f"Blob transformation failed: {e.message}", extra=e.context)
+                raise
+
+            # create dropdown transformer
+            self._dropdown_transformer = DropdownTransformer(schema, wrapper=self._wrapper)
+
+            #transform dropdown columns if they exist
+            try:
+                dataframe = self._dropdown_transformer.transform(dataframe)
+            except TransformationError as e:
+                logger.error(f"Dropdown transformation failed: {e.message}", extra=e.context)
+                raise
+
+            #create datetime converter
+            self._datetime_converter = DateTimeConverter(schema)
+
+            #convert date time if they exist
+            try:
+                dataframe = self._datetime_converter.transform(dataframe)
+            except TransformationError as e:
+                logger.error(f"Datetime transformation has failed: {e.message}", extra=e.context)
+                raise
+
+            # create result archiver
+            self._result_archiver = ResultArchiver(schema, wrapper=self._wrapper)
+
+            # archive results
+            dataframe = self._result_archiver.transform(dataframe, unique_identifiers, archive, compare_on)
+            if dataframe is None: #when all results have been ingested for this dataframe
+                logger.info(f"All results were already ingested for this dataframe, "
+                            f"the code will proceed to the next dataframe or exit.")
+                continue
+
+            #create blob handler
+            self._blob_handler = BlobHandler(schema, wrapper=self._wrapper)
+
+            #transform blobs if they exist
+            try:
+                dataframe = self._blob_handler.ingest(dataframe, dict_name_to_payload)
+            except HandlerError as e:
+                logger.error(f"Blob transformation: {e.message}", extra=e.context)
+                raise
+
+            #create results ingestor
+            self._result_ingestion = ResultIngestion(schema, wrapper=self._wrapper)
+
+            #ingest df
+            try:
+                self._result_ingestion.ingest(dataframe, project_id, table_id, commit_in_transaction)
+            except HandlerError as e:
+                logger.error(f"Ingestion has failed: {e.message}", extra=e.context)
+                raise
+
+        return list_missing_variables_per_df
+
+
+