tol-sdk 1.7.4__py3-none-any.whl → 1.7.5b2__py3-none-any.whl

tol/flows/converters/incoming_sample_to_ena_sample_converter.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+import re
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        ena_checklist_id: str
+        project_name: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        converting the samples DataObject into ENA format
+        """
+        s = data_object
+        attributes = {
+            'ENA-CHECKLIST': self.__config.ena_checklist_id,
+            'organism part': self.__replace_underscores(
+                s.attributes.get('ORGANISM_PART')),
+            'lifestage': (
+                'spore-bearing structure'
+                if s.attributes.get('LIFESTAGE') == 'SPORE_BEARING_STRUCTURE'
+                else self.__replace_underscores(
+                    s.attributes.get('LIFESTAGE'))
+            ),
+            'project name':
+                self.__config.project_name,
+            'collected by':
+                self.__replace_underscores(
+                    s.attributes.get('COLLECTED_BY')),
+            'collection date':
+                self.__replace_underscores(
+                    s.attributes.get('DATE_OF_COLLECTION')).lower(),
+            'geographic location (country and/or sea)':
+                self.__collection_country(s).replace('_', ' '),
+            'geographic location (latitude)':
+                self.__replace_underscores(
+                    s.attributes.get('DECIMAL_LATITUDE')).lower(),
+            'geographic location (latitude) units':
+                'DD',
+            'geographic location (longitude)':
+                self.__replace_underscores(
+                    s.attributes.get('DECIMAL_LONGITUDE')).lower(),
+            'geographic location (longitude) units':
+                'DD',
+            'geographic location (region and locality)':
+                self.__collection_region(s).replace('_', ' '),
+            'identified_by':
+                self.__replace_underscores(
+                    s.attributes.get('IDENTIFIED_BY')),
+            'habitat':
+                self.__replace_underscores(
+                    s.attributes.get('HABITAT')),
+            'identifier_affiliation':
+                self.__replace_underscores(
+                    s.attributes.get('IDENTIFIER_AFFILIATION')),
+            'sex':
+                self.__replace_underscores(
+                    s.attributes.get('SEX')),
+            'relationship':
+                self.__replace_underscores(
+                    s.attributes.get('RELATIONSHIP')),
+            'SYMBIONT':
+                'Y' if s.attributes.get('SYMBIONT') == 'SYMBIONT' else 'N',
+            'collecting institution':
+                self.__replace_underscores(
+                    s.attributes.get('COLLECTOR_AFFILIATION'))
+        }
+        if self.__sanitise(s.attributes.get('DEPTH')) != '':
+            attributes['geographic location (depth)'] = s.attributes.get('DEPTH')
+            attributes['geographic location (depth) units'] = 'm'
+        if self.__sanitise(s.attributes.get('ELEVATION')) != '':
+            attributes['geographic location (elevation)'] = s.attributes.get('ELEVATION')
+            attributes['geographic location (elevation) units'] = 'm'
+        if self.__sanitise(s.attributes.get('ORIGINAL_COLLECTION_DATE')) != '':
+            attributes['original collection date'] = s.attributes.get('ORIGINAL_COLLECTION_DATE')
+        if self.__sanitise(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION')) != '':
+            attributes['original geographic location'] = self.__replace_underscores(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION'))  # noqa
+        if s.attributes.get('GAL') is not None:
+            attributes['GAL'] = s.attributes.get('GAL')
+        if s.attributes.get('VOUCHER_ID') is not None:
+            attributes['specimen_voucher'] = s.attributes.get('VOUCHER_ID')
+        if s.attributes.get('SPECIMEN_ID') is not None:
+            attributes['specimen_id'] = s.attributes.get('SPECIMEN_ID')
+        if s.attributes.get('GAL_SAMPLE_ID') is not None:
+            attributes['GAL_sample_id'] = s.attributes.get('GAL_SAMPLE_ID')
+        if s.attributes.get('CULTURE_OR_STRAIN_ID') is not None:
+            attributes['culture_or_strain_id'] = s.attributes.get('CULTURE_OR_STRAIN_ID')
+
+        ret = self._data_object_factory(
+            'sample',
+            s.id,
+            attributes=attributes,
+        )
+        yield ret
+
+    def __collection_country(self, data_object: DataObject):
+        return re.split(
+            r'\s*\|\s*',
+            data_object.attributes.get('COLLECTION_LOCATION'))[0]
+
+    def __collection_region(self, data_object: DataObject):
+        return ' | '.join(re.split(
+            r'\s*\|\s*',
+            data_object.attributes.get('COLLECTION_LOCATION'))[1:])
+
+    def __replace_underscores(self, value):
+        if type(value) != str:
+            return value
+        return self.__sanitise(value, '').replace('_', ' ')
+
+    def __sanitise(self, value, default_value=''):
+        if value is None:
+            return default_value
+        return value

tol/flows/converters/incoming_sample_to_incoming_sample_with_lists_converter.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class IncomingSampleToIncomingSampleWithListsConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        fields_to_convert: str
+        separator: str = '|'
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        converting the samples DataObject into ENA format
+        """
+
+        ret = self._data_object_factory(
+            data_object.type,
+            data_object.id,
+            attributes={
+                k: v for k, v in data_object.attributes.items()
+                if k not in self.__config.fields_to_convert
+            } | {
+                field: self.__convert_to_list(data_object.get_field_by_name(field))
+                for field in self.__config.fields_to_convert
+            }
+        )
+        yield ret
+
+    def __convert_to_list(self, value: str | None) -> list[str]:
+        if not value:
+            return []
+        return [item.strip() for item in value.split(self.__config.separator)]

tol/s3/__init__.py

@@ -6,4 +6,3 @@ from .parser import Parser  # noqa F401
 from .converter import S3Converter  # noqa F401
 from .factory import create_s3_datasource  # noqa F401
 from .s3_datasource import S3DataSource  # noqa F401
-from .data_upload.blueprint import data_upload_blueprint # noqa F401

tol/sql/model.py

@@ -255,7 +255,7 @@ def model_base() -> Type[DefaultModel]:
         @property
         def instance_to_many_relations(self) -> dict[str, Iterable[Model]]:
             config = self.get_to_many_relationship_config()
-            return self.__get_attributes_map(config.keys())
+            return self.__get_attributes_map(config)
 
         @property
         def instance_id(self) -> Optional[str]:

tol/sql/pipeline_step/factory.py

@@ -140,7 +140,7 @@ def create_pipeline_step_models(
             autoincrement=True
         )
 
-        s3_url: Mapped[str] = mapped_column(nullable=False,)
+        s3_bucket: Mapped[str] = mapped_column(nullable=False,)
         s3_filename: Mapped[str] = mapped_column(nullable=False)
         spreadsheet_config: Mapped[str] = mapped_column(nullable=True)
 

tol/sql/sql_converter.py

@@ -83,7 +83,13 @@ class DefaultModelConverter(ModelConverter):
         return to_ones if to_ones else None
 
     def __convert_to_many_requested(self, model, tree):
-        return None
+        to_manys = {}
+        for rel_name in model.get_to_many_relationship_config():
+            if sub_tree := tree.get_sub_tree(rel_name):
+                to_manys[rel_name] = [
+                    self.__convert_requested(x, sub_tree) for x in getattr(model, rel_name)
+                ]
+        return to_manys if to_manys else None
 
 
 class DataObjectConverter(Converter[DataObject, Model], ABC):

tol/validators/__init__.py

@@ -2,8 +2,19 @@
 #
 # SPDX-License-Identifier: MIT
 
-from .allowed_values import AllowedValues, AllowedValuesValidator  # noqa
 from .allowed_keys import AllowedKeysValidator  # noqa
+from .allowed_values import AllowedValuesValidator  # noqa
+from .allowed_values_from_datasource import AllowedValuesFromDataSourceValidator  # noqa
+from .assert_on_condition import AssertOnConditionValidator  # noqa
+from .ena_checklist import EnaChecklistValidator # noqa
+from .mutually_exclusive import MutuallyExclusiveValidator  # noqa
+from .ena_submittable import EnaSubmittableValidator  # noqa
 from .regex import Regex, RegexValidator  # noqa
 from .regex_by_value import RegexByValueValidator  # noqa
+from .specimens_have_same_taxon import SpecimensHaveSameTaxonValidator # noqa
+from .sts_fields import StsFieldsValidator  # noqa
+from .tolid import TolidValidator  # noqa
 from .unique_values import UniqueValuesValidator  # noqa
+from .unique_whole_organisms import UniqueWholeOrganismsValidator  # noqa
+from .interfaces import Condition  # noqa
+from .min_one_valid_value import MinOneValidValueValidator   # noqa

tol/validators/allowed_keys.py

@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: MIT
 
+from dataclasses import dataclass
+from typing import List
+
 from tol.core import DataObject
 from tol.core.validate import Validator
 
@@ -12,20 +15,22 @@ class AllowedKeysValidator(Validator):
     ensuring that they only have attributes of the given
     allowed keys.
     """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        allowed_keys: List[str]
+        is_error: bool = True
+        detail: str = 'Key is not allowed'
+
+    __slots__ = ['__config']
+    __config: Config
 
     def __init__(
         self,
-        allowed_keys: list[str],
-        *,
-        is_error: bool = True,
-        detail: str = 'Key is not allowed'
+        config: Config
     ) -> None:
 
         super().__init__()
-
-        self.__keys = allowed_keys
-        self.__is_error = is_error
-        self.__detail = detail
+        self.__config = config
 
     def _validate_data_object(
         self,
@@ -33,7 +38,7 @@ class AllowedKeysValidator(Validator):
     ) -> None:
 
         for key in obj.attributes:
-            if key not in self.__keys:
+            if key not in self.__config.allowed_keys:
                 self.__add_result(
                     obj,
                     key,
@@ -45,15 +50,15 @@ class AllowedKeysValidator(Validator):
         key: str,
     ) -> None:
 
-        if self.__is_error:
+        if self.__config.is_error:
             self.add_error(
                 object_id=obj.id,
-                detail=self.__detail,
+                detail=self.__config.detail,
                 field=key,
             )
         else:
             self.add_warning(
                 object_id=obj.id,
-                detail=self.__detail,
+                detail=self.__config.detail,
                 field=key,
             )

tol/validators/allowed_values.py

@@ -3,103 +3,61 @@
 # SPDX-License-Identifier: MIT
 
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, List
 
 from tol.core import DataObject
 from tol.core.validate import Validator
 
 
-@dataclass(frozen=True, kw_only=True)
-class AllowedValues:
-    key: str
-    values: list[Any]
-
-    is_error: bool = True
-    detail: str = 'Value is not allowed for given key'
-
-    def is_allowed(self, __v: Any) -> bool:
-        return __v in self.values
-
-
-AllowedValuesDict = dict[
-    str,
-    str | bool | list[Any],
-]
-"""Can also specify `AllowedValues` as a `dict`"""
-
-
 class AllowedValuesValidator(Validator):
     """
     Validates an incoming stream of `DataObject` instances
     according to the specified allowed values for a given
     key.
     """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        field: str
+        allowed_values: List[Any]
+        is_error: bool = True
+        detail: str = 'Value is not allowed for the given key'
+
+    __slots__ = ['__config']
+    __config: Config
 
     def __init__(
         self,
-        config: list[AllowedValues | AllowedValuesDict]
+        config: Config
     ) -> None:
 
         super().__init__()
 
-        self.__config = self.__get_config(config)
+        self.__config = config
 
     def _validate_data_object(
         self,
         obj: DataObject
     ) -> None:
 
-        for k, v in obj.attributes.items():
-            self.__validate_attribute(obj, k, v)
-
-    def __get_config(
-        self,
-        config: list[AllowedValues | AllowedValuesDict],
-    ) -> list[AllowedValues]:
-
-        return [
-            c if isinstance(c, AllowedValues) else AllowedValues(**c)
-            for c in config
-        ]
-
-    def __validate_attribute(
-        self,
-        obj: DataObject,
-        key: str,
-        value: Any,
-    ) -> None:
-
-        config = self.__filter_config(key)
-
-        for c in config:
-            if not c.is_allowed(value):
-                self.__add_result(obj, c)
-
-    def __filter_config(
-        self,
-        key: str,
-    ) -> list[AllowedValues]:
-
-        return [
-            a for a in self.__config
-            if a.key == key
-        ]
+        for key, value in obj.attributes.items():
+            if key == self.__config.field and value not in self.__config.allowed_values:
+                self.__add_result(obj, key)
 
     def __add_result(
         self,
         obj: DataObject,
-        c: AllowedValues,
+        key: str,
     ) -> None:
 
-        if c.is_error:
+        if self.__config.is_error:
             self.add_error(
                 object_id=obj.id,
-                detail=c.detail,
-                field=c.key
+                detail=self.__config.detail,
+                field=key
             )
         else:
             self.add_warning(
                 object_id=obj.id,
-                detail=c.detail,
-                field=c.key,
+                detail=self.__config.detail,
+                field=key,
             )

tol/validators/allowed_values_from_datasource.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import List
+
+from tol.core import DataObject, DataSourceUtils
+from tol.core.validate import Validator
+from tol.sources.portaldb import portaldb
+
+
+class AllowedValuesFromDataSourceValidator(Validator):
+    """
+    Validates that a stream of `DataObject` instances
+    contains field that is part of a list.
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        datasource_instance_id: int
+        datasource_object_type: str
+        datasource_field_name: str
+        field_name: str
+
+    __slots__ = ['__config', '__cached_list']
+    __config: Config
+    __cached_list: List[str | int | float]
+
+    def __init__(
+        self,
+        config: Config,
+        allowed_values: List[str | int | float] | None = None  # For testing
+    ) -> None:
+
+        super().__init__()
+
+        self.__config = config
+        self.__cached_list = allowed_values \
+            or self.__initialize_list_from_datasource()
+
+    def __initialize_list_from_datasource(self) -> List[str | int | float]:
+        dsi = portaldb().get_one('data_source_instance', self.__config.datasource_instance_id)
+        ds = DataSourceUtils.get_data_source_by_data_source_instance(dsi)
+        return [
+            obj.get_field_by_name(
+                self.__config.datasource_field_name
+            ) for obj in ds.get_list(
+                object_type=self.__config.datasource_object_type
+            )
+        ]
+
+    def _validate_data_object(
+        self,
+        obj: DataObject
+    ) -> None:
+        field_value = obj.get_field_by_name(self.__config.field_name)
+        if not field_value:
+            return
+        if not isinstance(field_value, list):
+            field_value = [field_value]
+        if any(value not in self.__cached_list for value in field_value):
+            multiple_cached_values = len(self.__cached_list) > 1
+
+            cached_list_str = ''
+            if multiple_cached_values:
+                for index, field in enumerate(self.__cached_list):
+                    if index == 0:
+                        # First item in the list
+                        cached_list_str += f'{field}'
+                    elif index == len(self.__cached_list) - 1:
+                        # Last item in the list
+                        cached_list_str += f' or {field}'
+                    else:
+                        # Middle items
+                        cached_list_str += f', {field}'
+            else:  # Only one field
+                cached_list_str = self.__cached_list[0]
+
+            # This is extracted rather than being evaluated in the f-string
+            # because otherwise the linter doesn't like it!
+            pluralisation = ' one of' if multiple_cached_values else ''
+
+            self.add_error(
+                object_id=obj.id,
+                detail=f'The value of the field {self.__config.field_name} '
+                       f'must be{pluralisation} {cached_list_str} '
+                       f'(found value {field_value})',
+                field=self.__config.field_name,
+            )

tol/validators/assert_on_condition.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import List
+
+from tol.core import DataObject, Validator
+
+from .interfaces import Condition, ConditionDict, ConditionEvaluator
+
+
+class AssertOnConditionValidator(Validator, ConditionEvaluator):
+    """
+    Validates an incoming stream of `DataObject` instances,
+    using a condition to check a specific attrbiute. If this
+    condition passes, then the assertions will be run, which must
+    all pass.
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        condition: ConditionDict
+        assertions: List[ConditionDict]
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+
+        self.__config = config
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        # Check condition atribute
+        # (only perform the assertions if the condition passes)
+        if self._does_condition_pass(Condition.from_dict(self.__config.condition), obj):
+            # Perform each assertion
+            for assertion in self.__config.assertions:
+                self.__perform_assertion(obj, Condition.from_dict(assertion))
+
+    def __perform_assertion(self, obj: DataObject, assertion: Condition) -> None:
+        # There's only an error or warning if the assertion condition fails
+        condition_passed, found_value = self._evaluate_condition(assertion, obj)
+        if not condition_passed:
+            if assertion.is_error:
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Expected {assertion} (found value {found_value})',
+                    field=assertion.field,
+                )
+            else:
+                self.add_warning(
+                    object_id=obj.id,
+                    detail=f'Expected {assertion} (found value {found_value})',
+                    field=assertion.field,
+                )

tol/validators/ena_checklist.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+import re
+from dataclasses import dataclass
+
+from tol.core import DataSource
+from tol.core.data_object import DataObject
+from tol.core.validate import Validator
+from tol.sources.ena import ena
+
+
+class EnaChecklistValidator(Validator):
+    """
+    validates the ENA_CHECKLIST for each samples
+    """
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        ena_checklist_id: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, config: Config, datasource: DataSource = ena()) -> None:
+        super().__init__()
+        self.__config = config
+        self._datasource = datasource
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        ena_datasource = self._datasource
+        ena_checklist = ena_datasource.get_one('checklist', self.__config.ena_checklist_id)
+
+        validations = ena_checklist.attributes['checklist']
+        for key in validations:
+            field_name = key
+            if 'field' in validations[key]:
+                field_name = validations[key]['field']
+            if 'mandatory' in validations[key] and key not in obj.attributes:
+                self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
+                continue
+            if 'mandatory' in validations[key] and obj.attributes[key] is None:
+                self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
+                continue
+            if 'mandatory' in validations[key] and obj.attributes.get(key) == '':
+                self.add_error(
+                    object_id=obj.id,
+                    detail='Must not be empty', field=[field_name]
+                )
+
+            if 'restricted text' in validations[key] and key in obj.attributes:
+                for condition in validations[key]:
+                    if type(condition) == str and '(' in condition:
+                        regex = condition
+                compiled_re = re.compile(regex)
+                if not compiled_re.search(obj.attributes.get(key)):
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='Must match specific pattern', field=[field_name]
+                    )
+
+            # Check against allowed values
+            if 'text choice' in validations[key] and key in obj.attributes:
+                for condition in validations[key]:
+                    if type(condition) == list:
+                        allowed_values = condition
+                if obj.attributes.get(key).lower() not in \
+                        [x.lower() for x in allowed_values]:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='Must be in allowed values', field=[field_name]
+                    )