tol-sdk 1.7.5b0__tar.gz → 1.7.5b3__tar.gz

{tol_sdk-1.7.5b0/src/tol_sdk.egg-info → tol_sdk-1.7.5b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.5b0
+Version: 1.7.5b3
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tol-sdk"
-version = "1.7.5-beta"
+version = "1.7.5-beta3"
 description = "SDK for interaction with ToL, Sanger and external services"
 readme = "README.md"
 authors = [

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/core/operator/detail_getter.py

@@ -82,7 +82,7 @@ class DetailGetter(ABC):
         for id_ in object_ids:
             seekable_objects.seek(0)
             for obj in seekable_objects:
-                if obj.id == id_:
+                if str(obj.id) == str(id_):
                     yield obj
                     break
             else:

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/ena/client.py

@@ -116,12 +116,13 @@ class EnaApiClient(HttpClient):
         Returns the URL and parameters for a detail query.
         """
         if object_type == 'checklist':
-            ids = ','.join(object_ids)
+            ids = ','.join(str(id_) for id_ in object_ids)
             url = f'{self.__ena_url}/ena/browser/api/xml/{ids}'
             params = {}
             return url, params
         if object_type == 'submittable_taxon':
-            ids = ','.join(object_ids)
+            # This is actually called separately for each taxon id
+            ids = ','.join(str(id_) for id_ in object_ids)
             url = f'{self.__ena_url}/ena/taxonomy/rest/tax-id/{ids}'
             params = {}
             return url, params

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/flows/converters/__init__.py

@@ -34,6 +34,8 @@ from .gap_assembly_to_elastic_assembly_analysis_converter import GapAssemblyToEl
 from .genome_notes_genome_note_to_elastic_genome_note_converter import GenomeNotesGenomeNoteToElasticGenomeNoteConverter  # noqa F401
 from .goat_taxon_to_elastic_species_converter import GoatTaxonToElasticSpeciesConverter  # noqa F401
 from .grit_issue_to_elastic_curation_converter import GritIssueToElasticCurationConverter  # noqa F401
+from .incoming_sample_to_ena_sample_converter import IncomingSampleToEnaSampleConverter # noqa
+from .incoming_sample_to_incoming_sample_with_lists_converter import IncomingSampleToIncomingSampleWithListsConverter  # noqa F401
 from .informatics_tolid_to_elastic_tolid_converter import InformaticsTolidToElasticTolidConverter  # noqa F401
 from .labwhere_location_to_elastic_sample_update_converter import LabwhereLocationToElasticSampleUpdateConverter  # noqa F401
 from .labwhere_location_to_sts_tray_converter import LabwhereLocationToStsTrayConverter  # noqa F401

tol_sdk-1.7.5b3/src/tol/flows/converters/incoming_sample_to_ena_sample_converter.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+import re
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        ena_checklist_id: str
+        project_name: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        converting the samples DataObject into ENA format
+        """
+        s = data_object
+        attributes = {
+            'ENA-CHECKLIST': self.__config.ena_checklist_id,
+            'organism part': self.__replace_underscores(
+                s.attributes.get('ORGANISM_PART')),
+            'lifestage': (
+                'spore-bearing structure'
+                if s.attributes.get('LIFESTAGE') == 'SPORE_BEARING_STRUCTURE'
+                else self.__replace_underscores(
+                    s.attributes.get('LIFESTAGE'))
+            ),
+            'project name':
+                self.__config.project_name,
+            'collected by':
+                self.__replace_underscores(
+                    s.attributes.get('COLLECTED_BY')),
+            'collection date':
+                self.__replace_underscores(
+                    s.attributes.get('DATE_OF_COLLECTION')).lower(),
+            'geographic location (country and/or sea)':
+                self.__collection_country(s).replace('_', ' '),
+            'geographic location (latitude)':
+                self.__replace_underscores(
+                    s.attributes.get('DECIMAL_LATITUDE')).lower(),
+            'geographic location (latitude) units':
+                'DD',
+            'geographic location (longitude)':
+                self.__replace_underscores(
+                    s.attributes.get('DECIMAL_LONGITUDE')).lower(),
+            'geographic location (longitude) units':
+                'DD',
+            'geographic location (region and locality)':
+                self.__collection_region(s).replace('_', ' '),
+            'identified_by':
+                self.__replace_underscores(
+                    s.attributes.get('IDENTIFIED_BY')),
+            'habitat':
+                self.__replace_underscores(
+                    s.attributes.get('HABITAT')),
+            'identifier_affiliation':
+                self.__replace_underscores(
+                    s.attributes.get('IDENTIFIER_AFFILIATION')),
+            'sex':
+                self.__replace_underscores(
+                    s.attributes.get('SEX')),
+            'relationship':
+                self.__replace_underscores(
+                    s.attributes.get('RELATIONSHIP')),
+            'SYMBIONT':
+                'Y' if s.attributes.get('SYMBIONT') == 'SYMBIONT' else 'N',
+            'collecting institution':
+                self.__replace_underscores(
+                    s.attributes.get('COLLECTOR_AFFILIATION'))
+        }
+        if self.__sanitise(s.attributes.get('DEPTH')) != '':
+            attributes['geographic location (depth)'] = s.attributes.get('DEPTH')
+            attributes['geographic location (depth) units'] = 'm'
+        if self.__sanitise(s.attributes.get('ELEVATION')) != '':
+            attributes['geographic location (elevation)'] = s.attributes.get('ELEVATION')
+            attributes['geographic location (elevation) units'] = 'm'
+        if self.__sanitise(s.attributes.get('ORIGINAL_COLLECTION_DATE')) != '':
+            attributes['original collection date'] = s.attributes.get('ORIGINAL_COLLECTION_DATE')
+        if self.__sanitise(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION')) != '':
+            attributes['original geographic location'] = self.__replace_underscores(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION'))  # noqa
+        if s.attributes.get('GAL') is not None:
+            attributes['GAL'] = s.attributes.get('GAL')
+        if s.attributes.get('VOUCHER_ID') is not None:
+            attributes['specimen_voucher'] = s.attributes.get('VOUCHER_ID')
+        if s.attributes.get('SPECIMEN_ID') is not None:
+            attributes['specimen_id'] = s.attributes.get('SPECIMEN_ID')
+        if s.attributes.get('GAL_SAMPLE_ID') is not None:
+            attributes['GAL_sample_id'] = s.attributes.get('GAL_SAMPLE_ID')
+        if s.attributes.get('CULTURE_OR_STRAIN_ID') is not None:
+            attributes['culture_or_strain_id'] = s.attributes.get('CULTURE_OR_STRAIN_ID')
+
+        ret = self._data_object_factory(
+            'sample',
+            s.id,
+            attributes=attributes,
+        )
+        yield ret
+
+    def __collection_country(self, data_object: DataObject):
+        return re.split(
+            r'\s*\|\s*',
+            data_object.attributes.get('COLLECTION_LOCATION'))[0]
+
+    def __collection_region(self, data_object: DataObject):
+        return ' | '.join(re.split(
+            r'\s*\|\s*',
+            data_object.attributes.get('COLLECTION_LOCATION'))[1:])
+
+    def __replace_underscores(self, value):
+        if type(value) != str:
+            return value
+        return self.__sanitise(value, '').replace('_', ' ')
+
+    def __sanitise(self, value, default_value=''):
+        if value is None:
+            return default_value
+        return value

tol_sdk-1.7.5b3/src/tol/flows/converters/incoming_sample_to_incoming_sample_with_lists_converter.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class IncomingSampleToIncomingSampleWithListsConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        fields_to_convert: str
+        separator: str = '|'
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        converting the samples DataObject into ENA format
+        """
+
+        ret = self._data_object_factory(
+            data_object.type,
+            data_object.id,
+            attributes={
+                k: v for k, v in data_object.attributes.items()
+                if k not in self.__config.fields_to_convert
+            } | {
+                field: self.__convert_to_list(data_object.get_field_by_name(field))
+                for field in self.__config.fields_to_convert
+            }
+        )
+        yield ret
+
+    def __convert_to_list(self, value: str | None) -> list[str]:
+        if not value:
+            return []
+        return [item.strip() for item in value.split(self.__config.separator)]

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/sql/auth/models.py

@@ -19,6 +19,7 @@ from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import (
     Mapped,
     Session,
+    joinedload,
     mapped_column,
     relationship
 )
@@ -184,7 +185,7 @@ def create_models(
 
         id: Mapped[str] = mapped_column(  # noqa A003
             primary_key=True,
-            name=None if not prefix_with_name else 'state_id'
+            name='state_id' if prefix_with_name else None
         )
 
         created_at: Mapped[datetime] = mapped_column(
@@ -257,13 +258,13 @@ def create_models(
         id: Mapped[int] = mapped_column(  # noqa A003
             primary_key=True,
             autoincrement=True,
-            name=None if not prefix_with_name else 'user_id'
+            name='user_id' if prefix_with_name else None
         )
 
-        _tokens: Mapped[list['Token']] = relationship(
+        _tokens: Mapped[list[Token]] = relationship(
             back_populates='user'
         )
-        _role_bindings: Mapped[list['RoleBinding']] = relationship(
+        _role_bindings: Mapped[list[RoleBinding]] = relationship(
             back_populates='user'
         )
 
@@ -340,7 +341,7 @@ def create_models(
 
     class _TokenPKMixin:
         """
-        Has an integer has a primary key.
+        Has an integer as a primary key.
 
         This is all that is needed in most apps,
         and is the default.
@@ -403,11 +404,19 @@ def create_models(
             token: str
         ) -> Optional[Token]:
 
-            return sess.query(
-                cls
-            ).filter_by(
-                token=token
-            ).one_or_none()
+            return (
+                sess.query(cls)
+                .filter_by(token=token)
+                .options(
+                    # Join through to the Role table when fetching a Token so
+                    # that SQLAlchemy only fires one SELECT query instead of
+                    # four to get `token.user._role_bindings.role.name`
+                    joinedload(cls.user)
+                    .joinedload(User._role_bindings)
+                    .joinedload(RoleBinding.role)
+                )
+                .one_or_none()
+            )
 
         @classmethod
         def get_or_create(
@@ -475,14 +484,14 @@ def create_models(
         id: Mapped[int] = mapped_column(  # noqa A003
             primary_key=True,
             autoincrement=True,
-            name=None if not prefix_with_name else 'role_id'
+            name='role_id' if prefix_with_name else None
         )
 
         name: Mapped[str] = mapped_column(
             unique=True
         )
 
-        _role_bindings: Mapped[list['RoleBinding']] = relationship(
+        _role_bindings: Mapped[list[RoleBinding]] = relationship(
             back_populates='role'
         )
 

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/__init__.py

@@ -6,6 +6,8 @@ from .allowed_keys import AllowedKeysValidator  # noqa
 from .allowed_values import AllowedValuesValidator  # noqa
 from .allowed_values_from_datasource import AllowedValuesFromDataSourceValidator  # noqa
 from .assert_on_condition import AssertOnConditionValidator  # noqa
+from .converter_and_validate import ConverterAndValidateValidator  # noqa
+from .ena_checklist import EnaChecklistValidator # noqa
 from .mutually_exclusive import MutuallyExclusiveValidator  # noqa
 from .ena_submittable import EnaSubmittableValidator  # noqa
 from .regex import Regex, RegexValidator  # noqa
@@ -15,6 +17,5 @@ from .sts_fields import StsFieldsValidator  # noqa
 from .tolid import TolidValidator  # noqa
 from .unique_values import UniqueValuesValidator  # noqa
 from .unique_whole_organisms import UniqueWholeOrganismsValidator  # noqa
-# Conditions are used where validators are defined, not just within validators
 from .interfaces import Condition  # noqa
 from .min_one_valid_value import MinOneValidValueValidator   # noqa

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/allowed_keys.py

@@ -26,7 +26,8 @@ class AllowedKeysValidator(Validator):
 
     def __init__(
         self,
-        config: Config
+        config: Config,
+        **kwargs
     ) -> None:
 
         super().__init__()

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/allowed_values.py

@@ -27,7 +27,8 @@ class AllowedValuesValidator(Validator):
 
     def __init__(
         self,
-        config: Config
+        config: Config,
+        **kwargs
     ) -> None:
 
         super().__init__()

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/allowed_values_from_datasource.py

@@ -17,7 +17,6 @@ class AllowedValuesFromDataSourceValidator(Validator):
     """
     @dataclass(slots=True, frozen=True, kw_only=True)
     class Config:
-        allowed_values: List[str | int | float] | None = None
         datasource_instance_id: int
         datasource_object_type: str
         datasource_field_name: str
@@ -30,22 +29,19 @@ class AllowedValuesFromDataSourceValidator(Validator):
     def __init__(
         self,
         config: Config,
+        allowed_values: List[str | int | float] | None = None,  # For testing
+        **kwargs
     ) -> None:
 
         super().__init__()
 
         self.__config = config
-        self.__cached_list = self.__config.allowed_values \
+        self.__cached_list = allowed_values \
             or self.__initialize_list_from_datasource()
 
-        if self.__config.allowed_values is None:
-            self.__cached_list = self.__initialize_list_from_datasource()
-        else:
-            self.__cached_list = self.__config.allowed_values
-
     def __initialize_list_from_datasource(self) -> List[str | int | float]:
         dsi = portaldb().get_one('data_source_instance', self.__config.datasource_instance_id)
-        ds = DataSourceUtils.get_data_source_by_data_source_instance(dsi)
+        ds = DataSourceUtils.get_datasource_by_datasource_instance(dsi)
         return [
             obj.get_field_by_name(
                 self.__config.datasource_field_name
@@ -59,8 +55,11 @@ class AllowedValuesFromDataSourceValidator(Validator):
         obj: DataObject
     ) -> None:
         field_value = obj.get_field_by_name(self.__config.field_name)
-        if field_value not in self.__cached_list:
-            print(self.__cached_list)
+        if not field_value:
+            return
+        if not isinstance(field_value, list):
+            field_value = [field_value]
+        if any(value not in self.__cached_list for value in field_value):
             multiple_cached_values = len(self.__cached_list) > 1
 
             cached_list_str = ''

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/assert_on_condition.py

@@ -7,7 +7,7 @@ from typing import List
 
 from tol.core import DataObject, Validator
 
-from .interfaces import Condition, ConditionEvaluator
+from .interfaces import Condition, ConditionDict, ConditionEvaluator
 
 
 class AssertOnConditionValidator(Validator, ConditionEvaluator):
@@ -19,13 +19,13 @@ class AssertOnConditionValidator(Validator, ConditionEvaluator):
     """
     @dataclass(slots=True, frozen=True, kw_only=True)
     class Config:
-        condition: Condition
-        assertions: List[Condition]
+        condition: ConditionDict
+        assertions: List[ConditionDict]
 
     __slots__ = ['__config']
     __config: Config
 
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: Config, **kwargs) -> None:
         super().__init__()
 
         self.__config = config
@@ -33,10 +33,10 @@ class AssertOnConditionValidator(Validator, ConditionEvaluator):
     def _validate_data_object(self, obj: DataObject) -> None:
         # Check condition atribute
         # (only perform the assertions if the condition passes)
-        if self._does_condition_pass(self.__config.condition, obj):
+        if self._does_condition_pass(Condition.from_dict(self.__config.condition), obj):
             # Perform each assertion
             for assertion in self.__config.assertions:
-                self.__perform_assertion(obj, assertion)
+                self.__perform_assertion(obj, Condition.from_dict(assertion))
 
     def __perform_assertion(self, obj: DataObject, assertion: Condition) -> None:
         # There's only an error or warning if the assertion condition fails

tol_sdk-1.7.5b3/src/tol/validators/converter_and_validate.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+import importlib
+from dataclasses import dataclass
+
+from tol.core import DataObject
+from tol.core.factory import DataObjectFactory
+from tol.core.validate import ValidationResult, Validator
+
+
+class ConverterAndValidateValidator(Validator):
+    """
+    Convert DataObjects, validate the converted ones, and return the original
+    input unchanged. Inner validator results are not merged here.
+
+    {
+        "converters": [{
+            "module": "<path.to.module>",
+            "class_name": "<path.to.ConverterClass>",
+            "config": { ... }
+        }],
+        "validators": [{
+            "module": "<path.to.module>",
+            "class_name": "<path.to.ValidatorClass>",
+            "config": { ... }
+        }]
+    }
+
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        converters: list[dict]
+        validators: list[dict]
+
+    __slots__ = [
+        '__converters',
+        '__validators'
+    ]
+
+    def __init__(
+        self,
+        config: Config,
+        data_object_factory: DataObjectFactory,
+        **kwargs
+    ) -> None:
+        super().__init__()
+        self.__converters = []
+        self.__validators = []
+
+        for conv in config.converters:
+            __module = importlib.import_module(conv.get('module'))
+            converter_class = getattr(__module, conv.get('class_name'))
+
+            converter_conf = converter_class.Config(
+                **conv.get('config')
+            )
+            self.__converters.append(converter_class(
+                data_object_factory=data_object_factory,
+                config=converter_conf,
+            ))
+        for val in config.validators:
+            __module = importlib.import_module(val.get('module'))
+            validator_class = getattr(__module, val.get('class_name'))
+
+            validator_conf = validator_class.Config(
+                **val.get('config')
+            )
+            self.__validators.append(validator_class(
+                data_object_factory=data_object_factory,
+                config=validator_conf,
+            ))
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        converted_objs = [obj]
+        for converter in self.__converters:
+            converted_objs = converter.convert_iterable(converted_objs)
+        for obj in converted_objs:
+            for validator in self.__validators:
+                validator._validate_data_object(obj)
+
+    @property
+    def results(self) -> list[ValidationResult]:
+        return [result for validator in self.__validators for result in validator.results]
+
+    @property
+    def warnings(self) -> list[ValidationResult]:
+        return [warning for validator in self.__validators for warning in validator.warnings]
+
+    @property
+    def errors(self) -> list[ValidationResult]:
+        return [error for validator in self.__validators for error in validator.errors]

tol_sdk-1.7.5b3/src/tol/validators/ena_checklist.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+import re
+from dataclasses import dataclass
+
+from tol.core import DataSource
+from tol.core.data_object import DataObject
+from tol.core.validate import Validator
+from tol.sources.ena import ena
+
+
+class EnaChecklistValidator(Validator):
+    """
+    validates the ENA_CHECKLIST for each samples
+    """
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        ena_checklist_id: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, config: Config, datasource: DataSource = ena(), **kwargs) -> None:
+        super().__init__()
+        self.__config = config
+        self._datasource = datasource
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        ena_datasource = self._datasource
+        ena_checklist = ena_datasource.get_one('checklist', self.__config.ena_checklist_id)
+
+        validations = ena_checklist.attributes['checklist']
+        for key in validations:
+            field_name = key
+            if 'field' in validations[key]:
+                field_name = validations[key]['field']
+            if 'mandatory' in validations[key] and key not in obj.attributes:
+                self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
+                continue
+            if 'mandatory' in validations[key] and obj.attributes[key] is None:
+                self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
+                continue
+            if 'mandatory' in validations[key] and obj.attributes.get(key) == '':
+                self.add_error(
+                    object_id=obj.id,
+                    detail='Must not be empty', field=[field_name]
+                )
+
+            if 'restricted text' in validations[key] and key in obj.attributes:
+                for condition in validations[key]:
+                    if type(condition) == str and '(' in condition:
+                        regex = condition
+                compiled_re = re.compile(regex)
+                if not compiled_re.search(obj.attributes.get(key)):
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='Must match specific pattern', field=[field_name]
+                    )
+
+            # Check against allowed values
+            if 'text choice' in validations[key] and key in obj.attributes:
+                for condition in validations[key]:
+                    if type(condition) == list:
+                        allowed_values = condition
+                if obj.attributes.get(key).lower() not in \
+                        [x.lower() for x in allowed_values]:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='Must be in allowed values', field=[field_name]
+                    )

{tol_sdk-1.7.5b0 → tol_sdk-1.7.5b3}/src/tol/validators/ena_submittable.py

@@ -28,6 +28,7 @@ class EnaSubmittableValidator(Validator):
         self,
         config: Config,
         ena_datasource: DataSource | None = ena(),  # For testing
+        **kwargs
     ) -> None:
 
         super().__init__()

tol_sdk-1.7.5b3/src/tol/validators/interfaces/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from .condition_evaluator import Condition, ConditionDict, ConditionEvaluator  # noqa