tol-sdk 1.7.5b4__tar.gz → 1.8.0__tar.gz

{tol_sdk-1.7.5b4/src/tol_sdk.egg-info → tol_sdk-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.5b4
+Version: 1.8.0
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT
@@ -50,6 +50,8 @@ Requires-Dist: openpyxl>=3.0.10; extra == "sheets"
 Requires-Dist: XlsxWriter==3.1.9; extra == "sheets"
 Requires-Dist: xlrd==2.0.1; extra == "sheets"
 Requires-Dist: gspread>=5.12.0; extra == "sheets"
+Provides-Extra: s3
+Requires-Dist: minio==7.2.15; extra == "s3"
 Provides-Extra: all
 Requires-Dist: tol-sdk[api-base]; extra == "all"
 Requires-Dist: tol-sdk[benchling]; extra == "all"

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tol-sdk"
-version = "1.7.5-beta4"
+version = "1.8.0"
 description = "SDK for interaction with ToL, Sanger and external services"
 readme = "README.md"
 authors = [
@@ -53,6 +53,7 @@ sheets = [
     "xlrd == 2.0.1",
     "gspread >= 5.12.0"
 ]
+s3 = ["minio == 7.2.15"]
 all = [
     "tol-sdk[api-base]",
     "tol-sdk[benchling]",

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/core/data_object_converter.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MIT
 
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from typing import Iterable
 
 from more_itertools import flatten
@@ -95,9 +96,14 @@ class DefaultDataObjectToDataObjectConverter(DataObjectToDataObjectOrUpdateConve
 
 class SanitisingConverter(DataObjectToDataObjectOrUpdateConverter):
 
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        pass
+
     def __init__(
         self,
         data_object_factory: DataObjectFactory,
+        config: Config,
         **kwargs
     ):
         super().__init__(data_object_factory)

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/excel/excel_datasource.py

@@ -65,7 +65,7 @@ class ExcelDataSource(
     ) -> Iterable[DataObject]:
 
         return (
-            self.__marshal_row(row_index + 1, row)
+            self.__marshal_row(row_index + 2, row)  # Add 1 for header, 1 for 1-based ID
             for row_index, row
             in self.__df.iterrows()
         )
@@ -115,6 +115,10 @@ class ExcelDataSource(
         __v: Any,
     ) -> Any:
 
+        # Convert pandas Timestamp to Python datetime
+        if isinstance(__v, pd.Timestamp):
+            __v = datetime.fromtimestamp(__v.timestamp())
+
         if __k not in self.__mappings:
             return __v
 

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/flows/converters/incoming_sample_to_ena_sample_converter.py

@@ -3,6 +3,7 @@
 
 import re
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Iterable
 
 from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
@@ -30,8 +31,10 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
         s = data_object
         attributes = {
             'ENA-CHECKLIST': self.__config.ena_checklist_id,
-            'organism part': self.__replace_underscores(
-                s.attributes.get('ORGANISM_PART')),
+            'organism part': self.__join_list([
+                self.__replace_underscores(v)
+                for v in s.attributes.get('ORGANISM_PART', [])
+            ]),
             'lifestage': (
                 'spore-bearing structure'
                 if s.attributes.get('LIFESTAGE') == 'SPORE_BEARING_STRUCTURE'
@@ -40,35 +43,38 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
             ),
             'project name':
                 self.__config.project_name,
-            'collected by':
-                self.__replace_underscores(
-                    s.attributes.get('COLLECTED_BY')),
+            'collected_by': self.__join_list([
+                self.__replace_underscores(v)
+                for v in s.attributes.get('COLLECTED_BY', [])
+            ]),
             'collection date':
-                self.__replace_underscores(
-                    s.attributes.get('DATE_OF_COLLECTION')).lower(),
+                self.__format_date(
+                    s.attributes.get('DATE_OF_COLLECTION')),
             'geographic location (country and/or sea)':
                 self.__collection_country(s).replace('_', ' '),
             'geographic location (latitude)':
                 self.__replace_underscores(
-                    s.attributes.get('DECIMAL_LATITUDE')).lower(),
+                    str(s.attributes.get('DECIMAL_LATITUDE'))).lower(),
             'geographic location (latitude) units':
                 'DD',
             'geographic location (longitude)':
                 self.__replace_underscores(
-                    s.attributes.get('DECIMAL_LONGITUDE')).lower(),
+                    str(s.attributes.get('DECIMAL_LONGITUDE'))).lower(),
             'geographic location (longitude) units':
                 'DD',
             'geographic location (region and locality)':
                 self.__collection_region(s).replace('_', ' '),
-            'identified_by':
-                self.__replace_underscores(
-                    s.attributes.get('IDENTIFIED_BY')),
+            'identified_by': self.__join_list([
+                self.__replace_underscores(v)
+                for v in s.attributes.get('IDENTIFIED_BY', [])
+            ]),
             'habitat':
                 self.__replace_underscores(
                     s.attributes.get('HABITAT')),
-            'identifier_affiliation':
-                self.__replace_underscores(
-                    s.attributes.get('IDENTIFIER_AFFILIATION')),
+            'identifier_affiliation': self.__join_list([
+                self.__replace_underscores(v)
+                for v in s.attributes.get('IDENTIFIER_AFFILIATION', [])
+            ]),
             'sex':
                 self.__replace_underscores(
                     s.attributes.get('SEX')),
@@ -77,9 +83,10 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
                     s.attributes.get('RELATIONSHIP')),
             'SYMBIONT':
                 'Y' if s.attributes.get('SYMBIONT') == 'SYMBIONT' else 'N',
-            'collecting institution':
-                self.__replace_underscores(
-                    s.attributes.get('COLLECTOR_AFFILIATION'))
+            'collecting institution': self.__join_list([
+                self.__replace_underscores(v)
+                for v in s.attributes.get('COLLECTOR_AFFILIATION', [])
+            ]),
         }
         if self.__sanitise(s.attributes.get('DEPTH')) != '':
             attributes['geographic location (depth)'] = s.attributes.get('DEPTH')
@@ -88,9 +95,11 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
             attributes['geographic location (elevation)'] = s.attributes.get('ELEVATION')
             attributes['geographic location (elevation) units'] = 'm'
         if self.__sanitise(s.attributes.get('ORIGINAL_COLLECTION_DATE')) != '':
-            attributes['original collection date'] = s.attributes.get('ORIGINAL_COLLECTION_DATE')
+            attributes['original collection date'] = \
+                self.__format_date(s.attributes.get('ORIGINAL_COLLECTION_DATE'))
         if self.__sanitise(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION')) != '':
-            attributes['original geographic location'] = self.__replace_underscores(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION'))  # noqa
+            attributes['original geographic location'] = \
+                self.__replace_underscores(s.attributes.get('ORIGINAL_GEOGRAPHIC_LOCATION'))
         if s.attributes.get('GAL') is not None:
             attributes['GAL'] = s.attributes.get('GAL')
         if s.attributes.get('VOUCHER_ID') is not None:
@@ -103,7 +112,7 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
             attributes['culture_or_strain_id'] = s.attributes.get('CULTURE_OR_STRAIN_ID')
 
         ret = self._data_object_factory(
-            'sample',
+            data_object.type,
             s.id,
             attributes=attributes,
         )
@@ -128,3 +137,19 @@ class IncomingSampleToEnaSampleConverter(DataObjectToDataObjectOrUpdateConverter
         if value is None:
             return default_value
         return value
+
+    def __join_list(self, value_list):
+        if value_list is None:
+            return ''
+        if not isinstance(value_list, list):
+            return str(value_list)
+        return ' | '.join(str(v) for v in value_list)
+
+    def __format_date(self, value):
+        """Format date to YYYY-mm-dd format"""
+        if value is None:
+            return ''
+        if isinstance(value, datetime):
+            return value.strftime('%Y-%m-%d')
+
+        return str(value)

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/__init__.py

@@ -15,6 +15,7 @@ from .regex_by_value import RegexByValueValidator  # noqa
 from .specimens_have_same_taxon import SpecimensHaveSameTaxonValidator # noqa
 from .sts_fields import StsFieldsValidator  # noqa
 from .tolid import TolidValidator  # noqa
+from .types import TypesValidator  # noqa
 from .unique_values import UniqueValuesValidator  # noqa
 from .unique_whole_organisms import UniqueWholeOrganismsValidator  # noqa
 from .interfaces import Condition  # noqa

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/converter_and_validate.py

@@ -19,12 +19,12 @@ class ConverterAndValidateValidator(Validator):
         "converters": [{
             "module": "<path.to.module>",
             "class_name": "<path.to.ConverterClass>",
-            "config": { ... }
+            "config_details": { ... }
         }],
         "validators": [{
             "module": "<path.to.module>",
             "class_name": "<path.to.ValidatorClass>",
-            "config": { ... }
+            "config_details": { ... }
         }]
     }
 
@@ -54,7 +54,7 @@ class ConverterAndValidateValidator(Validator):
             converter_class = getattr(__module, conv.get('class_name'))
 
             converter_conf = converter_class.Config(
-                **conv.get('config')
+                **conv.get('config_details')
             )
             self.__converters.append(converter_class(
                 data_object_factory=data_object_factory,
@@ -65,7 +65,7 @@ class ConverterAndValidateValidator(Validator):
             validator_class = getattr(__module, val.get('class_name'))
 
             validator_conf = validator_class.Config(
-                **val.get('config')
+                **val.get('config_details')
             )
             self.__validators.append(validator_class(
                 data_object_factory=data_object_factory,

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/ena_checklist.py

@@ -27,31 +27,31 @@ class EnaChecklistValidator(Validator):
         super().__init__()
         self.__config = config
         self._datasource = datasource
+        self.__ena_checklist = datasource.get_one(
+            'checklist',
+            self.__config.ena_checklist_id
+        ).checklist
 
     def _validate_data_object(self, obj: DataObject) -> None:
-        ena_datasource = self._datasource
-        ena_checklist = ena_datasource.get_one('checklist', self.__config.ena_checklist_id)
-
-        validations = ena_checklist.attributes['checklist']
-        for key in validations:
+        for key, validation in self.__ena_checklist.items():
             field_name = key
-            if 'field' in validations[key]:
-                field_name = validations[key]['field']
-            if 'mandatory' in validations[key] and key not in obj.attributes:
+            if 'field' in validation:
+                field_name = validation['field']
+            if 'mandatory' in validation and key not in obj.attributes:
                 self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
                 continue
-            if 'mandatory' in validations[key] and obj.attributes[key] is None:
+            if 'mandatory' in validation and obj.attributes[key] is None:
                 self.add_error(object_id=obj.id, detail='Must be given', field=[field_name])
                 continue
-            if 'mandatory' in validations[key] and obj.attributes.get(key) == '':
+            if 'mandatory' in validation and obj.attributes.get(key) == '':
                 self.add_error(
                     object_id=obj.id,
                     detail='Must not be empty', field=[field_name]
                 )
 
-            if 'restricted text' in validations[key] and key in obj.attributes:
-                for condition in validations[key]:
-                    if type(condition) == str and '(' in condition:
+            if 'restricted text' in validation and key in obj.attributes:
+                for condition in validation:
+                    if isinstance(condition, str) and '(' in condition:
                         regex = condition
                 compiled_re = re.compile(regex)
                 if not compiled_re.search(obj.attributes.get(key)):
@@ -61,9 +61,9 @@ class EnaChecklistValidator(Validator):
                     )
 
             # Check against allowed values
-            if 'text choice' in validations[key] and key in obj.attributes:
-                for condition in validations[key]:
-                    if type(condition) == list:
+            if 'text choice' in validation and key in obj.attributes:
+                for condition in validation:
+                    if isinstance(condition, list):
                         allowed_values = condition
                 if obj.attributes.get(key).lower() not in \
                         [x.lower() for x in allowed_values]:

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/regex.py

@@ -26,9 +26,9 @@ class Regex:
 
     def is_allowed(self, __v: Any) -> bool:
         # Check regex
-        return bool(re.search(
+        return __v is None or __v == '' or bool(re.search(
             self.regex,
-            str(__v) if __v is not None else ''
+            str(__v)
         ))
 
 
@@ -90,7 +90,6 @@ class RegexValidator(Validator):
         obj: DataObject,
         c: Regex,
     ) -> None:
-
         if c.is_error:
             self.add_error(
                 object_id=obj.id,

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/sts_fields.py

@@ -53,36 +53,42 @@ class StsFieldsValidator(Validator):
         for field in self.__fields.values():
             # Get the value from the data object
             field_value = obj.get_field_by_name(field.get('data_input_key'))
-            if field.get('mandatory_input') and (field_value is None or field_value == ''):
+            if field.get('mandatory_validation') and (field_value is None or field_value == ''):
                 self.add_error(
                     object_id=obj.id,
                     detail=f'Field {field.get("data_input_key")} is required '
                            f'for project {self.__config.project_code}',
                     field=field.get('data_input_key'),
                 )
-            elif field.get('allowed_values') and field_value not in field.get('allowed_values'):
+            elif field.get('allowed_values'):
+                allowed_values = [
+                    value.get('value') for value in field.get('allowed_values', [])
+                ]
+                if field_value not in allowed_values:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail=f'Field {field.get("data_input_key")} value '
+                               f'"{field_value}" not found in allowed values '
+                               f'{allowed_values} for project '
+                               f'{self.__config.project_code}',
+                        field=field.get('data_input_key'),
+                    )
+            elif field.get('min') and field.get('type') == 'String' \
+                    and field_value is not None and len(field_value) < field.get('min'):
                 self.add_error(
                     object_id=obj.id,
                     detail=f'Field {field.get("data_input_key")} value '
-                           f'"{field_value}" not found in allowed values '
-                           f'{field.get("allowed_values")} for project '
-                           f'{self.__config.project_code}',
-                    field=field.get('data_input_key'),
-                )
-            elif field.get('min') and field_value < field.get('min'):
-                self.add_error(
-                    object_id=obj.id,
-                    detail=f'Field {field.get("data_input_key")} value '
-                           f'"{field_value}" is less than minimum value '
+                           f'"{field_value}" is shorter than minimum length '
                            f'"{field.get("min")}" for project '
                            f'{self.__config.project_code}',
                     field=field.get('data_input_key'),
                 )
-            elif field.get('max') and field_value > field.get('max'):
+            elif field.get('max') and field.get('type') == 'String' \
+                    and field_value is not None and len(field_value) > field.get('max'):
                 self.add_error(
                     object_id=obj.id,
                     detail=f'Field {field.get("data_input_key")} value '
-                           f'"{field_value}" is greater than maximum value '
+                           f'"{field_value}" is longer than maximum length '
                            f'"{field.get("max")}" for project '
                            f'{self.__config.project_code}',
                     field=field.get('data_input_key'),

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol/validators/tolid.py

@@ -103,7 +103,7 @@ class TolidValidator(Validator):
 
                 if str(obj.get_field_by_name(self.__config.species_id_field)) not in taxons:
                     self.add_error(
-                        object_id=obj.id,
+                        object_id=obj.id + 1,
                         detail=f'Specimen ID {specimen_id} does not match Taxon ID '
                                f'{obj.get_field_by_name(self.__config.species_id_field)}'
                                'in TolID source',

tol_sdk-1.8.0/src/tol/validators/types.py

@@ -0,0 +1,87 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from datetime import datetime, time
+
+from tol.core import DataObject
+from tol.core.validate import Validator
+
+
+class TypesValidator(Validator):
+    """
+    Validates an incoming stream of `DataObject` instances,
+    ensuring that they only have attributes of the given
+    allowed keys.
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        allowed_types: dict[str, str]
+        is_error: bool = True
+        detail: str = 'Value is of incorrect type'
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(
+        self,
+        config: Config,
+        **kwargs
+    ) -> None:
+
+        super().__init__()
+        self.__config = config
+
+    def _validate_data_object(
+        self,
+        obj: DataObject
+    ) -> None:
+
+        type_map = {
+            'str': str,
+            'int': int,
+            'float': float,
+            'bool': bool,
+            'list': list,
+            'dict': dict,
+            'datetime': datetime,
+            'time': time
+        }
+        for key, expected_type in self.__config.allowed_types.items():
+            if key in obj.attributes:
+                actual_value = obj.get_field_by_name(key)
+                if actual_value is None:
+                    continue
+                type_class = type_map.get(expected_type)
+                if type_class and not isinstance(actual_value, type_class):
+                    self.__add_result(
+                        obj,
+                        key,
+                    )
+                if type_class and isinstance(actual_value, type_class):
+                    # Special case for bool since isinstance(True, int) is True
+                    if expected_type == 'int' and isinstance(actual_value, bool):
+                        self.__add_result(
+                            obj,
+                            key,
+                        )
+
+    def __add_result(
+        self,
+        obj: DataObject,
+        key: str,
+    ) -> None:
+
+        if self.__config.is_error:
+            self.add_error(
+                object_id=obj.id,
+                detail=self.__config.detail,
+                field=key,
+            )
+        else:
+            self.add_warning(
+                object_id=obj.id,
+                detail=self.__config.detail,
+                field=key,
+            )

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0/src/tol_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.5b4
+Version: 1.8.0
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT
@@ -50,6 +50,8 @@ Requires-Dist: openpyxl>=3.0.10; extra == "sheets"
 Requires-Dist: XlsxWriter==3.1.9; extra == "sheets"
 Requires-Dist: xlrd==2.0.1; extra == "sheets"
 Requires-Dist: gspread>=5.12.0; extra == "sheets"
+Provides-Extra: s3
+Requires-Dist: minio==7.2.15; extra == "s3"
 Provides-Extra: all
 Requires-Dist: tol-sdk[api-base]; extra == "all"
 Requires-Dist: tol-sdk[benchling]; extra == "all"

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol_sdk.egg-info/SOURCES.txt

@@ -341,6 +341,7 @@ src/tol/validators/regex_by_value.py
 src/tol/validators/specimens_have_same_taxon.py
 src/tol/validators/sts_fields.py
 src/tol/validators/tolid.py
+src/tol/validators/types.py
 src/tol/validators/unique_values.py
 src/tol/validators/unique_whole_organisms.py
 src/tol/validators/interfaces/__init__.py

{tol_sdk-1.7.5b4 → tol_sdk-1.8.0}/src/tol_sdk.egg-info/requires.txt

@@ -57,6 +57,9 @@ psycopg2-binary==2.9.9
 prefect==2.20.18
 griffe<1.0.0
 
+[s3]
+minio==7.2.15
+
 [sheets]
 pandas==2.1.4
 openpyxl>=3.0.10