tol-sdk 1.7.3__py3-none-any.whl → 1.7.5b0__py3-none-any.whl

tol/validators/regex.py

@@ -4,7 +4,7 @@
 
 import re
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, List
 
 from tol.core import DataObject
 from tol.core.validate import Validator
@@ -23,28 +23,27 @@ class Regex:
         return re.search(self.regex, str(__v or ''))
 
 
-RegexDict = dict[
-    str,
-    str | bool | list[Any],
-]
-"""Can also specify `Regex` as a `dict`"""
-
-
 class RegexValidator(Validator):
     """
     Validates an incoming stream of `DataObject` instances
     according to the specified allowed values for a given
     key.
     """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        regexes: List[Regex]
+
+    __slots__ = ['__config']
+    __config: Config
 
     def __init__(
         self,
-        config: list[Regex | RegexDict]
+        config: Config
     ) -> None:
 
         super().__init__()
 
-        self.__config = self.__get_config(config)
+        self.__config = config
 
     def _validate_data_object(
         self,
@@ -54,19 +53,6 @@ class RegexValidator(Validator):
         for k, v in obj.attributes.items():
             self.__validate_attribute(obj, k, v)
 
-    def __get_config(
-        self,
-        config: list[Regex | RegexDict],
-    ) -> list[Regex]:
-
-        # Ensure config is in Regex format
-        # (as you can either pass in a list of Regex or a RegexDict,
-        # which can be used to initialize a Regex)
-        return [
-            c if isinstance(c, Regex) else Regex(**c)
-            for c in config
-        ]
-
     def __validate_attribute(
         self,
         obj: DataObject,
@@ -85,7 +71,7 @@ class RegexValidator(Validator):
         key: str,
     ) -> list[Regex]:
         return [
-            a for a in self.__config
+            a for a in self.__config.regexes
             if a.key == key
         ]
 

tol/validators/regex_by_value.py

@@ -2,21 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
-from typing import Any
+from dataclasses import dataclass
+from typing import Dict, List
 
 from tol.core import DataObject
 from tol.core.validate import Validator
 
 from .regex import Regex
 
-RegexDict = dict[
-    str,
-    str | bool | list[Any],
-]
-Config = dict[str, str | dict[str, list[Regex | RegexDict]]]
-
-"""Can also specify `Regex` as a `dict`"""
-
 
 class RegexByValueValidator(Validator):
     """
@@ -24,46 +17,34 @@ class RegexByValueValidator(Validator):
     according to the specified allowed values for a given
     key.
     """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        key_column: str
+        regexes: Dict[str, List[Regex]]
+
+    __slots__ = ['__config']
+    config: Config
 
     def __init__(
         self,
-        config: dict[str, str | list[str]]
+        config: Config
     ) -> None:
 
         super().__init__()
 
-        self.__config = self.__get_config(config)
-
-    def __get_config(
-        self,
-        config: Config,
-    ) -> Config:
-
-        return {
-            'key_column': config['key_column'],
-            'regexes': {
-                k: [
-                    # Ensure they're all in Regex format
-                    # (as you can either pass in a list of Regex or a RegexDict,
-                    # which can be used to initialize a Regex)
-                    c if isinstance(c, Regex) else Regex(**c)
-                    for c in v
-                ]
-                for k, v in config['regexes'].items()
-            }
-        }
+        self.__config = config
 
     def _validate_data_object(
         self,
         obj: DataObject
     ) -> None:
         # Pull out value of the 'key_column' attribute
-        key_column_value = obj.attributes.get(self.__config['key_column'])
+        key_column_value = obj.attributes.get(self.__config.key_column)
         if not key_column_value:
             return
 
         # Pull out relevant regex list based on this value: {[{'name': 'regex'}]}
-        regex_list = self.__config['regexes'].get(key_column_value)
+        regex_list = self.__config.regexes.get(key_column_value)
         if not regex_list:
             return
         self.__validate_attribute(obj, regex_list)
@@ -71,7 +52,7 @@ class RegexByValueValidator(Validator):
     def __validate_attribute(
         self,
         obj: DataObject,
-        regexes: list[Regex],
+        regexes: List[Regex],
     ) -> None:
         for r in regexes:
             attribute_name = r.key

tol/validators/specimens_have_same_taxon.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Dict
+
+from tol.core import Validator
+from tol.core.data_object import DataObject
+
+
+class SpecimensHaveSameTaxonValidator(Validator):
+    """
+    Validates an incoming stream of `DataObject` instances.
+    For each data object (sample) not a SYMBIONT, it checks that
+    there are no samples with SPECIMEN_ID which has different TAXON_ID
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        taxon_id_field: str
+        symbiont_field: str
+        specimen_id_field: str
+
+    __slots__ = ['__config', '__seen']
+    __config: Config
+    __seen: Dict[str, str]
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+        self.__seen = {}
+        self.__config = config
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        # Explaining the code concept using a standard example
+        # seen{}
+        # 1st Pass=>    element['specimen_id']  =   A
+        #               element['taxon_id']     =   AA
+        #               seen{ A:AA }
+        # 2nd pass=>    element['specimen_id']  =   A
+        #               element['taxon_id']     =   AB
+        #               AB != AA
+        #               Flag error
+        # From Nithin :)
+
+        # Ensure the data object is not a SYMBIONT
+        if obj.attributes.get(self.__config.symbiont_field) != 'SYMBIONT':
+            specimen_id = obj.attributes.get(self.__config.specimen_id_field)
+            if specimen_id is None:
+                return
+            taxon_id = obj.attributes.get(self.__config.taxon_id_field)
+            if taxon_id is None:
+                return
+            if specimen_id in self.__seen and taxon_id != self.__seen[specimen_id]:
+                self.add_error(
+                    object_id=obj.id,
+                    detail='A non-symbiont must have a matching Specimen ID and Taxon ID',
+                    field=self.__config.specimen_id_field,
+                )
+            if specimen_id not in self.__seen:
+                self.__seen[specimen_id] = taxon_id

tol/validators/sts_fields.py

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import List
+
+from tol.core import DataObject, DataSource
+from tol.core.validate import Validator
+from tol.sources.sts import sts
+
+
+class StsFieldsValidator(Validator):
+    """
+    Validates that a stream of `DataObject` instances
+    contains fields that observe the validations in STS
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        project_code: str
+
+    __slots__ = ['__config', '__datasource', '__fields']
+    __config: Config
+    __datasource: DataSource
+    __fields: List[str | int | float]
+
+    def __init__(
+        self,
+        config: Config,
+        datasource: DataSource = sts(),  # For testing
+    ) -> None:
+
+        super().__init__()
+
+        self.__config = config
+        self.__datasource = datasource
+        self.__fields = self.__initialize_fields_from_datasource()
+
+    def __initialize_fields_from_datasource(self) -> List[str | int | float]:
+        return {
+            field.get('data_input_key'): field
+            for field in self.__datasource.get_one(
+                'project', self.__config.project_code
+            ).template.get('data_fields', [])
+            if field.get('in_manifest')
+        }
+
+    def _validate_data_object(
+        self,
+        obj: DataObject
+    ) -> None:
+        for field in self.__fields.values():
+            # Get the value from the data object
+            field_value = obj.get_field_by_name(field.get('data_input_key'))
+            if field.get('mandatory_input') and (field_value is None or field_value == ''):
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Field {field.get("data_input_key")} is required '
+                           f'for project {self.__config.project_code}',
+                    field=field.get('data_input_key'),
+                )
+            elif field.get('allowed_values') and field_value not in field.get('allowed_values'):
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Field {field.get("data_input_key")} value '
+                           f'"{field_value}" not found in allowed values '
+                           f'{field.get("allowed_values")} for project '
+                           f'{self.__config.project_code}',
+                    field=field.get('data_input_key'),
+                )
+            elif field.get('min') and field_value < field.get('min'):
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Field {field.get("data_input_key")} value '
+                           f'"{field_value}" is less than minimum value '
+                           f'"{field.get("min")}" for project '
+                           f'{self.__config.project_code}',
+                    field=field.get('data_input_key'),
+                )
+            elif field.get('max') and field_value > field.get('max'):
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Field {field.get("data_input_key")} value '
+                           f'"{field_value}" is greater than maximum value '
+                           f'"{field.get("max")}" for project '
+                           f'{self.__config.project_code}',
+                    field=field.get('data_input_key'),
+                )

tol/validators/tolid.py

@@ -0,0 +1,110 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from tol.core import DataObject, DataSource
+from tol.core import DataSourceError, DataSourceFilter
+from tol.core.validate import Validator
+from tol.sources.tolid import tolid
+
+
+class TolidValidator(Validator):
+    """
+    Validates that a stream of `DataObject` instances
+    contains unique Tol IDs.
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        species_id_field: str
+        specimen_id_field: str
+        error_ignore_field: str
+        error_ignore_value: str
+        warning_detail: str = 'Species not found in Tol ID source'
+
+    __slots__ = ['__config', '__datasource', '__cached_species_id', '__cached_tolids']
+    __config: Config
+    __datasource: DataSource
+    __cached_species_ids: Dict[str, Any]
+    __cached_tolids: Dict[str, Any]
+
+    def __init__(
+        self,
+        config: Config,
+        datasource=tolid(),
+    ) -> None:
+
+        super().__init__()
+
+        self.__config = config
+        self.__datasource = datasource
+        self.__cached_species_ids = {}
+        self.__cached_tolids = {}
+
+    def _validate_data_object(
+        self,
+        obj: DataObject
+    ) -> None:
+        self.__warning_on_species_not_in_tolid(obj=obj)
+        self.__error_on_specimen_id_and_taxon_not_matching_tolid(obj=obj)
+
+    def __warning_on_species_not_in_tolid(
+        self,
+        obj: DataObject,
+    ) -> None:
+
+        obj_species_id = obj.get_field_by_name(self.__config.species_id_field)
+        if self.__config.species_id_field in obj.attributes:
+            try:
+                if obj_species_id not in self.__cached_species_ids:
+                    self.__cached_species_ids[obj_species_id] = (
+                        self.__datasource.get_one('species', obj_species_id) is not None)
+
+            except DataSourceError as e:
+                if e.status_code == 404:
+                    self.__cached_species_ids[obj_species_id] = False
+
+        species_in_tolid = self.__cached_species_ids[obj_species_id]
+        if species_in_tolid is False:
+            self.add_warning(
+                object_id=obj.id,
+                detail=self.__config.warning_detail,
+                field=self.__config.species_id_field,
+            )
+
+    def __error_on_specimen_id_and_taxon_not_matching_tolid(
+        self,
+        obj: DataObject,
+    ) -> None:
+
+        if (obj.get_field_by_name(self.__config.error_ignore_field) is
+                self.__config.error_ignore_value):
+            return
+
+        if self.__config.specimen_id_field in obj.attributes:
+            specimen_id = obj.get_field_by_name(self.__config.specimen_id_field)
+            if specimen_id not in self.__cached_tolids:
+                f = DataSourceFilter()
+                f.and_ = {'specimen_id': {'eq': {'value': specimen_id}}}
+                self.__cached_tolids[specimen_id] = list(self.__datasource.get_list(
+                    object_type='specimen',
+                    filters=f
+                ))
+
+            if (len(self.__cached_tolids[specimen_id]) == 0):
+                return
+            else:
+                taxons = set()
+                for tolid_ in self.__cached_tolids[specimen_id]:
+                    taxons.add(str(tolid_.species.id))
+
+                if str(obj.get_field_by_name(self.__config.species_id_field)) not in taxons:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail=f'Specimen ID {specimen_id} does not match Taxon ID '
+                               f'{obj.get_field_by_name(self.__config.species_id_field)}'
+                               'in TolID source',
+                        field=[self.__config.specimen_id_field, self.__config.species_id_field]
+                    )

tol/validators/unique_values.py

@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: MIT
 
+from dataclasses import dataclass
+from typing import Dict, List, Set
+
 from tol.core import DataObject
 from tol.core.validate import Validator
 
@@ -11,36 +14,69 @@ class UniqueValuesValidator(Validator):
     Validates that a stream of `DataObject` instances
     contains unique values for specified keys.
     """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        unique_keys: List[List[str] | str]
+        detail: str = 'Value is not unique'
+        is_error: bool = True
+
+    __slots__ = ['__config', '__duplicates', '__existing_values']
+    __config: Config
+    __duplicates: Dict[str, List[str]]
+    __existing_values: Dict[str, Set]
 
     def __init__(
         self,
-        unique_keys: list[str],
-        *,
-        detail: str = 'Value is not unique',
-        is_error: bool = True,
+        config: Config
     ) -> None:
 
         super().__init__()
 
-        self.__keys = unique_keys
-        self.__detail = detail
-        self.__is_error = is_error
-        self.__duplicates: dict[str, list[str]] = {}
-        self.__existing_values: dict[str, set] = {key: set() for key in unique_keys}
+        self.__config = config
+        self.__duplicates = {}
+        self.__existing_values = {}
+        for key in self.__config.unique_keys:
+            if isinstance(key, str):
+                self.__existing_values[key] = set()
+            elif isinstance(key, list):
+                concat_key = '/'.join(key)
+                self.__existing_values[concat_key] = set()
 
     def _validate_data_object(
         self,
         obj: DataObject
     ) -> None:
 
-        for key in obj.attributes:
-            if key in self.__keys:
-                if obj.attributes[key] in self.__existing_values[key]:
-                    if key not in self.__duplicates:
-                        self.__duplicates[key] = []
-                    self.__duplicates[key].append(obj.attributes[key])
+        for unique_key in self.__config.unique_keys:
+            if isinstance(unique_key, list):
+                concat = ''
+                for key in unique_key:
+                    concat = concat + '/' + (str(obj.attributes[key]))
+                if concat in self.__existing_values['/'.join(unique_key)]:
+                    self.__duplicate_checks(
+                        key=key,
+                        value=concat
+                    )
+                else:
+                    self.__existing_values['/'.join(unique_key)].add(concat)
+
+            else:
+                if obj.attributes[unique_key] in self.__existing_values[unique_key]:
+                    self.__duplicate_checks(
+                        key=unique_key,
+                        value=obj.attributes[unique_key]
+                    )
                 else:
-                    self.__existing_values[key].add(obj.attributes[key])
+                    self.__existing_values[unique_key].add(obj.attributes[unique_key])
+
+    def __duplicate_checks(
+        self,
+        key: str,
+        value: str
+    ):
+        if key not in self.__duplicates:
+            self.__duplicates[key] = []
+        self.__duplicates[key].append(value)
 
     def _post_validation(
         self,
@@ -58,15 +94,15 @@ class UniqueValuesValidator(Validator):
         key: str,
     ) -> None:
 
-        if self.__is_error:
+        if self.__config.is_error:
             self.add_error(
                 object_id=obj.id,
-                detail=self.__detail,
+                detail=self.__config.detail,
                 field=key,
             )
         else:
             self.add_warning(
                 object_id=obj.id,
-                detail=self.__detail,
+                detail=self.__config.detail,
                 field=key,
             )

tol/validators/unique_whole_organisms.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import List
+
+from tol.core import Validator
+from tol.core.data_object import DataObject
+
+
+class UniqueWholeOrganismsValidator(Validator):
+    """
+    Validates an incoming stream of `DataObject` instances.
+    For each data object (sample) not a SYMBIONT, it checks:
+    1. There are no two samples with organism part WHOLE_ORGANISM with the same SPECIMEN_ID
+    2. There are no samples with organism part *not* WHOLE_ORGANISM that have a SPECIMEN_ID
+       the same as a WHOLE_ORGANISM in the manifest.
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        symbiont_field: str
+        organism_part_field: str
+        specimen_id_field: str
+
+    __slots__ = ['__config', '__whole_organisms', '__part_organisms']
+    __config: Config
+    __whole_organisms: List[str]
+    __part_organisms: List[str]
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+        self.__whole_organisms = []
+        self.__part_organisms = []
+        self.__config = config
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        # This function uses a bit of a confusing method for its validation, so I'm going to
+        # leave an explanation here as to how it works for anyone who needs to modify it
+        # in the future!
+        #
+        # In the original code to be adapted, two loops were used. The first looped over each
+        # data object whose ORGANISM_PART was 'WHOLE_ORGANISM', adding them to a list. Before it
+        # did this though, it would check to see if the SPECIMEN_ID of this data object was already
+        # contained in said list (to ensure the specimen IDs were unique). In the second loop, the
+        # rest of the data objects (those whose ORGANISM PART was *not* 'WHOLE_ORGANISM') were
+        # looped over, each being checked to see if their SPECIMEN_ID was contained in the list
+        # (the one containing the whole organisms). In all, this ensures that all whole organisms
+        # have unique specimen IDs, and all part organisms do not share the specimen IDs of any of
+        # the whole organisms.
+        #
+        # The issue when adapting this to a Validator to be used in a pipeline, is that this
+        # function only takes in one data object at a time, via a generator (to save needing to
+        # load many into memory at once). The problem this left us with is that we could no longer
+        # achieve the same result by using two passes of the data, as only one pass was feasible.
+        #
+        # So here's what I ended up with. This validator stores the SPECIMEN_IDs of both all of the
+        # whole organisms *and* part organisms. From this, detecting duplicate whole organisms is
+        # the same, but detecting whether a part organism shared the SPECIMEN_ID of a whole
+        # organism now has two separate cases: when the data object passed into this function is
+        # a whole organism, or a part organism. In the case of it being a part organism, a very
+        # similar solution to before can be used: we simply check whether self.__whole_organisms
+        # conatins the same SPECIMEN_ID. However, for the case where the data object is a whole
+        # organism, effectively the inverse is done; it is the self.__part_organisms list that is
+        # checked. This covers all cases:
+        #   1. There are no duplicates, in which case there will never be a time when the same
+        #      SPECIMEN_ID will be in both lists.
+        #   2. A whole organism is checked, then a part organism with the same SPECIMEN_ID is
+        #      checked. In this case, self.__whole_organisms will contain the same SPECIMEN_ID,
+        #      so the duplicate is detected.
+        #   3. A part organism is checked, then a whole organism with the same SPECIMEN_ID is
+        #      checked. In this case, self.__part_organisms will contain the same SPECIMEN_ID,
+        #      so the duplicate is detected.
+        #
+        # From Thomas :)
+
+        # Ensure the data object is not a SYMBIONT, because organism part checks do not apply
+        if obj.attributes.get(self.__config.symbiont_field) != 'SYMBIONT':
+            specimen_id = obj.attributes.get(self.__config.specimen_id_field)
+            if specimen_id is None:
+                return
+
+            organism_part = obj.attributes.get(self.__config.organism_part_field)
+            if organism_part == 'WHOLE_ORGANISM':
+                if specimen_id in self.__whole_organisms:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='No two whole organisms can have the same Specimen ID',
+                        field=self.__config.specimen_id_field,
+                    )
+                if specimen_id in self.__part_organisms:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='A whole organism cannot have a Specimen ID already used for'
+                               'a non-whole organism',
+                        field=self.__config.specimen_id_field,
+                    )
+
+                self.__whole_organisms.append(specimen_id)
+            else:
+                if specimen_id in self.__whole_organisms:
+                    self.add_error(
+                        object_id=obj.id,
+                        detail='A non-whole organism cannot have a Specimen ID already used for'
+                               'a whole organism',
+                        field=self.__config.specimen_id_field,
+                    )
+
+                self.__part_organisms.append(specimen_id)

{tol_sdk-1.7.3.dist-info → tol_sdk-1.7.5b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.3
+Version: 1.7.5b0
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT