tol-sdk 1.8.8__py3-none-any.whl → 1.8.10__py3-none-any.whl

tol/core/datasource_utils.py

@@ -30,10 +30,13 @@ class DataSourceUtils:
     @classmethod
     def get_datasource_by_datasource_instance(
         cls,
-        datasource_instance: DataObject
+        datasource_instance: DataObject,
+        **kwargs
     ) -> DataSource:
         datasource_config = datasource_instance.data_source_config
-        kwargs = dict(datasource_instance.kwargs) if datasource_instance.kwargs else {}
+        new_kwargs = dict(datasource_instance.kwargs) if datasource_instance.kwargs else {}
+        if kwargs:
+            new_kwargs.update(kwargs)
         if datasource_config:
             relationship_config = cls.get_relationship_config_from_data_source_config(
                 datasource_config
@@ -44,14 +47,14 @@ class DataSourceUtils:
             runtime_fields = cls.get_runtime_fields_from_data_source_config(
                 datasource_config
             )
-            kwargs.update({
+            new_kwargs.update({
                 'relationship_cfg': relationship_config,
                 'attribute_metadata': amd,
                 'runtime_fields': runtime_fields
             })
         return DataSourceUtils.get_datasource_by_name(
             datasource_instance.builtin_name,
-            **kwargs
+            **new_kwargs
         )
 
     @classmethod
@@ -83,7 +86,6 @@ class DataSourceUtils:
         cls,
         datasource_config: DataObject
     ) -> dict:
-        from ..elastic.runtime_fields import RuntimeFields  # Break circular import cycle
         runtime_fields = {}
         f = DataSourceFilter()
         f.and_ = {
@@ -95,6 +97,7 @@ class DataSourceUtils:
             if dsa.object_type not in runtime_fields:
                 runtime_fields[dsa.object_type] = {}
             if 'function' in dsa.runtime_definition:
+                from ..elastic.runtime_fields import RuntimeFields  # Break circular import cycle
                 method = getattr(RuntimeFields, dsa.runtime_definition['function'])
                 runtime_fields[dsa.object_type][dsa.name] = \
                     method(**dsa.runtime_definition.get('function_kwargs', {}))

tol/ena/ena_datasource.py

@@ -11,6 +11,7 @@ from cachetools.func import ttl_cache
 
 import requests
 from requests.auth import HTTPBasicAuth
+from requests.exceptions import HTTPError
 
 
 from .client import EnaApiClient
@@ -118,9 +119,13 @@ class EnaDataSource(
         if object_type == 'submittable_taxon':
             ena_response = []
             for object_id in object_ids:
-                response = client.get_detail(object_type, [object_id])
-                if response and isinstance(response, list):
-                    ena_response.extend(response)
+                try:
+                    response = client.get_detail(object_type, [object_id])
+                    if response and isinstance(response, list):
+                        ena_response.extend(response)
+                except HTTPError as http_error:
+                    if http_error.response.status_code != 400:
+                        raise
         else:
             ena_response = client.get_detail(object_type, object_ids)
         # For a checklist we need to convert into a list of dicts

tol/sources/portaldb.py

@@ -4,6 +4,7 @@
 
 import os
 
+from .defaults import Defaults
 from ..api_client import (
     ApiDataSource,
     create_api_datasource
@@ -15,9 +16,10 @@ from ..core import (
 
 def portaldb(retries: int = 5, **kwargs) -> ApiDataSource:
     portaldb = create_api_datasource(
-        api_url=os.getenv('PORTAL_URL') + os.getenv('PORTAL_API_PATH') + '/local',
+        api_url=os.getenv('PORTAL_URL', Defaults.PORTAL_URL)
+        + os.getenv('PORTAL_API_PATH', Defaults.PORTAL_API_PATH),
         token=os.getenv('PORTAL_API_KEY'),
-        data_prefix='',
+        data_prefix='/local',
         retries=retries
     )
     core_data_object(portaldb)

tol/sources/sql.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: 2023 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from ..core import (
+    core_data_object
+)
+from ..sql import (
+    SqlDataSource,
+    create_sql_datasource
+)
+
+
+# Create a SQL datasource. This is also expecting the models to be passed in
+# so would most likely be used within an app
+def sql(models, db_uri, behind_api, database_factory, **kwargs) -> SqlDataSource:
+    sql_ds = create_sql_datasource(
+        models=models,
+        db_uri=db_uri,
+        behind_api=True,
+        database_factory=database_factory,
+        **kwargs
+    )
+    core_data_object(sql_ds)
+    return sql_ds

tol/sql/sql_datasource.py

@@ -86,6 +86,7 @@ class SqlDataSource(
         sorter_factory: SorterFactory,
         user_id_getter: Optional[UserIdGetter] = None,
         attribute_metadata: AttributeMetadata = DefaultAttributeMetadata,
+        **kwargs
     ) -> None:
         self.__db = db
         self.__type_tablename_map = type_tablename_map

tol/validators/__init__.py

@@ -19,8 +19,9 @@ from .types import TypesValidator  # noqa
 from .unique_values import UniqueValuesValidator  # noqa
 from .unique_whole_organisms import UniqueWholeOrganismsValidator  # noqa
 from .interfaces import Condition  # noqa
-from .min_one_valid_value import MinOneValidValueValidator   # noqa
+from .min_one_valid_value import MinOneValidValueValidator  # noqa
 from .value_check import ValueCheckValidator  # noqa
 from .branching import BranchingValidator  # noqa
 from .unique_value_check import UniqueValueCheckValidator  # noqa
-from .date_sorting import DateSortingValidator # noqa
+from .date_sorting import DateSortingValidator  # noqa
+from .taxon_matches_goat import TaxonMatchesGoatValidator  # noqa

tol/validators/allowed_keys.py

@@ -30,6 +30,7 @@ class AllowedKeysValidator(Validator):
         **kwargs
     ) -> None:
 
+        del kwargs
         super().__init__()
         self.__config = config
 

tol/validators/taxon_matches_goat.py

@@ -0,0 +1,86 @@
+# SPDX-FileCopyrightText: 2026 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+
+from tol.core import DataObject, Validator
+from tol.sources.goat import GoatDataSource, goat
+
+
+class TaxonMatchesGoatValidator(Validator):
+    """
+    Validates a stream of `DataObject` instances, checking whether its Taxonomy information
+    matches that in GoaT
+    """
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        species_field: str | None = None
+        genus_field: str | None = None
+        family_field: str | None = None
+        superfamily_field: str | None = None
+        phylum_field: str | None = None
+        kingdom_field: str | None = None
+        superkingdom_field: str | None = None
+        domain_field: str | None = None
+
+    __slots__ = ['__config', '__goat_datasource', '_cached_taxa']
+    __config: Config
+    __goat_datasource: GoatDataSource
+    _cached_taxa: dict[str, DataObject]
+
+    def __init__(self, config: Config, **kwargs) -> None:
+        super().__init__()
+        self.__config = config
+        self.__goat_datasource = goat()
+        self._cached_taxa = {}
+
+    def _validate_data_object(self, obj: DataObject) -> None:
+        taxon_id = obj.get_field_by_name('TAXON_ID')
+
+        # Check whether we already have the information for this id in the cache.
+        # If we don't, fetch it from GoaT and add it to the cache
+        taxon: DataObject | None
+        if taxon_id in self._cached_taxa:
+            taxon = self._cached_taxa[taxon_id]
+        else:
+            taxon = self.__goat_datasource.get_one('taxon', taxon_id)
+
+            # Add this taxon to cached taxa.
+            # Error if GoaT has no taxon with this id
+            if taxon is not None:
+                self._cached_taxa[taxon_id] = taxon
+            else:
+                self.add_error(
+                    object_id=obj.id,
+                    detail=f'Invalid Taxon ID: {taxon_id}',
+                    field='taxon_id'
+                )
+
+                # We can't validate a taxon that doesn't exist, so after this error move on
+                # to the next DataObject
+                return
+
+        # Check that each associated taxonomy rank for this taxon matches those in GoaT
+        taxonomic_ranks = ('species', 'genus', 'family', 'superfamily',
+                           'phylum', 'kingdom', 'superkingdom', 'domain')
+        for rank in taxonomic_ranks:
+            # From the rank to check, get the name of its field in the data object we're validating
+            # from the validator config. If this field name is `None`, then this taxonomic rank
+            # isn't being checked (likely because the data object does not have this field)
+            field_name: str | None = getattr(self.__config, f'{rank}_field')
+            if field_name is None:
+                continue
+
+            # Fetch the values of these taxonomic ranks
+            value_in_data_object = obj.get_field_by_name(field_name)
+            value_in_goat = taxon.get_field_by_name(f'{rank}.scientific_name')
+
+            # Ensure the value in the data object matches the one in GoaT
+            if value_in_data_object != value_in_goat:
+                self.add_warning(
+                    object_id=obj.id,
+                    detail=(f'Value for {field_name} ({value_in_data_object}) '
+                            f'does not match the value in GoaT ({value_in_goat})'),
+                    field=field_name,
+                )

{tol_sdk-1.8.8.dist-info → tol_sdk-1.8.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.8.8
+Version: 1.8.10
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.8.8.dist-info → tol_sdk-1.8.10.dist-info}/RECORD

@@ -96,7 +96,7 @@ tol/core/data_source_dict.py,sha256=d-hSmoWTwG6IOc0cQTLap1EBslsxYIWGUd3ScSoeH_Q,
 tol/core/datasource.py,sha256=e9GaeDPfO_Gs7cgQhmNxCiSDlRNf64reegzFebcMNkA,6303
 tol/core/datasource_error.py,sha256=TqfqaPANG0gishadhA7myCmTO1Fg9u7hVZOvsY6BdAo,1660
 tol/core/datasource_filter.py,sha256=RY2S9kTx0XwdrFRSE2n2GohB9__fKGzFVsZrkN5hzQk,726
-tol/core/datasource_utils.py,sha256=18mwvFmtJL73_mxtFb56rKXZCGCtZFoEb8sWFKn3Yf0,6232
+tol/core/datasource_utils.py,sha256=6J72RS49IO2quCaFUXBl4DnjZBlXx-aW3Zy9iuzyeQ8,6327
 tol/core/factory.py,sha256=qbLvp5mLTcHbxOjopqtnK-wbZrlskgXjRIREszZjSyE,9157
 tol/core/http_client.py,sha256=QyZarplEHVYIrqEfrySeHbawfbnBU4nN62TLt41x4tY,2242
 tol/core/relationship.py,sha256=etdyCjLbfi2tgkaqzE6cntpNtTzgT_jOPGeNKmPu5yc,4624
@@ -142,7 +142,7 @@ tol/eln/sanitise.py,sha256=fMj-VrQTnw4zn2X0wnjWQAI8gWAa8RYqNuv23LXQssI,406
 tol/ena/__init__.py,sha256=T3TCqaHpgi2Uk2PjPGu60GaG2V8cTrHJlVLtZfLFhTQ,174
 tol/ena/client.py,sha256=ldmm7Z9_auQf1zVWjsFLXYgbKvGtSHTsr88YO3zfv2Y,6731
 tol/ena/converter.py,sha256=nxbo4IFzzOvKNMq3Aeiw5iDqVWvY33nTngLppjHAoGY,1144
-tol/ena/ena_datasource.py,sha256=jEvyUaH4pfFxmdtn6O_PwOdjPz6u80uAT3SLlR2f5nM,8968
+tol/ena/ena_datasource.py,sha256=QRj4pXFeAaC0qPdmtPCrCD3DKVsXhu1jYZ97P3AHlnA,9184
 tol/ena/ena_methods.py,sha256=jgpLssZq-F-vgkO-fYu4jrXenmNkdFpFKAY3VKp5HHE,9209
 tol/ena/factory.py,sha256=3IJCmGLo59PWbGmNqmKho5WYG330OjL8SoZYstIwHt4,3423
 tol/ena/filter.py,sha256=UzOx5ivXvA0TY2QuNzFmS-zDPVNnaAx07DMVkAwVsAE,3370
@@ -290,9 +290,10 @@ tol/sources/labwhere.py,sha256=gbfpPNKEZEfvw9DXGshg1w4Kdo_tpjWoxndDza6Oa6E,541
 tol/sources/mlwh.py,sha256=wPs2yuU1A8v57jaDQbeIThraDqVtzzhm5DAa0qekApM,324
 tol/sources/portal.py,sha256=1mhX1c4fjbRoHJ86RuBgyXFbDJrVIV55pGwvnuOxVuA,761
 tol/sources/portal_attributes.py,sha256=j_ZIZ8RHpgbRzOdHSOnzhAM9xA0HcwiAxuVgybVtDrg,1926
-tol/sources/portaldb.py,sha256=A-eD_tpJ1cqyDEsnN_fEE67Zkl-WQnRcVlfodZzMcVw,545
+tol/sources/portaldb.py,sha256=fZNIUk5mZ3RuQiknZQlPxOK6Vdz_pzqz1-w_dW0L2yE,626
 tol/sources/prefect.py,sha256=16PrC_KfvxrsDzAcq7UvEBo4c6_n38NC23vHQeiljck,1046
 tol/sources/sciops.py,sha256=qbdoNEo-BfWGw16sjANV7dxFRVMDJQlvaSZgofE8604,1129
+tol/sources/sql.py,sha256=mP5hyNNoxi8yKY1Z-6G5EZ1VW24BW_3q5Hp-w5VpPmE,623
 tol/sources/sts.py,sha256=vd65QAYIM1lltHajqJPOrGg10ZCI7OUB2dbWNAlrAn8,749
 tol/sources/sts_legacy.py,sha256=jWDM1_Jicxhay9UEOGh4lzeO0TIxuH8sa34TCn-GDWQ,440
 tol/sources/tolid.py,sha256=ERSB5L_u7c4NtGYTzdGXWjiGiL58QJNC_0ij1MQMMEQ,663
@@ -314,7 +315,7 @@ tol/sql/relationship.py,sha256=EClMgVx_If5nZV9MV99TQk7Wr7uACWetwQdWAliM5XI,2891
 tol/sql/session.py,sha256=VmqTegr4L2X2zvaOJCpwSrkVRx8fc1RVL0drkL2MXu8,806
 tol/sql/sort.py,sha256=ENrjHGgj4fZtXKmkdlkv1HRi1X14SVlcl-tp8Pu7G0k,2553
 tol/sql/sql_converter.py,sha256=taD5FRwadvw2bBaUGrCIiUs0-ATAbBnRYI1M7xe3yEc,4618
-tol/sql/sql_datasource.py,sha256=RnPkfg4TSHaxfJG77eBrNgBAKexJaypc-fvfFzssdac,15598
+tol/sql/sql_datasource.py,sha256=2FUwGTIN5Dw2cNAEb9fWGsZ0PVdjhdjn8WajA_9LT_Y,15615
 tol/sql/action/__init__.py,sha256=T1zAsCza_lvsNtXF1ecSLt9OFGup8tGnIs68YylBmXI,142
 tol/sql/action/factory.py,sha256=HkareJp_57ud0_Bdd9Kwz3_Rnq2l211sGJgftohFAHg,3589
 tol/sql/auth/__init__.py,sha256=e3JuwugXmXobklqZ1Mt1w03qPgb1WdUaJVM7oblzHyk,202
@@ -335,8 +336,8 @@ tol/treeval/treeval_datasource.py,sha256=GzY6JwH67b5QdV-UVdCFJfgGAIuZ96J2nl53YxZ
 tol/utils/__init__.py,sha256=764-Na1OaNGUDWpMIu51ZtXG7n_nB5MccUFK6LmkWRI,138
 tol/utils/csv.py,sha256=mihww25fSn72c4h-RFeqD_pFIG6KHZP4v1_C0rx81ws,421
 tol/utils/s3.py,sha256=aoYCwJ-qcMqFrpxmViFqPa0O1jgp0phtztO3-0CSNjw,491
-tol/validators/__init__.py,sha256=_ETv6oGQ2bTH_6-foYFy9T5wP5OG3cl96zEjvrIS7zk,1399
-tol/validators/allowed_keys.py,sha256=RJcHBiguL84B8hjSRaXLNES21yZqaKFwJNp2Tz9zvh0,1506
+tol/validators/__init__.py,sha256=sF9i4rxi-NsJkqLvvxj2k_I_QPRgkhQZc9ErSqT7quk,1465
+tol/validators/allowed_keys.py,sha256=eLZnr6DVqK6ru6b-T-1hvzSdAxlWZkTNZuohrbO0vS8,1525
 tol/validators/allowed_values.py,sha256=-Yy3Sqo1WYacGKlot_dn3M2o7Oj5MXOioJrJmrWCCxs,1536
 tol/validators/allowed_values_from_datasource.py,sha256=9cVwllBbzfCls8UsojazfCInt9_AakA0_H9pBO1wSL4,3173
 tol/validators/assert_on_condition.py,sha256=eBGgSVfIQ6e45SheM-ZDg7daXJjyZxRVS5L8AWvbXag,2027
@@ -351,6 +352,7 @@ tol/validators/regex.py,sha256=dLAi_vQt9_DsT6wQZmbYC7X5-Wp15l0leUE6XkPaItg,2602
 tol/validators/regex_by_value.py,sha256=XM5EnT4vgD17rfpR3bUE9I56IemSw26BI9MZtMakd4E,2582
 tol/validators/specimens_have_same_taxon.py,sha256=BaJcZ38ZprPcuGTIorSxxC9uGN0_lj6HS6B54EObcuY,2183
 tol/validators/sts_fields.py,sha256=aYbzy15btEg4-ocDT1qrspe7-atoWRrOJ_KmuPU6J14,8936
+tol/validators/taxon_matches_goat.py,sha256=lYZ0qwPver9wXJm2ekv_6KcC49MugVXD9t1IAxPGG2Q,3492
 tol/validators/tolid.py,sha256=VOb6lNFz11H_0KaWX8_nvsw8xJEa6KrjB0p-5lkcqog,3885
 tol/validators/types.py,sha256=jMVpckRp8RS93f7usf58YH_K-5rKWgZIYs7bO9dHhQc,2914
 tol/validators/unique_value_check.py,sha256=sFvDooYkKeORvULGEOTsgIcxlbe0AXDWxY3Gbr3j0KI,1282
@@ -359,9 +361,9 @@ tol/validators/unique_whole_organisms.py,sha256=RdqA1GzIf3LTdrmNGGdxv0aW2udDY2P9
 tol/validators/value_check.py,sha256=DdNx_B1gns01zgBg5N6Bwia46Aukw6MAteM-M37Kv1k,1122
 tol/validators/interfaces/__init__.py,sha256=jtOxnwnwqV_29xjmmMcS_kvlt-pQiWwQYJn2YRP07_w,172
 tol/validators/interfaces/condition_evaluator.py,sha256=nj8Cb8hi47OBy6OVNfeLhF-Pjwtr8MiOSymYL6hfVes,3766
-tol_sdk-1.8.8.dist-info/licenses/LICENSE,sha256=RF9Jacy-9BpUAQQ20INhTgtaNBkmdTolYCHtrrkM2-8,1077
-tol_sdk-1.8.8.dist-info/METADATA,sha256=iyFf12GxWPRsRVPQgR5mrGzLDhAH-OrJ58-4T6CROLA,3142
-tol_sdk-1.8.8.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-tol_sdk-1.8.8.dist-info/entry_points.txt,sha256=jH3HfTwxjzog7E3lq8CKpUWGIRY9FSXbyL6CpUmv6D0,36
-tol_sdk-1.8.8.dist-info/top_level.txt,sha256=PwKMQLphyZNvagBoriVbl8uwHXQl8IC1niawVG0iXMM,10
-tol_sdk-1.8.8.dist-info/RECORD,,
+tol_sdk-1.8.10.dist-info/licenses/LICENSE,sha256=RF9Jacy-9BpUAQQ20INhTgtaNBkmdTolYCHtrrkM2-8,1077
+tol_sdk-1.8.10.dist-info/METADATA,sha256=8VkqLVdpb45ixAI3mGjagWB0lDmyJYxfrSp6tTr-NFg,3143
+tol_sdk-1.8.10.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+tol_sdk-1.8.10.dist-info/entry_points.txt,sha256=jH3HfTwxjzog7E3lq8CKpUWGIRY9FSXbyL6CpUmv6D0,36
+tol_sdk-1.8.10.dist-info/top_level.txt,sha256=PwKMQLphyZNvagBoriVbl8uwHXQl8IC1niawVG0iXMM,10
+tol_sdk-1.8.10.dist-info/RECORD,,

{tol_sdk-1.8.8.dist-info → tol_sdk-1.8.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any