tol-sdk 1.8.5__py3-none-any.whl → 1.8.7__py3-none-any.whl

tol/dummy/dummy_datasource.py

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: 2026 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import typing
+from functools import cache
+from typing import Callable, Iterable, Optional
+
+from .client import DummyClient
+from .converter import (
+    DummyConverter
+)
+from ..core import DataObject, DataSource, DataSourceError, DataSourceFilter
+from ..core.operator import (
+    DetailGetter,
+    ListGetter
+)
+
+if typing.TYPE_CHECKING:
+    from ..core.session import OperableSession
+
+ClientFactory = Callable[[], DummyClient]
+DummyConverterFactory = Callable[[], DummyConverter]
+
+
+class DummyDataSource(
+    DataSource,
+
+    # the supported operators
+    DetailGetter,
+    ListGetter
+):
+    """
+    A `DataSource` that outputs dummy data.
+
+    Developers should likely use `create_dummy_datasource`
+    instead of this directly.
+    """
+
+    def __init__(
+        self,
+        client_factory: ClientFactory,
+        dummy_converter_factory: DummyConverterFactory
+    ) -> None:
+
+        self.__client_factory = client_factory
+        self.__dummy_converter_factory = dummy_converter_factory
+        super().__init__({})
+
+    @property
+    @cache
+    def attribute_types(self) -> dict[str, dict[str, str]]:
+        return {
+            'record': {
+                'big_string': 'str',
+                'little_string': 'str',
+                'bool': 'bool',
+                'date': 'datetime',
+                'int': 'int',
+                'list': 'list[str]'
+            },
+            'category': {
+                'name': 'str'
+            }
+        }
+
+    @property
+    @cache
+    def supported_types(self) -> list[str]:
+        return list(
+            self.attribute_types.keys()
+        )
+
+    def get_by_id(
+        self,
+        object_type: str,
+        object_ids: Iterable[str],
+        **kwargs
+    ) -> Iterable[Optional[DataObject]]:
+        if object_type not in self.supported_types:
+            raise DataSourceError(f'{object_type} is not supported')
+
+        client = self.__client_factory()
+        dummy_response = client.get_detail(object_type, object_ids)
+        dummy_converter = self.__dummy_converter_factory()
+
+        converted_objects, _ = dummy_converter.convert_list(dummy_response) \
+            if dummy_response is not None else ([], 0)
+        yield from self.sort_by_id(converted_objects, object_ids)
+
+    def get_list(
+        self,
+        object_type: str,
+        object_filters: Optional[DataSourceFilter] = None,
+        session: Optional[OperableSession] = None
+    ) -> Iterable[DataObject]:
+        if object_filters:
+            raise DataSourceError('Filtering is not supported')
+        objects = self.__client_factory().get_list(
+            object_type
+        )
+        converted_objects, _ = self.__dummy_converter_factory().convert_list(objects)
+        return iter(converted_objects)

tol/dummy/factory.py

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: 2024 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from collections.abc import Mapping
+from typing import Callable, Iterator, Optional
+
+from .client import DummyClient
+from .converter import (
+    DummyConverter
+)
+from .dummy_datasource import (
+    DummyConverterFactory,
+    DummyDataSource
+)
+from .parser import DefaultParser
+from ..core import DataSource
+
+
+class _DummyDSDict(Mapping):
+    def __init__(self, api_ds: DummyDataSource) -> None:
+        self.__ds = api_ds
+
+    def __getitem__(self, __k: str) -> DummyDataSource:
+        if __k not in self.__ds.supported_types:
+            raise KeyError()
+        return self.__ds
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.__ds.supported_types)
+
+    def __len__(self) -> int:
+        return len(self.__ds.supported_types)
+
+
+class _ConverterFactory:
+    """
+    Manges the instantation of:
+
+    - `DummyConverter`
+    """
+
+    def __init__(self) -> None:
+        self.__data_source: Optional[DataSource] = None
+
+    @property
+    def data_source(self) -> Optional[DataSource]:
+        return self.__data_source
+
+    @data_source.setter
+    def data_source(
+        self,
+        ds: DataSource
+    ) -> None:
+
+        self.__data_source = ds
+
+    def dummy_converter_factory(self) -> DummyConverterFactory:
+        """
+        Returns an instantiated `DummyConverter`.
+        """
+
+        parser = DefaultParser(self.__ds_dict)
+        return DummyConverter(parser)
+
+    @property
+    def __ds_dict(self) -> dict[str, DataSource]:
+        return _DummyDSDict(self.data_source)
+
+
+def _get_client_factory() -> Callable[[], DummyClient]:
+    """
+    A resonable default for creating
+    a `DummyClient` instance
+    """
+
+    return lambda: DummyClient()
+
+
+def create_dummy_datasource() -> DummyDataSource:
+    """
+    Instantiates `DummyDataSource`
+    """
+
+    client_factory = _get_client_factory()
+    manager = _ConverterFactory()
+
+    dummy_ds = DummyDataSource(
+        client_factory,
+        manager.dummy_converter_factory
+    )
+
+    manager.data_source = dummy_ds
+
+    return dummy_ds

tol/dummy/parser.py

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: 2024 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import typing
+from abc import ABC, abstractmethod
+from typing import Any, Iterable
+
+from dateutil.parser import parse as dateutil_parse
+
+from ..core import DataObject
+
+if typing.TYPE_CHECKING:
+    from ..core import DataSource
+
+
+DummyResource = dict[str, Any]
+DummyDoc = dict[str, list[DummyResource]]
+
+
+class Parser(ABC):
+
+    def parse_iterable(
+        self,
+        transfers: Iterable[DummyResource]
+    ) -> Iterable[DataObject]:
+        """
+        Parses an `Iterable` of Dummy transfer resources
+        """
+
+        return (
+            self.parse(t) for t in transfers
+        )
+
+    @abstractmethod
+    def parse(self, transfer: DummyResource) -> DataObject:
+        """
+        Parses an individual Dummy transfer resource to a
+        `DataObject` instance
+        """
+
+
+class DefaultParser(Parser):
+
+    def __init__(self, data_source_dict: dict[str, DataSource]) -> None:
+        self.__dict = data_source_dict
+
+    def parse(self, transfer: DummyResource) -> DataObject:
+        ds = self.__dict[transfer.get('type')]
+        return ds.data_object_factory(
+            transfer.get('type'),
+            id_=transfer.get('id'),
+            attributes={
+                k: (
+                    dateutil_parse(v)
+                    if k in ['date'] and v is not None
+                    else v
+                )
+                for k, v in transfer.items()
+                if k not in ['id', 'type', 'category']
+            },
+            to_one={
+                'category': ds.data_object_factory(
+                    'category',
+                    transfer.get('category')
+                ) if 'category' in transfer else None
+            }
+        )

tol/flows/converters/__init__.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+from .benchling_extraction_container_to_elastic_extraction_container_converter import BenchlingExtractionContainerToElasticExtractionContainerConverter  # noqa F401
 from .benchling_extraction_to_elastic_extraction_converter import BenchlingExtractionToElasticExtractionConverter  # noqa F401
 from .benchling_extraction_to_elastic_sequencing_request_converter import BenchlingExtractionToElasticSequencingRequestConverter  # noqa F401
 from .benchling_sequencing_request_to_elastic_sequencing_request_converter import (  # noqa F401
@@ -56,4 +57,8 @@ from .sts_sample_to_casm_benchling_converter import StsSampleToCasmBenchlingConv
 from .treeofsex_species_to_treeofsexwh_species_converter import TreeofsexSpeciesToTreeofsexwhSpeciesConverter  # noqa F401
 from .treeofsex_upload_to_treeofsex_attribute_converter import TreeofsexUploadToTreeofsexAttributeConverter  # noqa F401
 from .skip_null_fields_converter import SkipNullFieldsConverter  # noqa F401
+from .default_field_value_if_missing_converter import DefaultFieldValueIfMissingConverter  # noqa F401
+from .prefix_field_converter import PrefixFieldConverter  # noqa F401
+from .combine_fields_converter import CombineFieldsConverter  # noqa F401
 from .auto_detect_manifest_type_converter import AutoDetectManifestTypeConverter # noqa F401
+from .time_string_to_time import TimeStringToTimeConverter # noqa F401

tol/flows/converters/benchling_extraction_container_to_elastic_extraction_container_converter.py

@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: 2026 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from typing import Iterable
+
+from ...core import (
+    DataObject,
+    DataObjectToDataObjectOrUpdateConverter
+)
+
+
+class BenchlingExtractionContainerToElasticExtractionContainerConverter(
+        DataObjectToDataObjectOrUpdateConverter):
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        if data_object.tissue_sts_id is not None:
+            ret = self._data_object_factory(
+                'extraction_container',
+                data_object.id,
+                attributes={
+                    **{k: v
+                       for k, v in data_object.attributes.items()
+                       if k not in ['tissue_sts_id', 'specimen_id', 'taxon_id',
+                                    'programme_id', 'eln_tissue_prep_id', 'extraction_id']}
+                },
+                to_one={
+                    'sample': self._data_object_factory(
+                        'sample',
+                        data_object.tissue_sts_id
+                    ) if data_object.tissue_sts_id is not None else None,
+                    'species': self._data_object_factory(
+                        'species',
+                        data_object.taxon_id
+                    ) if data_object.taxon_id is not None else None,
+                    'specimen': self._data_object_factory(
+                        'specimen',
+                        data_object.specimen_id
+                    ) if data_object.specimen_id is not None else None,
+                    'tolid': self._data_object_factory(
+                        'tolid',
+                        data_object.programme_id
+                    ) if data_object.programme_id is not None else None,
+                    'tissue_prep': self._data_object_factory(
+                        'tissue_prep',
+                        data_object.eln_tissue_prep_id
+                    ) if data_object.eln_tissue_prep_id is not None else None,
+                    'extraction': self._data_object_factory(
+                        'extraction',
+                        data_object.extraction_id
+                    ) if data_object.extraction_id is not None else None,
+                }
+            )
+            yield ret

tol/flows/converters/benchling_sequencing_request_to_elastic_sequencing_request_converter.py

@@ -15,12 +15,18 @@ class BenchlingSequencingRequestToElasticSequencingRequestConverter(
     def convert(self, data_object: DataObject) -> Iterable[DataObject]:
         if data_object.sts_id is not None:
             extraction = None
+            extraction_container = None
             tissue_prep = None
             if 'extraction_id' in data_object.attributes:
                 extraction = self._data_object_factory(
                     'extraction',
                     data_object.extraction_id
                 )
+            if 'fluidx_container_id' in data_object.attributes:
+                extraction_container = self._data_object_factory(
+                    'extraction_container',
+                    data_object.fluidx_container_id
+                )
             if 'tissue_prep_id' in data_object.attributes:
                 tissue_prep = self._data_object_factory(
                     'tissue_prep',
@@ -34,7 +40,7 @@ class BenchlingSequencingRequestToElasticSequencingRequestConverter(
                        for k, v in data_object.attributes.items()
                        if k not in ['sanger_sample_id', 'sts_id',
                                     'specimen_id', 'taxon_id', 'extraction_id',
-                                    'programme_id', 'tissue_prep_id']}
+                                    'programme_id', 'tissue_prep_id', 'fluidx_container_id']}
                 },
                 to_one={
                     'sample': self._data_object_factory(
@@ -54,6 +60,7 @@ class BenchlingSequencingRequestToElasticSequencingRequestConverter(
                         data_object.programme_id
                     ) if data_object.programme_id is not None else None,
                     'extraction': extraction,
+                    'extraction_container': extraction_container,
                     'tissue_prep': tissue_prep,
                 })
             yield ret

tol/flows/converters/combine_fields_converter.py

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class CombineFieldsConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        field1: str
+        field2: str
+        dest_field: str
+        lowercase_field1: bool
+
+    __slots__ = ('__config',)
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        Concatenates the values of two fields and stores the result in a new field.
+        The first field's value may be lowercased if specified in the configuration.
+        The destination field name is given by the configuration.
+        """
+
+        val1 = data_object.get_field_by_name(self.__config.field1)
+        val2 = data_object.get_field_by_name(self.__config.field2)
+        attributes = dict(data_object.attributes)
+
+        if val1 is not None and val2 is not None:
+            part1 = str(val1).lower() if self.__config.lowercase_field1 else str(val1)
+            attributes[self.__config.dest_field] = f'{part1}{val2}'
+
+        yield self._data_object_factory(
+            data_object.type,
+            data_object.id,
+            attributes=attributes,
+        )

tol/flows/converters/default_field_value_if_missing_converter.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class DefaultFieldValueIfMissingConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        field_name: str
+        default_value: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        Adds a default value for a configured field if missing, empty, or None
+        """
+
+        attributes_obj = data_object.attributes
+        if hasattr(attributes_obj, 'get_field_by_name'):
+            current_value = attributes_obj.get_field_by_name(self.__config.field_name)
+        else:
+            current_value = attributes_obj.get(self.__config.field_name)
+        attributes = dict(attributes_obj)
+        if not current_value:
+            attributes[self.__config.field_name] = self.__config.default_value
+
+        ret = self._data_object_factory(
+            data_object.type,
+            data_object.id,
+            attributes=attributes
+        )
+        yield ret

tol/flows/converters/elastic_sequencing_request_to_elastic_run_data_update_converter.py

@@ -29,5 +29,13 @@ class ElasticSequencingRequestToElasticRunDataUpdateConverter(
                     'extraction',
                     extraction.id
                 )
+        if 'benchling_extraction_container' in data_object.to_one_relationships:
+            extraction_container = \
+                data_object.to_one_relationships['benchling_extraction_container']
+            if extraction_container is not None:
+                to_ones['benchling_extraction_container'] = self._data_object_factory(
+                    'extraction_container',
+                    extraction_container.id
+                )
         yield (None, to_ones | {
             'mlwh_sequencing_request.id': data_object.id})  # The candidate key

tol/flows/converters/prefix_field_converter.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class PrefixFieldConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        field_name: str
+        prefix: str
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        Ensures the configured field value
+        starts with the configured prefix. If the field is None, it is
+        left as-is.
+        """
+
+        value = data_object.get_field_by_name(
+            self.__config.field_name
+        )
+
+        if value is not None:
+            value_str = str(value)
+            if not value_str.startswith(self.__config.prefix):
+                value = f'{self.__config.prefix}{value_str}'
+
+        ret = self._data_object_factory(
+            data_object.type,
+            data_object.id,
+            attributes={
+                **data_object.attributes,
+                self.__config.field_name: value
+            }
+        )
+        yield ret

tol/flows/converters/time_string_to_time.py

@@ -3,33 +3,42 @@
 # SPDX-License-Identifier: MIT
 
 import re
+from dataclasses import dataclass
 from datetime import time
+from typing import Iterable
 
-from tol.core import DataObject
-
-
-class Converter:
-    def convert(self, obj):
-        raise NotImplementedError()
-
-
-class TimeStringToTimeConverter(Converter):
-    """
-    Converts string fields representing time in HH:MM (24-hour) format to Python time objects.
-    If the string is not in HH:MM, tries to append ':00' and parse as HH:MM:SS.
-    """
-    def __init__(self, field: str):
-        self.field = field
-
-    def convert(self, obj: DataObject) -> DataObject:
-        value = obj.attributes.get(self.field)
-        if isinstance(value, str):
-            match = re.match(r'^(\d{1,2}):(\d{2})(?::(\d{2}))?$', value)
-            if match:
-                h, m = int(match.group(1)), int(match.group(2))
-                s = int(match.group(3)) if match.group(3) else 0
-                try:
-                    obj.attributes[self.field] = time(h, m, s)
-                except ValueError:
-                    pass
-        return obj
+from tol.core import DataObject, DataObjectToDataObjectOrUpdateConverter
+
+
+class TimeStringToTimeConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    @dataclass(slots=True, frozen=True, kw_only=True)
+    class Config:
+        field_names: list[str]
+
+    __slots__ = ['__config']
+    __config: Config
+
+    def __init__(self, data_object_factory, config: Config) -> None:
+        super().__init__(data_object_factory)
+        self.__config = config
+        self._data_object_factory = data_object_factory
+
+    def convert(self, data_object: DataObject) -> Iterable[DataObject]:
+        """
+        Converts string fields representing time in HH:MM (24-hour) format to Python time objects.
+        If the string is not in HH:MM, tries to append ':00' and parse as HH:MM:SS.
+        """
+
+        for field_name in self.__config.field_names:
+            value = data_object.attributes.get(field_name)
+            if isinstance(value, str):
+                match = re.match(r'^(\d{1,2}):(\d{2})(?::(\d{2}))?$', value)
+                if match:
+                    h, m = int(match.group(1)), int(match.group(2))
+                    s = int(match.group(3)) if match.group(3) else 0
+                    try:
+                        data_object.attributes[field_name] = time(h, m, s)
+                    except ValueError:
+                        pass
+        yield data_object

tol/sources/dummy.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2024 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from ..core import (
+    core_data_object
+)
+from ..dummy import (
+    DummyDataSource,
+    create_dummy_datasource
+)
+
+
+def dummy(**kwargs) -> DummyDataSource:
+    dummy = create_dummy_datasource()
+    core_data_object(dummy)
+    return dummy

tol/sql/auth/blueprint.py

@@ -8,7 +8,6 @@ from datetime import datetime, timedelta
 from typing import Any, Callable, Optional
 from urllib.parse import urlencode
 
-import requests
 from requests.auth import HTTPBasicAuth
 
 from .models import ModelClass, ModelTuple, create_models
@@ -228,9 +227,14 @@ class DbAuthManager(AuthManager):
         Raises:
             requests.HTTPError: If the user info request fails
         """
+
+        client = HttpClient()
+
+        session = client.get_session()
+
         headers = {'Authorization': f'Bearer {token}'}
 
-        r = requests.get(self.__config.user_info_url, headers=headers)
+        r = session.get(self.__config.user_info_url, headers=headers)
         r.raise_for_status()
 
         json_return = r.json()
@@ -401,13 +405,16 @@ class DbAuthManager(AuthManager):
         Raises:
             requests.HTTPError: If the token request fails
         """
-        r = requests.post(
+        client = HttpClient()
+
+        session = client.get_session()
+        r = session.post(
             self.__config.token_url,
-            auth=self.__basic_auth(),
             data=self.__token_post_data(code),
+            auth=self.__basic_auth()
         )
-        r.raise_for_status()
 
+        r.raise_for_status()
         return r.json()
 
     def __basic_auth(self) -> HTTPBasicAuth: