tol-sdk 1.7.4__py3-none-any.whl → 1.7.5b2__py3-none-any.whl

tol/core/data_object_converter.py

@@ -90,5 +90,40 @@ class DefaultDataObjectToDataObjectConverter(DataObjectToDataObjectOrUpdateConve
                     if k != self.__id_field
                 }
             )
-            return iter([ret])
-        return iter([])
+            yield ret
+
+
+class SanitisingConverter(DataObjectToDataObjectOrUpdateConverter):
+
+    def __init__(
+        self,
+        data_object_factory: DataObjectFactory
+    ):
+        super().__init__(data_object_factory)
+
+    def convert(
+        self,
+        data_object: DataObject
+    ) -> Iterable[DataObject]:
+        """
+        A converter that removes leading and trailing whitespace from
+        string attributes.
+        """
+        if data_object is not None and data_object.id is not None:
+            ret = self._data_object_factory(
+                id_=data_object.id,
+                type_=data_object.type,
+                attributes={
+                    k: self.__sanitise(v)
+                    for k, v in data_object.attributes.items()
+                }
+            )
+            yield ret
+
+    def __sanitise(
+        self,
+        value: object
+    ) -> object:
+        if isinstance(value, str):
+            return value.strip()
+        return value

tol/core/factory.py

@@ -4,19 +4,10 @@
 
 from __future__ import annotations
 
-import logging
 import typing
 from abc import ABC
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    Optional,
-    Protocol,
-    Type,
-    Union
-)
+from collections.abc import Callable, Iterable
+from typing import Any, Protocol
 
 from .data_object import DataDict, DataObject
 from .data_source_dict import DataSourceDict
@@ -28,7 +19,7 @@ if typing.TYPE_CHECKING:
     from .datasource import DataSource
 
 
-ToOne = dict[str, Optional[DataObject]]
+ToOne = dict[str, DataObject | None]
 ToMany = dict[str, Iterable[DataObject]]
 
 
@@ -42,8 +33,8 @@ class DataObjectFactory(Protocol):
         self,
         type_: str,
 
-        id_: Optional[str] = None,
-        attributes: Dict[str, Any] | None = None,
+        id_: str | None = None,
+        attributes: dict[str, Any] | None = None,
         to_one: ToOne | None = None,
         to_many: ToMany | None = None
     ) -> DataObject:
@@ -66,17 +57,17 @@ class DataSourceDictFactory(Protocol):
 
 OneDictFactory = Callable[
     [DataObject],
-    Dict[str, Optional[DataObject]]
+    dict[str, DataObject | None]
 ]
 """
 Takes a `DataObject` instance, returns a `dict` mapping
-to-one relationship names to its `Optional[DataObject]`
+to-one relationship names to its optional `DataObject`
 """
 
 
 ManyDictFactory = Callable[
     [DataObject],
-    Dict[str, Iterable[DataObject]]
+    dict[str, Iterable[DataObject]]
 ]
 """
 Takes a `DataObject` instance, returns a `dict` mapping
@@ -105,7 +96,7 @@ def core_data_object(
     one_dict_factory: OneDictFactory = lambda o: ToOneDict(o),
     many_dict_factory: ManyDictFactory = lambda o: ToManyDict(o),
     data_source_dict_factory: DataSourceDictFactory = lambda *d: DataSourceDict(*d)
-) -> Type[DataObject]:
+) -> type[DataObject]:
     """
     Takes a tuple of DataSource instances, and creates a CoreDataObject
     implementation that refers to all of them.
@@ -132,17 +123,18 @@ def core_data_object(
         def __init__(
             self,
             type_: str,
-            id_: Optional[str] = None,
+            id_: str | None = None,
             attributes: DataDict | None = None,
             to_one: ToOne | None = None,
             to_many: ToMany | None = None,
             stub: bool = False,
-            stub_types: Optional[Iterable[str]] = None
+            stub_types: Iterable[str] | None = None
         ):
             self.__id = id_
             self.__type = type_
-            self.__attributes = attributes if attributes is not None else {}
-            self.__to_one_objects = to_one if to_one is not None else {}
+            self.__attributes = {} if attributes is None else attributes
+            self.__to_one_objects = {} if to_one is None else to_one
+            self.__to_many_objects = {} if to_many is None else to_many
             if stub and id_ is None:
                 raise DataSourceError('ID must be set if stub is True')
             self.__stub_value = stub
@@ -151,11 +143,6 @@ def core_data_object(
                 self.__to_one_relations = one_dict_factory(self)
                 self.__to_many_relations = many_dict_factory(self)
 
-            if to_many:
-                logging.warning(
-                    'Setting of to_many relations is unsupported'
-                )
-
         def __str__(self) -> str:
             dump = f'type="{self.type}"'
 
@@ -164,36 +151,34 @@ def core_data_object(
 
             return f'CoreDataObject({dump})'
 
-        def __getattribute__(self, __name: str) -> Any:
-            if _local_name(__name):
-                return object.__getattribute__(self, __name)
+        def __getattribute__(self, name: str, /) -> Any:
+            if _local_name(name):
+                return object.__getattribute__(self, name)
 
             if self.__stub_value:
                 self.__unstub()
 
-            if __name in self.__to_one_names:
-                if __name in self._to_one_objects:
-                    return self._to_one_objects[__name]
-                return self.to_one_relationships.get(__name)
-
-            if __name in self.__to_many_names:
-                return self.to_many_relationships.get(__name, [])
-
-            return self.__attributes.get(__name)
-
-        def __setattr__(self, __name: str, __value: Any) -> None:
-            if _local_name(__name):
-                object.__setattr__(self, __name, __value)
-            elif __name in self.__to_one_names:
-                self._to_one_objects[__name] = __value
-            elif __name in self.__to_many_names:
-                raise DataSourceError(
-                    title='Read-only To-Many',
-                    detail='To-many relations are readonly',
-                    status_code=400
-                )
+            if name in self.__to_one_names:
+                if name in self._to_one_objects:
+                    return self._to_one_objects[name]
+                return self.to_one_relationships.get(name)
+
+            if name in self.__to_many_names:
+                if name in self._to_many_objects:
+                    return self._to_many_objects[name]
+                return self.to_many_relationships.get(name, [])
+
+            return self.__attributes.get(name)
+
+        def __setattr__(self, name: str, value: Any, /) -> None:
+            if _local_name(name):
+                object.__setattr__(self, name, value)
+            elif name in self.__to_one_names:
+                self._to_one_objects[name] = value
+            elif name in self.__to_many_names:
+                self._to_many_objects[name] = value
             else:
-                self.__attributes[__name] = __value
+                self.__attributes[name] = value
 
         def __unstub(self) -> None:
             self.__stub_value = False
@@ -214,7 +199,7 @@ def core_data_object(
             return self.__type
 
         @property
-        def id(self) -> Optional[str]:  # noqa
+        def id(self) -> str | None:  # noqa
             return self.__id
 
         @id.setter
@@ -222,27 +207,31 @@ def core_data_object(
             self.__id = new_id
 
         @property
-        def attributes(self) -> Dict[str, Any]:
+        def attributes(self) -> dict[str, Any]:
             if self.__stub_value:
                 self.__unstub()
             return self.__attributes
 
         @property
-        def to_one_relationships(self) -> Dict[str, Optional[DataObject]]:
+        def to_one_relationships(self) -> dict[str, DataObject | None]:
             if not self.__relational:
                 raise NotRelationalError(self)
             return self.__to_one_relations
 
         @property
-        def to_many_relationships(self) -> Dict[str, Iterable[DataObject]]:
+        def to_many_relationships(self) -> dict[str, list[DataObject]]:
             if not self.__relational:
                 raise NotRelationalError(self)
             return self.__to_many_relations
 
         @property
-        def _to_one_objects(self) -> Dict[str, DataObject]:
+        def _to_one_objects(self) -> dict[str, DataObject | None]:
             return self.__to_one_objects
 
+        @property
+        def _to_many_objects(self) -> dict[str, list[DataObject]]:
+            return self.__to_many_objects
+
         @property
         def __relational(self) -> bool:
             """Whether the hosting DataSource is relational or not"""
@@ -250,7 +239,7 @@ def core_data_object(
             return isinstance(self._host, Relational)
 
         @property
-        def __relationship_config(self) -> Optional[RelationshipConfig]:
+        def __relationship_config(self) -> RelationshipConfig | None:
             return self._host.relationship_config.get(self.type)
 
         @property
@@ -274,17 +263,17 @@ def core_data_object(
             )
 
         @property
-        def _host(self) -> Union[DataSource, Relational]:
+        def _host(self) -> DataSource | Relational:
             return data_source_dict[self.type]
 
     def core_data_object_factory(
         type_: str,
-        id_: Optional[str] = None,
-        attributes: Dict[str, Any] | None = None,
+        id_: str | None = None,
+        attributes: dict[str, Any] | None = None,
         to_one: ToOne | None = None,
         to_many: ToMany | None = None,
         stub: bool = False,  # Set stub if only type and id are given on creation
-        stub_types: Optional[Iterable[str]] = None
+        stub_types: Iterable[str] | None = None
     ) -> DataObject:
 
         return CoreDataObject(

tol/core/validate.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import Iterable
 
+
 from .data_object import DataObject, ErrorObject
 
 

tol/ena/client.py

@@ -7,9 +7,10 @@ from typing import Dict, Optional, Tuple
 import requests
 
 from .converter import EnaApiTransfer
+from ..core import HttpClient
 
 
-class EnaApiClient:
+class EnaApiClient(HttpClient):
     """
     Takes ENA API transfers and connects to a remote ENA API.
     """
@@ -22,6 +23,9 @@ class EnaApiClient:
         ena_contact_name: str,
         ena_contact_email: str,
     ) -> None:
+        super().__init__(
+            retries=5
+        )
         self.__ena_url = ena_url
         self.__ena_user = ena_user
         self.__ena_password = ena_password
@@ -46,7 +50,7 @@ class EnaApiClient:
         `object_type` and `object_id` or returns None if not found.
         """
         url, params = self.__detail_url(object_type, object_ids, filter_string)
-        return self.__fetch_detail(url, params)
+        return self.__fetch_detail(url, params, text=(object_type == 'checklist'))
 
     def get_list(
         self,
@@ -59,17 +63,27 @@ class EnaApiClient:
     def __fetch_detail(
         self,
         url: str,
-        params: Dict = {}
+        params: Dict = {},
+        text: bool = False
     ) -> Optional[EnaApiTransfer]:
         """
         Fetches data from the ENA API.
         """
+        session = self._get_session_with_retries()
         headers = {'Content-Type': 'application/json'}
-        r = requests.get(url, params=params, headers=headers)
+        r = session.get(url, params=params, headers=headers)
         if r.status_code == 404:
             return []
         r.raise_for_status()
-        return r.json() if r.json else []
+        if text:
+            return r.text
+        try:
+            data = r.json()
+            if isinstance(data, list):
+                return data
+            return [] if not data else [data]
+        except requests.exceptions.JSONDecodeError:
+            return []
 
     def __fetch_list(
         self,
@@ -79,13 +93,18 @@ class EnaApiClient:
         """
         Fetches data from the ENA API.
         """
+        session = self._get_session_with_retries()
         headers = {'Content-Type': 'application/json'}
-        r = requests.get(url, params=params, headers=headers)
+        r = session.get(url, params=params, headers=headers)
 
         if r.status_code == 404:
             return []
         r.raise_for_status()
-        return r.json() if r.json else []
+        try:
+            data = r.json()
+            return data if data else []
+        except requests.exceptions.JSONDecodeError:
+            return []
 
     def __detail_url(
         self,
@@ -96,6 +115,18 @@ class EnaApiClient:
         """
         Returns the URL and parameters for a detail query.
         """
+        if object_type == 'checklist':
+            ids = ','.join(str(id_) for id_ in object_ids)
+            url = f'{self.__ena_url}/ena/browser/api/xml/{ids}'
+            params = {}
+            return url, params
+        if object_type == 'submittable_taxon':
+            # This is actually called separately for each taxon id
+            ids = ','.join(str(id_) for id_ in object_ids)
+            url = f'{self.__ena_url}/ena/taxonomy/rest/tax-id/{ids}'
+            params = {}
+            return url, params
+
         url = f'{self.__ena_url}/ena/portal/api/search'
 
         if object_type == 'assembly':
@@ -150,14 +181,34 @@ class EnaApiClient:
         """
         Returns the fields for a given object type from the ENA portal API.
         """
-        response = requests.get(
-            self.__ena_url + '/ena/portal/api/returnFields?result=' + object_type + '&format=json',
+        if object_type == 'checklist':
+            return {'checklist': 'dict[str, Any]'}
+        if object_type == 'submittable_taxon':
+            return {
+                'scientific_name': 'str',
+                'formal_name': 'str',
+                'rank': 'str',
+                'division': 'str',
+                'lineage': 'str',
+                'genetic_code': 'str',
+                'mitochondrial_genetic_code': 'str',
+                'submittable': 'boolean',
+                'binomial': 'boolean',
+                'merged': 'str',
+                'authority': 'str',
+                'other_names': 'list[str]',
+                'metagenome': 'str'
+            }
+        session = self._get_session_with_retries()
+        r = session.get(
+            self.__ena_url + '/ena/portal/api/returnFields',
+            params={'result': object_type, 'format': 'json'},
             headers={
                 'Content-Type': 'application/json'
             }
         )
         fields = {}
-        for field in response.json():
+        for field in r.json():
 
             type_ = field['type'] if 'type' in field else 'string'
             ena_type = self.__type_mappings[type_] if type_ in self.__type_mappings else 'str'

tol/ena/ena_datasource.py

@@ -100,7 +100,10 @@ class EnaDataSource(
     @property
     @cache
     def supported_types(self) -> list[str]:
-        return ['assembly', 'read_run', 'sample', 'study', 'taxon']
+        return [
+            'assembly', 'read_run', 'sample', 'study', 'taxon', 'checklist',
+            'submittable_taxon'
+        ]
 
     def get_by_id(
         self,
@@ -111,7 +114,18 @@ class EnaDataSource(
         self.__validate_object_type(object_type)
 
         client = self.__client_factory()
-        ena_response = client.get_detail(object_type, object_ids)
+        # For a submittable_taxon we need to make multiple calls
+        if object_type == 'submittable_taxon':
+            ena_response = []
+            for object_id in object_ids:
+                response = client.get_detail(object_type, [object_id])
+                if response and isinstance(response, list):
+                    ena_response.extend(response)
+        else:
+            ena_response = client.get_detail(object_type, object_ids)
+        # For a checklist we need to convert into a list of dicts
+        if object_type == 'checklist':
+            ena_response = convert_checklist_xml_to_dict(ena_response)
         ena_converter = self.__ec_factory()
 
         converted_objects, _ = ena_converter.convert_list(object_type, ena_response) \
@@ -198,7 +212,6 @@ class EnaDataSource(
         return response
 
     def get_request(self, command: str, headers=None, params=None) -> requests.Response:
-
         response = requests.get(self.uri + command,
                                 params=params, headers=headers,
                                 auth=HTTPBasicAuth(self.user, self.password))
@@ -209,13 +222,6 @@ class EnaDataSource(
 
         return response
 
-    def get_xml_checklist(self, checklist_id: str) -> Dict[str, Tuple[str, str, object]]:
-        output = self.get_request(f'/ena/browser/api/xml/{checklist_id}')
-
-        checklist_dict = convert_checklist_xml_to_dict(output.text)
-
-        return checklist_dict
-
     def get_biosample_data_biosampleid(self, biosample_id: str):
         output = self.get_request(f'/ena/browser/api/xml/{biosample_id}')
 

tol/ena/ena_methods.py

@@ -30,51 +30,52 @@ submission_xml_template = """<?xml version="1.0" encoding="UTF-8"?>
 
 def convert_checklist_xml_to_dict(checklist_xml: str) -> Dict[str, Tuple[str, str, object]]:
     # key label, val [mandatory_status, ]
-
-    fields = {}
-
+    checklists = []
     root = ElementTree.fromstring(checklist_xml)
-    for field_group_node in root.findall('./CHECKLIST/DESCRIPTOR/FIELD_GROUP'):
-        for field_node in field_group_node.findall('./FIELD'):
-
-            label, mandatory_status = None, None
+    for checklist_node in root.findall('./CHECKLIST'):
+        checklist_id = checklist_node.get('accession')
+        fields = {}
+        for field_group_node in checklist_node.findall('./DESCRIPTOR/FIELD_GROUP'):
+            for field_node in field_group_node.findall('./FIELD'):
 
-            label_node = field_node.find('./LABEL')
+                label, mandatory_status = None, None
 
-            if label_node is not None:
-                label = label_node.text
+                label_node = field_node.find('./LABEL')
 
-            mandatory_node = field_node.find('./MANDATORY')
+                if label_node is not None:
+                    label = label_node.text
 
-            if mandatory_node is not None:
-                mandatory_status = mandatory_node.text
+                mandatory_node = field_node.find('./MANDATORY')
 
-            regex_node = field_node.find('./FIELD_TYPE/TEXT_FIELD/REGEX_VALUE')
-            if regex_node is not None:
-                regex_str = regex_node.text
-                fields[label] = [mandatory_status, 'restricted text', regex_str]
-                continue
+                if mandatory_node is not None:
+                    mandatory_status = mandatory_node.text
 
-            text_choice_node = field_node.find('./FIELD_TYPE/TEXT_CHOICE_FIELD')
+                regex_node = field_node.find('./FIELD_TYPE/TEXT_FIELD/REGEX_VALUE')
+                if regex_node is not None:
+                    regex_str = regex_node.text
+                    fields[label] = [mandatory_status, 'restricted text', regex_str]
+                    continue
 
-            if text_choice_node is not None:
-                text_options = []
-                for text_option_node in text_choice_node.findall('./TEXT_VALUE/VALUE'):
-                    text_options.append(text_option_node.text)
+                text_choice_node = field_node.find('./FIELD_TYPE/TEXT_CHOICE_FIELD')
 
-                fields[label] = [mandatory_status, 'text choice', text_options]
-                continue
+                if text_choice_node is not None:
+                    text_options = []
+                    for text_option_node in text_choice_node.findall('./TEXT_VALUE/VALUE'):
+                        text_options.append(text_option_node.text)
 
-            taxon_node = field_node.find('./FIELD_TYPE/TEXT_FIELD/TAXON_FIELD')
+                    fields[label] = [mandatory_status, 'text choice', text_options]
+                    continue
 
-            if taxon_node is not None:
-                regex_str = regex_node.text
-                fields[label] = [mandatory_status, 'valid taxonomy', '']
-                continue
+                taxon_node = field_node.find('./FIELD_TYPE/TEXT_FIELD/TAXON_FIELD')
 
-            fields[label] = [mandatory_status, 'free text', '']
+                if taxon_node is not None:
+                    regex_str = regex_node.text
+                    fields[label] = [mandatory_status, 'valid taxonomy', '']
+                    continue
 
-    return fields
+                fields[label] = [mandatory_status, 'free text', '']
+        checklists.append({'checklist_id': checklist_id, 'checklist': fields})
+    return checklists
 
 
 def convert_xml_to_list_of_sample_dict(response_xml: str) -> List[Dict[str, List[str]]]:

tol/ena/parser.py

@@ -8,6 +8,7 @@ import typing
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, Optional
 
+from caseconverter import snakecase
 
 from ..core import DataObject
 
@@ -88,6 +89,10 @@ class DefaultParser(Parser):
             return transfer['study_accession']
         elif type_ == 'taxon':
             return transfer['tax_id']
+        elif type_ == 'checklist':
+            return transfer['checklist_id']
+        elif type_ == 'submittable_taxon':
+            return transfer['taxId']
 
     def __convert_attributes(
         self,
@@ -99,6 +104,14 @@ class DefaultParser(Parser):
             return ret
 
         for k, v in attributes.items():
-            if k not in ['key'] and k in self.__dict[type_].attribute_types[type_]:
-                ret[k] = v
+            if k not in ['key'] and snakecase(k) in self.__dict[type_].attribute_types[type_]:
+                ret[snakecase(k)] = self.__convert_value(type_, snakecase(k), v)
         return ret
+
+    def __convert_value(self, type_: str, attribute_name: str, value: Any) -> Any:
+        attribute_type = self.__dict[type_].attribute_types[type_][attribute_name]
+        if attribute_type == 'boolean':
+            if isinstance(value, str):
+                return value.lower() == 'true'
+            return bool(value)
+        return value

tol/flows/converters/__init__.py

@@ -34,6 +34,8 @@ from .gap_assembly_to_elastic_assembly_analysis_converter import GapAssemblyToEl
 from .genome_notes_genome_note_to_elastic_genome_note_converter import GenomeNotesGenomeNoteToElasticGenomeNoteConverter  # noqa F401
 from .goat_taxon_to_elastic_species_converter import GoatTaxonToElasticSpeciesConverter  # noqa F401
 from .grit_issue_to_elastic_curation_converter import GritIssueToElasticCurationConverter  # noqa F401
+from .incoming_sample_to_ena_sample_converter import IncomingSampleToEnaSampleConverter # noqa
+from .incoming_sample_to_incoming_sample_with_lists_converter import IncomingSampleToIncomingSampleWithListsConverter  # noqa F401
 from .informatics_tolid_to_elastic_tolid_converter import InformaticsTolidToElasticTolidConverter  # noqa F401
 from .labwhere_location_to_elastic_sample_update_converter import LabwhereLocationToElasticSampleUpdateConverter  # noqa F401
 from .labwhere_location_to_sts_tray_converter import LabwhereLocationToStsTrayConverter  # noqa F401