tol-sdk 1.6.37__tar.gz → 1.7.1__tar.gz

{tol_sdk-1.6.37/src/tol_sdk.egg-info → tol_sdk-1.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.6.37
+Version: 1.7.1
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT
@@ -37,7 +37,7 @@ Requires-Dist: atlassian-python-api==3.41.14; extra == "jira"
 Provides-Extra: json
 Requires-Dist: minio==7.2.15; extra == "json"
 Provides-Extra: mysql
-Requires-Dist: mysql-connector-python; extra == "mysql"
+Requires-Dist: mysql-connector-python==9.5.0; extra == "mysql"
 Provides-Extra: postgresql
 Requires-Dist: SQLAlchemy==2.0.35; extra == "postgresql"
 Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tol-sdk"
-version = "1.6.37"
+version = "1.7.1"
 description = "SDK for interaction with ToL, Sanger and external services"
 readme = "README.md"
 authors = [
@@ -43,7 +43,7 @@ elastic = ["elasticsearch == 7.17.9"]
 irods = ["python-irodsclient == 3.1.0"]
 jira = ["atlassian-python-api==3.41.14"]
 json = ["minio == 7.2.15"]
-mysql = ["mysql-connector-python"]
+mysql = ["mysql-connector-python==9.5.0"]
 postgresql = ["SQLAlchemy == 2.0.35", "psycopg2-binary == 2.9.9"]
 prefect = ["prefect == 2.20.18", "griffe < 1.0.0"]
 sheets = [

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_base/blueprint.py

@@ -35,6 +35,7 @@ from ..core import DataSource, DataSourceError, OperableDataSource
 from ..core.data_source_dict import DataSourceDict
 from ..core.operator import Relational
 from ..core.operator.operator_config import DefaultOperatorConfig, OperatorConfig
+from ..core.requested_fields import ReqFieldsTree
 
 
 class DataBlueprint(Blueprint):
@@ -179,6 +180,7 @@ def _core_blueprint(
     data_source_dict: dict[str, DataSource],
     url_prefix: str,
     auth_inspector: Optional[AuthInspector] = None,
+    include_all_to_ones: bool = True,
 ) -> DataBlueprint:
     """
     Create the core blueprint responsible for managing DataSource endpoints.
@@ -193,6 +195,8 @@ def _core_blueprint(
         url_prefix (str): URL prefix for all data endpoints.
         auth_inspector (Optional[AuthInspector], optional): Authentication
             inspector for request authorisation.
+        include_all_to_ones (bool): Whether to fetch or store all to-one related objects
+            when fetching or serialising DataObjects.
 
     Returns:
         DataBlueprint: A configured blueprint with all data endpoints and error handlers.
@@ -231,18 +235,30 @@ def _core_blueprint(
         hop_limit = None if requested_fields else 1
 
         data_source = data_source_dict[object_type]
+
+        # Build a ReqFieldsTree template for the request
+        req_fields_tree = ReqFieldsTree(
+            object_type,
+            data_source,
+            requested_fields,
+            include_all_to_ones=include_all_to_ones,
+        )
+
         view = DefaultView(
+            requested_tree=req_fields_tree,
             prefix=url_prefix,
-            include_all_to_ones=True,
             hop_limit=hop_limit,
-            requested_fields=requested_fields,
         )
-        return Controller(data_source, view, auth_inspector=auth_inspector)
+        return Controller(data_source, view, req_fields_tree, auth_inspector=auth_inspector)
 
     @data_handler.route('/<object_type>/<path:object_id>', methods=['GET'])  # Allow slashes
     def get_detail(*, object_type: str, object_id: str):
         """Get details of a specific object by ID."""
-        controller = __new_controller(object_type)
+        request_args = ListGetParameters(request.args)
+        controller = __new_controller(
+            object_type,
+            requested_fields=request_args.requested_fields,
+        )
         object_id_unencoded = urllib.parse.unquote(object_id)
         return controller.get_detail(object_type, object_id_unencoded)
 
@@ -325,8 +341,11 @@ def _core_blueprint(
     @data_handler.post('/<object_type>:cursor')
     def get_cursor_page(*, object_type: str):
         """Get a page of results using cursor-based pagination."""
-        controller = __new_controller(object_type)
         request_args = ListGetParameters(request.args)
+        controller = __new_controller(
+            object_type,
+            requested_fields=request_args.requested_fields,
+        )
         search_after = request.json.get('search_after')
         return controller.get_cursor_page(object_type, request_args, search_after)
 
@@ -369,6 +388,7 @@ def data_blueprint(
     url_prefix: str = '/data',
     config_prefix: str = '/_config',
     auth_inspector: Optional[AuthInspector] = None,
+    include_all_to_ones: bool = True,
 ) -> DataBlueprint:
     """
     Create a complete data blueprint with both core and configuration endpoints.
@@ -405,7 +425,10 @@ def data_blueprint(
         config_prefix, data_sources, DefaultOperatorConfig(*data_sources)
     )
     core_bp = _core_blueprint(
-        DataSourceDict(*data_sources), url_prefix, auth_inspector=auth_inspector
+        DataSourceDict(*data_sources),
+        url_prefix,
+        auth_inspector=auth_inspector,
+        include_all_to_ones=include_all_to_ones,
     )
     core_bp.register_blueprint(config_bp)
 

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_base/controller.py

@@ -31,7 +31,7 @@ from ..api_client.exception import (
     UnsupportedOperationError,
 )
 from ..api_client.view import ResponseDict, View
-from ..core import DataObject, OperableDataSource
+from ..core import DataObject, OperableDataSource, ReqFieldsTree
 from ..core.datasource_filter import AndFilter, DataSourceFilter
 from ..core.operator import (
     Aggregator,
@@ -160,6 +160,7 @@ class Controller:
         self,
         data_source: OperableDataSource,
         view: View,
+        requested_tree: ReqFieldsTree | None = None,
         auth_inspector: Optional[AuthInspector] = None,
     ) -> None:
         """
@@ -173,6 +174,7 @@ class Controller:
         """
         self.__data_source = data_source
         self.__view = view
+        self.__requested_tree = requested_tree
         self.__inspector = auth_inspector
 
     @property
@@ -263,13 +265,13 @@ class Controller:
             page_size=query_args.page_size,
             object_filters=self.__combine_filters(query_args.filter, ext_and),
             sort_by=query_args.sort_by,
-            requested_fields=query_args.requested_fields,
+            requested_tree=self.__requested_tree,
         )
-        document_meta = {
+        meta = {
             'total': total,
             'types': self.__data_source.get_attribute_types(object_type),
         }
-        return self.__view.dump_bulk(data_objects, document_meta=document_meta)
+        return self.__view.dump_bulk(data_objects, document_meta=meta)
 
     @validate(Counter, 'get_count', OperatorMethod.COUNT)
     def get_count(
@@ -545,6 +547,7 @@ class Controller:
             query_args.page_size,
             self.__combine_filters(query_args.filter, ext_and),
             search_after,
+            requested_tree=self.__requested_tree,
         )
         meta = {'search_after': new_search_after}
 
@@ -657,7 +660,13 @@ class Controller:
         Raises:
             ObjectNotFoundByIdException: If no object with the given ID exists.
         """
-        data_objects = list(self.__data_source.get_by_id(object_type, [object_id]))
+        data_objects = list(
+            self.__data_source.get_by_id(
+                object_type,
+                [object_id],
+                requested_tree=self.__requested_tree,
+            )
+        )
         if len(data_objects) == 0 or data_objects[0] is None:
             raise ObjectNotFoundByIdException(object_type, object_id)
         return data_objects[0]

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_client/api_datasource.py

@@ -144,7 +144,11 @@ class ApiDataSource(
 
         client = self.__client_factory()
         json_responses = (
-            client.get_detail(object_type, id_)
+            client.get_detail(
+                object_type,
+                id_,
+                requested_fields=requested_fields,
+            )
             for id_ in object_ids
         )
         json_converter = self.__jc_factory()
@@ -184,16 +188,17 @@ class ApiDataSource(
         session: Optional[OperableSession] = None,
         requested_fields: list[str] | None = None,
     ) -> Iterable[DataObject]:
-
         if self.__can_cursor(object_type, object_filters):
             return self._get_list_by_cursor(
                 object_type,
-                object_filters
+                object_filters,
+                requested_fields=requested_fields,
             )
         else:
             return self.__get_list_regular(
                 object_type,
-                object_filters
+                object_filters,
+                requested_fields=requested_fields,
             )
 
     @validate('count')
@@ -266,7 +271,8 @@ class ApiDataSource(
             object_type,
             page_size,
             search_after,
-            filter_string=filter_string
+            filter_string=filter_string,
+            requested_fields=requested_fields,
         )
         return self.__jc_factory().convert_cursor_page(transfer)
 
@@ -429,7 +435,8 @@ class ApiDataSource(
     def __get_list_regular(
         self,
         object_type: str,
-        object_filters: Optional[DataSourceFilter]
+        object_filters: Optional[DataSourceFilter],
+        requested_fields: list[str] | None = None,
     ) -> Iterable[DataObject]:
 
         page = 1
@@ -443,7 +450,8 @@ class ApiDataSource(
                 object_type,
                 page,
                 page_size,
-                filter_string=filter_string
+                filter_string=filter_string,
+                requested_fields=requested_fields,
             )
             (results_page, _) = jc_converter.convert_list(transfer)
 

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_client/client.py

@@ -28,9 +28,13 @@ class JsonApiClient(HttpClient):
         config_prefix: str = '/_config',
         token_header: str = 'token',
         retries: int = 5,
+        status_forcelist: Optional[list[int]] = None,
         merge_collections: bool | None = None,
     ) -> None:
-        super().__init__(token=token, token_header=token_header, retries=retries)
+        kwargs = {'token': token, 'token_header': token_header, 'retries': retries}
+        if status_forcelist is not None:
+            kwargs['status_forcelist'] = status_forcelist
+        super().__init__(**kwargs)
         self.__data_url = f'{api_url}{data_prefix}'
         self.__config_url = f'{self.__data_url}{config_prefix}'
         self.__merge_collections = merge_collections
@@ -77,7 +81,7 @@ class JsonApiClient(HttpClient):
             page_size=page_size,
             filter=filter_string,
             sort_by=sort_string,
-            requested_field=requested_fields
+            requested_fields=requested_fields
         )
         headers = self._merge_headers()
         return self.__fetch_list(
@@ -455,7 +459,9 @@ class JsonApiClient(HttpClient):
         return f'{self.__config_url}/return_mode'
 
     def __no_none_value_dict(self, **kwargs) -> dict[str, Any]:
-        return {
-            k: v for k, v in kwargs.items()
-            if v is not None
-        }
+        str_params = {}
+        for k, v in kwargs.items():
+            if v is None:
+                continue
+            str_params[k] = ','.join([str(x) for x in v]) if isinstance(v, list) else str(v)
+        return str_params

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_client/converter.py

@@ -5,8 +5,8 @@
 from typing import Any, Dict, Optional, Union
 
 from .parser import Parser
-from .view import View
-from ..core import DataObject
+from .view import DefaultView
+from ..core import DataObject, DataSource, ReqFieldsTree
 from ..core.relationship import RelationshipConfig
 
 
@@ -25,7 +25,7 @@ JsonRelationshipConfig = dict[
 ]
 
 
-class JsonApiConverter():
+class JsonApiConverter:
 
     """
     Converts from JSON:API transfers to instances of
@@ -142,26 +142,40 @@ class JsonApiConverter():
         )
 
 
-class DataObjectConverter():
+class DataObjectConverter:
 
     """
     Converts from instances of `DataObject` to
     JSON:API transfers.
     """
 
-    def __init__(self, view: View) -> None:
-        self.__view = view
+    def __init__(
+        self,
+        data_source: DataSource,
+        prefix: str | None = None,
+    ) -> None:
+        self.__data_source = data_source
+        self.__prefix = prefix
+
+    def __build_view(self, object_type):
+        req_fields_tree = ReqFieldsTree(object_type, self.__data_source)
+        return DefaultView(req_fields_tree, self.__prefix)
 
     def convert(self, input_: DataObject) -> JsonApiTransfer:
         """
         Converts a single `DataObject` instance to a JsonApiTransfer
         """
 
-        return self.__view.dump(input_)
+        view = self.__build_view(input_.type)
+        return view.dump(input_)
 
     def convert_list(self, input_: list[DataObject]) -> JsonApiTransfer:
         """
         Converts a `list` of `DataObject` instances to a JsonApiTransfer
         """
 
-        return self.__view.dump_bulk(input_)
+        if not input_:
+            msg = 'Cannot convert empty list'
+            raise ValueError(msg)
+        view = self.__build_view(input_[0].type)
+        return view.dump_bulk(input_)

{tol_sdk-1.6.37 → tol_sdk-1.7.1}/src/tol/api_client/factory.py

@@ -17,7 +17,6 @@ from .converter import (
 )
 from .filter import DefaultApiFilter
 from .parser import DefaultParser
-from .view import DefaultView
 from ..core import DataSource
 
 
@@ -66,8 +65,7 @@ class _ConverterFactory:
         Returns an instantiated `DataObjectConverter`.
         """
 
-        view = DefaultView(prefix=self.__prefix)
-        return DataObjectConverter(view)
+        return DataObjectConverter(self.__data_source, prefix=self.__prefix)
 
     def json_converter_factory(self) -> JsonConverterFactory:
         """
@@ -87,6 +85,7 @@ def _get_client_factory(
     token: Optional[str],
     data_prefix: str,
     retries: int,
+    status_forcelist: list[int],
     merge_collections: bool | None,
 ) -> Callable[[], JsonApiClient]:
     """
@@ -99,6 +98,7 @@ def _get_client_factory(
         token=token,
         data_prefix=data_prefix,
         retries=retries,
+        status_forcelist=status_forcelist,
         merge_collections=merge_collections,
     )
 
@@ -113,6 +113,7 @@ def create_api_datasource(
 
     data_prefix: str = '/data',
     retries: int = 5,
+    status_forcelist: Optional[list[int]] = None,
     merge_collections: bool | None = None,
 ) -> ApiDataSource:
     """
@@ -127,6 +128,7 @@ def create_api_datasource(
         token=token,
         data_prefix=data_prefix,
         retries=retries,
+        status_forcelist=status_forcelist,
         merge_collections=merge_collections,
     )
     manager = _ConverterFactory(data_prefix)

tol_sdk-1.7.1/src/tol/api_client/view.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: 2022 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+import urllib
+from abc import ABC, abstractmethod
+from datetime import date
+from typing import Any, Dict, Iterable, List, Optional, Union
+
+from ..core import DataObject
+from ..core.requested_fields import ReqFieldsTree
+
+DocumentMeta = Dict[str, Any]
+DumpDict = Dict[str, Any]
+DumpDictMany = List[DumpDict]
+ResponseDict = Dict[str, Union[DumpDict, DumpDictMany]]
+
+
+class View(ABC):
+    """
+    Provides an MVC-esque View class. Can serialize both an individual
+    DataObject, as well as an Iterable of DataObject instances.
+    """
+
+    @abstractmethod
+    def dump(
+        self,
+        data_object: DataObject,
+        document_meta: Optional[DocumentMeta] = None,
+    ) -> ResponseDict:
+        """
+        Create a JSON:API response for an individual DataObject result
+        """
+
+    @abstractmethod
+    def dump_bulk(
+        self,
+        data_objects: Iterable[DataObject],
+        document_meta: Optional[DocumentMeta] = None,
+    ) -> ResponseDict:
+        """
+        Create a JSON:API response for an Iterable of DataObject results
+        """
+
+
+RelationshipDump = dict[str, dict[str, str]]
+AllRelationshipsDump = dict[str, RelationshipDump]
+
+
+class DefaultView(View):
+    """
+    Provides a default implementation of the View ABC.
+    """
+
+    def __init__(
+        self,
+        requested_tree: ReqFieldsTree,
+        prefix: str = '',
+        hop_limit: Optional[int] = None,
+    ) -> None:
+        """
+        Args:
+
+        - prefix                - the URL prefix on which the
+                                  data blueprint is served
+        - hop_limit             - the maximum recursion limit
+                                  on including related to-one
+                                  objects. Default no limit
+        - requested_tree        - a tree data structure of the
+                                  requested fields for the query
+        """
+
+        self.__prefix = prefix
+        self.__hop_limit = hop_limit
+        self.__requested_tree = requested_tree
+
+    def dump(
+        self,
+        data_object: DataObject,
+        document_meta: Optional[DocumentMeta] = None,
+    ) -> ResponseDict:
+        response = {
+            'data': self.__dump_object(
+                data_object,
+                tree=self.__requested_tree,
+            ),
+        }
+        if document_meta is not None:
+            response['meta'] = document_meta
+        return response
+
+    def dump_bulk(
+        self,
+        data_objects: Iterable[DataObject],
+        document_meta: Optional[DocumentMeta] = None,
+    ) -> ResponseDict:
+        dumped = [
+            self.__dump_object(
+                data_object,
+                tree=self.__requested_tree,
+            )
+            for data_object in data_objects
+        ]
+        response = {'data': dumped}
+        if document_meta is not None:
+            response['meta'] = document_meta
+        return response
+
+    def __dump_object(
+        self,
+        data_object: DataObject,
+        tree: ReqFieldsTree,
+    ) -> DumpDict:
+        dump = {'type': data_object.type, 'id': data_object.id}
+        # Stub trees are created by requested_fields paths ending in ".id"
+        if not tree.is_stub:
+            self.__add_attributes(data_object, dump, tree)
+        if tree.has_relationships:
+            self.__add_relationships(data_object, dump, tree)
+        return dump
+
+    def __add_attributes(
+        self,
+        data_object: DataObject,
+        dump: DumpDict,
+        tree: ReqFieldsTree | None,
+    ):
+        if tree and (attr_names := tree.attribute_names):
+            # Only add requested attributes
+            dump['attributes'] = self.__convert_attributes(
+                {name: getattr(data_object, name) for name in attr_names}
+            )
+        elif data_object.attributes:
+            # Default behaviour is to add all attributes
+            dump['attributes'] = self.__convert_attributes(data_object.attributes)
+
+    def __add_relationships(
+        self,
+        data_object: DataObject,
+        dump: DumpDict,
+        tree: ReqFieldsTree | None = None,
+    ) -> DumpDict:
+        rel_dict = self.__dump_to_one_relationships(
+            data_object, tree
+        ) | self.__dump_to_many_relationships(data_object, tree)
+        if rel_dict:
+            dump['relationships'] = rel_dict
+
+    def __dump_to_one_relationships(
+        self,
+        data_object: DataObject,
+        tree: ReqFieldsTree,
+    ) -> RelationshipDump:
+        to_ones = {}
+        for name in tree.to_one_names():
+            if name in data_object._to_one_objects:
+                one_dump = None
+                if one := data_object._to_one_objects.get(name):
+                    if sub_tree := tree.get_sub_tree(name):
+                        one_dump = {'data': self.__dump_object(one, tree=sub_tree)}
+                    else:
+                        one_dump = {'data': {'type': one.type, 'id': one.id}}
+                to_ones[name] = one_dump
+        return to_ones
+
+    def __dump_to_many_relationships(
+        self,
+        data_object: DataObject,
+        tree: ReqFieldsTree,
+    ) -> RelationshipDump:
+        quoted_id = urllib.parse.quote(str(data_object.id), safe='')
+        to_many = {}
+        for name in tree.to_many_names():
+            if sub_tree := tree.get_sub_tree(name):
+                to_many[name] = {
+                    'data': [
+                        self.__dump_object(x, tree=sub_tree) for x in getattr(data_object, name)
+                    ]
+                }
+            else:
+                link = f'{self.__prefix}/{data_object.type}/{quoted_id}/{name}'
+                to_many[name] = {'links': {'related': link}}
+        return to_many
+
+    def __convert_attributes(self, attributes: dict[str, Any]) -> dict[str, Any]:
+        return {k: self.__convert_value(v) for k, v in attributes.items()}
+
+    def __convert_value(self, __v: Any) -> Any:
+        if isinstance(__v, date):
+            # `datetime` is a subclass of `date`
+            return __v.isoformat()
+        return __v