tol-sdk 1.7.3__tar.gz → 1.7.5b0__tar.gz

{tol_sdk-1.7.3/src/tol_sdk.egg-info → tol_sdk-1.7.5b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.3
+Version: 1.7.5b0
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tol-sdk"
-version = "1.7.3"
+version = "1.7.5-beta"
 description = "SDK for interaction with ToL, Sanger and external services"
 readme = "README.md"
 authors = [

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_base/__init__.py

@@ -7,5 +7,6 @@ from .blueprint import (  # noqa
     custom_blueprint,
     data_blueprint
 )
+from .data_upload import data_upload_blueprint  # noqa
 from .pipeline_steps import pipeline_steps_blueprint # noqa
 from .system import system_blueprint  # noqa

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_base/blueprint.py

@@ -251,6 +251,18 @@ def _core_blueprint(
         )
         return Controller(data_source, view, req_fields_tree, auth_inspector=auth_inspector)
 
+    def __new_parser(
+        object_type: str,
+    ):
+        data_source = data_source_dict[object_type]
+        # Build a ReqFieldsTree template for the request
+        req_fields_tree = ReqFieldsTree(
+            object_type,
+            data_source,
+            include_all_to_ones=include_all_to_ones,
+        )
+        return DefaultParser(data_source_dict, requested_tree=req_fields_tree)
+
     @data_handler.route('/<object_type>/<path:object_id>', methods=['GET'])  # Allow slashes
     def get_detail(*, object_type: str, object_id: str):
         """Get details of a specific object by ID."""
@@ -311,19 +323,17 @@ def _core_blueprint(
     def post_inserts(*, object_type: str):
         """Insert new objects of the specified type."""
         controller = __new_controller(object_type)
-        request_body = JsonApiRequestBody(request.json)
-        parser = DefaultParser(data_source_dict)
-        objects = parser.parse_iterable(request_body.data)
+        parser = __new_parser(object_type)
+        objects = parser.parse_json_doc(request.json)
         return controller.post_inserts(object_type, objects)
 
     @data_handler.route('/<object_type>:upsert', methods=['POST'])
     def post_upserts(*, object_type: str):
         """Insert or update objects of the specified type."""
-        controller = __new_controller(object_type)
         request_args = ListGetParameters(request.args)
-        request_body = JsonApiRequestBody(request.json)
-        parser = DefaultParser(data_source_dict)
-        objects = parser.parse_iterable(request_body.data)
+        controller = __new_controller(object_type)
+        parser = __new_parser(object_type)
+        objects = parser.parse_json_doc(request.json)
         return controller.post_upserts(
             object_type,
             objects,
@@ -347,7 +357,8 @@ def _core_blueprint(
             requested_fields=request_args.requested_fields,
         )
         search_after = request.json.get('search_after')
-        return controller.get_cursor_page(object_type, request_args, search_after)
+        page = controller.get_cursor_page(object_type, request_args, search_after)
+        return page
 
     @data_handler.route('/<object_type>:to-one/<object_id>/<path:hops_suffix>', methods=['GET'])
     def get_to_one_relation(*, object_type: str, object_id: str, hops_suffix: str):

tol_sdk-1.7.5b0/src/tol/api_base/data_upload.py

@@ -0,0 +1,98 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+import os
+import uuid
+from tempfile import NamedTemporaryFile
+from typing import Any
+
+from flask import Blueprint, request, send_file
+
+from .blueprint import (
+    custom_blueprint,
+)
+from ..services.s3_client import S3Client
+
+
+ALLOWED_EXTENSIONS: set[str] = {'csv', 'json', 'xlsx'}
+
+
+def allowed_file(filename: str) -> bool:
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+def set_filename(filename: str) -> str:
+    environment = os.getenv('ENVIRONMENT', 'dev')
+    unique_id = str(uuid.uuid4())[:8]
+    return f'{environment}_{unique_id}_{filename}'
+
+
+def get_original_filename(s3_filename: str) -> str:
+    return s3_filename.split('_', maxsplit=2)[2]
+
+
+def data_upload_blueprint(
+    url_prefix: str = '/pipeline/data_upload',
+) -> Blueprint:
+
+    data_upload_blueprint = custom_blueprint(
+        name='data_upload',
+        url_prefix=url_prefix
+    )
+
+    @data_upload_blueprint.route('/upload', methods=['POST'])
+    def upload_file() -> tuple[dict[str, str], int]:
+        file = request.files['file']
+        s3_bucket: str = request.form.get('s3_bucket')
+
+        if not file:
+            return {'error': 'No file provided'}, 400
+
+        if not allowed_file(file.filename):
+            return {'error': 'File type not allowed'}, 400
+
+        if not s3_bucket:
+            return {'error': 'S3 bucket not specified'}, 400
+
+        try:
+            s3_client = S3Client()
+            new_filename = set_filename(file.filename)
+
+            with NamedTemporaryFile() as temp_file:
+                file.save(temp_file.name)
+                s3_client.put_object(s3_bucket, new_filename, temp_file.name)
+
+            return {'message': 'File uploaded successfully', 'file_name': new_filename}, 200
+
+        except Exception as e:
+            return {'error': f'Failed to upload file: {str(e)}'}, 500
+
+    @data_upload_blueprint.route('/download', methods=['POST'])
+    def download_file() -> tuple[dict[str, str], int]:
+        body: dict[str, Any] = request.json.get('data', {})
+
+        if 's3_bucket' not in body or 'file_name' not in body:
+            return {'error': 'S3 bucket or file name not specified'}, 400
+
+        bucket_name = body['s3_bucket']
+        file_name = body['file_name']
+
+        try:
+            s3_client = S3Client()
+
+            with NamedTemporaryFile() as temp_file:
+                s3_client.get_object(bucket_name, file_name, temp_file.name)
+
+                original = get_original_filename(file_name)
+
+                return send_file(
+                    temp_file.name,
+                    as_attachment=True,
+                    download_name=original,
+                    mimetype='application/octet-stream'
+                )
+        except Exception as e:
+            return {'error': f'Failed to download file: {str(e)}'}, 500
+
+    return data_upload_blueprint

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_base/pipeline_steps.py

@@ -27,7 +27,7 @@ if typing.TYPE_CHECKING:
 
 @dataclass
 class UploadData:
-    s3_url: str
+    s3_bucket: str
     s3_filename: str
     spreadsheet_config: str
     pipeline_id: int
@@ -37,7 +37,7 @@ class UploadData:
 
 
 REQUIRED_FIELDS: List = [
-    's3_url',
+    's3_bucket',
     's3_filename',
     'spreadsheet_config',
     'pipeline_id',
@@ -49,7 +49,7 @@ REQUIRED_FIELDS: List = [
 def pipeline_steps_blueprint(
     sql_ds: SqlDataSource,
     prefect_ds: PrefectDataSource,
-    role: str | None = 'exporter',
+    role: str | None = None,
     url_prefix: str = '/run-pipeline',
 
     ctx_getter: CtxGetter = default_ctx_getter,
@@ -105,7 +105,7 @@ def pipeline_steps_blueprint(
         upload = sql_ds.data_object_factory(
             'upload',
             attributes={
-                's3_url': upload_data.s3_url,
+                's3_bucket': upload_data.s3_bucket,
                 's3_filename': upload_data.s3_filename,
                 'spreadsheet_config': upload_data.spreadsheet_config,
                 'pipeline_id': upload_data.pipeline_id,
@@ -155,7 +155,7 @@ def pipeline_steps_blueprint(
                 'deployment_name': flow_name,
                 'parameters': flow_params,
                 'tags': [
-                    'app_name:treeofsex',
+                    f'app_name: {os.environ.get("APP_NAME", "tol")}',
                 ],
             }
         )
@@ -190,10 +190,13 @@ def pipeline_steps_blueprint(
     def run_pipeline_steps() -> tuple[dict[str, Any], int]:
 
         ctx = ctx_getter()
-        user_id = ctx.user_id
+        if role is not None:
+            if not ctx or not ctx.authenticated:
+                raise ForbiddenError()
+            if role not in ctx.roles:
+                raise ForbiddenError()
 
-        if role is not None and role not in ctx.roles:
-            raise ForbiddenError()
+        user_id = ctx.user_id if ctx and ctx.authenticated else None
 
         body: dict[str, Any] = request.json.get('data', {})
 
@@ -204,7 +207,7 @@ def pipeline_steps_blueprint(
         __get_pipeline(pipeline_id)
 
         upload_data = UploadData(
-            s3_url=body['s3_url'],
+            s3_bucket=body['s3_bucket'],
             s3_filename=body['s3_filename'],
             spreadsheet_config=body['spreadsheet_config'],
             pipeline_id=pipeline_id,

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_client/api_datasource.py

@@ -151,7 +151,7 @@ class ApiDataSource(
             )
             for id_ in object_ids
         )
-        json_converter = self.__jc_factory()
+        json_converter = self.__jc_factory(object_type, requested_fields)
         return (
             json_converter.convert(r)
             if r is not None else None
@@ -179,7 +179,7 @@ class ApiDataSource(
             sort_string=sort_by,
             requested_fields=requested_fields,
         )
-        return self.__jc_factory().convert_list(transfer)
+        return self.__jc_factory(object_type, requested_fields).convert_list(transfer)
 
     def get_list(
         self,
@@ -274,7 +274,7 @@ class ApiDataSource(
             filter_string=filter_string,
             requested_fields=requested_fields,
         )
-        return self.__jc_factory().convert_cursor_page(transfer)
+        return self.__jc_factory(object_type, requested_fields).convert_cursor_page(transfer)
 
     @validate('delete')
     def delete(
@@ -304,7 +304,7 @@ class ApiDataSource(
             merge_collections=merge_collections,
         )
         if self.return_mode[object_type] == ReturnMode.POPULATED:
-            converted, _ = self.__jc_factory().convert_list(returned)
+            converted, _ = self.__jc_factory(object_type).convert_list(returned)
             return converted
         return []  # when the underlying DataSource doesn't return anything
 
@@ -325,7 +325,7 @@ class ApiDataSource(
         )
         if transfer is None:
             return None
-        return self.__jc_factory().convert(transfer)
+        return self.__jc_factory(source.type).convert(transfer)
 
     @validate('relational', direct_object=True)
     @validate_id
@@ -359,7 +359,7 @@ class ApiDataSource(
             page,
             page_size
         )
-        return self.__jc_factory().convert_list(transfer)
+        return self.__jc_factory(source.type).convert_list(transfer)
 
     @validate('relational', direct_object=True)
     @validate_id
@@ -406,7 +406,7 @@ class ApiDataSource(
             transfer
         )
         if self.return_mode[object_type] == ReturnMode.POPULATED:
-            converted, _ = self.__jc_factory().convert_list(returned)
+            converted, _ = self.__jc_factory(object_type).convert_list(returned)
             return converted
 
     @property
@@ -442,7 +442,7 @@ class ApiDataSource(
         page = 1
         page_size = self.get_page_size()
         client = self.__client_factory()
-        jc_converter = self.__jc_factory()
+        jc_converter = self.__jc_factory(object_type, requested_fields)
         filter_string = self.__get_filter_string(object_filters)
 
         while True:

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_client/converter.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from typing import Any, Dict, Optional, Union
+from typing import Any
 
 from .parser import Parser
 from .view import DefaultView
@@ -11,22 +11,18 @@ from ..core.relationship import RelationshipConfig
 
 
 JsonApiObject = dict[str, Any]
-JsonApiTransfer = dict[
-    str,
-    Union[JsonApiObject, list[JsonApiObject]]
-]
+JsonApiTransfer = dict[str, JsonApiObject | list[JsonApiObject]]
 JsonRelationship = dict[
     str,  # "one" or "many"
-    dict[str, str]  # relationship_name:target_type
+    dict[str, str],  # relationship_name:target_type
 ]
 JsonRelationshipConfig = dict[
     str,  # the object_type
-    JsonRelationship
+    JsonRelationship,
 ]
 
 
 class JsonApiConverter:
-
     """
     Converts from JSON:API transfers to instances of
     `DataObject`.
@@ -35,89 +31,82 @@ class JsonApiConverter:
     def __init__(
         self,
         parser: Parser,
-        data_key: str = 'data',
-        meta_key: str = 'meta'
     ) -> None:
-
         self.__parser = parser
-        self.__data_key = data_key
-        self.__meta_key = meta_key
 
-    def convert(self, input_: JsonApiTransfer) -> DataObject:
+    def convert(
+        self,
+        json_xfer: JsonApiTransfer,
+    ) -> DataObject:
         """
         Converts a JsonApiTransfer containing a detail (single) result
         """
 
-        json_obj = input_[self.__data_key]
-        return self.__parser.parse(json_obj)
+        obj_list = self.__parser.parse_json_doc(json_xfer)
+        return obj_list[0] if obj_list else None
 
     def convert_list(
         self,
-        input_: JsonApiTransfer
-    ) -> tuple[list[DataObject], Optional[int]]:
+        json_xfer: JsonApiTransfer,
+    ) -> tuple[list[DataObject], int | None]:
         """
         Converts a JsonApiTransfer containing a list of results. Also
         returns a count of the total results meeting.
         """
 
-        json_obj_list = input_[self.__data_key]
-        total_count = input_.get('meta', {}).get('total', None)
-        return [
-            self.__parser.parse(json_obj)
-            for json_obj in json_obj_list
-        ], total_count
+        objs = self.__parser.parse_json_doc(json_xfer)
+        total_count = json_xfer.get('meta', {}).get('total', None)
+        return objs, total_count
 
     def convert_count(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return stats['total']
 
     def convert_stats(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return self.__parser.parse_stats(stats)
 
     def convert_group_stats(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of grouped stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return self.__parser.parse_group_stats(stats)
 
     def convert_cursor_page(
         self,
-        input_: JsonApiTransfer
+        json_xfer: JsonApiTransfer,
     ) -> tuple[list[DataObject], list[str] | None]:
         """
         Converts a `JsonApiTransfer` of a cursor-page
         """
 
-        objs = self.__parser.parse_iterable(
-            input_[self.__data_key]
-        )
-        search_after = input_.get('meta', {}).get('search_after')
+        objs = self.__parser.parse_json_doc(json_xfer)
+        search_after = json_xfer.get('meta', {}).get('search_after')
 
         return objs, search_after
 
     def convert_relationship_config(
         self,
-        config_transfer: JsonRelationshipConfig
+        config_transfer: JsonRelationshipConfig,
     ) -> dict[str, RelationshipConfig]:
         """
         Converts a `JsonRelationshipConfig` dict, returned from
@@ -127,23 +116,20 @@ class JsonApiConverter:
 
         return {
             type_: self.__convert_relationship(rel)
-            for type_, rel
-            in config_transfer.items()
+            for type_, rel in config_transfer.items()
         }
 
     def __convert_relationship(
         self,
-        rel: JsonRelationship
+        rel: JsonRelationship,
     ) -> RelationshipConfig:
-
         return RelationshipConfig(
             to_one=rel.get('one'),
-            to_many=rel.get('many')
+            to_many=rel.get('many'),
         )
 
 
 class DataObjectConverter:
-
     """
     Converts from instances of `DataObject` to
     JSON:API transfers.
@@ -161,21 +147,21 @@ class DataObjectConverter:
         req_fields_tree = ReqFieldsTree(object_type, self.__data_source)
         return DefaultView(req_fields_tree, self.__prefix)
 
-    def convert(self, input_: DataObject) -> JsonApiTransfer:
+    def convert(self, data_obj: DataObject) -> JsonApiTransfer:
         """
         Converts a single `DataObject` instance to a JsonApiTransfer
         """
 
-        view = self.__build_view(input_.type)
-        return view.dump(input_)
+        view = self.__build_view(data_obj.type)
+        return view.dump(data_obj)
 
-    def convert_list(self, input_: list[DataObject]) -> JsonApiTransfer:
+    def convert_list(self, data_obj_list: list[DataObject]) -> JsonApiTransfer:
         """
         Converts a `list` of `DataObject` instances to a JsonApiTransfer
         """
 
-        if not input_:
+        if not data_obj_list:
             msg = 'Cannot convert empty list'
             raise ValueError(msg)
-        view = self.__build_view(input_[0].type)
-        return view.dump_bulk(input_)
+        view = self.__build_view(data_obj_list[0].type)
+        return view.dump_bulk(data_obj_list)

{tol_sdk-1.7.3 → tol_sdk-1.7.5b0}/src/tol/api_client/factory.py

@@ -5,19 +5,12 @@
 from collections.abc import Mapping
 from typing import Callable, Iterator, Optional
 
-from .api_datasource import (
-    ApiDataSource,
-    DOConverterFactory,
-    JsonConverterFactory
-)
+from .api_datasource import ApiDataSource, DOConverterFactory, JsonConverterFactory
 from .client import JsonApiClient
-from .converter import (
-    DataObjectConverter,
-    JsonApiConverter
-)
+from .converter import DataObjectConverter, JsonApiConverter
 from .filter import DefaultApiFilter
 from .parser import DefaultParser
-from ..core import DataSource
+from ..core import DataSource, ReqFieldsTree
 
 
 class _ApiDSDict(Mapping):
@@ -53,11 +46,7 @@ class _ConverterFactory:
         return self.__data_source
 
     @data_source.setter
-    def data_source(
-        self,
-        ds: DataSource
-    ) -> None:
-
+    def data_source(self, ds: DataSource) -> None:
         self.__data_source = ds
 
     def do_converter_factory(self) -> DOConverterFactory:
@@ -67,12 +56,26 @@ class _ConverterFactory:
 
         return DataObjectConverter(self.__data_source, prefix=self.__prefix)
 
-    def json_converter_factory(self) -> JsonConverterFactory:
+    def json_converter_factory(
+        self,
+        object_type: str | None = None,
+        requested_fields: list[str] | None = None,
+    ) -> JsonConverterFactory:
         """
         Returns an instantiated `JsonApiConverter`.
         """
 
-        parser = DefaultParser(self.__ds_dict)
+        req_fields_tree = (
+            ReqFieldsTree(
+                object_type,
+                self.__data_source,
+                requested_fields=requested_fields,
+            )
+            if object_type
+            else None
+        )
+
+        parser = DefaultParser(self.__ds_dict, req_fields_tree)
         return JsonApiConverter(parser)
 
     @property
@@ -110,7 +113,6 @@ def _filter_factory() -> DefaultApiFilter:
 def create_api_datasource(
     api_url: str,
     token: Optional[str] = None,
-
     data_prefix: str = '/data',
     retries: int = 5,
     status_forcelist: Optional[list[int]] = None,
@@ -137,7 +139,7 @@ def create_api_datasource(
         client_factory,
         manager.json_converter_factory,
         manager.do_converter_factory,
-        _filter_factory
+        _filter_factory,
     )
 
     manager.data_source = api_ds