tol-sdk 1.7.4__py3-none-any.whl → 1.7.5b0__py3-none-any.whl

tol/api_base/__init__.py

@@ -7,5 +7,6 @@ from .blueprint import (  # noqa
     custom_blueprint,
     data_blueprint
 )
+from .data_upload import data_upload_blueprint  # noqa
 from .pipeline_steps import pipeline_steps_blueprint # noqa
 from .system import system_blueprint  # noqa

tol/api_base/blueprint.py

@@ -251,6 +251,18 @@ def _core_blueprint(
         )
         return Controller(data_source, view, req_fields_tree, auth_inspector=auth_inspector)
 
+    def __new_parser(
+        object_type: str,
+    ):
+        data_source = data_source_dict[object_type]
+        # Build a ReqFieldsTree template for the request
+        req_fields_tree = ReqFieldsTree(
+            object_type,
+            data_source,
+            include_all_to_ones=include_all_to_ones,
+        )
+        return DefaultParser(data_source_dict, requested_tree=req_fields_tree)
+
     @data_handler.route('/<object_type>/<path:object_id>', methods=['GET'])  # Allow slashes
     def get_detail(*, object_type: str, object_id: str):
         """Get details of a specific object by ID."""
@@ -311,19 +323,17 @@ def _core_blueprint(
     def post_inserts(*, object_type: str):
         """Insert new objects of the specified type."""
         controller = __new_controller(object_type)
-        request_body = JsonApiRequestBody(request.json)
-        parser = DefaultParser(data_source_dict)
-        objects = parser.parse_iterable(request_body.data)
+        parser = __new_parser(object_type)
+        objects = parser.parse_json_doc(request.json)
         return controller.post_inserts(object_type, objects)
 
     @data_handler.route('/<object_type>:upsert', methods=['POST'])
     def post_upserts(*, object_type: str):
         """Insert or update objects of the specified type."""
-        controller = __new_controller(object_type)
         request_args = ListGetParameters(request.args)
-        request_body = JsonApiRequestBody(request.json)
-        parser = DefaultParser(data_source_dict)
-        objects = parser.parse_iterable(request_body.data)
+        controller = __new_controller(object_type)
+        parser = __new_parser(object_type)
+        objects = parser.parse_json_doc(request.json)
         return controller.post_upserts(
             object_type,
             objects,
@@ -347,7 +357,8 @@ def _core_blueprint(
             requested_fields=request_args.requested_fields,
         )
         search_after = request.json.get('search_after')
-        return controller.get_cursor_page(object_type, request_args, search_after)
+        page = controller.get_cursor_page(object_type, request_args, search_after)
+        return page
 
     @data_handler.route('/<object_type>:to-one/<object_id>/<path:hops_suffix>', methods=['GET'])
     def get_to_one_relation(*, object_type: str, object_id: str, hops_suffix: str):

tol/{s3/data_upload/blueprint.py → api_base/data_upload.py}

@@ -2,15 +2,17 @@
 #
 # SPDX-License-Identifier: MIT
 
+import os
+import uuid
 from tempfile import NamedTemporaryFile
 from typing import Any
 
 from flask import Blueprint, request, send_file
 
-from ...api_base import (
+from .blueprint import (
     custom_blueprint,
 )
-from ...services.s3_client import S3Client
+from ..services.s3_client import S3Client
 
 
 ALLOWED_EXTENSIONS: set[str] = {'csv', 'json', 'xlsx'}
@@ -20,6 +22,16 @@ def allowed_file(filename: str) -> bool:
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
+def set_filename(filename: str) -> str:
+    environment = os.getenv('ENVIRONMENT', 'dev')
+    unique_id = str(uuid.uuid4())[:8]
+    return f'{environment}_{unique_id}_{filename}'
+
+
+def get_original_filename(s3_filename: str) -> str:
+    return s3_filename.split('_', maxsplit=2)[2]
+
+
 def data_upload_blueprint(
     url_prefix: str = '/pipeline/data_upload',
 ) -> Blueprint:
@@ -45,11 +57,13 @@ def data_upload_blueprint(
 
         try:
             s3_client = S3Client()
+            new_filename = set_filename(file.filename)
+
             with NamedTemporaryFile() as temp_file:
                 file.save(temp_file.name)
-                s3_client.put_object(s3_bucket, file.filename, temp_file.name)
+                s3_client.put_object(s3_bucket, new_filename, temp_file.name)
 
-            return {'message': 'File uploaded successfully'}, 200
+            return {'message': 'File uploaded successfully', 'file_name': new_filename}, 200
 
         except Exception as e:
             return {'error': f'Failed to upload file: {str(e)}'}, 500
@@ -69,12 +83,13 @@ def data_upload_blueprint(
 
             with NamedTemporaryFile() as temp_file:
                 s3_client.get_object(bucket_name, file_name, temp_file.name)
-                download_name = request.json.get('download_name', file_name)
+
+                original = get_original_filename(file_name)
 
                 return send_file(
                     temp_file.name,
                     as_attachment=True,
-                    download_name=download_name,
+                    download_name=original,
                     mimetype='application/octet-stream'
                 )
         except Exception as e:

tol/api_base/pipeline_steps.py

@@ -27,7 +27,7 @@ if typing.TYPE_CHECKING:
 
 @dataclass
 class UploadData:
-    s3_url: str
+    s3_bucket: str
     s3_filename: str
     spreadsheet_config: str
     pipeline_id: int
@@ -37,7 +37,7 @@ class UploadData:
 
 
 REQUIRED_FIELDS: List = [
-    's3_url',
+    's3_bucket',
     's3_filename',
     'spreadsheet_config',
     'pipeline_id',
@@ -105,7 +105,7 @@ def pipeline_steps_blueprint(
         upload = sql_ds.data_object_factory(
             'upload',
             attributes={
-                's3_url': upload_data.s3_url,
+                's3_bucket': upload_data.s3_bucket,
                 's3_filename': upload_data.s3_filename,
                 'spreadsheet_config': upload_data.spreadsheet_config,
                 'pipeline_id': upload_data.pipeline_id,
@@ -207,7 +207,7 @@ def pipeline_steps_blueprint(
         __get_pipeline(pipeline_id)
 
         upload_data = UploadData(
-            s3_url=body['s3_url'],
+            s3_bucket=body['s3_bucket'],
             s3_filename=body['s3_filename'],
             spreadsheet_config=body['spreadsheet_config'],
             pipeline_id=pipeline_id,

tol/api_client/api_datasource.py

@@ -151,7 +151,7 @@ class ApiDataSource(
             )
             for id_ in object_ids
         )
-        json_converter = self.__jc_factory()
+        json_converter = self.__jc_factory(object_type, requested_fields)
         return (
             json_converter.convert(r)
             if r is not None else None
@@ -179,7 +179,7 @@ class ApiDataSource(
             sort_string=sort_by,
             requested_fields=requested_fields,
         )
-        return self.__jc_factory().convert_list(transfer)
+        return self.__jc_factory(object_type, requested_fields).convert_list(transfer)
 
     def get_list(
         self,
@@ -274,7 +274,7 @@ class ApiDataSource(
             filter_string=filter_string,
             requested_fields=requested_fields,
         )
-        return self.__jc_factory().convert_cursor_page(transfer)
+        return self.__jc_factory(object_type, requested_fields).convert_cursor_page(transfer)
 
     @validate('delete')
     def delete(
@@ -304,7 +304,7 @@ class ApiDataSource(
             merge_collections=merge_collections,
         )
         if self.return_mode[object_type] == ReturnMode.POPULATED:
-            converted, _ = self.__jc_factory().convert_list(returned)
+            converted, _ = self.__jc_factory(object_type).convert_list(returned)
             return converted
         return []  # when the underlying DataSource doesn't return anything
 
@@ -325,7 +325,7 @@ class ApiDataSource(
         )
         if transfer is None:
             return None
-        return self.__jc_factory().convert(transfer)
+        return self.__jc_factory(source.type).convert(transfer)
 
     @validate('relational', direct_object=True)
     @validate_id
@@ -359,7 +359,7 @@ class ApiDataSource(
             page,
             page_size
         )
-        return self.__jc_factory().convert_list(transfer)
+        return self.__jc_factory(source.type).convert_list(transfer)
 
     @validate('relational', direct_object=True)
     @validate_id
@@ -406,7 +406,7 @@ class ApiDataSource(
             transfer
         )
         if self.return_mode[object_type] == ReturnMode.POPULATED:
-            converted, _ = self.__jc_factory().convert_list(returned)
+            converted, _ = self.__jc_factory(object_type).convert_list(returned)
             return converted
 
     @property
@@ -442,7 +442,7 @@ class ApiDataSource(
         page = 1
         page_size = self.get_page_size()
         client = self.__client_factory()
-        jc_converter = self.__jc_factory()
+        jc_converter = self.__jc_factory(object_type, requested_fields)
         filter_string = self.__get_filter_string(object_filters)
 
         while True:

tol/api_client/converter.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from typing import Any, Dict, Optional, Union
+from typing import Any
 
 from .parser import Parser
 from .view import DefaultView
@@ -11,22 +11,18 @@ from ..core.relationship import RelationshipConfig
 
 
 JsonApiObject = dict[str, Any]
-JsonApiTransfer = dict[
-    str,
-    Union[JsonApiObject, list[JsonApiObject]]
-]
+JsonApiTransfer = dict[str, JsonApiObject | list[JsonApiObject]]
 JsonRelationship = dict[
     str,  # "one" or "many"
-    dict[str, str]  # relationship_name:target_type
+    dict[str, str],  # relationship_name:target_type
 ]
 JsonRelationshipConfig = dict[
     str,  # the object_type
-    JsonRelationship
+    JsonRelationship,
 ]
 
 
 class JsonApiConverter:
-
     """
     Converts from JSON:API transfers to instances of
     `DataObject`.
@@ -35,89 +31,82 @@ class JsonApiConverter:
     def __init__(
         self,
         parser: Parser,
-        data_key: str = 'data',
-        meta_key: str = 'meta'
     ) -> None:
-
         self.__parser = parser
-        self.__data_key = data_key
-        self.__meta_key = meta_key
 
-    def convert(self, input_: JsonApiTransfer) -> DataObject:
+    def convert(
+        self,
+        json_xfer: JsonApiTransfer,
+    ) -> DataObject:
         """
         Converts a JsonApiTransfer containing a detail (single) result
         """
 
-        json_obj = input_[self.__data_key]
-        return self.__parser.parse(json_obj)
+        obj_list = self.__parser.parse_json_doc(json_xfer)
+        return obj_list[0] if obj_list else None
 
     def convert_list(
         self,
-        input_: JsonApiTransfer
-    ) -> tuple[list[DataObject], Optional[int]]:
+        json_xfer: JsonApiTransfer,
+    ) -> tuple[list[DataObject], int | None]:
         """
         Converts a JsonApiTransfer containing a list of results. Also
         returns a count of the total results meeting.
         """
 
-        json_obj_list = input_[self.__data_key]
-        total_count = input_.get('meta', {}).get('total', None)
-        return [
-            self.__parser.parse(json_obj)
-            for json_obj in json_obj_list
-        ], total_count
+        objs = self.__parser.parse_json_doc(json_xfer)
+        total_count = json_xfer.get('meta', {}).get('total', None)
+        return objs, total_count
 
     def convert_count(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return stats['total']
 
     def convert_stats(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return self.__parser.parse_stats(stats)
 
     def convert_group_stats(
         self,
-        input_: JsonApiTransfer
-    ) -> Dict[str, Any]:
+        json_xfer: JsonApiTransfer,
+    ) -> dict[str, Any]:
         """
         Converts a JsonApiTransfer containing a list of grouped stats.
         """
 
-        stats = input_[self.__meta_key]
+        stats = json_xfer['meta']
         return self.__parser.parse_group_stats(stats)
 
     def convert_cursor_page(
         self,
-        input_: JsonApiTransfer
+        json_xfer: JsonApiTransfer,
     ) -> tuple[list[DataObject], list[str] | None]:
         """
         Converts a `JsonApiTransfer` of a cursor-page
         """
 
-        objs = self.__parser.parse_iterable(
-            input_[self.__data_key]
-        )
-        search_after = input_.get('meta', {}).get('search_after')
+        objs = self.__parser.parse_json_doc(json_xfer)
+        search_after = json_xfer.get('meta', {}).get('search_after')
 
         return objs, search_after
 
     def convert_relationship_config(
         self,
-        config_transfer: JsonRelationshipConfig
+        config_transfer: JsonRelationshipConfig,
     ) -> dict[str, RelationshipConfig]:
         """
         Converts a `JsonRelationshipConfig` dict, returned from
@@ -127,23 +116,20 @@ class JsonApiConverter:
 
         return {
             type_: self.__convert_relationship(rel)
-            for type_, rel
-            in config_transfer.items()
+            for type_, rel in config_transfer.items()
         }
 
     def __convert_relationship(
         self,
-        rel: JsonRelationship
+        rel: JsonRelationship,
     ) -> RelationshipConfig:
-
         return RelationshipConfig(
             to_one=rel.get('one'),
-            to_many=rel.get('many')
+            to_many=rel.get('many'),
         )
 
 
 class DataObjectConverter:
-
     """
     Converts from instances of `DataObject` to
     JSON:API transfers.
@@ -161,21 +147,21 @@ class DataObjectConverter:
         req_fields_tree = ReqFieldsTree(object_type, self.__data_source)
         return DefaultView(req_fields_tree, self.__prefix)
 
-    def convert(self, input_: DataObject) -> JsonApiTransfer:
+    def convert(self, data_obj: DataObject) -> JsonApiTransfer:
         """
         Converts a single `DataObject` instance to a JsonApiTransfer
         """
 
-        view = self.__build_view(input_.type)
-        return view.dump(input_)
+        view = self.__build_view(data_obj.type)
+        return view.dump(data_obj)
 
-    def convert_list(self, input_: list[DataObject]) -> JsonApiTransfer:
+    def convert_list(self, data_obj_list: list[DataObject]) -> JsonApiTransfer:
         """
         Converts a `list` of `DataObject` instances to a JsonApiTransfer
         """
 
-        if not input_:
+        if not data_obj_list:
             msg = 'Cannot convert empty list'
             raise ValueError(msg)
-        view = self.__build_view(input_[0].type)
-        return view.dump_bulk(input_)
+        view = self.__build_view(data_obj_list[0].type)
+        return view.dump_bulk(data_obj_list)

tol/api_client/factory.py

@@ -5,19 +5,12 @@
 from collections.abc import Mapping
 from typing import Callable, Iterator, Optional
 
-from .api_datasource import (
-    ApiDataSource,
-    DOConverterFactory,
-    JsonConverterFactory
-)
+from .api_datasource import ApiDataSource, DOConverterFactory, JsonConverterFactory
 from .client import JsonApiClient
-from .converter import (
-    DataObjectConverter,
-    JsonApiConverter
-)
+from .converter import DataObjectConverter, JsonApiConverter
 from .filter import DefaultApiFilter
 from .parser import DefaultParser
-from ..core import DataSource
+from ..core import DataSource, ReqFieldsTree
 
 
 class _ApiDSDict(Mapping):
@@ -53,11 +46,7 @@ class _ConverterFactory:
         return self.__data_source
 
     @data_source.setter
-    def data_source(
-        self,
-        ds: DataSource
-    ) -> None:
-
+    def data_source(self, ds: DataSource) -> None:
         self.__data_source = ds
 
     def do_converter_factory(self) -> DOConverterFactory:
@@ -67,12 +56,26 @@ class _ConverterFactory:
 
         return DataObjectConverter(self.__data_source, prefix=self.__prefix)
 
-    def json_converter_factory(self) -> JsonConverterFactory:
+    def json_converter_factory(
+        self,
+        object_type: str | None = None,
+        requested_fields: list[str] | None = None,
+    ) -> JsonConverterFactory:
         """
         Returns an instantiated `JsonApiConverter`.
         """
 
-        parser = DefaultParser(self.__ds_dict)
+        req_fields_tree = (
+            ReqFieldsTree(
+                object_type,
+                self.__data_source,
+                requested_fields=requested_fields,
+            )
+            if object_type
+            else None
+        )
+
+        parser = DefaultParser(self.__ds_dict, req_fields_tree)
         return JsonApiConverter(parser)
 
     @property
@@ -110,7 +113,6 @@ def _filter_factory() -> DefaultApiFilter:
 def create_api_datasource(
     api_url: str,
     token: Optional[str] = None,
-
     data_prefix: str = '/data',
     retries: int = 5,
     status_forcelist: Optional[list[int]] = None,
@@ -137,7 +139,7 @@ def create_api_datasource(
         client_factory,
         manager.json_converter_factory,
         manager.do_converter_factory,
-        _filter_factory
+        _filter_factory,
     )
 
     manager.data_source = api_ds