tol-sdk 1.7.2__py3-none-any.whl → 1.7.4__py3-none-any.whl

tol/api_base/pipeline_steps.py

@@ -49,7 +49,7 @@ REQUIRED_FIELDS: List = [
 def pipeline_steps_blueprint(
     sql_ds: SqlDataSource,
     prefect_ds: PrefectDataSource,
-    role: str | None = 'exporter',
+    role: str | None = None,
     url_prefix: str = '/run-pipeline',
 
     ctx_getter: CtxGetter = default_ctx_getter,
@@ -155,7 +155,7 @@ def pipeline_steps_blueprint(
                 'deployment_name': flow_name,
                 'parameters': flow_params,
                 'tags': [
-                    'app_name:treeofsex',
+                    f'app_name: {os.environ.get("APP_NAME", "tol")}',
                 ],
             }
         )
@@ -190,10 +190,13 @@ def pipeline_steps_blueprint(
     def run_pipeline_steps() -> tuple[dict[str, Any], int]:
 
         ctx = ctx_getter()
-        user_id = ctx.user_id
+        if role is not None:
+            if not ctx or not ctx.authenticated:
+                raise ForbiddenError()
+            if role not in ctx.roles:
+                raise ForbiddenError()
 
-        if role is not None and role not in ctx.roles:
-            raise ForbiddenError()
+        user_id = ctx.user_id if ctx and ctx.authenticated else None
 
         body: dict[str, Any] = request.json.get('data', {})
 

tol/s3/__init__.py

@@ -6,3 +6,4 @@ from .parser import Parser  # noqa F401
 from .converter import S3Converter  # noqa F401
 from .factory import create_s3_datasource  # noqa F401
 from .s3_datasource import S3DataSource  # noqa F401
+from .data_upload.blueprint import data_upload_blueprint # noqa F401

tol/{sql/summary → s3/data_upload}/__init__.py

@@ -1,6 +1,3 @@
 # SPDX-FileCopyrightText: 2025 Genome Research Ltd.
 #
 # SPDX-License-Identifier: MIT
-
-
-from .factory import create_summary_models  # noqa F401

tol/s3/data_upload/blueprint.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from tempfile import NamedTemporaryFile
+from typing import Any
+
+from flask import Blueprint, request, send_file
+
+from ...api_base import (
+    custom_blueprint,
+)
+from ...services.s3_client import S3Client
+
+
+ALLOWED_EXTENSIONS: set[str] = {'csv', 'json', 'xlsx'}
+
+
+def allowed_file(filename: str) -> bool:
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+def data_upload_blueprint(
+    url_prefix: str = '/pipeline/data_upload',
+) -> Blueprint:
+
+    data_upload_blueprint = custom_blueprint(
+        name='data_upload',
+        url_prefix=url_prefix
+    )
+
+    @data_upload_blueprint.route('/upload', methods=['POST'])
+    def upload_file() -> tuple[dict[str, str], int]:
+        file = request.files['file']
+        s3_bucket: str = request.form.get('s3_bucket')
+
+        if not file:
+            return {'error': 'No file provided'}, 400
+
+        if not allowed_file(file.filename):
+            return {'error': 'File type not allowed'}, 400
+
+        if not s3_bucket:
+            return {'error': 'S3 bucket not specified'}, 400
+
+        try:
+            s3_client = S3Client()
+            with NamedTemporaryFile() as temp_file:
+                file.save(temp_file.name)
+                s3_client.put_object(s3_bucket, file.filename, temp_file.name)
+
+            return {'message': 'File uploaded successfully'}, 200
+
+        except Exception as e:
+            return {'error': f'Failed to upload file: {str(e)}'}, 500
+
+    @data_upload_blueprint.route('/download', methods=['POST'])
+    def download_file() -> tuple[dict[str, str], int]:
+        body: dict[str, Any] = request.json.get('data', {})
+
+        if 's3_bucket' not in body or 'file_name' not in body:
+            return {'error': 'S3 bucket or file name not specified'}, 400
+
+        bucket_name = body['s3_bucket']
+        file_name = body['file_name']
+
+        try:
+            s3_client = S3Client()
+
+            with NamedTemporaryFile() as temp_file:
+                s3_client.get_object(bucket_name, file_name, temp_file.name)
+                download_name = request.json.get('download_name', file_name)
+
+                return send_file(
+                    temp_file.name,
+                    as_attachment=True,
+                    download_name=download_name,
+                    mimetype='application/octet-stream'
+                )
+        except Exception as e:
+            return {'error': f'Failed to download file: {str(e)}'}, 500
+
+    return data_upload_blueprint

tol/services/s3_client.py

@@ -26,7 +26,7 @@ class S3Client:
             secure = os.getenv('S3_SECURE', 'true').lower() == 'true'
 
         self.client = Minio(
-            self.s3_uri,
+            endpoint=self.s3_uri,
             access_key=self.access_key,
             secret_key=self.secret_key,
             secure=secure
@@ -39,7 +39,8 @@ class S3Client:
         file_path: str
     ) -> None:
 
-        self.client.fget_object(bucket_name, object_name, file_path)
+        self.client.fget_object(bucket_name=bucket_name,
+                                object_name=object_name, file_path=file_path)
 
     def list_objects(
         self,
@@ -55,4 +56,5 @@ class S3Client:
         file_path: str
     ) -> None:
 
-        self.client.fput_object(bucket_name, object_name, file_path)
+        self.client.fput_object(bucket_name=bucket_name,
+                                object_name=object_name, file_path=file_path)

tol/sql/pipeline_step/factory.py

@@ -146,7 +146,7 @@ def create_pipeline_step_models(
 
         user_id: Mapped[int] = mapped_column(
             ForeignKey('user.id'),
-            nullable=False
+            nullable=True
         )
 
         pipeline_id: Mapped[int] = mapped_column(

tol/sql/standard/factory.py

@@ -37,6 +37,7 @@ class StandardModels(IterableABC[type[Model]]):
     data_source_config: type[Model]
     data_source_config_attribute: type[Model]
     data_source_config_relationship: type[Model]
+    data_source_config_summary: type[Model]
     component: type[Model]
     component_zone: type[Model]
     zone: type[Model]
@@ -61,6 +62,7 @@ class StandardModels(IterableABC[type[Model]]):
                 self.view_board,
                 self.view,
                 self.board,
+                self.data_source_config_summary,
                 self.data_source_config_relationship,
                 self.data_source_config_attribute,
                 self.loader_instance,
@@ -458,7 +460,6 @@ def create_standard_models(
 
         name: Mapped[str] = mapped_column(nullable=False)
         object_type: Mapped[str] = mapped_column(nullable=False)
-        name: Mapped[str] = mapped_column(nullable=False)
         display_name: Mapped[str] = mapped_column(nullable=True)
         description: Mapped[str] = mapped_column(nullable=True)
         available_on_relationships: Mapped[bool] = mapped_column(nullable=False, default=True)
@@ -484,6 +485,53 @@ def create_standard_models(
             foreign_keys=[data_source_config_id]
         )
 
+    class DataSourceConfigSummary(base_model_class):
+        __tablename__ = 'data_source_config_summary'
+
+        id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)  # noqa A003
+
+        source_object_type: Mapped[str] = mapped_column(nullable=False)
+        destination_object_type: Mapped[str] = mapped_column(nullable=True)
+
+        object_filters: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default={}
+        )
+
+        group_by: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        stats_fields: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        stats: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        prefix: Mapped[str] = mapped_column(
+            nullable=False,
+            default=''
+        )
+
+        data_source_config_id: Mapped[int] = mapped_column(
+            ForeignKey('data_source_config.id'),
+            nullable=False
+        )
+
+        data_source_config: Mapped['DataSourceConfig'] = relationship(  # noqa F821
+            back_populates='data_source_config_summaries',
+            foreign_keys=[data_source_config_id]
+        )
+
     class DataSourceConfig(base_model_class):
         __tablename__ = 'data_source_config'
 
@@ -504,6 +552,10 @@ def create_standard_models(
             back_populates='data_source_config',
             foreign_keys=[DataSourceConfigRelationship.data_source_config_id]
         )
+        data_source_config_summaries: Mapped[list['DataSourceConfigSummary']] = relationship(
+            back_populates='data_source_config',
+            foreign_keys=[DataSourceConfigSummary.data_source_config_id]
+        )
 
     class _UserMixin:
 
@@ -538,6 +590,7 @@ def create_standard_models(
         data_source_config=DataSourceConfig,
         data_source_config_attribute=DataSourceConfigAttribute,
         data_source_config_relationship=DataSourceConfigRelationship,
+        data_source_config_summary=DataSourceConfigSummary,
         component=Component,
         component_zone=ComponentZone,
         zone=Zone,

tol/validators/unique_values.py

@@ -14,7 +14,7 @@ class UniqueValuesValidator(Validator):
 
     def __init__(
         self,
-        unique_keys: list[str],
+        unique_keys: list[list[str] | str],
         *,
         detail: str = 'Value is not unique',
         is_error: bool = True,
@@ -26,21 +26,49 @@ class UniqueValuesValidator(Validator):
         self.__detail = detail
         self.__is_error = is_error
         self.__duplicates: dict[str, list[str]] = {}
-        self.__existing_values: dict[str, set] = {key: set() for key in unique_keys}
+        self.__existing_values: dict[str, set] = {}
+        for key in self.__keys:
+            if isinstance(key, str):
+                self.__existing_values[key] = set()
+            elif isinstance(key, list):
+                concat_key = '/'.join(key)
+                self.__existing_values[concat_key] = set()
 
     def _validate_data_object(
         self,
         obj: DataObject
     ) -> None:
 
-        for key in obj.attributes:
-            if key in self.__keys:
-                if obj.attributes[key] in self.__existing_values[key]:
-                    if key not in self.__duplicates:
-                        self.__duplicates[key] = []
-                    self.__duplicates[key].append(obj.attributes[key])
+        for unique_key in self.__keys:
+            if isinstance(unique_key, list):
+                concat = ''
+                for key in unique_key:
+                    concat = concat + '/' + (str(obj.attributes[key]))
+                if concat in self.__existing_values['/'.join(unique_key)]:
+                    self._duplicate_checks(
+                        key=key,
+                        value=concat
+                    )
                 else:
-                    self.__existing_values[key].add(obj.attributes[key])
+                    self.__existing_values['/'.join(unique_key)].add(concat)
+
+            else:
+                if obj.attributes[unique_key] in self.__existing_values[unique_key]:
+                    self._duplicate_checks(
+                        key=unique_key,
+                        value=obj.attributes[unique_key]
+                    )
+                else:
+                    self.__existing_values[unique_key].add(obj.attributes[unique_key])
+
+    def _duplicate_checks(
+        self,
+        key: str,
+        value: str
+    ):
+        if key not in self.__duplicates:
+            self.__duplicates[key] = []
+        self.__duplicates[key].append(value)
 
     def _post_validation(
         self,

{tol_sdk-1.7.2.dist-info → tol_sdk-1.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.2
+Version: 1.7.4
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.7.2.dist-info → tol_sdk-1.7.4.dist-info}/RECORD

@@ -8,7 +8,7 @@ tol/api_base/__init__.py,sha256=xv9kkAyb1Q5vhKgq6f-JpY7lIAla51U-gIpA5U1OfkA,312
 tol/api_base/action.py,sha256=l3MzD3jSDdYiZs-HCT4YEsXkp34X26k6yWugXrRB2XU,6575
 tol/api_base/blueprint.py,sha256=lyNXgyyZ7kEL5uuIuSNkardFy8ZcDIfftt8c5xiRjyA,19061
 tol/api_base/controller.py,sha256=5Fp2FobjXXPlongu1j7adGJKlSVufW0_jpk_wZIMRa4,26818
-tol/api_base/pipeline_steps.py,sha256=lz888M2cF-5z-EkGK_sWfZf1otnYRmgnNuC0WUqI-po,5667
+tol/api_base/pipeline_steps.py,sha256=Y7tOwaBBanxAqiUrTpmF6yaLkEJI6o0ELnM-2CaCvHs,5832
 tol/api_base/system.py,sha256=c47sbcNaCd3teiJLO-Zwg1cUNvfCvRO4GmKzAMNxZrw,1219
 tol/api_base/auth/__init__.py,sha256=LNbVtQDlzKWzLoCmxAPSIAcM6mlVMnn0Aj9HFUudseo,460
 tol/api_base/auth/asserts.py,sha256=3kHP2yW6HZxevIgeO8Rl3iKovi8WHZcguVbPSFUjawQ,2765
@@ -237,11 +237,13 @@ tol/prefect/factory.py,sha256=mO4KVnaEYMv-ojGJuiencNQMq6PAMU8cIc4QN5Kq8Gw,2208
 tol/prefect/filter.py,sha256=aNVTSmcrbb_4EllcFtBOSPuUGgwE-f-Kd5_an6lREtM,7044
 tol/prefect/prefect_datasource.py,sha256=JXLS4odWNWUu4mgojQogwE28mBTAbj-OQYFxhYNYHnM,8865
 tol/prefect/prefect_object.py,sha256=Nn3Y6ErQnXubX7M-GzRsfLHEzdHwKr5RWCjIJ4FA28U,495
-tol/s3/__init__.py,sha256=L2Aa12NAYjTRr8W_ArJi3z3ME2bUtCf4CG3JDw1MoMk,282
+tol/s3/__init__.py,sha256=AnYM6rWRDJ-AkUBPuIFGrF3wyeh7euwUCiv2-gPy9J4,351
 tol/s3/converter.py,sha256=KylGAQvu1jRYJl4dcYK1IgbFsmHP3JXRI22MX8UzHz4,1023
 tol/s3/factory.py,sha256=rFcRVy4B8KVsUaaHhtpLUE90_hRnixkO2IjnB6-arkc,2163
 tol/s3/parser.py,sha256=EOiPYJ8ZLigAxKGL4cC8r7OGyjGdSMHmJMnDqHSiG_E,1907
 tol/s3/s3_datasource.py,sha256=tSUc78FN1OudpJcQ-7FI4S_LPT5j-BYIcwFaQPDy_VA,2072
+tol/s3/data_upload/__init__.py,sha256=4dkK7MPB0CzkkujFSVj0FzoG5ri_usQ7XaBB81Vc_qE,85
+tol/s3/data_upload/blueprint.py,sha256=rwZAXsHe_AJXAdRZVdUhtJg3P3SxOs_G_0F1eewPuT4,2574
 tol/sciops/__init__.py,sha256=PaC1_5wZhsZlqeYQRMpDeIf9L6Hi0eXs_pHXaWfscAo,131
 tol/sciops/configuration.py,sha256=21ddgHOmhZE1tcE-5_f4KiV-7CUqkF8yXi4iBYPtfmc,1279
 tol/sciops/consumer.py,sha256=Ikl8glrZ0yQKBhegC84vwNj43iK1Ca_SAn_gcyAhbKQ,2137
@@ -250,7 +252,7 @@ tol/sciops/messages.py,sha256=0kFsa9pIyWV6rADHjcqVa2q2D61U1AmtFK9Lq5mfjbU,1750
 tol/sciops/response_processors.py,sha256=QDBYT__3tmWArTzeVR29k6yfYQdXanvWSy5VI7nrpio,2658
 tol/sciops/sequencing_datasource.py,sha256=JT5eenYjrN-pJmaReSGigMq5PvmE2NSDI4bUMHfeMz4,7553
 tol/services/__init__.py,sha256=FgW9Rb3vp-bfG7OYGyyNdh0JC420opH0gKtr89z6WmE,125
-tol/services/s3_client.py,sha256=kMs49KJXyh9gwEAePAPaEAhOESoiguGXdr-Y0hHl_mg,1424
+tol/services/s3_client.py,sha256=OR2PDz7EiU2i__UUIwtvd18MpMEOlPGJqoTGYxIT4Mo,1565
 tol/sources/__init__.py,sha256=4LbvDIZOOG7p-ebbvivP7NvrJeApUvGEIcDL58ahQJE,85
 tol/sources/benchling.py,sha256=C1mDri10F6DJ3PYOHHnrf8sJI2WF3xwMoFGLZw0nkKI,585
 tol/sources/benchling_warehouse.py,sha256=8i91fBSYWy0YvYj031BTGUL4bdi4vzQzcOBkhHM2s_Q,694
@@ -303,11 +305,9 @@ tol/sql/auth/__init__.py,sha256=e3JuwugXmXobklqZ1Mt1w03qPgb1WdUaJVM7oblzHyk,202
 tol/sql/auth/blueprint.py,sha256=u0vT_TC9IMKrg08QFa9W29_83mT0y0jzLj3DvXy1BBw,25906
 tol/sql/auth/models.py,sha256=4xNWBvyGiI3mwRyDY5ty48Bv9178ApQXPR-YjIdCsvk,11879
 tol/sql/pipeline_step/__init__.py,sha256=O7u4RrLfuoB0mwLcPxFoUrdTBZGB_4bE1vWCn5ho-qw,147
-tol/sql/pipeline_step/factory.py,sha256=SFJ-BE1I_qmEsqrJ10evdGCdoUHjdiJq6iqrzIlX2jU,5132
+tol/sql/pipeline_step/factory.py,sha256=hrv9iAjdUu3sq6R94PXKclatZNMKY8YMgBDwLWKuvR0,5131
 tol/sql/standard/__init__.py,sha256=2NbLXFk0rneGZosZ2ESIRcT0WMK0KncmPWaLPqvX-i4,142
-tol/sql/standard/factory.py,sha256=4bg6omWsa223y5nEqz_DDp1om6Ip-NlYm3JoT1EQGMM,18361
-tol/sql/summary/__init__.py,sha256=5fNYAeOJ-XOdG9cgu7cUgDBwzG89cuc6RGGw_ZOjURk,143
-tol/sql/summary/factory.py,sha256=F01du9HYl4kmUBHGxgpibSi651GAI5wsRUdk7LBZzzI,2009
+tol/sql/standard/factory.py,sha256=yY8iWmZRMvUqphwnzBeOIQtKGgxsU6AcA7YTz53UYvc,20010
 tol/status/__init__.py,sha256=sBo-j1wCmberl89uryVCBEJk8ohbfsYhaNpIp_brR9Y,146
 tol/status/status_datasource.py,sha256=UYU2vB561XRWY8y2dY96qHiWXy15xaHxsbGCVCIUnqs,1916
 tol/sts/__init__.py,sha256=1nb_lBWDwxJo3hutxSid2rqMIpfZ4GHxDS6cfj-FKv4,187
@@ -324,10 +324,10 @@ tol/validators/allowed_keys.py,sha256=BJMomJtaQdxsdGsueDtLewv75TlwdIXiQipLGFcJ7_
 tol/validators/allowed_values.py,sha256=yJ5SdiUlV7PSKORtsBJ9hYSqwvlx_esbFmFL_Gxh-p4,2262
 tol/validators/regex.py,sha256=dKodGH0sv6DbqWeV6QXE6-GYjnG4rMO0rg8IEIaQG60,2364
 tol/validators/regex_by_value.py,sha256=o99NJlWPgQ0GrpVnep8-cHfjWnc9F2rChmXHIxjrMrk,2543
-tol/validators/unique_values.py,sha256=stI-1i006WEbERcjSMapRggJkEF-RFDzw2uUtXBAE_M,1885
-tol_sdk-1.7.2.dist-info/licenses/LICENSE,sha256=RF9Jacy-9BpUAQQ20INhTgtaNBkmdTolYCHtrrkM2-8,1077
-tol_sdk-1.7.2.dist-info/METADATA,sha256=FOII5eZYn_0x2XpUiv_dTnTUBz7_ZArof7jrD5NZHms,3079
-tol_sdk-1.7.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tol_sdk-1.7.2.dist-info/entry_points.txt,sha256=jH3HfTwxjzog7E3lq8CKpUWGIRY9FSXbyL6CpUmv6D0,36
-tol_sdk-1.7.2.dist-info/top_level.txt,sha256=PwKMQLphyZNvagBoriVbl8uwHXQl8IC1niawVG0iXMM,10
-tol_sdk-1.7.2.dist-info/RECORD,,
+tol/validators/unique_values.py,sha256=tEXDNJj95XUJdhsZlO-pkXdQz-EYaSG-mcp4MDTL7eY,2835
+tol_sdk-1.7.4.dist-info/licenses/LICENSE,sha256=RF9Jacy-9BpUAQQ20INhTgtaNBkmdTolYCHtrrkM2-8,1077
+tol_sdk-1.7.4.dist-info/METADATA,sha256=1gosjFEvdmraYpLVv81dHvwCKBFGXDUdz__wwvImdLo,3079
+tol_sdk-1.7.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tol_sdk-1.7.4.dist-info/entry_points.txt,sha256=jH3HfTwxjzog7E3lq8CKpUWGIRY9FSXbyL6CpUmv6D0,36
+tol_sdk-1.7.4.dist-info/top_level.txt,sha256=PwKMQLphyZNvagBoriVbl8uwHXQl8IC1niawVG0iXMM,10
+tol_sdk-1.7.4.dist-info/RECORD,,

tol/sql/summary/factory.py

@@ -1,91 +0,0 @@
-# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
-#
-# SPDX-License-Identifier: MIT
-
-from __future__ import annotations
-
-from collections.abc import Iterable as IterableABC
-from dataclasses import dataclass
-from typing import Iterator
-
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import (
-    Mapped,
-    mapped_column
-)
-
-from ..model import Model
-
-
-@dataclass(frozen=True, kw_only=True)
-class SummaryModels(IterableABC[type[Model]]):
-    """
-    Contains the needed models for summarys.
-
-    """
-
-    summary: type[Model]
-
-    def __iter__(self) -> Iterator[type[Model]]:
-        """
-        Returns in order they should be deleted
-        """
-
-        return iter(
-            [
-                self.summary
-            ]
-        )
-
-
-def create_summary_models(
-    base_model_class: type[Model]
-) -> SummaryModels:
-    """
-    Creates all needed models (and joining tables) for
-    summaries.
-
-    Returns a `SummaryModels` instance that functions like an
-    `Iterable`.
-    """
-
-    class Summary(base_model_class):
-        __tablename__ = 'summary'
-
-        id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)  # noqa A003
-
-        source_object_type: Mapped[str] = mapped_column(nullable=False)
-        destination_object_type: Mapped[str] = mapped_column(nullable=True)
-
-        object_filters: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default={}
-        )
-
-        group_by: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        stats_fields: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        stats: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        prefix: Mapped[str] = mapped_column(
-            nullable=False,
-            default=''
-        )
-
-    return SummaryModels(
-        summary=Summary,
-    )