tol-sdk 1.7.2__tar.gz → 1.7.4__tar.gz

{tol_sdk-1.7.2/src/tol_sdk.egg-info → tol_sdk-1.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.2
+Version: 1.7.4
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tol-sdk"
-version = "1.7.2"
+version = "1.7.4"
 description = "SDK for interaction with ToL, Sanger and external services"
 readme = "README.md"
 authors = [

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol/api_base/pipeline_steps.py

@@ -49,7 +49,7 @@ REQUIRED_FIELDS: List = [
 def pipeline_steps_blueprint(
     sql_ds: SqlDataSource,
     prefect_ds: PrefectDataSource,
-    role: str | None = 'exporter',
+    role: str | None = None,
     url_prefix: str = '/run-pipeline',
 
     ctx_getter: CtxGetter = default_ctx_getter,
@@ -155,7 +155,7 @@ def pipeline_steps_blueprint(
                 'deployment_name': flow_name,
                 'parameters': flow_params,
                 'tags': [
-                    'app_name:treeofsex',
+                    f'app_name: {os.environ.get("APP_NAME", "tol")}',
                 ],
             }
         )
@@ -190,10 +190,13 @@ def pipeline_steps_blueprint(
     def run_pipeline_steps() -> tuple[dict[str, Any], int]:
 
         ctx = ctx_getter()
-        user_id = ctx.user_id
+        if role is not None:
+            if not ctx or not ctx.authenticated:
+                raise ForbiddenError()
+            if role not in ctx.roles:
+                raise ForbiddenError()
 
-        if role is not None and role not in ctx.roles:
-            raise ForbiddenError()
+        user_id = ctx.user_id if ctx and ctx.authenticated else None
 
         body: dict[str, Any] = request.json.get('data', {})
 

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol/s3/__init__.py

@@ -6,3 +6,4 @@ from .parser import Parser  # noqa F401
 from .converter import S3Converter  # noqa F401
 from .factory import create_s3_datasource  # noqa F401
 from .s3_datasource import S3DataSource  # noqa F401
+from .data_upload.blueprint import data_upload_blueprint # noqa F401

{tol_sdk-1.7.2/src/tol/sql/summary → tol_sdk-1.7.4/src/tol/s3/data_upload}/__init__.py

@@ -1,6 +1,3 @@
 # SPDX-FileCopyrightText: 2025 Genome Research Ltd.
 #
 # SPDX-License-Identifier: MIT
-
-
-from .factory import create_summary_models  # noqa F401

tol_sdk-1.7.4/src/tol/s3/data_upload/blueprint.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from tempfile import NamedTemporaryFile
+from typing import Any
+
+from flask import Blueprint, request, send_file
+
+from ...api_base import (
+    custom_blueprint,
+)
+from ...services.s3_client import S3Client
+
+
+ALLOWED_EXTENSIONS: set[str] = {'csv', 'json', 'xlsx'}
+
+
+def allowed_file(filename: str) -> bool:
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+def data_upload_blueprint(
+    url_prefix: str = '/pipeline/data_upload',
+) -> Blueprint:
+
+    data_upload_blueprint = custom_blueprint(
+        name='data_upload',
+        url_prefix=url_prefix
+    )
+
+    @data_upload_blueprint.route('/upload', methods=['POST'])
+    def upload_file() -> tuple[dict[str, str], int]:
+        file = request.files['file']
+        s3_bucket: str = request.form.get('s3_bucket')
+
+        if not file:
+            return {'error': 'No file provided'}, 400
+
+        if not allowed_file(file.filename):
+            return {'error': 'File type not allowed'}, 400
+
+        if not s3_bucket:
+            return {'error': 'S3 bucket not specified'}, 400
+
+        try:
+            s3_client = S3Client()
+            with NamedTemporaryFile() as temp_file:
+                file.save(temp_file.name)
+                s3_client.put_object(s3_bucket, file.filename, temp_file.name)
+
+            return {'message': 'File uploaded successfully'}, 200
+
+        except Exception as e:
+            return {'error': f'Failed to upload file: {str(e)}'}, 500
+
+    @data_upload_blueprint.route('/download', methods=['POST'])
+    def download_file() -> tuple[dict[str, str], int]:
+        body: dict[str, Any] = request.json.get('data', {})
+
+        if 's3_bucket' not in body or 'file_name' not in body:
+            return {'error': 'S3 bucket or file name not specified'}, 400
+
+        bucket_name = body['s3_bucket']
+        file_name = body['file_name']
+
+        try:
+            s3_client = S3Client()
+
+            with NamedTemporaryFile() as temp_file:
+                s3_client.get_object(bucket_name, file_name, temp_file.name)
+                download_name = request.json.get('download_name', file_name)
+
+                return send_file(
+                    temp_file.name,
+                    as_attachment=True,
+                    download_name=download_name,
+                    mimetype='application/octet-stream'
+                )
+        except Exception as e:
+            return {'error': f'Failed to download file: {str(e)}'}, 500
+
+    return data_upload_blueprint

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol/services/s3_client.py

@@ -26,7 +26,7 @@ class S3Client:
             secure = os.getenv('S3_SECURE', 'true').lower() == 'true'
 
         self.client = Minio(
-            self.s3_uri,
+            endpoint=self.s3_uri,
             access_key=self.access_key,
             secret_key=self.secret_key,
             secure=secure
@@ -39,7 +39,8 @@ class S3Client:
         file_path: str
     ) -> None:
 
-        self.client.fget_object(bucket_name, object_name, file_path)
+        self.client.fget_object(bucket_name=bucket_name,
+                                object_name=object_name, file_path=file_path)
 
     def list_objects(
         self,
@@ -55,4 +56,5 @@ class S3Client:
         file_path: str
     ) -> None:
 
-        self.client.fput_object(bucket_name, object_name, file_path)
+        self.client.fput_object(bucket_name=bucket_name,
+                                object_name=object_name, file_path=file_path)

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol/sql/pipeline_step/factory.py

@@ -146,7 +146,7 @@ def create_pipeline_step_models(
 
         user_id: Mapped[int] = mapped_column(
             ForeignKey('user.id'),
-            nullable=False
+            nullable=True
         )
 
         pipeline_id: Mapped[int] = mapped_column(

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol/sql/standard/factory.py

@@ -37,6 +37,7 @@ class StandardModels(IterableABC[type[Model]]):
     data_source_config: type[Model]
     data_source_config_attribute: type[Model]
     data_source_config_relationship: type[Model]
+    data_source_config_summary: type[Model]
     component: type[Model]
     component_zone: type[Model]
     zone: type[Model]
@@ -61,6 +62,7 @@ class StandardModels(IterableABC[type[Model]]):
                 self.view_board,
                 self.view,
                 self.board,
+                self.data_source_config_summary,
                 self.data_source_config_relationship,
                 self.data_source_config_attribute,
                 self.loader_instance,
@@ -458,7 +460,6 @@ def create_standard_models(
 
         name: Mapped[str] = mapped_column(nullable=False)
         object_type: Mapped[str] = mapped_column(nullable=False)
-        name: Mapped[str] = mapped_column(nullable=False)
         display_name: Mapped[str] = mapped_column(nullable=True)
         description: Mapped[str] = mapped_column(nullable=True)
         available_on_relationships: Mapped[bool] = mapped_column(nullable=False, default=True)
@@ -484,6 +485,53 @@ def create_standard_models(
             foreign_keys=[data_source_config_id]
         )
 
+    class DataSourceConfigSummary(base_model_class):
+        __tablename__ = 'data_source_config_summary'
+
+        id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)  # noqa A003
+
+        source_object_type: Mapped[str] = mapped_column(nullable=False)
+        destination_object_type: Mapped[str] = mapped_column(nullable=True)
+
+        object_filters: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default={}
+        )
+
+        group_by: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        stats_fields: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        stats: Mapped[dict] = mapped_column(
+            JSONB,
+            nullable=False,
+            default=[]
+        )
+
+        prefix: Mapped[str] = mapped_column(
+            nullable=False,
+            default=''
+        )
+
+        data_source_config_id: Mapped[int] = mapped_column(
+            ForeignKey('data_source_config.id'),
+            nullable=False
+        )
+
+        data_source_config: Mapped['DataSourceConfig'] = relationship(  # noqa F821
+            back_populates='data_source_config_summaries',
+            foreign_keys=[data_source_config_id]
+        )
+
     class DataSourceConfig(base_model_class):
         __tablename__ = 'data_source_config'
 
@@ -504,6 +552,10 @@ def create_standard_models(
             back_populates='data_source_config',
             foreign_keys=[DataSourceConfigRelationship.data_source_config_id]
         )
+        data_source_config_summaries: Mapped[list['DataSourceConfigSummary']] = relationship(
+            back_populates='data_source_config',
+            foreign_keys=[DataSourceConfigSummary.data_source_config_id]
+        )
 
     class _UserMixin:
 
@@ -538,6 +590,7 @@ def create_standard_models(
         data_source_config=DataSourceConfig,
         data_source_config_attribute=DataSourceConfigAttribute,
         data_source_config_relationship=DataSourceConfigRelationship,
+        data_source_config_summary=DataSourceConfigSummary,
         component=Component,
         component_zone=ComponentZone,
         zone=Zone,

tol_sdk-1.7.4/src/tol/validators/unique_values.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
+#
+# SPDX-License-Identifier: MIT
+
+from tol.core import DataObject
+from tol.core.validate import Validator
+
+
+class UniqueValuesValidator(Validator):
+    """
+    Validates that a stream of `DataObject` instances
+    contains unique values for specified keys.
+    """
+
+    def __init__(
+        self,
+        unique_keys: list[list[str] | str],
+        *,
+        detail: str = 'Value is not unique',
+        is_error: bool = True,
+    ) -> None:
+
+        super().__init__()
+
+        self.__keys = unique_keys
+        self.__detail = detail
+        self.__is_error = is_error
+        self.__duplicates: dict[str, list[str]] = {}
+        self.__existing_values: dict[str, set] = {}
+        for key in self.__keys:
+            if isinstance(key, str):
+                self.__existing_values[key] = set()
+            elif isinstance(key, list):
+                concat_key = '/'.join(key)
+                self.__existing_values[concat_key] = set()
+
+    def _validate_data_object(
+        self,
+        obj: DataObject
+    ) -> None:
+
+        for unique_key in self.__keys:
+            if isinstance(unique_key, list):
+                concat = ''
+                for key in unique_key:
+                    concat = concat + '/' + (str(obj.attributes[key]))
+                if concat in self.__existing_values['/'.join(unique_key)]:
+                    self._duplicate_checks(
+                        key=key,
+                        value=concat
+                    )
+                else:
+                    self.__existing_values['/'.join(unique_key)].add(concat)
+
+            else:
+                if obj.attributes[unique_key] in self.__existing_values[unique_key]:
+                    self._duplicate_checks(
+                        key=unique_key,
+                        value=obj.attributes[unique_key]
+                    )
+                else:
+                    self.__existing_values[unique_key].add(obj.attributes[unique_key])
+
+    def _duplicate_checks(
+        self,
+        key: str,
+        value: str
+    ):
+        if key not in self.__duplicates:
+            self.__duplicates[key] = []
+        self.__duplicates[key].append(value)
+
+    def _post_validation(
+        self,
+        obj: DataObject,
+    ) -> None:
+        for key in self.__duplicates:
+            self.__add_result(
+                obj=obj,
+                key=key,
+            )
+
+    def __add_result(
+        self,
+        obj: DataObject,
+        key: str,
+    ) -> None:
+
+        if self.__is_error:
+            self.add_error(
+                object_id=obj.id,
+                detail=self.__detail,
+                field=key,
+            )
+        else:
+            self.add_warning(
+                object_id=obj.id,
+                detail=self.__detail,
+                field=key,
+            )

{tol_sdk-1.7.2 → tol_sdk-1.7.4/src/tol_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tol-sdk
-Version: 1.7.2
+Version: 1.7.4
 Summary: SDK for interaction with ToL, Sanger and external services
 Author-email: ToL Platforms Team <tol-platforms@sanger.ac.uk>
 License: MIT

{tol_sdk-1.7.2 → tol_sdk-1.7.4}/src/tol_sdk.egg-info/SOURCES.txt

@@ -247,6 +247,8 @@ src/tol/s3/converter.py
 src/tol/s3/factory.py
 src/tol/s3/parser.py
 src/tol/s3/s3_datasource.py
+src/tol/s3/data_upload/__init__.py
+src/tol/s3/data_upload/blueprint.py
 src/tol/sciops/__init__.py
 src/tol/sciops/configuration.py
 src/tol/sciops/consumer.py
@@ -311,8 +313,6 @@ src/tol/sql/pipeline_step/__init__.py
 src/tol/sql/pipeline_step/factory.py
 src/tol/sql/standard/__init__.py
 src/tol/sql/standard/factory.py
-src/tol/sql/summary/__init__.py
-src/tol/sql/summary/factory.py
 src/tol/status/__init__.py
 src/tol/status/status_datasource.py
 src/tol/sts/__init__.py

tol_sdk-1.7.2/src/tol/sql/summary/factory.py

@@ -1,91 +0,0 @@
-# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
-#
-# SPDX-License-Identifier: MIT
-
-from __future__ import annotations
-
-from collections.abc import Iterable as IterableABC
-from dataclasses import dataclass
-from typing import Iterator
-
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import (
-    Mapped,
-    mapped_column
-)
-
-from ..model import Model
-
-
-@dataclass(frozen=True, kw_only=True)
-class SummaryModels(IterableABC[type[Model]]):
-    """
-    Contains the needed models for summarys.
-
-    """
-
-    summary: type[Model]
-
-    def __iter__(self) -> Iterator[type[Model]]:
-        """
-        Returns in order they should be deleted
-        """
-
-        return iter(
-            [
-                self.summary
-            ]
-        )
-
-
-def create_summary_models(
-    base_model_class: type[Model]
-) -> SummaryModels:
-    """
-    Creates all needed models (and joining tables) for
-    summaries.
-
-    Returns a `SummaryModels` instance that functions like an
-    `Iterable`.
-    """
-
-    class Summary(base_model_class):
-        __tablename__ = 'summary'
-
-        id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)  # noqa A003
-
-        source_object_type: Mapped[str] = mapped_column(nullable=False)
-        destination_object_type: Mapped[str] = mapped_column(nullable=True)
-
-        object_filters: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default={}
-        )
-
-        group_by: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        stats_fields: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        stats: Mapped[dict] = mapped_column(
-            JSONB,
-            nullable=False,
-            default=[]
-        )
-
-        prefix: Mapped[str] = mapped_column(
-            nullable=False,
-            default=''
-        )
-
-    return SummaryModels(
-        summary=Summary,
-    )

tol_sdk-1.7.2/src/tol/validators/unique_values.py

@@ -1,72 +0,0 @@
-# SPDX-FileCopyrightText: 2025 Genome Research Ltd.
-#
-# SPDX-License-Identifier: MIT
-
-from tol.core import DataObject
-from tol.core.validate import Validator
-
-
-class UniqueValuesValidator(Validator):
-    """
-    Validates that a stream of `DataObject` instances
-    contains unique values for specified keys.
-    """
-
-    def __init__(
-        self,
-        unique_keys: list[str],
-        *,
-        detail: str = 'Value is not unique',
-        is_error: bool = True,
-    ) -> None:
-
-        super().__init__()
-
-        self.__keys = unique_keys
-        self.__detail = detail
-        self.__is_error = is_error
-        self.__duplicates: dict[str, list[str]] = {}
-        self.__existing_values: dict[str, set] = {key: set() for key in unique_keys}
-
-    def _validate_data_object(
-        self,
-        obj: DataObject
-    ) -> None:
-
-        for key in obj.attributes:
-            if key in self.__keys:
-                if obj.attributes[key] in self.__existing_values[key]:
-                    if key not in self.__duplicates:
-                        self.__duplicates[key] = []
-                    self.__duplicates[key].append(obj.attributes[key])
-                else:
-                    self.__existing_values[key].add(obj.attributes[key])
-
-    def _post_validation(
-        self,
-        obj: DataObject,
-    ) -> None:
-        for key in self.__duplicates:
-            self.__add_result(
-                obj=obj,
-                key=key,
-            )
-
-    def __add_result(
-        self,
-        obj: DataObject,
-        key: str,
-    ) -> None:
-
-        if self.__is_error:
-            self.add_error(
-                object_id=obj.id,
-                detail=self.__detail,
-                field=key,
-            )
-        else:
-            self.add_warning(
-                object_id=obj.id,
-                detail=self.__detail,
-                field=key,
-            )