UncountablePythonSDK 0.0.47__py3-none-any.whl → 0.0.49__py3-none-any.whl

{UncountablePythonSDK-0.0.47.dist-info → UncountablePythonSDK-0.0.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.47
+Version: 0.0.49
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -28,6 +28,7 @@ Requires-Dist: google-api-python-client ==2.*
 Requires-Dist: tqdm ==4.*
 Requires-Dist: pysftp ==0.*
 Requires-Dist: paramiko ==3.*
+Requires-Dist: boto3 ==1.*
 Provides-Extra: test
 Requires-Dist: mypy ==1.* ; extra == 'test'
 Requires-Dist: ruff ==0.* ; extra == 'test'

{UncountablePythonSDK-0.0.47.dist-info → UncountablePythonSDK-0.0.49.dist-info}/RECORD

@@ -17,7 +17,7 @@ docs/static/favicons/safari-pinned-tab.svg,sha256=S84fRnz0ZxLnQrKtmmFZytiRyu1xLt
 examples/async_batch.py,sha256=CffQ8O9drJ-Mdd6S5DnMIOBsHv5aVkTZrD3l3xBnB4s,1094
 examples/create_entity.py,sha256=noZdtJ5f9Wfiob3zUH-8bDVbrCPJnFtXFk_W9pSjvUA,664
 examples/edit_recipe_inputs.py,sha256=SFHhccktlpWlz4ujh30KZ-hZTFneRWuEp9b3nJSbNn8,1647
-examples/invoke_uploader.py,sha256=NOHsvObRGLrOYaSKtwk1xkhsdLEdbUmrseUX6JzC4Q0,635
+examples/invoke_uploader.py,sha256=Rn5sSZowW6_0yX6843q2HOSIECfK4cOCB1Hmco3iWVo,625
 examples/upload_files.py,sha256=tUfKFqiqwnw08OL5Y8_e4j5pSRhp94cFex8XTuVa_ig,487
 pkgs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pkgs/argument_parser/__init__.py,sha256=CsQ6QoPKSLLRVl-z6URAmPkiUL9ZPZoV4rJHgy_-RjA,385
@@ -25,11 +25,12 @@ pkgs/argument_parser/_is_enum.py,sha256=Gw6jJa8nBwYGqXwwCZbSnWL8Rvr5alkg5lSVAqXt
 pkgs/argument_parser/_is_namedtuple.py,sha256=Rjc1bKanIPPogl3qG5JPBxglG1TqWYOo1nxxhBASQWY,265
 pkgs/argument_parser/argument_parser.py,sha256=S5x4yDpaBqTRkmcOyX2UuBWw9iCE4j2Po5LZPg9jhe4,17308
 pkgs/argument_parser/case_convert.py,sha256=NuJLJUJRbyVb6_Slen4uqaStEHbcOS1d-hBBfDrrw-c,605
-pkgs/filesystem_utils/__init__.py,sha256=yxfwtYFvqq_fjMl-tg2jwa6eNPNZUF4ykqsSzALyNdw,1143
+pkgs/filesystem_utils/__init__.py,sha256=NSsQrUCoGISBCqCCyq6_583sYHTVEQeDjDO8hvZn3ag,1261
 pkgs/filesystem_utils/_gdrive_session.py,sha256=OZudNoP2HikolnpurVJhJdh5fgzqbaZQvn53ReGGXx4,11015
 pkgs/filesystem_utils/_local_session.py,sha256=xFEYhAvNqrOYqwt4jrEYOuYkjJn0zclZhTelW_Q1-rw,2325
+pkgs/filesystem_utils/_s3_session.py,sha256=q4q0MTWXWu5RNRVZ5ibv4M4UXXxWl_J6xCnitvngIMM,3957
 pkgs/filesystem_utils/_sftp_session.py,sha256=gNoUD_b4MuVqWj31nU-FpfpXZlyWkwdEHtX1S8W6gpQ,4727
-pkgs/filesystem_utils/file_type_utils.py,sha256=CC2c7127TE84vlRTy7bpE4Qvy7ytxuRNA3dwP91PXA8,1257
+pkgs/filesystem_utils/file_type_utils.py,sha256=Xd-mg35mAENUgNJVz5uK8nEfrUp-NQld_gnXFEq3K-8,1487
 pkgs/filesystem_utils/filesystem_session.py,sha256=BQ2Go8Mu9-GcnaWh2Pm4x7ugLVsres6XrOQ8RoiEpcE,1045
 pkgs/serialization/__init__.py,sha256=LifasRW0a50A3qRFmo2bf3FQ6TXhZWOTz2-CVTgPjcQ,753
 pkgs/serialization/missing_sentry.py,sha256=aM_9KxbCk9dVvXvcOKgkIQBqFWvLhv8QlIUCiuFEXMo,806
@@ -86,13 +87,13 @@ uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=axGpIfF0IMHilGzFawDizWoDhYffeqYXuhfyYkQhWF4,8754
+uncountable/integration/executors/generic_upload_executor.py,sha256=HR5bZE1DMVLRB-lwmsfZnoLEsaL66aHZM0xCEO_D3-4,10238
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=M0qXVJpD8hMYIFypHFeyh598sqmIDX8ZOyXK23CluF0,1323
 uncountable/types/__init__.py,sha256=95iOd3WXWoI_4a461IS2ieWRic3zRyNaCYzfTpX764o,8162
 uncountable/types/async_batch.py,sha256=ihCv5XWSTTPmuO-GMPn1EACGI2CBUIJTATZ3aPgsNBA,523
-uncountable/types/async_batch_processor.py,sha256=BNEWdRIcuJHGeY4KVCudf4gG-kV6_dZR7aeSWupCPjk,8581
+uncountable/types/async_batch_processor.py,sha256=VO0P-oSLAvzBK_Rk-KPS74AhSyrkj5ILZrAu-gs7Lpc,8564
 uncountable/types/async_batch_t.py,sha256=9jp9rOyetRdD5aQVyijzQggTyYU4021PBVGXk0ooBCQ,1911
 uncountable/types/base.py,sha256=xVSjWvA_fUUnkCg83EjoYEFvAfmskinKFMeYFOxNc9E,359
 uncountable/types/base_t.py,sha256=XXjZXexx0xWFUxMMhW8i9nIL6n8dsZVsHwdgnhZ0zJ4,2714
@@ -100,7 +101,7 @@ uncountable/types/calculations.py,sha256=FFO_D3BbKoGDZnqWvTKpW4KF359i2vrKjpdFCLY
 uncountable/types/calculations_t.py,sha256=7GTSi2L8NYjzjUJJx3cmtVkK9uD-uhfYvIFK-ffQj-8,556
 uncountable/types/chemical_structure.py,sha256=E-LnikTFDoVQ1b2zKaVUIO_PAKm-7aZZYJi8I8SDSic,302
 uncountable/types/chemical_structure_t.py,sha256=aFsTkkbzy6Gvyde3qrrEYD95gcYhxkgKMiDRaRE0o-Y,760
-uncountable/types/client_base.py,sha256=foD_tf3-FniOHIDO7AdPpPr5FdIDhUNVceXlyo-pA2M,65257
+uncountable/types/client_base.py,sha256=3CKr_v3bXez7ZjUg41q09pju5tgYGMsbgJAoYMffh-c,65240
 uncountable/types/client_config.py,sha256=4h5Liko9uKCo9_0gdbPhoK6Jr2Kv7tioLiQ8iKeq-_4,301
 uncountable/types/client_config_t.py,sha256=_HdS37gMSTIiD4qLnW9dIgt8_Rt5A6xhwMGGga7vnLg,625
 uncountable/types/curves.py,sha256=W6uMpG5SyW1MS82szNpxkFEn1MnxNpBFyFbQb2Ysfng,366
@@ -114,7 +115,7 @@ uncountable/types/field_values_t.py,sha256=_JYzmHlPczEnROz_tUbY0r3mZWffCB8m2_a9k
 uncountable/types/fields.py,sha256=GUY5ne8Zp2_Lalikr0zcbdJrin8dG81eyS8fKWJ9yf8,266
 uncountable/types/fields_t.py,sha256=9pDroeNvj4hQQ4QB4-2ioTAFR30YTuW_364Q-Ka9SoQ,564
 uncountable/types/generic_upload.py,sha256=hbtuOVMJhCRfPQM_BwtLMV509--iv8NXjtlTfCeRqdU,395
-uncountable/types/generic_upload_t.py,sha256=3Sv6oddGthbZXT77x_Soy8CClysAbNZ4McUpoazIKA4,1237
+uncountable/types/generic_upload_t.py,sha256=euOF_oFwT62a---oBjZeGVvU67TpXh8da9njrhlg46w,1202
 uncountable/types/id_source.py,sha256=wGLA0fMl-5IqBG_fg2UDC7fY-8CWRBNFJOokejOoN_w,567
 uncountable/types/id_source_t.py,sha256=ReOuIHIuFbkE2mQ58x6UP4jCV4_JUxMCvAU9bxUxG84,1369
 uncountable/types/identifier.py,sha256=6ziONd__L07ijhVwpIehUUDvxgKTtHbunk-6CivJqxQ,503
@@ -123,8 +124,8 @@ uncountable/types/input_attributes.py,sha256=IrIKQnHqHdS1Utdfzr9GnOe17a8riaqYcO1
 uncountable/types/input_attributes_t.py,sha256=wE1ekiQfb72Z9VpF5SHipKJkgaJFUHJrNkkJdmuhF9w,820
 uncountable/types/inputs.py,sha256=6RIEFfCxLqpeHEGOpu63O4i8zPogjGeB7wiV_rPBw_g,404
 uncountable/types/inputs_t.py,sha256=RW7gF9zTOwByu-nMTMVuBabLOuWKx4O1nvfgvx_R55o,1611
-uncountable/types/job_definition.py,sha256=W2wth9vPRBVL0temwnSmvfxxPMKFxZqbQtsMXn8BU3U,1637
-uncountable/types/job_definition_t.py,sha256=1VZO3l5oGnvtG-ikQB6YuukblHhEwtmJfQpGmo0Gi40,5735
+uncountable/types/job_definition.py,sha256=sCQqtyHI3hksc5pVpk5tqbG55F91ST4FoDwr2TmTOuQ,1787
+uncountable/types/job_definition_t.py,sha256=DpqTYDocJ6n6kVq5PBFhFb5Dlb2vpHdp9IZfFb1Qcpo,6590
 uncountable/types/outputs.py,sha256=sUZx_X-TKCZtLm1YCEH8OISX9DdPlv9ZuUfM3-askCc,281
 uncountable/types/outputs_t.py,sha256=2aORUOr0ls1ZYo-ddkWax3D1ZndmQsWtHfJxpYozlhg,656
 uncountable/types/permissions.py,sha256=1mRnSsmRgjuLgp6pylTwwACD_YRIcmlqxHkufwZtMns,297
@@ -236,8 +237,8 @@ uncountable/types/api/recipes/unlock_recipes.py,sha256=AvzQeZCLs9i7CuhMs3Xltdi4n
 uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/uploader/invoke_uploader.py,sha256=2RCvnccwEm6J5dt7UJVLUKxfk49vfpGXjBLCWIBD3Ck,1006
-UncountablePythonSDK-0.0.47.dist-info/METADATA,sha256=GNA-6K_SnpEbHAp-TfhxvSwjxMjceKun1_M3twl0aW8,1707
-UncountablePythonSDK-0.0.47.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-UncountablePythonSDK-0.0.47.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.47.dist-info/RECORD,,
+uncountable/types/api/uploader/invoke_uploader.py,sha256=w7NVSpTpKxsF2TgLzVrji1Ql5Z8QWTz6d5OvXpRtyDo,992
+UncountablePythonSDK-0.0.49.dist-info/METADATA,sha256=YzHISP7OjFmFRJWuVPWl6jgdlvsE6W7glCma0mhU7sc,1734
+UncountablePythonSDK-0.0.49.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+UncountablePythonSDK-0.0.49.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.49.dist-info/RECORD,,

examples/invoke_uploader.py

@@ -10,14 +10,14 @@ client = Client(
         api_secret_key=os.environ["UNC_API_SECRET_KEY"],
     ),
 )
-uploaded = client.upload_files(
+uploaded_file = client.upload_files(
     file_uploads=[
         MediaFileUpload(path="~/Downloads/my_file_to_upload.csv"),
     ]
-)
+)[0]
 
 client.invoke_uploader(
-    file_ids=[file.file_id for file in uploaded],
+    file_id=uploaded_file.file_id,
     uploader_key=IdentifierKeyId(id=48),
-    material_family_keys=[IdentifierKeyId(id=1)],
+    material_family_key=IdentifierKeyId(id=1),
 )

pkgs/filesystem_utils/__init__.py

@@ -5,12 +5,14 @@ from ._gdrive_session import list_gdrive_files as list_gdrive_files
 from ._gdrive_session import move_gdrive_file as move_gdrive_file
 from ._gdrive_session import upload_file_gdrive as upload_file_gdrive
 from ._local_session import LocalSession as LocalSession
+from ._s3_session import S3Session as S3Session
 from ._sftp_session import SFTPSession as SFTPSession
 from ._sftp_session import list_sftp_files as list_sftp_files
 from ._sftp_session import move_sftp_files as move_sftp_files
 from .file_type_utils import FileObjectData as FileObjectData
 from .file_type_utils import FileSystemFileReference as FileSystemFileReference
 from .file_type_utils import FileSystemObject as FileSystemObject
+from .file_type_utils import FileSystemS3Config as FileSystemS3Config
 from .file_type_utils import FileSystemSFTPConfig as FileSystemSFTPConfig
 from .file_type_utils import FileTransfer as FileTransfer
 from .file_type_utils import IncompatibleFileReference as IncompatibleFileReference

pkgs/filesystem_utils/_s3_session.py

@@ -0,0 +1,116 @@
+from io import BytesIO
+
+from boto3.session import Session
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileSystemS3Config,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+def _add_slash(prefix: str) -> str:
+    if len(prefix) > 0 and prefix[-1] != "/":
+        prefix = prefix + "/"
+    return prefix
+
+
+class S3Session(FileSystemSession):
+    config: FileSystemS3Config
+
+    def __init__(self, s3_config: FileSystemS3Config) -> None:
+        super().__init__()
+        self.config = s3_config
+
+    def start(self) -> None:
+        session = Session(region_name=self.config.region_name)
+        s3_resource = session.resource(
+            service_name="s3",
+            endpoint_url=self.config.endpoint_url,
+            aws_access_key_id=self.config.access_key_id,
+            aws_secret_access_key=self.config.secret_access_key,
+            aws_session_token=self.config.session_token,
+        )
+
+        self.bucket = s3_resource.Bucket(self.config.bucket_name)
+
+    def __enter__(self) -> "S3Session":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.bucket = None
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = False,
+        valid_extensions: list[str] | None = None,
+    ) -> list[FileSystemObject]:
+        if recursive:
+            raise NotImplementedError("recursive file listings not implemented for s3")
+
+        if not isinstance(dir_path, FileSystemFileReference):
+            raise IncompatibleFileReference()
+
+        assert self.bucket is not None, "call to list_files on uninitialized s3 session"
+
+        filesystem_references: list[FileSystemObject] = []
+        for obj in self.bucket.objects.filter(Prefix=dir_path.filepath):
+            if valid_extensions is None or any(
+                obj.key.endswith(valid_extension) for valid_extension in valid_extensions
+            ):
+                filesystem_references.append(FileSystemFileReference(obj.key))
+
+        return filesystem_references
+
+    def download_files(
+        self,
+        filepaths: list[FileSystemObject],
+    ) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        assert (
+            self.bucket is not None
+        ), "call to download_files on uninitialized s3 session"
+
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+            s3_file_obj = self.bucket.Object(_add_slash(file_object.filepath))
+            response = s3_file_obj.get()
+            file_obj_bytes = response["Body"].read()
+            downloaded_files.append(
+                FileObjectData(
+                    file_data=file_obj_bytes,
+                    file_IO=BytesIO(file_obj_bytes),
+                    filename=file_object.filename,
+                    filepath=file_object.filepath,
+                )
+            )
+
+        return downloaded_files
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        assert self.bucket is not None, "call to move_files on uninitialized s3 session"
+
+        for src_file, dest_file in file_mappings:
+            if not isinstance(src_file, FileSystemFileReference) or not isinstance(
+                dest_file, FileSystemFileReference
+            ):
+                raise IncompatibleFileReference()
+            self.bucket.Object(_add_slash(dest_file.filepath)).copy_from(
+                CopySource={
+                    "Bucket": self.bucket.name,
+                    "Key": _add_slash(src_file.filepath),
+                }
+            )
+            self.bucket.Object(_add_slash(src_file.filepath)).delete()

pkgs/filesystem_utils/file_type_utils.py

@@ -59,3 +59,13 @@ class FileSystemSFTPConfig:
     password: str | None = None
     valid_extensions: Optional[tuple[str]] = None
     recursive: bool = True
+
+
+@dataclass(kw_only=True)
+class FileSystemS3Config:
+    endpoint_url: str
+    bucket_name: str
+    region_name: Optional[str]
+    access_key_id: Optional[str]
+    secret_access_key: Optional[str]
+    session_token: Optional[str]

uncountable/integration/executors/generic_upload_executor.py

@@ -9,12 +9,13 @@ from pkgs.filesystem_utils import (
     FileObjectData,
     FileSystemFileReference,
     FileSystemObject,
+    FileSystemS3Config,
     FileSystemSFTPConfig,
     FileTransfer,
+    S3Session,
     SFTPSession,
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
-from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.file_upload import DataFileUpload, FileUpload
 from uncountable.integration.job import Job, JobArguments, JobLogger
 from uncountable.integration.secret_retrieval import retrieve_secret
@@ -24,8 +25,10 @@ from uncountable.types.generic_upload_t import (
 )
 from uncountable.types.job_definition_t import (
     GenericUploadDataSource,
+    GenericUploadDataSourceS3,
     GenericUploadDataSourceSFTP,
     JobResult,
+    S3CloudProvider,
 )
 
 
@@ -186,12 +189,48 @@ class GenericUploadJob(Job):
                     pem_key=pem_key,
                 )
                 return SFTPSession(sftp_config=sftp_config)
+            case GenericUploadDataSourceS3():
+                if self.data_source.access_key_secret is not None:
+                    secret_access_key = retrieve_secret(
+                        self.data_source.access_key_secret,
+                        profile_metadata=args.profile_metadata,
+                    )
+                else:
+                    secret_access_key = None
+
+                if self.data_source.endpoint_url is None:
+                    assert (
+                        self.data_source.cloud_provider is not None
+                    ), "either cloud_provider or endpoint_url must be specified"
+                    match self.data_source.cloud_provider:
+                        case S3CloudProvider.AWS:
+                            endpoint_url = "https://s3.amazonaws.com"
+                        case S3CloudProvider.OVH:
+                            assert (
+                                self.data_source.region_name is not None
+                            ), "region_name must be specified for cloud_provider OVH"
+                            endpoint_url = (
+                                f"https://s3.{self.data_source.region_name}.cloud.ovh.net"
+                            )
+                else:
+                    endpoint_url = self.data_source.endpoint_url
+
+                s3_config = FileSystemS3Config(
+                    endpoint_url=endpoint_url,
+                    bucket_name=self.data_source.bucket_name,
+                    region_name=self.data_source.region_name,
+                    access_key_id=self.data_source.access_key_id,
+                    secret_access_key=secret_access_key,
+                    session_token=None,
+                )
+
+                return S3Session(s3_config=s3_config)
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client
+        batch_processor = args.batch_processor
         logger = args.logger
 
-        batch_executor = AsyncBatchProcessor(client=client)
         with self._construct_filesystem_session(args) as filesystem_session:
             files_to_upload: list[FileUpload] = []
             for remote_directory in self.remote_directories:
@@ -227,20 +266,12 @@ class GenericUploadJob(Job):
 
         file_ids = [file.file_id for file in uploaded_files]
 
-        if self.upload_strategy.parse_files_individually:
+        for material_family_key in self.upload_strategy.material_family_keys:
             for file_id in file_ids:
-                batch_executor.invoke_uploader(
-                    file_ids=[file_id],
+                batch_processor.invoke_uploader(
+                    file_id=file_id,
                     uploader_key=self.upload_strategy.uploader_key,
-                    material_family_keys=self.upload_strategy.material_family_keys,
+                    material_family_key=material_family_key,
                 )
-        else:
-            batch_executor.invoke_uploader(
-                file_ids=file_ids,
-                uploader_key=self.upload_strategy.uploader_key,
-                material_family_keys=self.upload_strategy.material_family_keys,
-            )
-
-        batch_executor.send()
 
         return JobResult(success=True)

uncountable/types/api/uploader/invoke_uploader.py

@@ -26,9 +26,9 @@ ENDPOINT_PATH = "api/external/uploader/invoke_uploader"
 # DO NOT MODIFY -- This file is generated by type_spec
 @dataclasses.dataclass(kw_only=True)
 class Arguments:
-    file_ids: list[base_t.ObjectId]
+    file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
-    material_family_keys: list[identifier_t.IdentifierKey]
+    material_family_key: identifier_t.IdentifierKey
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/async_batch_processor.py

@@ -160,19 +160,19 @@ class AsyncBatchProcessorBase(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         :param depends_on: A list of batch reference keys to process before processing this request
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         json_data = serialize_for_api(args)
 

uncountable/types/client_base.py

@@ -854,17 +854,17 @@ class ClientMethods(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
     ) -> invoke_uploader_t.Data:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

uncountable/types/generic_upload_t.py

@@ -36,6 +36,5 @@ class GenericRemoteDirectoryScope:
 class GenericUploadStrategy:
     uploader_key: identifier_t.IdentifierKey
     material_family_keys: list[identifier_t.IdentifierKey]
-    parse_files_individually: bool
     skip_moving_files: bool = False
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/job_definition.py

@@ -12,6 +12,8 @@ from .job_definition_t import JobExecutorBase as JobExecutorBase
 from .job_definition_t import JobExecutorScript as JobExecutorScript
 from .job_definition_t import GenericUploadDataSourceBase as GenericUploadDataSourceBase
 from .job_definition_t import GenericUploadDataSourceSFTP as GenericUploadDataSourceSFTP
+from .job_definition_t import S3CloudProvider as S3CloudProvider
+from .job_definition_t import GenericUploadDataSourceS3 as GenericUploadDataSourceS3
 from .job_definition_t import GenericUploadDataSource as GenericUploadDataSource
 from .job_definition_t import JobExecutorGenericUpload as JobExecutorGenericUpload
 from .job_definition_t import JobExecutor as JobExecutor

uncountable/types/job_definition_t.py

@@ -23,6 +23,7 @@ __all__: list[str] = [
     "CronJobDefinition",
     "GenericUploadDataSource",
     "GenericUploadDataSourceBase",
+    "GenericUploadDataSourceS3",
     "GenericUploadDataSourceSFTP",
     "GenericUploadDataSourceType",
     "JobDefinition",
@@ -36,6 +37,7 @@ __all__: list[str] = [
     "JobResult",
     "ProfileDefinition",
     "ProfileMetadata",
+    "S3CloudProvider",
 ]
 
 
@@ -58,6 +60,7 @@ class AuthRetrievalType(StrEnum):
 # DO NOT MODIFY -- This file is generated by type_spec
 class GenericUploadDataSourceType(StrEnum):
     SFTP = "sftp"
+    S3 = "s3"
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -94,13 +97,35 @@ class GenericUploadDataSourceSFTP(GenericUploadDataSourceBase):
     pem_secret: secret_retrieval_t.SecretRetrieval
 
 
+# DO NOT MODIFY -- This file is generated by type_spec
+class S3CloudProvider(StrEnum):
+    OVH = "ovh"
+    AWS = "aws"
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class GenericUploadDataSourceS3(GenericUploadDataSourceBase):
+    type: typing.Literal[GenericUploadDataSourceType.S3] = GenericUploadDataSourceType.S3
+    bucket_name: str
+    cloud_provider: typing.Optional[S3CloudProvider] = None
+    endpoint_url: typing.Optional[str] = None
+    region_name: typing.Optional[str] = None
+    access_key_id: typing.Optional[str] = None
+    access_key_secret: typing.Optional[secret_retrieval_t.SecretRetrieval] = None
+
+
 # DO NOT MODIFY -- This file is generated by type_spec
 GenericUploadDataSource = typing.Annotated[
-    typing.Union[GenericUploadDataSourceSFTP],
+    typing.Union[GenericUploadDataSourceSFTP, GenericUploadDataSourceS3],
     serial_union_annotation(
         discriminator="type",
         discriminator_map={
             "sftp": GenericUploadDataSourceSFTP,
+            "s3": GenericUploadDataSourceS3,
         },
     ),
 ]