UncountablePythonSDK 0.0.47__tar.gz → 0.0.49__tar.gz

{uncountablepythonsdk-0.0.47/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.47
+Version: 0.0.49
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -28,6 +28,7 @@ Requires-Dist: google-api-python-client==2.*
 Requires-Dist: tqdm==4.*
 Requires-Dist: pysftp==0.*
 Requires-Dist: paramiko==3.*
+Requires-Dist: boto3==1.*
 Provides-Extra: test
 Requires-Dist: mypy==1.*; extra == "test"
 Requires-Dist: ruff==0.*; extra == "test"

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.47
+Version: 0.0.49
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -28,6 +28,7 @@ Requires-Dist: google-api-python-client==2.*
 Requires-Dist: tqdm==4.*
 Requires-Dist: pysftp==0.*
 Requires-Dist: paramiko==3.*
+Requires-Dist: boto3==1.*
 Provides-Extra: test
 Requires-Dist: mypy==1.*; extra == "test"
 Requires-Dist: ruff==0.*; extra == "test"

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ pkgs/argument_parser/case_convert.py
 pkgs/filesystem_utils/__init__.py
 pkgs/filesystem_utils/_gdrive_session.py
 pkgs/filesystem_utils/_local_session.py
+pkgs/filesystem_utils/_s3_session.py
 pkgs/filesystem_utils/_sftp_session.py
 pkgs/filesystem_utils/file_type_utils.py
 pkgs/filesystem_utils/filesystem_session.py

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/UncountablePythonSDK.egg-info/requires.txt

@@ -10,6 +10,7 @@ google-api-python-client==2.*
 tqdm==4.*
 pysftp==0.*
 paramiko==3.*
+boto3==1.*
 
 [test]
 mypy==1.*

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/examples/invoke_uploader.py

@@ -10,14 +10,14 @@ client = Client(
         api_secret_key=os.environ["UNC_API_SECRET_KEY"],
     ),
 )
-uploaded = client.upload_files(
+uploaded_file = client.upload_files(
     file_uploads=[
         MediaFileUpload(path="~/Downloads/my_file_to_upload.csv"),
     ]
-)
+)[0]
 
 client.invoke_uploader(
-    file_ids=[file.file_id for file in uploaded],
+    file_id=uploaded_file.file_id,
     uploader_key=IdentifierKeyId(id=48),
-    material_family_keys=[IdentifierKeyId(id=1)],
+    material_family_key=IdentifierKeyId(id=1),
 )

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/pkgs/filesystem_utils/__init__.py

@@ -5,12 +5,14 @@ from ._gdrive_session import list_gdrive_files as list_gdrive_files
 from ._gdrive_session import move_gdrive_file as move_gdrive_file
 from ._gdrive_session import upload_file_gdrive as upload_file_gdrive
 from ._local_session import LocalSession as LocalSession
+from ._s3_session import S3Session as S3Session
 from ._sftp_session import SFTPSession as SFTPSession
 from ._sftp_session import list_sftp_files as list_sftp_files
 from ._sftp_session import move_sftp_files as move_sftp_files
 from .file_type_utils import FileObjectData as FileObjectData
 from .file_type_utils import FileSystemFileReference as FileSystemFileReference
 from .file_type_utils import FileSystemObject as FileSystemObject
+from .file_type_utils import FileSystemS3Config as FileSystemS3Config
 from .file_type_utils import FileSystemSFTPConfig as FileSystemSFTPConfig
 from .file_type_utils import FileTransfer as FileTransfer
 from .file_type_utils import IncompatibleFileReference as IncompatibleFileReference

uncountablepythonsdk-0.0.49/pkgs/filesystem_utils/_s3_session.py

@@ -0,0 +1,116 @@
+from io import BytesIO
+
+from boto3.session import Session
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileSystemS3Config,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+def _add_slash(prefix: str) -> str:
+    if len(prefix) > 0 and prefix[-1] != "/":
+        prefix = prefix + "/"
+    return prefix
+
+
+class S3Session(FileSystemSession):
+    config: FileSystemS3Config
+
+    def __init__(self, s3_config: FileSystemS3Config) -> None:
+        super().__init__()
+        self.config = s3_config
+
+    def start(self) -> None:
+        session = Session(region_name=self.config.region_name)
+        s3_resource = session.resource(
+            service_name="s3",
+            endpoint_url=self.config.endpoint_url,
+            aws_access_key_id=self.config.access_key_id,
+            aws_secret_access_key=self.config.secret_access_key,
+            aws_session_token=self.config.session_token,
+        )
+
+        self.bucket = s3_resource.Bucket(self.config.bucket_name)
+
+    def __enter__(self) -> "S3Session":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.bucket = None
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = False,
+        valid_extensions: list[str] | None = None,
+    ) -> list[FileSystemObject]:
+        if recursive:
+            raise NotImplementedError("recursive file listings not implemented for s3")
+
+        if not isinstance(dir_path, FileSystemFileReference):
+            raise IncompatibleFileReference()
+
+        assert self.bucket is not None, "call to list_files on uninitialized s3 session"
+
+        filesystem_references: list[FileSystemObject] = []
+        for obj in self.bucket.objects.filter(Prefix=dir_path.filepath):
+            if valid_extensions is None or any(
+                obj.key.endswith(valid_extension) for valid_extension in valid_extensions
+            ):
+                filesystem_references.append(FileSystemFileReference(obj.key))
+
+        return filesystem_references
+
+    def download_files(
+        self,
+        filepaths: list[FileSystemObject],
+    ) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        assert (
+            self.bucket is not None
+        ), "call to download_files on uninitialized s3 session"
+
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+            s3_file_obj = self.bucket.Object(_add_slash(file_object.filepath))
+            response = s3_file_obj.get()
+            file_obj_bytes = response["Body"].read()
+            downloaded_files.append(
+                FileObjectData(
+                    file_data=file_obj_bytes,
+                    file_IO=BytesIO(file_obj_bytes),
+                    filename=file_object.filename,
+                    filepath=file_object.filepath,
+                )
+            )
+
+        return downloaded_files
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        assert self.bucket is not None, "call to move_files on uninitialized s3 session"
+
+        for src_file, dest_file in file_mappings:
+            if not isinstance(src_file, FileSystemFileReference) or not isinstance(
+                dest_file, FileSystemFileReference
+            ):
+                raise IncompatibleFileReference()
+            self.bucket.Object(_add_slash(dest_file.filepath)).copy_from(
+                CopySource={
+                    "Bucket": self.bucket.name,
+                    "Key": _add_slash(src_file.filepath),
+                }
+            )
+            self.bucket.Object(_add_slash(src_file.filepath)).delete()

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/pkgs/filesystem_utils/file_type_utils.py

@@ -59,3 +59,13 @@ class FileSystemSFTPConfig:
     password: str | None = None
     valid_extensions: Optional[tuple[str]] = None
     recursive: bool = True
+
+
+@dataclass(kw_only=True)
+class FileSystemS3Config:
+    endpoint_url: str
+    bucket_name: str
+    region_name: Optional[str]
+    access_key_id: Optional[str]
+    secret_access_key: Optional[str]
+    session_token: Optional[str]

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/pyproject.toml

@@ -59,7 +59,8 @@ dependencies = [
     "google-api-python-client == 2.*",
     "tqdm == 4.*",
     "pysftp == 0.*",
-    "paramiko == 3.*"
+    "paramiko == 3.*",
+    "boto3 == 1.*"
 ]
 dynamic = ["version"]
 

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/integration/executors/generic_upload_executor.py

@@ -9,12 +9,13 @@ from pkgs.filesystem_utils import (
     FileObjectData,
     FileSystemFileReference,
     FileSystemObject,
+    FileSystemS3Config,
     FileSystemSFTPConfig,
     FileTransfer,
+    S3Session,
     SFTPSession,
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
-from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.file_upload import DataFileUpload, FileUpload
 from uncountable.integration.job import Job, JobArguments, JobLogger
 from uncountable.integration.secret_retrieval import retrieve_secret
@@ -24,8 +25,10 @@ from uncountable.types.generic_upload_t import (
 )
 from uncountable.types.job_definition_t import (
     GenericUploadDataSource,
+    GenericUploadDataSourceS3,
     GenericUploadDataSourceSFTP,
     JobResult,
+    S3CloudProvider,
 )
 
 
@@ -186,12 +189,48 @@ class GenericUploadJob(Job):
                     pem_key=pem_key,
                 )
                 return SFTPSession(sftp_config=sftp_config)
+            case GenericUploadDataSourceS3():
+                if self.data_source.access_key_secret is not None:
+                    secret_access_key = retrieve_secret(
+                        self.data_source.access_key_secret,
+                        profile_metadata=args.profile_metadata,
+                    )
+                else:
+                    secret_access_key = None
+
+                if self.data_source.endpoint_url is None:
+                    assert (
+                        self.data_source.cloud_provider is not None
+                    ), "either cloud_provider or endpoint_url must be specified"
+                    match self.data_source.cloud_provider:
+                        case S3CloudProvider.AWS:
+                            endpoint_url = "https://s3.amazonaws.com"
+                        case S3CloudProvider.OVH:
+                            assert (
+                                self.data_source.region_name is not None
+                            ), "region_name must be specified for cloud_provider OVH"
+                            endpoint_url = (
+                                f"https://s3.{self.data_source.region_name}.cloud.ovh.net"
+                            )
+                else:
+                    endpoint_url = self.data_source.endpoint_url
+
+                s3_config = FileSystemS3Config(
+                    endpoint_url=endpoint_url,
+                    bucket_name=self.data_source.bucket_name,
+                    region_name=self.data_source.region_name,
+                    access_key_id=self.data_source.access_key_id,
+                    secret_access_key=secret_access_key,
+                    session_token=None,
+                )
+
+                return S3Session(s3_config=s3_config)
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client
+        batch_processor = args.batch_processor
         logger = args.logger
 
-        batch_executor = AsyncBatchProcessor(client=client)
         with self._construct_filesystem_session(args) as filesystem_session:
             files_to_upload: list[FileUpload] = []
             for remote_directory in self.remote_directories:
@@ -227,20 +266,12 @@ class GenericUploadJob(Job):
 
         file_ids = [file.file_id for file in uploaded_files]
 
-        if self.upload_strategy.parse_files_individually:
+        for material_family_key in self.upload_strategy.material_family_keys:
             for file_id in file_ids:
-                batch_executor.invoke_uploader(
-                    file_ids=[file_id],
+                batch_processor.invoke_uploader(
+                    file_id=file_id,
                     uploader_key=self.upload_strategy.uploader_key,
-                    material_family_keys=self.upload_strategy.material_family_keys,
+                    material_family_key=material_family_key,
                 )
-        else:
-            batch_executor.invoke_uploader(
-                file_ids=file_ids,
-                uploader_key=self.upload_strategy.uploader_key,
-                material_family_keys=self.upload_strategy.material_family_keys,
-            )
-
-        batch_executor.send()
 
         return JobResult(success=True)

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/api/uploader/invoke_uploader.py

@@ -26,9 +26,9 @@ ENDPOINT_PATH = "api/external/uploader/invoke_uploader"
 # DO NOT MODIFY -- This file is generated by type_spec
 @dataclasses.dataclass(kw_only=True)
 class Arguments:
-    file_ids: list[base_t.ObjectId]
+    file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
-    material_family_keys: list[identifier_t.IdentifierKey]
+    material_family_key: identifier_t.IdentifierKey
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/async_batch_processor.py

@@ -160,19 +160,19 @@ class AsyncBatchProcessorBase(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         :param depends_on: A list of batch reference keys to process before processing this request
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         json_data = serialize_for_api(args)
 

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/client_base.py

@@ -854,17 +854,17 @@ class ClientMethods(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
     ) -> invoke_uploader_t.Data:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/generic_upload_t.py

@@ -36,6 +36,5 @@ class GenericRemoteDirectoryScope:
 class GenericUploadStrategy:
     uploader_key: identifier_t.IdentifierKey
     material_family_keys: list[identifier_t.IdentifierKey]
-    parse_files_individually: bool
     skip_moving_files: bool = False
 # DO NOT MODIFY -- This file is generated by type_spec

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/job_definition.py

@@ -12,6 +12,8 @@ from .job_definition_t import JobExecutorBase as JobExecutorBase
 from .job_definition_t import JobExecutorScript as JobExecutorScript
 from .job_definition_t import GenericUploadDataSourceBase as GenericUploadDataSourceBase
 from .job_definition_t import GenericUploadDataSourceSFTP as GenericUploadDataSourceSFTP
+from .job_definition_t import S3CloudProvider as S3CloudProvider
+from .job_definition_t import GenericUploadDataSourceS3 as GenericUploadDataSourceS3
 from .job_definition_t import GenericUploadDataSource as GenericUploadDataSource
 from .job_definition_t import JobExecutorGenericUpload as JobExecutorGenericUpload
 from .job_definition_t import JobExecutor as JobExecutor

{uncountablepythonsdk-0.0.47 → uncountablepythonsdk-0.0.49}/uncountable/types/job_definition_t.py

@@ -23,6 +23,7 @@ __all__: list[str] = [
     "CronJobDefinition",
     "GenericUploadDataSource",
     "GenericUploadDataSourceBase",
+    "GenericUploadDataSourceS3",
     "GenericUploadDataSourceSFTP",
     "GenericUploadDataSourceType",
     "JobDefinition",
@@ -36,6 +37,7 @@ __all__: list[str] = [
     "JobResult",
     "ProfileDefinition",
     "ProfileMetadata",
+    "S3CloudProvider",
 ]
 
 
@@ -58,6 +60,7 @@ class AuthRetrievalType(StrEnum):
 # DO NOT MODIFY -- This file is generated by type_spec
 class GenericUploadDataSourceType(StrEnum):
     SFTP = "sftp"
+    S3 = "s3"
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -94,13 +97,35 @@ class GenericUploadDataSourceSFTP(GenericUploadDataSourceBase):
     pem_secret: secret_retrieval_t.SecretRetrieval
 
 
+# DO NOT MODIFY -- This file is generated by type_spec
+class S3CloudProvider(StrEnum):
+    OVH = "ovh"
+    AWS = "aws"
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class GenericUploadDataSourceS3(GenericUploadDataSourceBase):
+    type: typing.Literal[GenericUploadDataSourceType.S3] = GenericUploadDataSourceType.S3
+    bucket_name: str
+    cloud_provider: typing.Optional[S3CloudProvider] = None
+    endpoint_url: typing.Optional[str] = None
+    region_name: typing.Optional[str] = None
+    access_key_id: typing.Optional[str] = None
+    access_key_secret: typing.Optional[secret_retrieval_t.SecretRetrieval] = None
+
+
 # DO NOT MODIFY -- This file is generated by type_spec
 GenericUploadDataSource = typing.Annotated[
-    typing.Union[GenericUploadDataSourceSFTP],
+    typing.Union[GenericUploadDataSourceSFTP, GenericUploadDataSourceS3],
     serial_union_annotation(
         discriminator="type",
         discriminator_map={
             "sftp": GenericUploadDataSourceSFTP,
+            "s3": GenericUploadDataSourceS3,
         },
     ),
 ]