UncountablePythonSDK 0.0.47__py3-none-any.whl → 0.0.48__py3-none-any.whl

{UncountablePythonSDK-0.0.47.dist-info → UncountablePythonSDK-0.0.48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.47
+Version: 0.0.48
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -28,6 +28,7 @@ Requires-Dist: google-api-python-client ==2.*
 Requires-Dist: tqdm ==4.*
 Requires-Dist: pysftp ==0.*
 Requires-Dist: paramiko ==3.*
+Requires-Dist: boto3 ==1.*
 Provides-Extra: test
 Requires-Dist: mypy ==1.* ; extra == 'test'
 Requires-Dist: ruff ==0.* ; extra == 'test'

{UncountablePythonSDK-0.0.47.dist-info → UncountablePythonSDK-0.0.48.dist-info}/RECORD

@@ -17,7 +17,7 @@ docs/static/favicons/safari-pinned-tab.svg,sha256=S84fRnz0ZxLnQrKtmmFZytiRyu1xLt
 examples/async_batch.py,sha256=CffQ8O9drJ-Mdd6S5DnMIOBsHv5aVkTZrD3l3xBnB4s,1094
 examples/create_entity.py,sha256=noZdtJ5f9Wfiob3zUH-8bDVbrCPJnFtXFk_W9pSjvUA,664
 examples/edit_recipe_inputs.py,sha256=SFHhccktlpWlz4ujh30KZ-hZTFneRWuEp9b3nJSbNn8,1647
-examples/invoke_uploader.py,sha256=NOHsvObRGLrOYaSKtwk1xkhsdLEdbUmrseUX6JzC4Q0,635
+examples/invoke_uploader.py,sha256=Rn5sSZowW6_0yX6843q2HOSIECfK4cOCB1Hmco3iWVo,625
 examples/upload_files.py,sha256=tUfKFqiqwnw08OL5Y8_e4j5pSRhp94cFex8XTuVa_ig,487
 pkgs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pkgs/argument_parser/__init__.py,sha256=CsQ6QoPKSLLRVl-z6URAmPkiUL9ZPZoV4rJHgy_-RjA,385
@@ -28,8 +28,9 @@ pkgs/argument_parser/case_convert.py,sha256=NuJLJUJRbyVb6_Slen4uqaStEHbcOS1d-hBB
 pkgs/filesystem_utils/__init__.py,sha256=yxfwtYFvqq_fjMl-tg2jwa6eNPNZUF4ykqsSzALyNdw,1143
 pkgs/filesystem_utils/_gdrive_session.py,sha256=OZudNoP2HikolnpurVJhJdh5fgzqbaZQvn53ReGGXx4,11015
 pkgs/filesystem_utils/_local_session.py,sha256=xFEYhAvNqrOYqwt4jrEYOuYkjJn0zclZhTelW_Q1-rw,2325
+pkgs/filesystem_utils/_s3_session.py,sha256=q4q0MTWXWu5RNRVZ5ibv4M4UXXxWl_J6xCnitvngIMM,3957
 pkgs/filesystem_utils/_sftp_session.py,sha256=gNoUD_b4MuVqWj31nU-FpfpXZlyWkwdEHtX1S8W6gpQ,4727
-pkgs/filesystem_utils/file_type_utils.py,sha256=CC2c7127TE84vlRTy7bpE4Qvy7ytxuRNA3dwP91PXA8,1257
+pkgs/filesystem_utils/file_type_utils.py,sha256=Xd-mg35mAENUgNJVz5uK8nEfrUp-NQld_gnXFEq3K-8,1487
 pkgs/filesystem_utils/filesystem_session.py,sha256=BQ2Go8Mu9-GcnaWh2Pm4x7ugLVsres6XrOQ8RoiEpcE,1045
 pkgs/serialization/__init__.py,sha256=LifasRW0a50A3qRFmo2bf3FQ6TXhZWOTz2-CVTgPjcQ,753
 pkgs/serialization/missing_sentry.py,sha256=aM_9KxbCk9dVvXvcOKgkIQBqFWvLhv8QlIUCiuFEXMo,806
@@ -86,13 +87,13 @@ uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=axGpIfF0IMHilGzFawDizWoDhYffeqYXuhfyYkQhWF4,8754
+uncountable/integration/executors/generic_upload_executor.py,sha256=Ixxg01LLzc9pBpptBqJG6-jG82eltZTF7igFqORujyo,8393
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=M0qXVJpD8hMYIFypHFeyh598sqmIDX8ZOyXK23CluF0,1323
 uncountable/types/__init__.py,sha256=95iOd3WXWoI_4a461IS2ieWRic3zRyNaCYzfTpX764o,8162
 uncountable/types/async_batch.py,sha256=ihCv5XWSTTPmuO-GMPn1EACGI2CBUIJTATZ3aPgsNBA,523
-uncountable/types/async_batch_processor.py,sha256=BNEWdRIcuJHGeY4KVCudf4gG-kV6_dZR7aeSWupCPjk,8581
+uncountable/types/async_batch_processor.py,sha256=VO0P-oSLAvzBK_Rk-KPS74AhSyrkj5ILZrAu-gs7Lpc,8564
 uncountable/types/async_batch_t.py,sha256=9jp9rOyetRdD5aQVyijzQggTyYU4021PBVGXk0ooBCQ,1911
 uncountable/types/base.py,sha256=xVSjWvA_fUUnkCg83EjoYEFvAfmskinKFMeYFOxNc9E,359
 uncountable/types/base_t.py,sha256=XXjZXexx0xWFUxMMhW8i9nIL6n8dsZVsHwdgnhZ0zJ4,2714
@@ -100,7 +101,7 @@ uncountable/types/calculations.py,sha256=FFO_D3BbKoGDZnqWvTKpW4KF359i2vrKjpdFCLY
 uncountable/types/calculations_t.py,sha256=7GTSi2L8NYjzjUJJx3cmtVkK9uD-uhfYvIFK-ffQj-8,556
 uncountable/types/chemical_structure.py,sha256=E-LnikTFDoVQ1b2zKaVUIO_PAKm-7aZZYJi8I8SDSic,302
 uncountable/types/chemical_structure_t.py,sha256=aFsTkkbzy6Gvyde3qrrEYD95gcYhxkgKMiDRaRE0o-Y,760
-uncountable/types/client_base.py,sha256=foD_tf3-FniOHIDO7AdPpPr5FdIDhUNVceXlyo-pA2M,65257
+uncountable/types/client_base.py,sha256=3CKr_v3bXez7ZjUg41q09pju5tgYGMsbgJAoYMffh-c,65240
 uncountable/types/client_config.py,sha256=4h5Liko9uKCo9_0gdbPhoK6Jr2Kv7tioLiQ8iKeq-_4,301
 uncountable/types/client_config_t.py,sha256=_HdS37gMSTIiD4qLnW9dIgt8_Rt5A6xhwMGGga7vnLg,625
 uncountable/types/curves.py,sha256=W6uMpG5SyW1MS82szNpxkFEn1MnxNpBFyFbQb2Ysfng,366
@@ -114,7 +115,7 @@ uncountable/types/field_values_t.py,sha256=_JYzmHlPczEnROz_tUbY0r3mZWffCB8m2_a9k
 uncountable/types/fields.py,sha256=GUY5ne8Zp2_Lalikr0zcbdJrin8dG81eyS8fKWJ9yf8,266
 uncountable/types/fields_t.py,sha256=9pDroeNvj4hQQ4QB4-2ioTAFR30YTuW_364Q-Ka9SoQ,564
 uncountable/types/generic_upload.py,sha256=hbtuOVMJhCRfPQM_BwtLMV509--iv8NXjtlTfCeRqdU,395
-uncountable/types/generic_upload_t.py,sha256=3Sv6oddGthbZXT77x_Soy8CClysAbNZ4McUpoazIKA4,1237
+uncountable/types/generic_upload_t.py,sha256=euOF_oFwT62a---oBjZeGVvU67TpXh8da9njrhlg46w,1202
 uncountable/types/id_source.py,sha256=wGLA0fMl-5IqBG_fg2UDC7fY-8CWRBNFJOokejOoN_w,567
 uncountable/types/id_source_t.py,sha256=ReOuIHIuFbkE2mQ58x6UP4jCV4_JUxMCvAU9bxUxG84,1369
 uncountable/types/identifier.py,sha256=6ziONd__L07ijhVwpIehUUDvxgKTtHbunk-6CivJqxQ,503
@@ -236,8 +237,8 @@ uncountable/types/api/recipes/unlock_recipes.py,sha256=AvzQeZCLs9i7CuhMs3Xltdi4n
 uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/uploader/invoke_uploader.py,sha256=2RCvnccwEm6J5dt7UJVLUKxfk49vfpGXjBLCWIBD3Ck,1006
-UncountablePythonSDK-0.0.47.dist-info/METADATA,sha256=GNA-6K_SnpEbHAp-TfhxvSwjxMjceKun1_M3twl0aW8,1707
-UncountablePythonSDK-0.0.47.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-UncountablePythonSDK-0.0.47.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.47.dist-info/RECORD,,
+uncountable/types/api/uploader/invoke_uploader.py,sha256=w7NVSpTpKxsF2TgLzVrji1Ql5Z8QWTz6d5OvXpRtyDo,992
+UncountablePythonSDK-0.0.48.dist-info/METADATA,sha256=j72rX6737WefzshiwZNBrM8nqCM55gDs2DH6dH79C24,1734
+UncountablePythonSDK-0.0.48.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+UncountablePythonSDK-0.0.48.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.48.dist-info/RECORD,,

examples/invoke_uploader.py

@@ -10,14 +10,14 @@ client = Client(
         api_secret_key=os.environ["UNC_API_SECRET_KEY"],
     ),
 )
-uploaded = client.upload_files(
+uploaded_file = client.upload_files(
     file_uploads=[
         MediaFileUpload(path="~/Downloads/my_file_to_upload.csv"),
     ]
-)
+)[0]
 
 client.invoke_uploader(
-    file_ids=[file.file_id for file in uploaded],
+    file_id=uploaded_file.file_id,
     uploader_key=IdentifierKeyId(id=48),
-    material_family_keys=[IdentifierKeyId(id=1)],
+    material_family_key=IdentifierKeyId(id=1),
 )

pkgs/filesystem_utils/_s3_session.py

@@ -0,0 +1,116 @@
+from io import BytesIO
+
+from boto3.session import Session
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileSystemS3Config,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+def _add_slash(prefix: str) -> str:
+    if len(prefix) > 0 and prefix[-1] != "/":
+        prefix = prefix + "/"
+    return prefix
+
+
+class S3Session(FileSystemSession):
+    config: FileSystemS3Config
+
+    def __init__(self, s3_config: FileSystemS3Config) -> None:
+        super().__init__()
+        self.config = s3_config
+
+    def start(self) -> None:
+        session = Session(region_name=self.config.region_name)
+        s3_resource = session.resource(
+            service_name="s3",
+            endpoint_url=self.config.endpoint_url,
+            aws_access_key_id=self.config.access_key_id,
+            aws_secret_access_key=self.config.secret_access_key,
+            aws_session_token=self.config.session_token,
+        )
+
+        self.bucket = s3_resource.Bucket(self.config.bucket_name)
+
+    def __enter__(self) -> "S3Session":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.bucket = None
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = False,
+        valid_extensions: list[str] | None = None,
+    ) -> list[FileSystemObject]:
+        if recursive:
+            raise NotImplementedError("recursive file listings not implemented for s3")
+
+        if not isinstance(dir_path, FileSystemFileReference):
+            raise IncompatibleFileReference()
+
+        assert self.bucket is not None, "call to list_files on uninitialized s3 session"
+
+        filesystem_references: list[FileSystemObject] = []
+        for obj in self.bucket.objects.filter(Prefix=dir_path.filepath):
+            if valid_extensions is None or any(
+                obj.key.endswith(valid_extension) for valid_extension in valid_extensions
+            ):
+                filesystem_references.append(FileSystemFileReference(obj.key))
+
+        return filesystem_references
+
+    def download_files(
+        self,
+        filepaths: list[FileSystemObject],
+    ) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        assert (
+            self.bucket is not None
+        ), "call to download_files on uninitialized s3 session"
+
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+            s3_file_obj = self.bucket.Object(_add_slash(file_object.filepath))
+            response = s3_file_obj.get()
+            file_obj_bytes = response["Body"].read()
+            downloaded_files.append(
+                FileObjectData(
+                    file_data=file_obj_bytes,
+                    file_IO=BytesIO(file_obj_bytes),
+                    filename=file_object.filename,
+                    filepath=file_object.filepath,
+                )
+            )
+
+        return downloaded_files
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        assert self.bucket is not None, "call to move_files on uninitialized s3 session"
+
+        for src_file, dest_file in file_mappings:
+            if not isinstance(src_file, FileSystemFileReference) or not isinstance(
+                dest_file, FileSystemFileReference
+            ):
+                raise IncompatibleFileReference()
+            self.bucket.Object(_add_slash(dest_file.filepath)).copy_from(
+                CopySource={
+                    "Bucket": self.bucket.name,
+                    "Key": _add_slash(src_file.filepath),
+                }
+            )
+            self.bucket.Object(_add_slash(src_file.filepath)).delete()

pkgs/filesystem_utils/file_type_utils.py

@@ -59,3 +59,13 @@ class FileSystemSFTPConfig:
     password: str | None = None
     valid_extensions: Optional[tuple[str]] = None
     recursive: bool = True
+
+
+@dataclass(kw_only=True)
+class FileSystemS3Config:
+    endpoint_url: str
+    bucket_name: str
+    region_name: Optional[str]
+    access_key_id: Optional[str]
+    secret_access_key: Optional[str]
+    session_token: Optional[str]

uncountable/integration/executors/generic_upload_executor.py

@@ -14,7 +14,6 @@ from pkgs.filesystem_utils import (
     SFTPSession,
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
-from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.file_upload import DataFileUpload, FileUpload
 from uncountable.integration.job import Job, JobArguments, JobLogger
 from uncountable.integration.secret_retrieval import retrieve_secret
@@ -189,9 +188,9 @@ class GenericUploadJob(Job):
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client
+        batch_processor = args.batch_processor
         logger = args.logger
 
-        batch_executor = AsyncBatchProcessor(client=client)
         with self._construct_filesystem_session(args) as filesystem_session:
             files_to_upload: list[FileUpload] = []
             for remote_directory in self.remote_directories:
@@ -227,20 +226,12 @@ class GenericUploadJob(Job):
 
         file_ids = [file.file_id for file in uploaded_files]
 
-        if self.upload_strategy.parse_files_individually:
+        for material_family_key in self.upload_strategy.material_family_keys:
             for file_id in file_ids:
-                batch_executor.invoke_uploader(
-                    file_ids=[file_id],
+                batch_processor.invoke_uploader(
+                    file_id=file_id,
                     uploader_key=self.upload_strategy.uploader_key,
-                    material_family_keys=self.upload_strategy.material_family_keys,
+                    material_family_key=material_family_key,
                 )
-        else:
-            batch_executor.invoke_uploader(
-                file_ids=file_ids,
-                uploader_key=self.upload_strategy.uploader_key,
-                material_family_keys=self.upload_strategy.material_family_keys,
-            )
-
-        batch_executor.send()
 
         return JobResult(success=True)

uncountable/types/api/uploader/invoke_uploader.py

@@ -26,9 +26,9 @@ ENDPOINT_PATH = "api/external/uploader/invoke_uploader"
 # DO NOT MODIFY -- This file is generated by type_spec
 @dataclasses.dataclass(kw_only=True)
 class Arguments:
-    file_ids: list[base_t.ObjectId]
+    file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
-    material_family_keys: list[identifier_t.IdentifierKey]
+    material_family_key: identifier_t.IdentifierKey
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/async_batch_processor.py

@@ -160,19 +160,19 @@ class AsyncBatchProcessorBase(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         :param depends_on: A list of batch reference keys to process before processing this request
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         json_data = serialize_for_api(args)
 

uncountable/types/client_base.py

@@ -854,17 +854,17 @@ class ClientMethods(ABC):
     def invoke_uploader(
         self,
         *,
-        file_ids: list[base_t.ObjectId],
+        file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
-        material_family_keys: list[identifier_t.IdentifierKey],
+        material_family_key: identifier_t.IdentifierKey,
     ) -> invoke_uploader_t.Data:
-        """Runs files through an uploader.
+        """Runs a file through an uploader.
 
         """
         args = invoke_uploader_t.Arguments(
-            file_ids=file_ids,
+            file_id=file_id,
             uploader_key=uploader_key,
-            material_family_keys=material_family_keys,
+            material_family_key=material_family_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

uncountable/types/generic_upload_t.py

@@ -36,6 +36,5 @@ class GenericRemoteDirectoryScope:
 class GenericUploadStrategy:
     uploader_key: identifier_t.IdentifierKey
     material_family_keys: list[identifier_t.IdentifierKey]
-    parse_files_individually: bool
     skip_moving_files: bool = False
 # DO NOT MODIFY -- This file is generated by type_spec