UncountablePythonSDK 0.0.43__py3-none-any.whl → 0.0.45__py3-none-any.whl

{UncountablePythonSDK-0.0.43.dist-info → UncountablePythonSDK-0.0.45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.43
+Version: 0.0.45
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{UncountablePythonSDK-0.0.43.dist-info → UncountablePythonSDK-0.0.45.dist-info}/RECORD

@@ -28,8 +28,8 @@ pkgs/argument_parser/case_convert.py,sha256=NuJLJUJRbyVb6_Slen4uqaStEHbcOS1d-hBB
 pkgs/filesystem_utils/__init__.py,sha256=yxfwtYFvqq_fjMl-tg2jwa6eNPNZUF4ykqsSzALyNdw,1143
 pkgs/filesystem_utils/_gdrive_session.py,sha256=OZudNoP2HikolnpurVJhJdh5fgzqbaZQvn53ReGGXx4,11015
 pkgs/filesystem_utils/_local_session.py,sha256=xFEYhAvNqrOYqwt4jrEYOuYkjJn0zclZhTelW_Q1-rw,2325
-pkgs/filesystem_utils/_sftp_session.py,sha256=jtvxmMs3H1RneuJhztx1pCSbX9hBq3srFV1IVNqUlBA,4729
-pkgs/filesystem_utils/file_type_utils.py,sha256=F6P2chbXIDwAUAiMc_ja1_rxrhGy3fA1-ymAnIK3gVU,1259
+pkgs/filesystem_utils/_sftp_session.py,sha256=gNoUD_b4MuVqWj31nU-FpfpXZlyWkwdEHtX1S8W6gpQ,4727
+pkgs/filesystem_utils/file_type_utils.py,sha256=CC2c7127TE84vlRTy7bpE4Qvy7ytxuRNA3dwP91PXA8,1257
 pkgs/filesystem_utils/filesystem_session.py,sha256=BQ2Go8Mu9-GcnaWh2Pm4x7ugLVsres6XrOQ8RoiEpcE,1045
 pkgs/serialization/__init__.py,sha256=LifasRW0a50A3qRFmo2bf3FQ6TXhZWOTz2-CVTgPjcQ,753
 pkgs/serialization/missing_sentry.py,sha256=aM_9KxbCk9dVvXvcOKgkIQBqFWvLhv8QlIUCiuFEXMo,806
@@ -50,7 +50,7 @@ pkgs/type_spec/config.py,sha256=IQyo2Vj11uNt7_d6jQxvominAOU-oPB8ldEmuGzJLpU,4644
 pkgs/type_spec/emit_io_ts.py,sha256=Ghd8XYqyNYldHQDepwa9GLfHXcoi48ztBw84K28ETic,5707
 pkgs/type_spec/emit_open_api.py,sha256=92POd3j4nrrROrw9M-bgEmK4ukYbI0TW6E9IigJzoTE,24512
 pkgs/type_spec/emit_open_api_util.py,sha256=y2slouAflUJmyTPH_d4CbXql9zpOoD1uTOELqL3NX-M,2448
-pkgs/type_spec/emit_python.py,sha256=R5CZcApOdTzbA4R5klFOxXt4AQtTI72GTxp4Non8q2E,46509
+pkgs/type_spec/emit_python.py,sha256=bQUqVmfVgAXw8rGcqlg93YbUsUvawgR2o_NrV_c9Zio,46873
 pkgs/type_spec/emit_typescript.py,sha256=cdr5h8N70PuwORcvhURUujzwH9r1LVwJB8V2EoipGkw,17917
 pkgs/type_spec/emit_typescript_util.py,sha256=sR7ys3Ilnh6SQiXJbfYk4pxfOu0bDjbUFTEYEW-ud6c,863
 pkgs/type_spec/load_types.py,sha256=BOLyndtxPqqhUqZAh-lIbN5IZBaW_m-bdYpKGsbPyXM,3654
@@ -72,21 +72,21 @@ pkgs/type_spec/value_spec/types.py,sha256=a2zxbbCRWepY1l8OtjeCDKgBKFPFHVgV99oP6p
 uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
 uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/core/__init__.py,sha256=RFv0kO6rKFf1PtBPu83hCGmxqkJamRtsgQ9_-ztw7tA,341
-uncountable/core/async_batch.py,sha256=Zo02TICZ7AK81Qa02NQ8fp8uaijQph1FEta28K3cKb0,749
+uncountable/core/async_batch.py,sha256=QTEnazSEO8GtMvI9qnSXIcgWxk3dbAVg2HijoXW5ddQ,1173
 uncountable/core/client.py,sha256=50vR11GwwyTc3qOIahUxCJK77zCM7LOXwbZdHoZCZgY,10168
 uncountable/core/file_upload.py,sha256=TkQ0fKbbYrPgns1Jh51JU35DUqZHB3ljOaVgjSlBx9Y,3149
 uncountable/core/types.py,sha256=s2CjqYJpsmbC7xMwxxT7kJ_V9bwokrjjWVVjpMcQpKI,333
 uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/construct_client.py,sha256=y8x65wzG-_84uLlnSVQG4WuMhyVGUfdJVmhy8gs8v4A,1082
-uncountable/integration/cron.py,sha256=czW-RY2T5lD7yJwyVZcFg9240mXURrzeExd1EL6zHeg,1179
+uncountable/integration/cron.py,sha256=n4HjW-KNruYQ_09iwNBd9i4-Juy_9ddzbf71_wnydHA,1674
 uncountable/integration/entrypoint.py,sha256=Hs-XUTTxjrxsb31lytN7yVViiBYc10x96W_Vy6iq95A,1382
-uncountable/integration/job.py,sha256=ZaVm77v2Pkhyy0FA8PJPp4hd7yNshXs1T3NlZjNi03U,1398
+uncountable/integration/job.py,sha256=5HJcjtwj_pGFYAZ6SK7xa9RqDXCoBb5rAb_OHdTR61Q,1463
 uncountable/integration/server.py,sha256=OlRVxrpnpJ_QXJ27kQt4z_U5HnI0pfcm5OkxhN_Sa-Y,3347
 uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=lCHGIMCe3t_UvEgHMBoOneWR2zbYaGXztt5F2zDRBLw,8796
+uncountable/integration/executors/generic_upload_executor.py,sha256=axGpIfF0IMHilGzFawDizWoDhYffeqYXuhfyYkQhWF4,8754
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=M0qXVJpD8hMYIFypHFeyh598sqmIDX8ZOyXK23CluF0,1323
@@ -121,8 +121,8 @@ uncountable/types/input_attributes.py,sha256=IrIKQnHqHdS1Utdfzr9GnOe17a8riaqYcO1
 uncountable/types/input_attributes_t.py,sha256=wE1ekiQfb72Z9VpF5SHipKJkgaJFUHJrNkkJdmuhF9w,820
 uncountable/types/inputs.py,sha256=6RIEFfCxLqpeHEGOpu63O4i8zPogjGeB7wiV_rPBw_g,404
 uncountable/types/inputs_t.py,sha256=RW7gF9zTOwByu-nMTMVuBabLOuWKx4O1nvfgvx_R55o,1611
-uncountable/types/job_definition.py,sha256=ImmUw8y4_nEsz_pBEceZdbnJDhFZRfm9BtW4hHhe9ik,1584
-uncountable/types/job_definition_t.py,sha256=oqF988ub1UZDRdMrg2cndTMn7oml102LQQKZ-vLUAVI,5406
+uncountable/types/job_definition.py,sha256=W2wth9vPRBVL0temwnSmvfxxPMKFxZqbQtsMXn8BU3U,1637
+uncountable/types/job_definition_t.py,sha256=wz1DE0L9BMGKwTVCvOo8DSv1QZlvwkNpiEnivSo_o3A,5552
 uncountable/types/outputs.py,sha256=sUZx_X-TKCZtLm1YCEH8OISX9DdPlv9ZuUfM3-askCc,281
 uncountable/types/outputs_t.py,sha256=2aORUOr0ls1ZYo-ddkWax3D1ZndmQsWtHfJxpYozlhg,656
 uncountable/types/permissions.py,sha256=1mRnSsmRgjuLgp6pylTwwACD_YRIcmlqxHkufwZtMns,297
@@ -136,7 +136,7 @@ uncountable/types/recipe_identifiers_t.py,sha256=suumLdp_8SZD7oqIxSEaV9D1irMeLJ5
 uncountable/types/recipe_inputs.py,sha256=5ThNFEOGbADqqfj1V3hNvZUcO5pn1Gm17LmzQkWDrtI,351
 uncountable/types/recipe_inputs_t.py,sha256=t5nFzuF1AU6SUHpya6xKHSlcckmt3XxAuk9ofjZ2oGU,746
 uncountable/types/recipe_links.py,sha256=YRiu6t7FWr7ZWycIaOPXBtQFqbY_q5unaP6KQzh1LzE,343
-uncountable/types/recipe_links_t.py,sha256=RPS-Q4FAvKlio7pwufIdERQhlNr27IZGVxMPaoTVqrI,1400
+uncountable/types/recipe_links_t.py,sha256=ZaTeP8AsIT9-gDRi_uQreDsAzEmqF34YTesh9uydYa0,1417
 uncountable/types/recipe_metadata.py,sha256=Zpcrupq_mlT2UhXpMJup5XjtubmvgZ8SbJNq7da2pCs,441
 uncountable/types/recipe_metadata_t.py,sha256=y2O2rIanS6WG0j1UDSsRSwyEKYNDeNaixlR3PmkYnkU,1455
 uncountable/types/recipe_output_metadata.py,sha256=83jKZCvogG9QjZeJpQptdSam_MnnUj-tqh9EVIrTWjE,322
@@ -235,7 +235,7 @@ uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/uploader/invoke_uploader.py,sha256=2RCvnccwEm6J5dt7UJVLUKxfk49vfpGXjBLCWIBD3Ck,1006
-UncountablePythonSDK-0.0.43.dist-info/METADATA,sha256=vhbJVX4f-c2phTYrI3H2MJ4NCXfkOHVeHa3NHq-SEGk,1707
-UncountablePythonSDK-0.0.43.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-UncountablePythonSDK-0.0.43.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.43.dist-info/RECORD,,
+UncountablePythonSDK-0.0.45.dist-info/METADATA,sha256=WAhZRRAEVBAG_NLepXUkHuR71qY-2uRlIWNrIsN7JJY,1707
+UncountablePythonSDK-0.0.45.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+UncountablePythonSDK-0.0.45.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.45.dist-info/RECORD,,

{UncountablePythonSDK-0.0.43.dist-info → UncountablePythonSDK-0.0.45.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.1.0)
+Generator: setuptools (72.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

pkgs/filesystem_utils/_sftp_session.py

@@ -63,7 +63,7 @@ class SFTPSession(FileSystemSession):
         super().__init__()
         self.host: str = sftp_config.ip
         self.username: str = sftp_config.username
-        self.key_file: str | paramiko.AgentKey | None = (
+        self.key_file: str | paramiko.RSAKey | None = (
             sftp_config.pem_path
             if sftp_config.pem_path is not None
             else sftp_config.pem_key

pkgs/filesystem_utils/file_type_utils.py

@@ -55,7 +55,7 @@ class FileSystemSFTPConfig:
     ip: str
     username: str
     pem_path: str | None
-    pem_key: paramiko.AgentKey | None = None
+    pem_key: paramiko.RSAKey | None = None
     password: str | None = None
     valid_extensions: Optional[tuple[str]] = None
     recursive: bool = True

pkgs/type_spec/emit_python.py

@@ -719,6 +719,14 @@ def _emit_properties(
                 default = "None"
             elif prop.has_default:
                 default = _emit_value(ctx, prop.spec_type, prop.default)
+                if (
+                    isinstance(prop.spec_type, builder.SpecTypeInstance)
+                    and (
+                        prop.spec_type.defn_type.is_base_type(builder.BaseTypeName.s_list)
+                    )
+                    and default == "[]"
+                ):
+                    default = "dataclasses.field(default_factory=list)"
             class_out.write(f"{INDENT * num_indent}{py_name}: {ref_type}")
             if default:
                 class_out.write(f" = {default}")

uncountable/core/async_batch.py

@@ -4,9 +4,14 @@ from uncountable.types.async_batch import AsyncBatchRequest
 from uncountable.types.async_batch_processor import AsyncBatchProcessorBase
 
 
+class AsyncBatchSubmissionError(BaseException):
+    pass
+
+
 class AsyncBatchProcessor(AsyncBatchProcessorBase):
     _client: Client
     _queue: list[AsyncBatchRequest]
+    _submitted_job_ids: list[base_t.ObjectId]
 
     def __init__(self, *, client: Client) -> None:
         super().__init__()
@@ -16,7 +21,16 @@ class AsyncBatchProcessor(AsyncBatchProcessorBase):
     def _enqueue(self, req: async_batch_t.AsyncBatchRequest) -> None:
         self._queue.append(req)
 
+    def current_queue_size(self) -> int:
+        return len(self._queue)
+
     def send(self) -> base_t.ObjectId:
+        if len(self._queue) == 0:
+            raise AsyncBatchSubmissionError("queue is empty")
         job_id = self._client.execute_batch_load_async(requests=self._queue).job_id
+        self._submitted_job_ids.append(job_id)
         self._queue = []
         return job_id
+
+    def get_submitted_job_ids(self) -> list[base_t.ObjectId]:
+        return self._submitted_job_ids

uncountable/integration/cron.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 
 from pkgs.argument_parser import CachedParser
+from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.integration.construct_client import construct_uncountable_client
 from uncountable.integration.executors.executors import resolve_executor
 from uncountable.integration.job import CronJobArguments, JobLogger
@@ -18,9 +19,12 @@ cron_args_parser = CachedParser(CronJobArgs)
 
 def cron_job_executor(**kwargs: dict) -> None:
     args_passed = cron_args_parser.parse_storage(kwargs)
+    client = construct_uncountable_client(profile_meta=args_passed.profile_metadata)
+    batch_processor = AsyncBatchProcessor(client=client)
     args = CronJobArguments(
         job_definition=args_passed.definition,
-        client=construct_uncountable_client(profile_meta=args_passed.profile_metadata),
+        client=client,
+        batch_processor=batch_processor,
         profile_metadata=args_passed.profile_metadata,
         logger=JobLogger(
             profile_metadata=args_passed.profile_metadata,
@@ -33,3 +37,11 @@ def cron_job_executor(**kwargs: dict) -> None:
     print(f"running job {args_passed.definition.name}")
 
     job.run(args=args)
+
+    if batch_processor.current_queue_size() != 0:
+        batch_processor.send()
+
+    print(f"completed job {args_passed.definition.name}")
+    submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
+    if len(submitted_batch_job_ids) != 0:
+        print("submitted batch jobs", submitted_batch_job_ids)

uncountable/integration/executors/generic_upload_executor.py

@@ -2,7 +2,8 @@ import io
 import os
 import re
 from datetime import datetime, timezone
-from tempfile import NamedTemporaryFile
+
+import paramiko
 
 from pkgs.filesystem_utils import (
     FileObjectData,
@@ -15,7 +16,7 @@ from pkgs.filesystem_utils import (
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.file_upload import DataFileUpload, FileUpload
-from uncountable.integration.job import Job, JobArguments, JobLogger, JobResult
+from uncountable.integration.job import Job, JobArguments, JobLogger
 from uncountable.integration.secret_retrieval import retrieve_secret
 from uncountable.types.generic_upload_t import (
     GenericRemoteDirectoryScope,
@@ -24,6 +25,7 @@ from uncountable.types.generic_upload_t import (
 from uncountable.types.job_definition_t import (
     GenericUploadDataSource,
     GenericUploadDataSourceSFTP,
+    JobResult,
 )
 
 
@@ -176,14 +178,14 @@ class GenericUploadJob(Job):
                 pem_secret = retrieve_secret(
                     self.data_source.pem_secret, profile_metadata=args.profile_metadata
                 )
-                with NamedTemporaryFile() as pem_file:
-                    pem_file.write(pem_secret.encode())
-                    sftp_config = FileSystemSFTPConfig(
-                        ip=self.data_source.host,
-                        username=self.data_source.username,
-                        pem_path=pem_file.name,
-                    )
-                    return SFTPSession(sftp_config=sftp_config)
+                pem_key = paramiko.RSAKey.from_private_key(io.StringIO(pem_secret))
+                sftp_config = FileSystemSFTPConfig(
+                    ip=self.data_source.host,
+                    username=self.data_source.username,
+                    pem_path=None,
+                    pem_key=pem_key,
+                )
+                return SFTPSession(sftp_config=sftp_config)
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client

uncountable/integration/job.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
+from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
-from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
+from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
 class JobLogger:
@@ -25,6 +26,7 @@ class JobArgumentsBase:
     job_definition: JobDefinition
     profile_metadata: ProfileMetadata
     client: Client
+    batch_processor: AsyncBatchProcessor
     logger: JobLogger
 
 
@@ -37,11 +39,6 @@ class CronJobArguments(JobArgumentsBase):
 JobArguments = CronJobArguments
 
 
-@dataclass
-class JobResult:
-    success: bool
-
-
 class Job(ABC):
     _unc_job_registered: bool = False
 

uncountable/types/job_definition.py

@@ -23,4 +23,5 @@ from .job_definition_t import AuthRetrievalEnv as AuthRetrievalEnv
 from .job_definition_t import AuthRetrieval as AuthRetrieval
 from .job_definition_t import ProfileDefinition as ProfileDefinition
 from .job_definition_t import ProfileMetadata as ProfileMetadata
+from .job_definition_t import JobResult as JobResult
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/job_definition_t.py

@@ -32,6 +32,7 @@ __all__: list[str] = [
     "JobExecutorGenericUpload",
     "JobExecutorScript",
     "JobExecutorType",
+    "JobResult",
     "ProfileDefinition",
     "ProfileMetadata",
 ]
@@ -200,4 +201,10 @@ class ProfileMetadata:
     name: str
     base_url: str
     auth_retrieval: AuthRetrieval
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class JobResult:
+    success: bool
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/recipe_links_t.py

@@ -46,5 +46,5 @@ class RecipeLink:
     recipe_id_from: base_t.ObjectId
     recipe_id_to: base_t.ObjectId
     link_type: str
-    name: str
+    name: typing.Optional[str]
 # DO NOT MODIFY -- This file is generated by type_spec