UncountablePythonSDK 0.0.43__tar.gz → 0.0.45__tar.gz

{uncountablepythonsdk-0.0.43/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.43
+Version: 0.0.45
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.43
+Version: 0.0.45
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/pkgs/filesystem_utils/_sftp_session.py

@@ -63,7 +63,7 @@ class SFTPSession(FileSystemSession):
         super().__init__()
         self.host: str = sftp_config.ip
         self.username: str = sftp_config.username
-        self.key_file: str | paramiko.AgentKey | None = (
+        self.key_file: str | paramiko.RSAKey | None = (
             sftp_config.pem_path
             if sftp_config.pem_path is not None
             else sftp_config.pem_key

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/pkgs/filesystem_utils/file_type_utils.py

@@ -55,7 +55,7 @@ class FileSystemSFTPConfig:
     ip: str
     username: str
     pem_path: str | None
-    pem_key: paramiko.AgentKey | None = None
+    pem_key: paramiko.RSAKey | None = None
     password: str | None = None
     valid_extensions: Optional[tuple[str]] = None
     recursive: bool = True

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/pkgs/type_spec/emit_python.py

@@ -719,6 +719,14 @@ def _emit_properties(
                 default = "None"
             elif prop.has_default:
                 default = _emit_value(ctx, prop.spec_type, prop.default)
+                if (
+                    isinstance(prop.spec_type, builder.SpecTypeInstance)
+                    and (
+                        prop.spec_type.defn_type.is_base_type(builder.BaseTypeName.s_list)
+                    )
+                    and default == "[]"
+                ):
+                    default = "dataclasses.field(default_factory=list)"
             class_out.write(f"{INDENT * num_indent}{py_name}: {ref_type}")
             if default:
                 class_out.write(f" = {default}")

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/core/async_batch.py

@@ -4,9 +4,14 @@ from uncountable.types.async_batch import AsyncBatchRequest
 from uncountable.types.async_batch_processor import AsyncBatchProcessorBase
 
 
+class AsyncBatchSubmissionError(BaseException):
+    pass
+
+
 class AsyncBatchProcessor(AsyncBatchProcessorBase):
     _client: Client
     _queue: list[AsyncBatchRequest]
+    _submitted_job_ids: list[base_t.ObjectId]
 
     def __init__(self, *, client: Client) -> None:
         super().__init__()
@@ -16,7 +21,16 @@ class AsyncBatchProcessor(AsyncBatchProcessorBase):
     def _enqueue(self, req: async_batch_t.AsyncBatchRequest) -> None:
         self._queue.append(req)
 
+    def current_queue_size(self) -> int:
+        return len(self._queue)
+
     def send(self) -> base_t.ObjectId:
+        if len(self._queue) == 0:
+            raise AsyncBatchSubmissionError("queue is empty")
         job_id = self._client.execute_batch_load_async(requests=self._queue).job_id
+        self._submitted_job_ids.append(job_id)
         self._queue = []
         return job_id
+
+    def get_submitted_job_ids(self) -> list[base_t.ObjectId]:
+        return self._submitted_job_ids

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/integration/cron.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 
 from pkgs.argument_parser import CachedParser
+from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.integration.construct_client import construct_uncountable_client
 from uncountable.integration.executors.executors import resolve_executor
 from uncountable.integration.job import CronJobArguments, JobLogger
@@ -18,9 +19,12 @@ cron_args_parser = CachedParser(CronJobArgs)
 
 def cron_job_executor(**kwargs: dict) -> None:
     args_passed = cron_args_parser.parse_storage(kwargs)
+    client = construct_uncountable_client(profile_meta=args_passed.profile_metadata)
+    batch_processor = AsyncBatchProcessor(client=client)
     args = CronJobArguments(
         job_definition=args_passed.definition,
-        client=construct_uncountable_client(profile_meta=args_passed.profile_metadata),
+        client=client,
+        batch_processor=batch_processor,
         profile_metadata=args_passed.profile_metadata,
         logger=JobLogger(
             profile_metadata=args_passed.profile_metadata,
@@ -33,3 +37,11 @@ def cron_job_executor(**kwargs: dict) -> None:
     print(f"running job {args_passed.definition.name}")
 
     job.run(args=args)
+
+    if batch_processor.current_queue_size() != 0:
+        batch_processor.send()
+
+    print(f"completed job {args_passed.definition.name}")
+    submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
+    if len(submitted_batch_job_ids) != 0:
+        print("submitted batch jobs", submitted_batch_job_ids)

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/integration/executors/generic_upload_executor.py

@@ -2,7 +2,8 @@ import io
 import os
 import re
 from datetime import datetime, timezone
-from tempfile import NamedTemporaryFile
+
+import paramiko
 
 from pkgs.filesystem_utils import (
     FileObjectData,
@@ -15,7 +16,7 @@ from pkgs.filesystem_utils import (
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.file_upload import DataFileUpload, FileUpload
-from uncountable.integration.job import Job, JobArguments, JobLogger, JobResult
+from uncountable.integration.job import Job, JobArguments, JobLogger
 from uncountable.integration.secret_retrieval import retrieve_secret
 from uncountable.types.generic_upload_t import (
     GenericRemoteDirectoryScope,
@@ -24,6 +25,7 @@ from uncountable.types.generic_upload_t import (
 from uncountable.types.job_definition_t import (
     GenericUploadDataSource,
     GenericUploadDataSourceSFTP,
+    JobResult,
 )
 
 
@@ -176,14 +178,14 @@ class GenericUploadJob(Job):
                 pem_secret = retrieve_secret(
                     self.data_source.pem_secret, profile_metadata=args.profile_metadata
                 )
-                with NamedTemporaryFile() as pem_file:
-                    pem_file.write(pem_secret.encode())
-                    sftp_config = FileSystemSFTPConfig(
-                        ip=self.data_source.host,
-                        username=self.data_source.username,
-                        pem_path=pem_file.name,
-                    )
-                    return SFTPSession(sftp_config=sftp_config)
+                pem_key = paramiko.RSAKey.from_private_key(io.StringIO(pem_secret))
+                sftp_config = FileSystemSFTPConfig(
+                    ip=self.data_source.host,
+                    username=self.data_source.username,
+                    pem_path=None,
+                    pem_key=pem_key,
+                )
+                return SFTPSession(sftp_config=sftp_config)
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/integration/job.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
+from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
-from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
+from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
 class JobLogger:
@@ -25,6 +26,7 @@ class JobArgumentsBase:
     job_definition: JobDefinition
     profile_metadata: ProfileMetadata
     client: Client
+    batch_processor: AsyncBatchProcessor
     logger: JobLogger
 
 
@@ -37,11 +39,6 @@ class CronJobArguments(JobArgumentsBase):
 JobArguments = CronJobArguments
 
 
-@dataclass
-class JobResult:
-    success: bool
-
-
 class Job(ABC):
     _unc_job_registered: bool = False
 

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/types/job_definition.py

@@ -23,4 +23,5 @@ from .job_definition_t import AuthRetrievalEnv as AuthRetrievalEnv
 from .job_definition_t import AuthRetrieval as AuthRetrieval
 from .job_definition_t import ProfileDefinition as ProfileDefinition
 from .job_definition_t import ProfileMetadata as ProfileMetadata
+from .job_definition_t import JobResult as JobResult
 # DO NOT MODIFY -- This file is generated by type_spec

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/types/job_definition_t.py

@@ -32,6 +32,7 @@ __all__: list[str] = [
     "JobExecutorGenericUpload",
     "JobExecutorScript",
     "JobExecutorType",
+    "JobResult",
     "ProfileDefinition",
     "ProfileMetadata",
 ]
@@ -200,4 +201,10 @@ class ProfileMetadata:
     name: str
     base_url: str
     auth_retrieval: AuthRetrieval
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class JobResult:
+    success: bool
 # DO NOT MODIFY -- This file is generated by type_spec

{uncountablepythonsdk-0.0.43 → uncountablepythonsdk-0.0.45}/uncountable/types/recipe_links_t.py

@@ -46,5 +46,5 @@ class RecipeLink:
     recipe_id_from: base_t.ObjectId
     recipe_id_to: base_t.ObjectId
     link_type: str
-    name: str
+    name: typing.Optional[str]
 # DO NOT MODIFY -- This file is generated by type_spec