UncountablePythonSDK 0.0.41__tar.gz → 0.0.42__tar.gz

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42}/.github/workflows/documentation.yml

@@ -11,10 +11,9 @@ jobs:
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
       - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
-      - uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2
+      - uses: extractions/setup-crate@4993624604c307fbca528d28a3c8b60fa5ecc859 # v1
         with:
-            just-version: '1.8.0'
-            github-token: ${{ secrets.GITHUB_TOKEN }}
+          repo: casey/just@0.10
       - name: Install dependencies
         run: |
           # install the dependencies for the package

{uncountablepythonsdk-0.0.41/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.41
+Version: 0.0.42
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -24,6 +24,10 @@ Requires-Dist: APScheduler==3.*
 Requires-Dist: python-dateutil==2.*
 Requires-Dist: shelljob==0.*
 Requires-Dist: PyYAML==6.*
+Requires-Dist: google-api-python-client==2.*
+Requires-Dist: tqdm==4.*
+Requires-Dist: pysftp==0.*
+Requires-Dist: paramiko==3.*
 Provides-Extra: test
 Requires-Dist: mypy==1.*; extra == "test"
 Requires-Dist: ruff==0.*; extra == "test"

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.41
+Version: 0.0.42
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -24,6 +24,10 @@ Requires-Dist: APScheduler==3.*
 Requires-Dist: python-dateutil==2.*
 Requires-Dist: shelljob==0.*
 Requires-Dist: PyYAML==6.*
+Requires-Dist: google-api-python-client==2.*
+Requires-Dist: tqdm==4.*
+Requires-Dist: pysftp==0.*
+Requires-Dist: paramiko==3.*
 Provides-Extra: test
 Requires-Dist: mypy==1.*; extra == "test"
 Requires-Dist: ruff==0.*; extra == "test"

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ docs/static/favicons/safari-pinned-tab.svg
 examples/async_batch.py
 examples/create_entity.py
 examples/edit_recipe_inputs.py
+examples/invoke_uploader.py
 examples/upload_files.py
 pkgs/__init__.py
 pkgs/argument_parser/__init__.py
@@ -34,6 +35,12 @@ pkgs/argument_parser/_is_enum.py
 pkgs/argument_parser/_is_namedtuple.py
 pkgs/argument_parser/argument_parser.py
 pkgs/argument_parser/case_convert.py
+pkgs/filesystem_utils/__init__.py
+pkgs/filesystem_utils/_gdrive_session.py
+pkgs/filesystem_utils/_local_session.py
+pkgs/filesystem_utils/_sftp_session.py
+pkgs/filesystem_utils/file_type_utils.py
+pkgs/filesystem_utils/filesystem_session.py
 pkgs/serialization/__init__.py
 pkgs/serialization/missing_sentry.py
 pkgs/serialization/opaque_key.py
@@ -85,11 +92,14 @@ uncountable/integration/cron.py
 uncountable/integration/entrypoint.py
 uncountable/integration/job.py
 uncountable/integration/server.py
-uncountable/integration/types.py
 uncountable/integration/db/__init__.py
 uncountable/integration/db/connect.py
 uncountable/integration/executors/__init__.py
+uncountable/integration/executors/executors.py
+uncountable/integration/executors/generic_upload_executor.py
 uncountable/integration/executors/script_executor.py
+uncountable/integration/secret_retrieval/__init__.py
+uncountable/integration/secret_retrieval/retrieve_secret.py
 uncountable/types/__init__.py
 uncountable/types/async_batch.py
 uncountable/types/async_batch_processor.py
@@ -111,6 +121,8 @@ uncountable/types/field_values.py
 uncountable/types/field_values_t.py
 uncountable/types/fields.py
 uncountable/types/fields_t.py
+uncountable/types/generic_upload.py
+uncountable/types/generic_upload_t.py
 uncountable/types/id_source.py
 uncountable/types/id_source_t.py
 uncountable/types/identifier.py
@@ -119,6 +131,8 @@ uncountable/types/input_attributes.py
 uncountable/types/input_attributes_t.py
 uncountable/types/inputs.py
 uncountable/types/inputs_t.py
+uncountable/types/job_definition.py
+uncountable/types/job_definition_t.py
 uncountable/types/outputs.py
 uncountable/types/outputs_t.py
 uncountable/types/permissions.py
@@ -145,6 +159,8 @@ uncountable/types/recipes.py
 uncountable/types/recipes_t.py
 uncountable/types/response.py
 uncountable/types/response_t.py
+uncountable/types/secret_retrieval.py
+uncountable/types/secret_retrieval_t.py
 uncountable/types/units.py
 uncountable/types/units_t.py
 uncountable/types/users.py
@@ -226,4 +242,6 @@ uncountable/types/api/recipes/set_recipe_tags.py
 uncountable/types/api/recipes/unarchive_recipes.py
 uncountable/types/api/recipes/unlock_recipes.py
 uncountable/types/api/triggers/__init__.py
-uncountable/types/api/triggers/run_trigger.py
+uncountable/types/api/triggers/run_trigger.py
+uncountable/types/api/uploader/__init__.py
+uncountable/types/api/uploader/invoke_uploader.py

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42}/UncountablePythonSDK.egg-info/requires.txt

@@ -6,6 +6,10 @@ APScheduler==3.*
 python-dateutil==2.*
 shelljob==0.*
 PyYAML==6.*
+google-api-python-client==2.*
+tqdm==4.*
+pysftp==0.*
+paramiko==3.*
 
 [test]
 mypy==1.*

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42}/docs/requirements.txt

@@ -1,7 +1,7 @@
-furo==2024.5.6
+furo==2024.7.18
 myst-parser==3.0.1
-sphinx-autoapi==3.1.0
+sphinx-autoapi==3.2.0
 sphinx-copybutton==0.5.2
-Sphinx==7.3.7
+Sphinx==7.4.4
 sphinx_design==0.6.0
 sphinx-favicon==1.0.1

uncountablepythonsdk-0.0.42/examples/invoke_uploader.py

@@ -0,0 +1,23 @@
+import os
+
+from uncountable.core import AuthDetailsApiKey, Client, MediaFileUpload
+from uncountable.types.identifier_t import IdentifierKeyId
+
+client = Client(
+    base_url="http://localhost:5000",
+    auth_details=AuthDetailsApiKey(
+        api_id=os.environ["UNC_API_ID"],
+        api_secret_key=os.environ["UNC_API_SECRET_KEY"],
+    ),
+)
+uploaded = client.upload_files(
+    file_uploads=[
+        MediaFileUpload(path="~/Downloads/my_file_to_upload.csv"),
+    ]
+)
+
+client.invoke_uploader(
+    file_ids=[file.file_id for file in uploaded],
+    uploader_key=IdentifierKeyId(id=48),
+    material_family_keys=[IdentifierKeyId(id=1)],
+)

{uncountablepythonsdk-0.0.41 → uncountablepythonsdk-0.0.42}/pkgs/argument_parser/argument_parser.py

@@ -218,7 +218,7 @@ def _build_parser_inner(
         arg_parsers = [_build_parser_inner(arg, context) for arg in sorted_args]
         return lambda value: _invoke_fallback_parsers(parsed_type, arg_parsers, value)
 
-    if parsed_type is typing.Any:
+    if parsed_type is typing.Any:  # type: ignore[comparison-overlap]
         return lambda value: value
 
     if origin in (list, set):

uncountablepythonsdk-0.0.42/pkgs/filesystem_utils/__init__.py

@@ -0,0 +1,17 @@
+from ._gdrive_session import GDriveSession as GDriveSession
+from ._gdrive_session import delete_gdrive_file as delete_gdrive_file
+from ._gdrive_session import download_gdrive_file as download_gdrive_file
+from ._gdrive_session import list_gdrive_files as list_gdrive_files
+from ._gdrive_session import move_gdrive_file as move_gdrive_file
+from ._gdrive_session import upload_file_gdrive as upload_file_gdrive
+from ._local_session import LocalSession as LocalSession
+from ._sftp_session import SFTPSession as SFTPSession
+from ._sftp_session import list_sftp_files as list_sftp_files
+from ._sftp_session import move_sftp_files as move_sftp_files
+from .file_type_utils import FileObjectData as FileObjectData
+from .file_type_utils import FileSystemFileReference as FileSystemFileReference
+from .file_type_utils import FileSystemObject as FileSystemObject
+from .file_type_utils import FileSystemSFTPConfig as FileSystemSFTPConfig
+from .file_type_utils import FileTransfer as FileTransfer
+from .file_type_utils import IncompatibleFileReference as IncompatibleFileReference
+from .file_type_utils import RemoteObjectReference as RemoteObjectReference

uncountablepythonsdk-0.0.42/pkgs/filesystem_utils/_gdrive_session.py

@@ -0,0 +1,306 @@
+import os
+from io import BytesIO
+from typing import Any, Optional
+
+from google.oauth2 import service_account
+from googleapiclient.discovery import build as build_gdrive_connection
+from googleapiclient.errors import HttpError
+from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload
+from tqdm import tqdm
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileTransfer,
+    IncompatibleFileReference,
+    RemoteObjectReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+# NOTE: google apis do not have static types
+GDriveResource = Any
+
+
+def download_gdrive_file(
+    gdrive_connection: GDriveResource,
+    file_id: str,
+    filename: str,
+    mime_type: str,
+    *,
+    verbose: bool = False,
+) -> Optional[FileObjectData]:
+    if "folder" in mime_type:
+        if verbose:
+            print(f"{filename} is a folder and will not be downloaded.")
+        return None
+    elif "google-apps" in mime_type:
+        # Handle google workspace doc
+        if "spreadsheet" in mime_type:
+            if verbose:
+                print(f"{filename} is a Google Sheet, exporting.")
+            file_request = gdrive_connection.files().export_media(
+                fileId=file_id, mimeType="text/csv"
+            )
+            filename += ".csv"
+        elif "document" in mime_type:
+            if verbose:
+                print(f"{filename} is a Google Doc, exporting.")
+            file_request = gdrive_connection.files().export_media(
+                fileId=file_id, mimeType="application/msword"
+            )
+            filename += ".doc"
+        else:
+            if verbose:
+                print(f"{filename} is an unsupported google workspace filetype.")
+                print(f"Skipping. mimeType: {mime_type}.")
+            return None
+    else:
+        file_request = gdrive_connection.files().get_media(fileId=file_id)
+
+    file_handler = BytesIO()
+    downloader = MediaIoBaseDownload(file_handler, file_request)
+    download_complete = False
+    while not download_complete:
+        status, download_complete = downloader.next_chunk()
+
+    file_handler.seek(0)
+    file_data = file_handler.read()
+    return FileObjectData(
+        file_data=file_data,
+        file_IO=BytesIO(file_data),
+        filename=filename,
+        filepath=file_id,
+        metadata={"id": file_id},
+        mime_type=mime_type,
+    )
+
+
+def list_gdrive_files(
+    gdrive_connection: GDriveResource, gdrive_folder_id: str, *, recurse: bool = False
+) -> list[dict[str, str]]:
+    query = f"parents = '{gdrive_folder_id}'"
+    print("Listing files", end="", flush=True)
+    paginated_files_in_folder = [
+        (
+            gdrive_connection.files()
+            .list(
+                q=query,
+                corpora="allDrives",
+                includeItemsFromAllDrives=True,
+                supportsAllDrives=True,
+            )
+            .execute()
+        )
+    ]
+    while paginated_files_in_folder[-1].get("nextPageToken") is not None:
+        print(".", end="", flush=True)
+        paginated_files_in_folder.append(
+            gdrive_connection.files()
+            .list(
+                q=query,
+                corpora="allDrives",
+                includeItemsFromAllDrives=True,
+                supportsAllDrives=True,
+                pageToken=paginated_files_in_folder[-1]["nextPageToken"],
+            )
+            .execute()
+        )
+    print()
+    # Get available files: https://developers.google.com/drive/api/v3/manage-downloads#python
+    files: list[dict[str, str]] = []
+    for files_in_folder in paginated_files_in_folder:
+        files.extend(files_in_folder.get("files", []))
+    subfiles: list[dict[str, str]] = []
+    if recurse:
+        for file in files:
+            if file["mimeType"] == "application/vnd.google-apps.folder":
+                subfiles.extend(
+                    list_gdrive_files(
+                        gdrive_connection=gdrive_connection,
+                        gdrive_folder_id=file["id"],
+                        recurse=True,
+                    )
+                )
+    return [*files, *subfiles]
+
+
+def upload_file_gdrive(
+    gdrive_connection: GDriveResource,
+    src_file: BytesIO,
+    mime_type: str,
+    dest_folder_id: str,
+    dest_filename: str,
+) -> None:
+    file_metadata = {"name": dest_filename, "parents": [dest_folder_id]}
+    media = MediaIoBaseUpload(src_file, mimetype=mime_type)
+    try:
+        gdrive_connection.files().create(
+            body=file_metadata, media_body=media, fields="id", supportsAllDrives=True
+        ).execute()
+    except HttpError:
+        print("FileSystemObject Upload to GDrive Unsuccessful")
+
+
+def move_gdrive_file(
+    gdrive_connection: GDriveResource,
+    src_file_id: str,
+    dest_folder_id: str,
+    *,
+    dest_filename: Optional[str] = None,
+) -> None:
+    # Retrieve the existing parents to remove
+    file = (
+        gdrive_connection.files()
+        .get(fileId=src_file_id, fields="parents, name", supportsTeamDrives=True)
+        .execute()
+    )
+
+    new_filename = file["name"]
+    if dest_filename is not None:
+        new_filename = dest_filename
+    previous_parents = ",".join(file.get("parents"))
+    metadata = {"name": new_filename}
+    gdrive_connection.files().update(
+        fileId=src_file_id, body=metadata, fields="name", supportsTeamDrives=True
+    ).execute()
+    gdrive_connection.files().update(
+        fileId=src_file_id,
+        addParents=dest_folder_id,
+        removeParents=previous_parents,
+        fields="id, parents",
+        supportsTeamDrives=True,
+    ).execute()
+
+
+def delete_gdrive_file(gdrive_connection: GDriveResource, file_id: str) -> None:
+    gdrive_connection.files().delete(fileId=file_id, supportsAllDrives=True).execute()
+
+
+class GDriveSession(FileSystemSession):
+    def __init__(self, service_account_json_path: str) -> None:
+        super().__init__()
+        self.service_account_json_path = service_account_json_path
+
+    def start(self) -> None:
+        credentials = service_account.Credentials.from_service_account_file(  # type: ignore[no-untyped-call]
+            self.service_account_json_path
+        )
+        gdrive_connection = build_gdrive_connection(
+            "drive", "v3", credentials=credentials
+        )
+        self.connection = gdrive_connection
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = False,
+        valid_file_extensions: Optional[tuple[str, ...]] = None,
+    ) -> list[FileSystemObject]:
+        if not isinstance(dir_path, RemoteObjectReference):
+            raise IncompatibleFileReference(
+                "Incompatible FileSystemObject to GDriveSession.list_files"
+            )
+        if not dir_path.is_dir:
+            raise IncompatibleFileReference(
+                "FileSystemObject does not reference a directory"
+            )
+        files = list_gdrive_files(self.connection, dir_path.file_id, recurse=recursive)
+        gdrive_files: list[FileSystemObject] = []
+        for file_context in files:
+            if (
+                valid_file_extensions is not None
+                and os.path.splitext(file_context["name"])[1] not in valid_file_extensions
+            ):
+                continue
+            gdrive_files.append(
+                RemoteObjectReference(
+                    file_id=file_context["id"],
+                    mime_type=file_context["mimeType"],
+                    filename=file_context["name"],
+                )
+            )
+        return gdrive_files
+
+    def delete_files(self, filepaths: list[FileSystemObject]) -> None:
+        """Warning:
+        Security account must have sufficient permissions to perform delete!
+        https://developers.google.com/drive/api/v3/reference/files/delete?hl=en
+        https://developers.google.com/drive/api/v3/ref-roles
+        """
+        for file_object in filepaths:
+            if not isinstance(file_object, RemoteObjectReference):
+                raise IncompatibleFileReference(
+                    "Incompatible FileSystemObject provided to GDriveSession.delete_files"
+                )
+            delete_gdrive_file(self.connection, file_object.file_id)
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        for src_file, dest_file in file_mappings:
+            if (
+                isinstance(src_file, FileSystemFileReference)
+                or not isinstance(dest_file, RemoteObjectReference)
+                or not dest_file.is_dir
+                or (isinstance(src_file, RemoteObjectReference) and src_file.is_dir)
+            ):
+                continue
+            new_filename = dest_file.filename
+            if isinstance(src_file, RemoteObjectReference):
+                if new_filename is not None:
+                    move_gdrive_file(
+                        self.connection,
+                        src_file.file_id,
+                        dest_file.file_id,
+                        dest_filename=new_filename,
+                    )
+                else:
+                    move_gdrive_file(self.connection, src_file.file_id, dest_file.file_id)
+            elif isinstance(src_file, FileObjectData):
+                if src_file.mime_type is None:
+                    raise IncompatibleFileReference(
+                        "No mime_type present on source file data."
+                    )
+                new_filename = src_file.filename
+                if dest_file.filename is not None:
+                    new_filename = dest_file.filename
+                upload_file_gdrive(
+                    self.connection,
+                    src_file.file_IO,
+                    src_file.mime_type,
+                    dest_file.file_id,
+                    new_filename,
+                )
+            else:
+                raise IncompatibleFileReference(
+                    "Unrecognized file reference in FileTransfer object"
+                )
+
+    def download_files(self, filepaths: list[FileSystemObject]) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        print(f"Downloading {len(filepaths)} files")
+        for file_object in tqdm(filepaths):
+            if (
+                not isinstance(file_object, RemoteObjectReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference(
+                    "Incompatible FileSystemObject included in filepaths"
+                )
+            downloaded_file = download_gdrive_file(
+                self.connection,
+                file_object.file_id,
+                file_object.filename,
+                file_object.mime_type,
+            )
+            if downloaded_file is not None:
+                downloaded_files.append(downloaded_file)
+        return downloaded_files
+
+    def __enter__(self) -> "GDriveSession":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.connection.close()

uncountablepythonsdk-0.0.42/pkgs/filesystem_utils/_local_session.py

@@ -0,0 +1,69 @@
+import os
+from io import BytesIO
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+class LocalSession(FileSystemSession):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def start(self) -> None:
+        return None
+
+    def __enter__(self) -> "LocalSession":
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        return None
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        for src_file, dest_file in file_mappings:
+            if not (
+                isinstance(src_file, FileSystemFileReference)
+                and isinstance(dest_file, FileSystemFileReference)
+            ):
+                raise IncompatibleFileReference()
+            os.rename(src_file.filepath, dest_file.filepath)
+
+    def download_files(self, filepaths: list[FileSystemObject]) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+            with open(file_object.filepath, "rb") as file_data:
+                file_bytes = file_data.read()
+            downloaded_files.append(
+                FileObjectData(
+                    file_bytes,
+                    BytesIO(file_bytes),
+                    file_object.filename,
+                    filepath=file_object.filepath,
+                )
+            )
+        return downloaded_files
+
+    def list_files(
+        self, dir_path: FileSystemObject, *, recursive: bool = False
+    ) -> list[FileSystemObject]:
+        if not isinstance(dir_path, FileSystemFileReference) or not os.path.isdir(
+            dir_path.filepath
+        ):
+            raise IncompatibleFileReference()
+        if recursive:
+            raise NotImplementedError("recursive not implemented for local session")
+        return [
+            FileSystemFileReference(os.path.join(dir_path.filepath, filename))
+            for filename in os.listdir(dir_path.filepath)
+        ]