divbase-cli 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev3__py3-none-any.whl

divbase_cli/pre_signed_urls.py

@@ -1,169 +0,0 @@
-"""
-Module responsible for taking pre-signed urls and using them to do file download and upload.
-
-TODO: Consider adding retries, error handling, progress bars, etc.
-"""
-
-import logging
-from dataclasses import dataclass
-from pathlib import Path
-
-import httpx
-
-from divbase_lib.api_schemas.s3 import PreSignedDownloadResponse, PreSignedUploadResponse
-from divbase_lib.exceptions import ChecksumVerificationError
-from divbase_lib.s3_checksums import verify_downloaded_checksum
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class SuccessfulDownload:
-    """Represents a successfully downloaded file."""
-
-    file_path: Path
-    object_name: str
-
-
-@dataclass
-class FailedDownload:
-    """Represents a failed download attempt."""
-
-    object_name: str
-    file_path: Path
-    exception: Exception
-
-
-@dataclass
-class DownloadOutcome:
-    """Outcome of attempting to download multiple files."""
-
-    successful: list[SuccessfulDownload]
-    failed: list[FailedDownload]
-
-
-def download_multiple_pre_signed_urls(
-    pre_signed_urls: list[PreSignedDownloadResponse], verify_checksums: bool, download_dir: Path
-) -> DownloadOutcome:
-    """
-    Download files using pre-signed URLs.
-    Returns a DownloadResults object containing all successful and failed downloads.
-    """
-    successful_downloads, failed_downloads = [], []
-    with httpx.Client(timeout=30.0) as client:
-        for obj in pre_signed_urls:
-            result = _download_single_pre_signed_url(
-                httpx_client=client,
-                pre_signed_url=obj.pre_signed_url,
-                verify_checksums=verify_checksums,
-                output_file_path=download_dir / obj.name,
-                object_name=obj.name,
-            )
-            if isinstance(result, SuccessfulDownload):
-                successful_downloads.append(result)
-            else:
-                failed_downloads.append(result)
-
-    return DownloadOutcome(successful=successful_downloads, failed=failed_downloads)
-
-
-def _download_single_pre_signed_url(
-    httpx_client: httpx.Client, pre_signed_url: str, verify_checksums: bool, output_file_path: Path, object_name: str
-) -> SuccessfulDownload | FailedDownload:
-    """
-    Download a single file using a pre-signed URL.
-    Helper function, do not call directly from outside this module.
-    """
-    with httpx_client.stream("GET", pre_signed_url) as response:
-        try:
-            response.raise_for_status()
-        except httpx.HTTPError as err:
-            return FailedDownload(object_name=object_name, file_path=output_file_path, exception=err)
-
-        server_checksum = response.headers.get("ETag", "").strip('"')
-
-        with open(output_file_path, "wb") as file:
-            for chunk in response.iter_bytes(chunk_size=8192):
-                file.write(chunk)
-
-    if verify_checksums:
-        try:
-            verify_downloaded_checksum(file_path=output_file_path, expected_checksum=server_checksum)
-        except ChecksumVerificationError as err:
-            return FailedDownload(object_name=object_name, file_path=output_file_path, exception=err)
-
-    return SuccessfulDownload(file_path=output_file_path, object_name=object_name)
-
-
-@dataclass
-class SuccessfulUpload:
-    """Represents a successfully uploaded file."""
-
-    file_path: Path
-    object_name: str
-
-
-@dataclass
-class FailedUpload:
-    """Represents a failed upload attempt."""
-
-    object_name: str
-    file_path: Path
-    exception: Exception
-
-
-@dataclass
-class UploadOutcome:
-    """Outcome of attempting to upload multiple files."""
-
-    successful: list[SuccessfulUpload]
-    failed: list[FailedUpload]
-
-
-def upload_multiple_pre_signed_urls(
-    pre_signed_urls: list[PreSignedUploadResponse], all_files: list[Path]
-) -> UploadOutcome:
-    """
-    Upload files using pre-signed PUT URLs.
-    Returns a UploadResults object containing the results of the upload attempts.
-    """
-    file_map = {file.name: file for file in all_files}
-
-    successful_uploads, failed_uploads = [], []
-    with httpx.Client(timeout=30.0) as client:
-        for obj in pre_signed_urls:
-            result = _upload_single_pre_signed_url(
-                httpx_client=client,
-                pre_signed_url=obj.pre_signed_url,
-                file_path=file_map[obj.name],
-                object_name=obj.name,
-                headers=obj.put_headers,
-            )
-
-            if isinstance(result, SuccessfulUpload):
-                successful_uploads.append(result)
-            else:
-                failed_uploads.append(result)
-
-    return UploadOutcome(successful=successful_uploads, failed=failed_uploads)
-
-
-def _upload_single_pre_signed_url(
-    httpx_client: httpx.Client,
-    pre_signed_url: str,
-    file_path: Path,
-    object_name: str,
-    headers: dict[str, str],
-) -> SuccessfulUpload | FailedUpload:
-    """
-    Upload a single file using a pre-signed PUT URL.
-    Helper function, do not call directly from outside this module.
-    """
-    with open(file_path, "rb") as file:
-        try:
-            response = httpx_client.put(pre_signed_url, content=file, headers=headers)
-            response.raise_for_status()
-        except httpx.HTTPError as err:
-            return FailedUpload(object_name=object_name, file_path=file_path, exception=err)
-
-    return SuccessfulUpload(file_path=file_path, object_name=object_name)

divbase_cli/services.py

@@ -1,219 +0,0 @@
-"""
-Service layer for DivBase CLI project version and S3 file operations.
-"""
-
-from pathlib import Path
-
-from divbase_cli.cli_exceptions import (
-    FileDoesNotExistInSpecifiedVersionError,
-    FilesAlreadyInProjectError,
-)
-from divbase_cli.pre_signed_urls import (
-    DownloadOutcome,
-    UploadOutcome,
-    download_multiple_pre_signed_urls,
-    upload_multiple_pre_signed_urls,
-)
-from divbase_cli.user_auth import make_authenticated_request
-from divbase_lib.api_schemas.project_versions import (
-    AddVersionRequest,
-    AddVersionResponse,
-    DeleteVersionRequest,
-    DeleteVersionResponse,
-    ProjectVersionDetailResponse,
-    ProjectVersionInfo,
-)
-from divbase_lib.api_schemas.s3 import ExistingFileResponse, PreSignedDownloadResponse, PreSignedUploadResponse
-from divbase_lib.s3_checksums import MD5CheckSumFormat, calculate_md5_checksum, convert_checksum_hex_to_base64
-
-
-def add_version_command(project_name: str, divbase_base_url: str, name: str, description: str) -> AddVersionResponse:
-    """Add a new version to the project versions table stored on the divbase server"""
-    request_data = AddVersionRequest(name=name, description=description)
-
-    response = make_authenticated_request(
-        method="PATCH",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/project-versions/add?project_name={project_name}",
-        json=request_data.model_dump(),
-    )
-
-    return AddVersionResponse(**response.json())
-
-
-def list_versions_command(project_name: str, include_deleted: bool, divbase_base_url: str) -> list[ProjectVersionInfo]:
-    """
-    List all versions in the project versions table stored on the divbase server.
-    Returns a dict of version names (keys) to details about the versions.
-    """
-    response = make_authenticated_request(
-        method="GET",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/project-versions/list?project_name={project_name}&include_deleted={str(include_deleted).lower()}",
-    )
-
-    project_versions = []
-    response_data = response.json()
-    for version in response_data:
-        project_versions.append(ProjectVersionInfo(**version))
-
-    return project_versions
-
-
-def get_version_details_command(
-    project_name: str, divbase_base_url: str, version_name: str
-) -> ProjectVersionDetailResponse:
-    """Get details about a specific project version, including all files and their version IDs at that version."""
-    response = make_authenticated_request(
-        method="GET",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/project-versions/version_details?project_name={project_name}&version_name={version_name}",
-    )
-
-    return ProjectVersionDetailResponse(**response.json())
-
-
-def delete_version_command(project_name: str, divbase_base_url: str, version_name: str) -> DeleteVersionResponse:
-    """
-    Delete a version from the project versions table stored on the divbase server.
-    This marks the version as (soft) deleted server side,
-    and it will eventually be permanently deleted (after some grace period).
-    """
-    request_data = DeleteVersionRequest(version_name=version_name)
-
-    response = make_authenticated_request(
-        method="DELETE",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/project-versions/delete?project_name={project_name}",
-        json=request_data.model_dump(),
-    )
-
-    return DeleteVersionResponse(**response.json())
-
-
-def list_files_command(divbase_base_url: str, project_name: str) -> list[str]:
-    """List all files in a project."""
-    response = make_authenticated_request(
-        method="GET",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/s3/?project_name={project_name}",
-    )
-
-    return response.json()
-
-
-def download_files_command(
-    divbase_base_url: str,
-    project_name: str,
-    all_files: list[str],
-    download_dir: Path,
-    verify_checksums: bool,
-    project_version: str | None = None,
-) -> DownloadOutcome:
-    """
-    Download files from the given project's S3 bucket.
-    """
-    if not download_dir.is_dir():
-        raise NotADirectoryError(
-            f"The specified download directory '{download_dir}' is not a directory. Please create it or specify a valid directory before continuing."
-        )
-
-    if project_version:
-        project_version_details = get_version_details_command(
-            project_name=project_name, divbase_base_url=divbase_base_url, version_name=project_version
-        )
-
-        # check if all files specified exist for download exist at this project version
-        missing_objects = [f for f in all_files if f not in project_version_details.files]
-        if missing_objects:
-            raise FileDoesNotExistInSpecifiedVersionError(
-                project_name=project_name,
-                project_version=project_version,
-                missing_files=missing_objects,
-            )
-        to_download = {file: project_version_details.files[file] for file in all_files}
-        json_data = [{"name": obj, "version_id": to_download[obj]} for obj in all_files]
-    else:
-        json_data = [{"name": obj, "version_id": None} for obj in all_files]
-
-    response = make_authenticated_request(
-        method="POST",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/s3/download?project_name={project_name}",
-        json=json_data,
-    )
-    pre_signed_urls = [PreSignedDownloadResponse(**item) for item in response.json()]
-
-    download_results = download_multiple_pre_signed_urls(
-        pre_signed_urls=pre_signed_urls, download_dir=download_dir, verify_checksums=verify_checksums
-    )
-    return download_results
-
-
-def upload_files_command(
-    project_name: str, divbase_base_url: str, all_files: list[Path], safe_mode: bool
-) -> UploadOutcome:
-    """
-    Upload files to the project's S3 bucket.
-    Returns an UploadOutcome object containing details of which files were successfully uploaded and which failed.
-
-    - Safe mode:
-        1. checks if any of the files that are to be uploaded already exist in the bucket (by comparing checksums)
-        2. Adds checksum to upload request to allow server to verify upload.
-    """
-    file_checksums_hex = {}
-    if safe_mode:
-        for file in all_files:
-            file_checksums_hex[file.name] = calculate_md5_checksum(file_path=file, output_format=MD5CheckSumFormat.HEX)
-
-        files_to_check = []
-        for file in all_files:
-            files_to_check.append({"object_name": file.name, "md5_checksum": file_checksums_hex[file.name]})
-
-        response = make_authenticated_request(
-            method="POST",
-            divbase_base_url=divbase_base_url,
-            api_route=f"v1/s3/check-exists?project_name={project_name}",
-            json=files_to_check,
-        )
-        existing_files = response.json()
-
-        if existing_files:
-            existing_object_names = [ExistingFileResponse(**file) for file in existing_files]
-            raise FilesAlreadyInProjectError(existing_files=existing_object_names, project_name=project_name)
-
-    objects_to_upload = []
-    for file in all_files:
-        upload_object = {
-            "name": file.name,
-            "content_length": file.stat().st_size,
-        }
-        if safe_mode:
-            hex_checksum = file_checksums_hex[file.name]
-            base64_checksum = convert_checksum_hex_to_base64(hex_checksum)
-            upload_object["md5_hash"] = base64_checksum
-
-        objects_to_upload.append(upload_object)
-
-    response = make_authenticated_request(
-        method="POST",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/s3/upload?project_name={project_name}",
-        json=objects_to_upload,
-    )
-    pre_signed_urls = [PreSignedUploadResponse(**item) for item in response.json()]
-    return upload_multiple_pre_signed_urls(pre_signed_urls=pre_signed_urls, all_files=all_files)
-
-
-def soft_delete_objects_command(divbase_base_url: str, project_name: str, all_files: list[str]) -> list[str]:
-    """
-    Soft delete objects from the project's bucket.
-    Returns a list of the soft deleted objects
-    """
-    response = make_authenticated_request(
-        method="DELETE",
-        divbase_base_url=divbase_base_url,
-        api_route=f"v1/s3/?project_name={project_name}",
-        json=all_files,
-    )
-    return response.json()

divbase_cli-0.1.0.dev2.dist-info/RECORD

@@ -1,22 +0,0 @@
-divbase_cli/__init__.py,sha256=njbWM3TEZYvcuYvwka0xuGnHkcz7YX1ZvGC9gdbzzkI,27
-divbase_cli/cli_config.py,sha256=zMAFg-GWdrBIgJicVYDms-UhipPIqQO2z0Q2-GNmwc4,1541
-divbase_cli/cli_exceptions.py,sha256=lzKVKhkWEtK-GPFwM5ADvrtkRPwAVmaOeY5IvBZgiLo,5555
-divbase_cli/config_resolver.py,sha256=Yd4JCa1Hul4oJFTBeSRdaPJzmYZ8-7iu5v7Q8Wdl6Kg,3159
-divbase_cli/display_task_history.py,sha256=k9V4YeWsqv6HTZn2Ns6sl4vngiWANFiPPj7E5kE0_K4,7628
-divbase_cli/divbase_cli.py,sha256=16Qa0p8NB_vQVO3yl9PQQ7BG85zHgzSZ_hnzRK-nrNo,2358
-divbase_cli/pre_signed_urls.py,sha256=ypQixtG03xEhMa3R_RpVUQ4thUaAJ1sIqRUy9HO35pc,5312
-divbase_cli/services.py,sha256=CTkadqDqg5XbEJJRdfG5QPIQehK3iWIrFje03oRYNJE,8286
-divbase_cli/user_auth.py,sha256=N3niQaU-4xIMqIHAwovp0h4wKHkjebU_u3q4KODBJfM,9603
-divbase_cli/user_config.py,sha256=tXsRXuJYkSwTvY89dzC3JF4Aw-6kp87rPrWYdmbsRYk,5884
-divbase_cli/cli_commands/__init__.py,sha256=K_2r8V1QGpEmxDcD2QOyWlXR4HPoc16yytmZwGkIyLw,166
-divbase_cli/cli_commands/auth_cli.py,sha256=SQUAU-34HYcCA7Qenbrtq454GoaQM3SMxXXCx0uMctk,3104
-divbase_cli/cli_commands/dimensions_cli.py,sha256=JPeQrFY6eBXmIXu5DsmOKI27m40KytRbS-kZAXd7dB0,5104
-divbase_cli/cli_commands/file_cli.py,sha256=W4SsEQE87gD1UqYMqI44B3iBgmuTD4Xx72xeWtDhxOY,10038
-divbase_cli/cli_commands/query_cli.py,sha256=-ncdyFkHTSblToWLlKjh6GEdBfo57sZcJCZLvHLSG0w,5778
-divbase_cli/cli_commands/task_history_cli.py,sha256=2A46RCST8iKWNaOjubHo7HxYcExxXkWk_fdG40sEC4U,4715
-divbase_cli/cli_commands/user_config_cli.py,sha256=eRw5bHDBuSEEjQQPZhI3SseplK4elvAvrtkIEMHdnIY,6463
-divbase_cli/cli_commands/version_cli.py,sha256=GnHsgLWGi7xOraFatAUnVBgUWdeYyQZ2C6qjpO9WTJ0,6469
-divbase_cli-0.1.0.dev2.dist-info/METADATA,sha256=MsUISyphFyqD0SGSdO_Q3UDY5FhDlG-wLYnLcPJtEmo,1390
-divbase_cli-0.1.0.dev2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-divbase_cli-0.1.0.dev2.dist-info/entry_points.txt,sha256=vaRJvvGmfesTaMMCCy3kcBhYzf51wZoEPuLqas5LDMg,100
-divbase_cli-0.1.0.dev2.dist-info/RECORD,,