deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/__init__.py

@@ -1,23 +1,49 @@
 import logging
 
-import deltacat.logs
-from deltacat.types.media import ContentType, ContentEncoding, TableType
-from deltacat.types.tables import TableWriteMode
-from deltacat.storage import ListResult, Namespace, LifecycleState, \
-    SchemaConsistencyType, LocalTable, LocalDataset, DistributedDataset
-from deltacat.compute.compactor import SortKey, SortOrder
+import deltacat.logs  # noqa: F401
+from deltacat.catalog.delegate import (
+    alter_namespace,
+    alter_table,
+    create_namespace,
+    create_table,
+    default_namespace,
+    drop_namespace,
+    drop_table,
+    get_namespace,
+    get_table,
+    list_namespaces,
+    list_tables,
+    namespace_exists,
+    read_table,
+    refresh_table,
+    rename_table,
+    table_exists,
+    truncate_table,
+    write_to_table,
+)
+from deltacat.catalog.model.catalog import (  # noqa: F401
+    Catalog,
+    Catalogs,
+    all_catalogs,
+    init,
+)
 from deltacat.catalog.model.table_definition import TableDefinition
-from deltacat.catalog.model.catalog import Catalog, Catalogs, all_catalogs, \
-    init
-from deltacat.catalog.delegate import alter_table, create_table, drop_table, \
-    refresh_table, list_tables, get_table, truncate_table, rename_table, \
-    table_exists, list_namespaces, alter_namespace, create_namespace, \
-    drop_namespace, default_namespace, get_namespace, namespace_exists, \
-    write_to_table, read_table
+from deltacat.compute.compactor import SortKey, SortOrder
+from deltacat.storage import (
+    DistributedDataset,
+    LifecycleState,
+    ListResult,
+    LocalDataset,
+    LocalTable,
+    Namespace,
+    SchemaConsistencyType,
+)
+from deltacat.types.media import ContentEncoding, ContentType, TableType
+from deltacat.types.tables import TableWriteMode
 
-logs.configure_deltacat_logger(logging.getLogger(__name__))
+deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "0.1.10.dev"
+__version__ = "0.1.12"
 
 
 __all__ = [

deltacat/aws/clients.py

@@ -1,28 +1,21 @@
-import boto3
 import logging
+from functools import lru_cache
+from typing import Optional
+
+import boto3
+from boto3.exceptions import ResourceNotExistsError
 from boto3.resources.base import ServiceResource
 from botocore.client import BaseClient
 from botocore.config import Config
-from boto3.exceptions import ResourceNotExistsError
-from functools import lru_cache
+
 from deltacat import logs
 from deltacat.aws.constants import BOTO_MAX_RETRIES
 
-from typing import Optional
-
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-def _resource(
-        name: str,
-        region: Optional[str],
-        **kwargs) -> ServiceResource:
-    boto_config = Config(
-        retries={
-            "max_attempts": BOTO_MAX_RETRIES,
-            "mode": 'standard'
-        }
-    )
+def _resource(name: str, region: Optional[str], **kwargs) -> ServiceResource:
+    boto_config = Config(retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "standard"})
     return boto3.resource(
         name,
         region,
@@ -31,20 +24,14 @@ def _resource(
     )
 
 
-def _client(
-        name: str,
-        region: Optional[str],
-        **kwargs) -> BaseClient:
+def _client(name: str, region: Optional[str], **kwargs) -> BaseClient:
     try:
         # try to re-use a client from the resource cache first
         return resource_cache(name, region, **kwargs).meta.client
     except ResourceNotExistsError:
         # fall back for clients without an associated resource
         boto_config = Config(
-            retries={
-                "max_attempts": BOTO_MAX_RETRIES,
-                "mode": 'standard'
-            }
+            retries={"max_attempts": BOTO_MAX_RETRIES, "mode": "standard"}
         )
         return boto3.client(
             name,
@@ -54,19 +41,13 @@ def _client(
         )
 
 
-def resource_cache(
-        name: str,
-        region: Optional[str],
-        **kwargs) -> ServiceResource:
+def resource_cache(name: str, region: Optional[str], **kwargs) -> ServiceResource:
     # we don't use the @lru_cache decorator because Ray can't pickle it
     cached_function = lru_cache()(_resource)
     return cached_function(name, region, **kwargs)
 
 
-def client_cache(
-        name: str,
-        region: Optional[str],
-        **kwargs) -> BaseClient:
+def client_cache(name: str, region: Optional[str], **kwargs) -> BaseClient:
     # we don't use the @lru_cache decorator because Ray can't pickle it
     cached_function = lru_cache()(_client)
     return cached_function(name, region, **kwargs)

deltacat/aws/constants.py

@@ -1,6 +1,6 @@
-from deltacat.utils.common import env_integer
 from typing import List
 
+from deltacat.utils.common import env_integer
 
 BOTO_MAX_RETRIES = env_integer("BOTO_MAX_RETRIES", 15)
 TIMEOUT_ERROR_CODES: List[str] = ["ReadTimeoutError", "ConnectTimeoutError"]

deltacat/aws/redshift/__init__.py

@@ -1,5 +1,10 @@
-from deltacat.aws.redshift.model.manifest import Manifest, ManifestAuthor, \
-    ManifestEntry, ManifestMeta, ManifestEntryList
+from deltacat.aws.redshift.model.manifest import (
+    Manifest,
+    ManifestAuthor,
+    ManifestEntry,
+    ManifestEntryList,
+    ManifestMeta,
+)
 
 __all__ = [
     "Manifest",

deltacat/aws/redshift/model/manifest.py

@@ -1,12 +1,12 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-import logging
 import itertools
+import logging
+from typing import Any, Dict, List, Optional
+from uuid import uuid4
 
 from deltacat import logs
-from uuid import uuid4
-from typing import Any, Dict, List, Optional
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -14,10 +14,11 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 class Manifest(dict):
     @staticmethod
     def _build_manifest(
-            meta: Optional[ManifestMeta],
-            entries: Optional[ManifestEntryList],
-            author: Optional[ManifestAuthor] = None,
-            uuid: str = None) -> Manifest:
+        meta: Optional[ManifestMeta],
+        entries: Optional[ManifestEntryList],
+        author: Optional[ManifestAuthor] = None,
+        uuid: str = None,
+    ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
         manifest = Manifest()
@@ -31,9 +32,11 @@ class Manifest(dict):
         return manifest
 
     @staticmethod
-    def of(entries: ManifestEntryList,
-           author: Optional[ManifestAuthor] = None,
-           uuid: str = None) -> Manifest:
+    def of(
+        entries: ManifestEntryList,
+        author: Optional[ManifestAuthor] = None,
+        uuid: str = None,
+    ) -> Manifest:
         if not uuid:
             uuid = str(uuid4())
         total_record_count = 0
@@ -52,15 +55,19 @@ class Manifest(dict):
                     content_encoding = None
                 entry_content_type = meta.content_type
                 if entry_content_type != content_type:
-                    msg = f"Expected all manifest entries to have content " \
-                          f"type '{content_type}' but found " \
-                          f"'{entry_content_type}'"
+                    msg = (
+                        f"Expected all manifest entries to have content "
+                        f"type '{content_type}' but found "
+                        f"'{entry_content_type}'"
+                    )
                     raise ValueError(msg)
                 entry_content_encoding = meta["content_encoding"]
                 if entry_content_encoding != content_encoding:
-                    msg = f"Expected all manifest entries to have content " \
-                          f"encoding '{content_encoding}' but found " \
-                          f"'{entry_content_encoding}'"
+                    msg = (
+                        f"Expected all manifest entries to have content "
+                        f"encoding '{content_encoding}' but found "
+                        f"'{entry_content_encoding}'"
+                    )
                     raise ValueError(msg)
                 total_record_count += meta.record_count or 0
                 total_content_length += meta.content_length or 0
@@ -70,25 +77,19 @@ class Manifest(dict):
             total_content_length,
             content_type,
             content_encoding,
-            total_source_content_length
-        )
-        manifest = Manifest._build_manifest(
-            meta,
-            entries,
-            author,
-            uuid
+            total_source_content_length,
         )
+        manifest = Manifest._build_manifest(meta, entries, author, uuid)
         return manifest
 
     @staticmethod
     def merge_manifests(
-            manifests: List[Manifest],
-            author: Optional[ManifestAuthor] = None) -> Manifest:
+        manifests: List[Manifest], author: Optional[ManifestAuthor] = None
+    ) -> Manifest:
         all_entries = ManifestEntryList(
-            itertools.chain(*[m.entries for m in manifests]))
-        merged_manifest = Manifest.of(
-            all_entries,
-            author)
+            itertools.chain(*[m.entries for m in manifests])
+        )
+        merged_manifest = Manifest.of(all_entries, author)
         return merged_manifest
 
     @property
@@ -119,14 +120,15 @@ class Manifest(dict):
 
 class ManifestMeta(dict):
     @staticmethod
-    def of(record_count: Optional[int],
-           content_length: Optional[int],
-           content_type: Optional[str],
-           content_encoding: Optional[str],
-           source_content_length: Optional[int] = None,
-           credentials: Optional[Dict[str, str]] = None,
-           content_type_parameters: Optional[List[Dict[str, str]]] = None) \
-            -> ManifestMeta:
+    def of(
+        record_count: Optional[int],
+        content_length: Optional[int],
+        content_type: Optional[str],
+        content_encoding: Optional[str],
+        source_content_length: Optional[int] = None,
+        credentials: Optional[Dict[str, str]] = None,
+        content_type_parameters: Optional[List[Dict[str, str]]] = None,
+    ) -> ManifestMeta:
         manifest_meta = ManifestMeta()
         if record_count is not None:
             manifest_meta["record_count"] = record_count
@@ -175,8 +177,7 @@ class ManifestMeta(dict):
 
 class ManifestAuthor(dict):
     @staticmethod
-    def of(name: Optional[str],
-           version: Optional[str]) -> ManifestAuthor:
+    def of(name: Optional[str], version: Optional[str]) -> ManifestAuthor:
         manifest_author = ManifestAuthor()
         if name is not None:
             manifest_author["name"] = name
@@ -195,15 +196,16 @@ class ManifestAuthor(dict):
 
 class ManifestEntry(dict):
     @staticmethod
-    def of(url: Optional[str],
-           meta: Optional[ManifestMeta],
-           mandatory: bool = True,
-           uri: Optional[str] = None,
-           uuid: Optional[str] = None) -> ManifestEntry:
+    def of(
+        url: Optional[str],
+        meta: Optional[ManifestMeta],
+        mandatory: bool = True,
+        uri: Optional[str] = None,
+        uuid: Optional[str] = None,
+    ) -> ManifestEntry:
         manifest_entry = ManifestEntry()
         if not (uri or url):
-            raise ValueError(
-                "No URI or URL specified for manifest entry contents.")
+            raise ValueError("No URI or URL specified for manifest entry contents.")
         if (uri and url) and (uri != url):
             raise ValueError(f"Manifest entry URI ({uri}) != URL ({url})")
         if url:
@@ -220,11 +222,13 @@ class ManifestEntry(dict):
 
     @staticmethod
     def from_s3_obj_url(
-            url: str,
-            record_count: int,
-            source_content_length: Optional[int] = None,
-            **s3_client_kwargs) -> ManifestEntry:
+        url: str,
+        record_count: int,
+        source_content_length: Optional[int] = None,
+        **s3_client_kwargs,
+    ) -> ManifestEntry:
         from deltacat.aws import s3u as s3_utils
+
         s3_obj = s3_utils.get_object_at_url(url, **s3_client_kwargs)
         logger.debug(f"Building manifest entry from {url}: {s3_obj}")
         manifest_entry_meta = ManifestMeta.of(