deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/catalog/interface.py

@@ -1,19 +1,32 @@
-import pyarrow as pa
-from deltacat import SortKey, TableWriteMode, ContentType, ListResult, \
-    Namespace, LifecycleState, SchemaConsistencyType, LocalTable, \
-    LocalDataset, DistributedDataset, TableDefinition
 from typing import Any, Dict, List, Optional, Set, Union
 
+import pyarrow as pa
+
+from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.compute.compactor.model.sort_key import SortKey
+from deltacat.storage.model.list_result import ListResult
+from deltacat.storage.model.namespace import Namespace
+from deltacat.storage.model.types import (
+    DistributedDataset,
+    LifecycleState,
+    LocalDataset,
+    LocalTable,
+    SchemaConsistencyType,
+)
+from deltacat.types.media import ContentType
+from deltacat.types.tables import TableWriteMode
+
 
 # table functions
 def write_to_table(
-        data: Union[LocalTable, LocalDataset, DistributedDataset],
-        table: str,
-        namespace: Optional[str] = None,
-        mode: TableWriteMode = TableWriteMode.AUTO,
-        content_type: ContentType = ContentType.PARQUET,
-        *args,
-        **kwargs) -> None:
+    data: Union[LocalTable, LocalDataset, DistributedDataset],
+    table: str,
+    namespace: Optional[str] = None,
+    mode: TableWriteMode = TableWriteMode.AUTO,
+    content_type: ContentType = ContentType.PARQUET,
+    *args,
+    **kwargs
+) -> None:
     """Write local or distributed data to a table. Raises an error if the
     table does not exist and the table write mode is not CREATE or AUTO.
 
@@ -25,168 +38,136 @@ def write_to_table(
 
 
 def read_table(
-        table: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> DistributedDataset:
+    table: str, namespace: Optional[str] = None, *args, **kwargs
+) -> DistributedDataset:
     """Read a table into a distributed dataset."""
     raise NotImplementedError("read_table not implemented")
 
 
 def alter_table(
-        table: str,
-        namespace: Optional[str] = None,
-        lifecycle_state: Optional[LifecycleState] = None,
-        schema_updates: Optional[Dict[str, Any]] = None,
-        partition_updates: Optional[Dict[str, Any]] = None,
-        primary_keys: Optional[Set[str]] = None,
-        sort_keys: Optional[List[SortKey]] = None,
-        description: Optional[str] = None,
-        properties: Optional[Dict[str, str]] = None,
-        *args,
-        **kwargs) -> None:
+    table: str,
+    namespace: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = None,
+    schema_updates: Optional[Dict[str, Any]] = None,
+    partition_updates: Optional[Dict[str, Any]] = None,
+    primary_keys: Optional[Set[str]] = None,
+    sort_keys: Optional[List[SortKey]] = None,
+    description: Optional[str] = None,
+    properties: Optional[Dict[str, str]] = None,
+    *args,
+    **kwargs
+) -> None:
     """Alter table definition."""
     raise NotImplementedError("alter_table not implemented")
 
 
 def create_table(
-        table: str,
-        namespace: Optional[str] = None,
-        lifecycle_state: Optional[LifecycleState] = None,
-        schema: Optional[Union[pa.Schema, str, bytes]] = None,
-        schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
-        partition_keys: Optional[List[Dict[str, Any]]] = None,
-        primary_keys: Optional[Set[str]] = None,
-        sort_keys: Optional[List[SortKey]] = None,
-        description: Optional[str] = None,
-        properties: Optional[Dict[str, str]] = None,
-        permissions: Optional[Dict[str, Any]] = None,
-        content_types: Optional[List[ContentType]] = None,
-        replace_existing_table: bool = False,
-        *args,
-        **kwargs) -> TableDefinition:
+    table: str,
+    namespace: Optional[str] = None,
+    lifecycle_state: Optional[LifecycleState] = None,
+    schema: Optional[Union[pa.Schema, str, bytes]] = None,
+    schema_consistency: Optional[Dict[str, SchemaConsistencyType]] = None,
+    partition_keys: Optional[List[Dict[str, Any]]] = None,
+    primary_keys: Optional[Set[str]] = None,
+    sort_keys: Optional[List[SortKey]] = None,
+    description: Optional[str] = None,
+    properties: Optional[Dict[str, str]] = None,
+    permissions: Optional[Dict[str, Any]] = None,
+    content_types: Optional[List[ContentType]] = None,
+    replace_existing_table: bool = False,
+    *args,
+    **kwargs
+) -> TableDefinition:
     """Create an empty table. Raises an error if the table already exists and
     `replace_existing_table` is False."""
     raise NotImplementedError("create_table not implemented")
 
 
 def drop_table(
-        table: str,
-        namespace: Optional[str] = None,
-        purge: bool = False,
-        *args,
-        **kwargs) -> None:
+    table: str, namespace: Optional[str] = None, purge: bool = False, *args, **kwargs
+) -> None:
     """Drop a table from the catalog and optionally purge it. Raises an error
     if the table does not exist."""
     raise NotImplementedError("drop_table not implemented")
 
 
-def refresh_table(
-        table: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> None:
+def refresh_table(table: str, namespace: Optional[str] = None, *args, **kwargs) -> None:
     """Refresh metadata cached on the Ray cluster for the given table."""
     raise NotImplementedError("refresh_table not implemented")
 
 
 def list_tables(
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> ListResult[TableDefinition]:
+    namespace: Optional[str] = None, *args, **kwargs
+) -> ListResult[TableDefinition]:
     """List a page of table definitions. Raises an error if the given namespace
     does not exist."""
     raise NotImplementedError("list_tables not implemented")
 
 
 def get_table(
-        table: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> Optional[TableDefinition]:
+    table: str, namespace: Optional[str] = None, *args, **kwargs
+) -> Optional[TableDefinition]:
     """Get table definition metadata. Returns None if the given table does not
     exist."""
     raise NotImplementedError("get_table not implemented")
 
 
 def truncate_table(
-        table: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> None:
+    table: str, namespace: Optional[str] = None, *args, **kwargs
+) -> None:
     """Truncate table data. Raises an error if the table does not exist."""
     raise NotImplementedError("truncate_table not implemented")
 
 
 def rename_table(
-        table: str,
-        new_name: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> None:
+    table: str, new_name: str, namespace: Optional[str] = None, *args, **kwargs
+) -> None:
     """Rename a table."""
     raise NotImplementedError("rename_table not implemented")
 
 
-def table_exists(
-        table: str,
-        namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> bool:
+def table_exists(table: str, namespace: Optional[str] = None, *args, **kwargs) -> bool:
     """Returns True if the given table exists, False if not."""
     raise NotImplementedError("table_exists not implemented")
 
 
 # namespace functions
-def list_namespaces(
-        *args,
-        **kwargs) -> ListResult[Namespace]:
+def list_namespaces(*args, **kwargs) -> ListResult[Namespace]:
     """List a page of table namespaces."""
     raise NotImplementedError("list_namespaces not implemented")
 
 
-def get_namespace(
-        namespace: str,
-        *args,
-        **kwargs) -> Optional[Namespace]:
+def get_namespace(namespace: str, *args, **kwargs) -> Optional[Namespace]:
     """Gets table namespace metadata for the specified table namespace. Returns
     None if the given namespace does not exist."""
     raise NotImplementedError("get_namespace not implemented")
 
 
-def namespace_exists(
-        namespace: str,
-        *args,
-        **kwargs) -> bool:
+def namespace_exists(namespace: str, *args, **kwargs) -> bool:
     """Returns True if the given table namespace exists, False if not."""
     raise NotImplementedError("namespace_exists not implemented")
 
 
 def create_namespace(
-        namespace: str,
-        permissions: Dict[str, Any],
-        *args,
-        **kwargs) -> Namespace:
+    namespace: str, permissions: Dict[str, Any], *args, **kwargs
+) -> Namespace:
     """Creates a table namespace with the given name and permissions. Returns
     the created namespace. Raises an error if the namespace already exists."""
     raise NotImplementedError("create_namespace not implemented")
 
 
 def alter_namespace(
-        namespace: str,
-        permissions: Optional[Dict[str, Any]] = None,
-        new_namespace: Optional[str] = None,
-        *args,
-        **kwargs) -> None:
+    namespace: str,
+    permissions: Optional[Dict[str, Any]] = None,
+    new_namespace: Optional[str] = None,
+    *args,
+    **kwargs
+) -> None:
     """Alter table namespace definition."""
     raise NotImplementedError("alter_namespace not implemented")
 
 
-def drop_namespace(
-        namespace: str,
-        purge: bool = False,
-        *args,
-        **kwargs) -> None:
+def drop_namespace(namespace: str, purge: bool = False, *args, **kwargs) -> None:
     """Drop the given namespace and all of its tables from the catalog,
     optionally purging them."""
     raise NotImplementedError("drop_namespace not implemented")

deltacat/catalog/model/catalog.py

@@ -1,21 +1,17 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Any, Dict, List, Optional
+
 import ray
 
 from deltacat.catalog import interface as catalog_interface
-from typing import Any, Dict, List, Optional
-
 
 all_catalogs: Optional[Catalogs] = None
 
 
 class Catalog:
-    def __init__(
-            self,
-            impl=catalog_interface,
-            *args,
-            **kwargs):
+    def __init__(self, impl=catalog_interface, *args, **kwargs):
         self._impl = impl
         self._impl.initialize(*args, **kwargs)
 
@@ -27,19 +23,22 @@ class Catalog:
 @ray.remote
 class Catalogs:
     def __init__(
-            self,
-            catalogs: Dict[str, Catalog],
-            default_catalog_name: str = None,
-            *args,
-            **kwargs):
+        self,
+        catalogs: Dict[str, Catalog],
+        default_catalog_name: str = None,
+        *args,
+        **kwargs,
+    ):
         if default_catalog_name and default_catalog_name not in catalogs:
             raise ValueError(
                 f"Catalog {default_catalog_name} not found "
-                f"in catalogs to register: {catalogs}")
+                f"in catalogs to register: {catalogs}"
+            )
         if not catalogs:
             raise ValueError(
                 f"No catalogs given to register. "
-                f"Please specify one or more catalogs.")
+                f"Please specify one or more catalogs."
+            )
         self.catalogs: Dict[str, Catalog] = catalogs
         if default_catalog_name:
             self.default_catalog = self.catalogs[default_catalog_name]
@@ -65,11 +64,12 @@ class Catalogs:
 
 
 def init(
-        catalogs: Dict[str, Catalog],
-        default_catalog_name: str = None,
-        ray_init_args: Dict[str, Any] = None,
-        *args,
-        **kwargs) -> None:
+    catalogs: Dict[str, Catalog],
+    default_catalog_name: str = None,
+    ray_init_args: Dict[str, Any] = None,
+    *args,
+    **kwargs,
+) -> None:
 
     if not ray.is_initialized():
         if ray_init_args:
@@ -79,5 +79,5 @@ def init(
 
     global all_catalogs
     all_catalogs = Catalogs.remote(
-        catalogs=catalogs,
-        default_catalog_name=default_catalog_name)
+        catalogs=catalogs, default_catalog_name=default_catalog_name
+    )

deltacat/catalog/model/table_definition.py

@@ -1,19 +1,21 @@
 # Allow self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from deltacat.storage import Table, TableVersion, Stream
+from deltacat.storage import Stream, Table, TableVersion
 
 
 class TableDefinition(dict):
     @staticmethod
-    def of(table: Table,
-           table_version: TableVersion,
-           stream: Stream) -> TableDefinition:
-        return TableDefinition({
-            "table": table,
-            "tableVersion": table_version,
-            "stream": stream,
-        })
+    def of(
+        table: Table, table_version: TableVersion, stream: Stream
+    ) -> TableDefinition:
+        return TableDefinition(
+            {
+                "table": table,
+                "tableVersion": table_version,
+                "stream": stream,
+            }
+        )
 
     @property
     def table(self) -> Table:

deltacat/compute/compactor/__init__.py

@@ -1,20 +1,16 @@
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
-from deltacat.compute.compactor.model.delta_file_envelope import \
-    DeltaFileEnvelope
-from deltacat.compute.compactor.model.delta_file_locator import \
-    DeltaFileLocator
-from deltacat.compute.compactor.model.materialize_result import \
-   MaterializeResult
-from deltacat.compute.compactor.model.primary_key_index import \
-    PrimaryKeyIndexLocator, PrimaryKeyIndexMeta, \
-    PrimaryKeyIndexVersionLocator, PrimaryKeyIndexVersionMeta
-from deltacat.compute.compactor.model.pyarrow_write_result import \
-    PyArrowWriteResult
-from deltacat.compute.compactor.model.round_completion_info import \
-   RoundCompletionInfo
-from deltacat.compute.compactor.model.sort_key import \
-   SortKey, SortOrder
-
+from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
+from deltacat.compute.compactor.model.delta_file_locator import DeltaFileLocator
+from deltacat.compute.compactor.model.materialize_result import MaterializeResult
+from deltacat.compute.compactor.model.primary_key_index import (
+    PrimaryKeyIndexLocator,
+    PrimaryKeyIndexMeta,
+    PrimaryKeyIndexVersionLocator,
+    PrimaryKeyIndexVersionMeta,
+)
+from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
+from deltacat.compute.compactor.model.round_completion_info import RoundCompletionInfo
+from deltacat.compute.compactor.model.sort_key import SortKey, SortOrder
 
 __all__ = [
     "DeltaAnnotated",