deltacat 1.1.36__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/storage/__init__.py

@@ -1,24 +1,63 @@
-from deltacat.aws.redshift import (
+from deltacat.storage.model.manifest import (
+    EntryType,
+    EntryParams,
     Manifest,
     ManifestAuthor,
     ManifestEntry,
     ManifestEntryList,
     ManifestMeta,
 )
-from deltacat.storage.model.delta import Delta, DeltaLocator
+from deltacat.storage.model.delta import (
+    Delta,
+    DeltaLocator,
+    DeltaProperties,
+)
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.locator import Locator
-from deltacat.storage.model.namespace import Namespace, NamespaceLocator
-from deltacat.storage.model.partition import Partition, PartitionLocator
-from deltacat.storage.model.stream import Stream, StreamLocator
-from deltacat.storage.model.table import Table, TableLocator
-from deltacat.storage.model.table_version import TableVersion, TableVersionLocator
-from deltacat.storage.model.delete_parameters import DeleteParameters
-from deltacat.storage.model.partition_spec import (
-    PartitionFilter,
+from deltacat.storage.model.metafile import (
+    Metafile,
+)
+from deltacat.storage.model.transaction import (
+    TransactionOperation,
+    Transaction,
+)
+from deltacat.storage.model.namespace import (
+    Namespace,
+    NamespaceLocator,
+    NamespaceProperties,
+)
+from deltacat.storage.model.partition import (
+    Partition,
+    PartitionLocator,
+    PartitionLocatorAlias,
+    PartitionKey,
+    PartitionScheme,
+    PartitionSchemeList,
     PartitionValues,
-    DeltaPartitionSpec,
-    StreamPartitionSpec,
+)
+from deltacat.storage.model.schema import (
+    Field,
+    FieldId,
+    FieldLocator,
+    FieldName,
+    NestedFieldName,
+    Schema,
+    SchemaList,
+)
+from deltacat.storage.model.stream import (
+    Stream,
+    StreamLocator,
+    StreamLocatorAlias,
+)
+from deltacat.storage.model.table import (
+    Table,
+    TableLocator,
+    TableProperties,
+)
+from deltacat.storage.model.table_version import (
+    TableVersion,
+    TableVersionLocator,
+    TableVersionProperties,
 )
 from deltacat.storage.model.transform import (
     Transform,
@@ -26,9 +65,17 @@ from deltacat.storage.model.transform import (
     TransformParameters,
     BucketingStrategy,
     BucketTransformParameters,
-    IdentityTransformParameters,
+    TruncateTransformParameters,
+    BucketTransform,
+    IdentityTransform,
+    VoidTransform,
+    UnknownTransform,
+    HourTransform,
+    DayTransform,
+    MonthTransform,
+    YearTransform,
+    TruncateTransform,
 )
-
 from deltacat.storage.model.types import (
     CommitState,
     DeltaType,
@@ -36,18 +83,39 @@ from deltacat.storage.model.types import (
     LifecycleState,
     LocalDataset,
     LocalTable,
+    NullOrder,
     SchemaConsistencyType,
+    StreamFormat,
+    SortOrder,
+    TransactionType,
+    TransactionOperationType,
+)
+from deltacat.storage.model.sort_key import (
+    SortKey,
+    SortScheme,
+    SortSchemeList,
 )
-from deltacat.storage.model.sort_key import SortKey, SortOrder
+from deltacat.storage.main import impl as metastore
 
 __all__ = [
+    "BucketingStrategy",
+    "BucketTransform",
+    "BucketTransformParameters",
     "CommitState",
+    "DayTransform",
     "Delta",
     "DeltaLocator",
-    "Partition",
-    "DeleteParameters",
+    "DeltaProperties",
     "DeltaType",
     "DistributedDataset",
+    "EntryType",
+    "EntryParams",
+    "Field",
+    "FieldId",
+    "FieldLocator",
+    "FieldName",
+    "HourTransform",
+    "IdentityTransform",
     "LifecycleState",
     "ListResult",
     "LocalDataset",
@@ -56,28 +124,50 @@ __all__ = [
     "Manifest",
     "ManifestAuthor",
     "ManifestEntry",
-    "ManifestMeta",
     "ManifestEntryList",
+    "ManifestMeta",
+    "Metafile",
+    "metastore",
+    "MonthTransform",
     "Namespace",
     "NamespaceLocator",
+    "NamespaceProperties",
+    "NestedFieldName",
+    "NullOrder",
+    "Partition",
+    "PartitionKey",
     "PartitionLocator",
-    "Stream",
+    "PartitionLocatorAlias",
+    "PartitionScheme",
+    "PartitionSchemeList",
+    "PartitionValues",
+    "Schema",
+    "SchemaList",
     "SchemaConsistencyType",
+    "SortKey",
+    "SortOrder",
+    "SortScheme",
+    "SortSchemeList",
+    "Stream",
+    "StreamFormat",
     "StreamLocator",
+    "StreamLocatorAlias",
     "Table",
     "TableLocator",
+    "TableProperties",
     "TableVersion",
     "TableVersionLocator",
-    "SortKey",
-    "SortOrder",
-    "PartitionFilter",
-    "PartitionValues",
-    "DeltaPartitionSpec",
-    "StreamPartitionSpec",
+    "TableVersionProperties",
+    "Transaction",
+    "TransactionOperation",
+    "TransactionOperationType",
+    "TransactionType",
     "Transform",
     "TransformName",
     "TransformParameters",
-    "BucketingStrategy",
-    "BucketTransformParameters",
-    "IdentityTransformParameters",
+    "TruncateTransform",
+    "TruncateTransformParameters",
+    "UnknownTransform",
+    "VoidTransform",
+    "YearTransform",
 ]

deltacat/storage/iceberg/iceberg_scan_planner.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+from pyiceberg.catalog import Catalog
+from deltacat.storage.model.scan.push_down import Pushdown
+from deltacat.storage.model.scan.scan_plan import ScanPlan
+from deltacat.storage.model.scan.scan_task import FileScanTask, DataFile
+from deltacat.storage.util.scan_planner import ScanPlanner
+from deltacat.storage.iceberg.impl import _try_load_iceberg_table
+
+
+class IcebergScanPlanner(ScanPlanner):
+    def __init__(self, catalog: Catalog):
+        self.catalog = catalog
+
+    def create_scan_plan(
+        self,
+        table_name: str,
+        namespace: Optional[str] = None,
+        pushdown: Optional[Pushdown] = None,
+    ) -> ScanPlan:
+        iceberg_table = _try_load_iceberg_table(
+            self.catalog, namespace=namespace, table_name=table_name
+        )
+        file_scan_tasks = []
+        # TODO: implement predicate pushdown to Iceberg
+        for scan_task in iceberg_table.scan().plan_files():
+            file_scan_tasks.append(FileScanTask([DataFile(scan_task.file.file_path)]))
+        return ScanPlan(file_scan_tasks)