deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/scan/push_down.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Optional
+
+from deltacat.storage.model.expression import Expression
+
+
+class RowFilter:
+    ...
+
+
+class ColumnFilter:
+    ...
+
+
+@dataclass
+class PartitionFilter:
+    expr: Expression
+
+    @staticmethod
+    def of(expr: Expression) -> PartitionFilter:
+        return PartitionFilter(expr)
+
+
+@dataclass
+class Pushdown:
+    """Represents pushdown predicates to be applied for DeltaCAT Tables"""
+
+    row_filter: Optional[RowFilter]
+    column_filter: Optional[ColumnFilter]
+    partition_filter: Optional[PartitionFilter]
+    limit: Optional[int]
+
+    @staticmethod
+    def of(
+        row_filter: Optional[RowFilter],
+        column_filter: Optional[ColumnFilter],
+        partition_filter: Optional[PartitionFilter],
+        limit: Optional[int],
+    ) -> Pushdown:
+        return Pushdown(
+            row_filter=row_filter,
+            column_filter=column_filter,
+            partition_filter=partition_filter,
+            limit=limit,
+        )

deltacat/storage/model/scan/scan_plan.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+from deltacat.storage.model.scan.scan_task import ScanTask
+
+
+@dataclass
+class ScanPlan:
+    """Represents collection of ScanTasks to be passed to compute engine for query planning"""
+
+    scan_tasks: list[ScanTask]

deltacat/storage/model/scan/scan_task.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class DataFile:
+    """Represents a data file, e.g. a S3 object, or a local file."""
+
+    file_path: str
+
+
+class ScanTask(ABC):
+    """Base class representing a unit of data to be read by a compute worker"""
+
+    @abstractmethod
+    def data_files(self) -> list[DataFile]:
+        pass
+
+
+@dataclass
+class FileScanTask(ScanTask):
+    """A unit of data in the form of data files"""
+
+    data_file_list: list[DataFile]
+
+    def data_files(self) -> list[DataFile]:
+        return self.data_file_list
+
+
+class ShardedScanTask(ScanTask):
+    """A unit of data in the form of shards (e.g. shard 1-10 each represents 1/10 of all data in a Table)"""
+
+    def data_files(self) -> list[DataFile]:
+        raise NotImplementedError("data_files is not implemented for ShardedScanTask")