deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/examples/indexer/job_runner.py

@@ -0,0 +1,198 @@
+import argparse
+import pathlib
+
+from deltacat.compute import (
+    job_client,
+    JobStatus,
+)
+
+
+def run_async(
+    source: str,
+    dest: str,
+    jobs_to_submit: int,
+    job_timeout: int,
+    cloud: str,
+    restart_ray: bool,
+):
+    # print package version info
+    working_dir = pathlib.Path(__file__).parent
+    cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
+    job_number = 0
+    client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
+    job_ids = []
+    while jobs_to_submit > 0:
+        jobs_to_submit -= 1
+        job_dest = dest + f".{job_number}"
+        job_id = client.submit_job(
+            # Entrypoint shell command to execute
+            entrypoint=f"python3 indexer.py --source '{source}' --dest '{job_dest}'",
+            # Path to the local directory that contains the indexer.py file
+            # This entire directory will be zipped into a job package, so keep
+            # it small.
+            runtime_env={"working_dir": working_dir},
+        )
+        job_ids.append(job_id)
+        job_number += 1
+
+    print("Waiting for all jobs to complete...")
+    job_number = 0
+    all_job_logs = ""
+    for job_id in job_ids:
+        job_status = client.await_job(job_id, timeout_seconds=job_timeout)
+        if job_status != JobStatus.SUCCEEDED:
+            print(f"Job `{job_id}` logs: ")
+            print(client.get_job_logs(job_id))
+            raise RuntimeError(f"Job `{job_id}` terminated with status: {job_status}")
+        all_job_logs += f"\nJob #{job_number} logs: \n"
+        all_job_logs += client.get_job_logs(job_id)
+        job_number += 1
+    print("All jobs completed!")
+    print("Job Logs: ")
+    print(all_job_logs)
+
+
+def run_sync(
+    source: str,
+    dest: str,
+    jobs_to_submit: int,
+    job_timeout: int,
+    cloud: str,
+    restart_ray: bool,
+):
+    working_dir = pathlib.Path(__file__).parent
+    cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
+    client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
+    job_number = 0
+    while job_number < jobs_to_submit:
+        job_dest = dest + f".{job_number}"
+        job_run_result = client.run_job(
+            # Entrypoint shell command to execute
+            entrypoint=f"python3 indexer.py --source '{source}' --dest '{job_dest}'",
+            # Path to the local directory that contains the indexer.py file
+            # This entire directory will be zipped into a job package, so keep
+            # it small.
+            runtime_env={"working_dir": working_dir},
+            timeout_seconds=job_timeout,
+        )
+        print(
+            f"Job ID {job_run_result.job_id} terminal state: {job_run_result.job_status}"
+        )
+        print(f"Job ID {job_run_result.job_id} logs: ")
+        print(job_run_result.job_logs)
+        job_number += 1
+
+
+def run(
+    source: str,
+    dest: str,
+    restart_ray: bool,
+    jobs_to_submit: int,
+    job_timeout: int,
+    asynchronous: bool,
+    cloud_provider: str,
+):
+    run_func = run_async if asynchronous else run_sync
+    run_func(
+        source=source,
+        dest=dest,
+        jobs_to_submit=jobs_to_submit,
+        job_timeout=job_timeout,
+        cloud=cloud_provider,
+        restart_ray=restart_ray,
+    )
+
+
+if __name__ == "__main__":
+    """
+    # Run this example through a command of the form:
+    $ python ./deltacat/examples/job_runner.py -- \
+    $ --source text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31 \
+    $ --dest parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet \
+    $ --asynchronous \
+    $ --jobs-to-submit 100 \
+    $ --job-timeout 90 \
+    $ --cloud-provider aws
+    """
+    script_args = [
+        (
+            [
+                "--source",
+            ],
+            {
+                "help": "Source DeltaCAT URL to index.",
+                "type": str,
+                "default": "text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31",
+            },
+        ),
+        (
+            [
+                "--dest",
+            ],
+            {
+                "help": "Destination DeltaCAT URL to store the indexed file.",
+                "type": str,
+                "default": "parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet",
+            },
+        ),
+        (
+            [
+                "--restart-ray",
+            ],
+            {
+                "help": "Restart Ray on an existing cluster.",
+                "action": "store_true",
+                "default": False,
+            },
+        ),
+        (
+            [
+                "--asynchronous",
+            ],
+            {
+                "help": "Run jobs asynchronously.",
+                "action": "store_true",
+                "default": False,
+            },
+        ),
+        (
+            [
+                "--jobs-to-submit",
+            ],
+            {
+                "help": "Number of indexer jobs to submit for execution.",
+                "type": int,
+                "default": 1,
+            },
+        ),
+        (
+            [
+                "--job-timeout",
+            ],
+            {
+                "help": "Job timeout in seconds.",
+                "type": int,
+                "default": 300,
+            },
+        ),
+        (
+            [
+                "--cloud-provider",
+            ],
+            {
+                "help": "Ray Cluster Cloud Provider ('aws' or 'gcp')",
+                "type": str,
+                "default": "aws",
+            },
+        ),
+    ]
+
+    # parse CLI input arguments
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+
+    # run the example using os.environ as kwargs
+    run(**vars(args))

deltacat/exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from enum import Enum
-from typing import Callable
+from typing import Callable, Optional, TYPE_CHECKING
 import logging
 
 import tenacity
@@ -28,6 +28,9 @@ from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
 )
 
+if TYPE_CHECKING:
+    from deltacat.storage.model.schema import FieldLocator
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 DELTACAT_STORAGE_PARAM = "deltacat_storage"
@@ -74,9 +77,18 @@ class DeltaCatErrorNames(str, Enum):
     TABLE_NOT_FOUND_ERROR = "TableNotFoundError"
     TABLE_VERSION_NOT_FOUND_ERROR = "TableVersionNotFoundError"
     STREAM_NOT_FOUND_ERROR = "StreamNotFoundError"
+    PARTITION_NOT_FOUND_ERROR = "PartitionNotFoundError"
     DELTA_NOT_FOUND_ERROR = "DeltaNotFoundError"
     TABLE_ALREADY_EXISTS_ERROR = "TableAlreadyExistsError"
+    TABLE_VERSION_ALREADY_EXISTS_ERROR = "TableVersionAlreadyExistsError"
     NAMESPACE_ALREADY_EXISTS_ERROR = "NamespaceAlreadyExistsError"
+    SCHEMA_COMPATIBILITY_ERROR = "SchemaCompatibilityError"
+    SCHEMA_VALIDATION_ERROR = "SchemaValidationError"
+    TABLE_VALIDATION_ERROR = "TableValidationError"
+    CONCURRENT_MODIFICATION_ERROR = "ConcurrentModificationError"
+    OBJECT_NOT_FOUND_ERROR = "ObjectNotFoundError"
+    OBJECT_DELETED_ERROR = "ObjectDeletedError"
+    OBJECT_ALREADY_EXISTS_ERROR = "ObjectAlreadyExistsError"
 
 
 class DeltaCatError(Exception):
@@ -87,9 +99,12 @@ class DeltaCatError(Exception):
         super().__init__(*args, **kwargs)
 
     def _get_ray_task_id_and_node_ip(self):
-        task_id = get_current_ray_task_id()
-        node_ip = ray.util.get_node_ip_address()
-        return task_id, node_ip
+        if ray.is_initialized():
+            task_id = get_current_ray_task_id()
+            node_ip = ray.util.get_node_ip_address()
+            return task_id, node_ip
+        else:
+            return None, None
 
 
 class NonRetryableError(DeltaCatError):
@@ -232,6 +247,10 @@ class TableVersionNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_VERSION_NOT_FOUND_ERROR.value
 
 
+class PartitionNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.PARTITION_NOT_FOUND_ERROR.value
+
+
 class StreamNotFoundError(NonRetryableError):
     error_name = DeltaCatErrorNames.STREAM_NOT_FOUND_ERROR.value
 
@@ -244,10 +263,53 @@ class TableAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class TableVersionAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VERSION_ALREADY_EXISTS_ERROR.value
+
+
 class NamespaceAlreadyExistsError(NonRetryableError):
     error_name = DeltaCatErrorNames.TABLE_ALREADY_EXISTS_ERROR.value
 
 
+class ObjectNotFoundError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_NOT_FOUND_ERROR.value
+
+
+class ObjectDeletedError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_DELETED_ERROR.value
+
+
+class ObjectAlreadyExistsError(NonRetryableError):
+    error_name = DeltaCatErrorNames.OBJECT_ALREADY_EXISTS_ERROR.value
+
+
+class ConcurrentModificationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.CONCURRENT_MODIFICATION_ERROR.value
+
+
+class SchemaValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_VALIDATION_ERROR.value
+
+
+class TableValidationError(NonRetryableError):
+    error_name = DeltaCatErrorNames.TABLE_VALIDATION_ERROR.value
+
+
+class SchemaCompatibilityError(NonRetryableError):
+    error_name = DeltaCatErrorNames.SCHEMA_COMPATIBILITY_ERROR.value
+    """Raised when a schema update would break backward compatibility."""
+
+    def __init__(
+        self,
+        message: str,
+        field_locator: Optional[FieldLocator] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(message, *args, **kwargs)
+        self.field_locator = field_locator
+
+
 def categorize_errors(func: Callable):
     def wrapper(*args, **kwargs):
         try:

deltacat/experimental/catalog/iceberg/__init__.py

@@ -0,0 +1,6 @@
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
+import deltacat.experimental.catalog.iceberg.impl as IcebergCatalog
+
+__all__ = ["IcebergCatalogConfig", "IcebergCatalog"]

deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py

@@ -15,7 +15,7 @@ class IcebergCatalogConfig:
 
     This configuration is passed through to PyIceberg by invoking load_catalog.
     The Properties provided must match properties accepted by PyIceberg for each catalog type
-    See: :func:`deltacat.catalog.iceberg.initialize`
+    See: :func:`deltacat.experimental.catalog.iceberg.initialize`
 
     Attributes:
         type: The PyIceberg Catalog instance

deltacat/{catalog → experimental/catalog}/iceberg/impl.py

@@ -1,16 +1,26 @@
 import logging
+import sys
 
 from typing import Any, Dict, List, Optional, Union
 
-from daft import DataFrame
+from daft import DataFrame, context
+from daft.daft import ScanOperatorHandle, StorageConfig
+from daft.logical.builder import LogicalPlanBuilder
 
 from deltacat import logs
+from deltacat.catalog.model.catalog import Catalog
 from deltacat.catalog.model.table_definition import TableDefinition
+from deltacat.utils.daft import DeltaCatScanOperator
 from deltacat.exceptions import TableAlreadyExistsError
-from deltacat.storage.iceberg.iceberg_scan_planner import IcebergScanPlanner
-from deltacat.storage.iceberg.model import PartitionSchemeMapper, SchemaMapper
+from deltacat.experimental.storage.iceberg.iceberg_scan_planner import (
+    IcebergScanPlanner,
+)
+from deltacat.experimental.storage.iceberg.model import (
+    PartitionSchemeMapper,
+    SchemaMapper,
+)
 from deltacat.storage.model.partition import PartitionScheme
-from deltacat.storage.iceberg.impl import _get_native_catalog
+from deltacat.experimental.storage.iceberg.impl import _get_native_catalog
 from deltacat.storage.model.sort_key import SortScheme
 from deltacat.storage.model.list_result import ListResult
 from deltacat.storage.model.namespace import Namespace, NamespaceProperties
@@ -23,20 +33,31 @@ from deltacat.storage.model.types import (
     LocalTable,
     StreamFormat,
 )
-from deltacat.storage.iceberg import impl as IcebergStorage
+from deltacat.experimental.storage.iceberg import impl as IcebergStorage
 from deltacat.types.media import ContentType
 from deltacat.types.tables import TableWriteMode
 from deltacat.constants import DEFAULT_NAMESPACE
-from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
+from deltacat.experimental.catalog.iceberg.iceberg_catalog_config import (
+    IcebergCatalogConfig,
+)
 
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog as PyIcebergCatalog, load_catalog
 from pyiceberg.transforms import BucketTransform
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
+IcebergCatalog = sys.modules[__name__]
+
+
+def from_config(config: IcebergCatalogConfig, *args, **kwargs) -> Catalog:
+    """
+    Factory method to construct a catalog from Iceberg catalog configuration.
+    """
+    return Catalog(config, impl=IcebergCatalog, *args, **kwargs)
+
 
 # catalog functions
-def initialize(*args, config: IcebergCatalogConfig, **kwargs) -> Catalog:
+def initialize(config: IcebergCatalogConfig, **kwargs) -> PyIcebergCatalog:
     """
     Initializes an Iceberg catalog with the given config.
 
@@ -120,7 +141,7 @@ def write_to_table(
                     )
                     # TODO(pdames): only append s3:// to output file paths when writing to S3!
                     out_file_paths = [f"s3://{val}" for val in out_df.to_arrow()[0]]
-                    from deltacat.catalog.iceberg import overrides
+                    from deltacat.experimental.catalog.iceberg import overrides
 
                     overrides.append(
                         table_definition.table.native_object,
@@ -144,7 +165,17 @@ def read_table(
     table: str, *args, namespace: Optional[str] = None, **kwargs
 ) -> DistributedDataset:
     """Read a table into a distributed dataset."""
-    raise NotImplementedError("read_table not implemented")
+    # TODO: more proper IO configuration
+    io_config = context.get_context().daft_planning_config.default_io_config
+    multithreaded_io = context.get_context().get_or_create_runner().name != "ray"
+
+    storage_config = StorageConfig(multithreaded_io, io_config)
+
+    dc_table = get_table(name=table, namespace=namespace, **kwargs)
+    dc_scan_operator = DeltaCatScanOperator(dc_table, storage_config)
+    handle = ScanOperatorHandle.from_python_scan_operator(dc_scan_operator)
+    builder = LogicalPlanBuilder.from_tabular_scan(scan_operator=handle)
+    return DataFrame(builder)
 
 
 def alter_table(
@@ -167,7 +198,7 @@ def create_table(
     name: str,
     *args,
     namespace: Optional[str] = None,
-    version: Optional[str] = None,
+    table_version: Optional[str] = None,
     lifecycle_state: Optional[LifecycleState] = None,
     schema: Optional[Schema] = None,
     partition_scheme: Optional[PartitionScheme] = None,
@@ -211,7 +242,7 @@ def create_table(
     IcebergStorage.create_table_version(
         namespace=namespace,
         table_name=name,
-        table_version=version,
+        table_version=table_version,
         schema=schema,
         partition_scheme=partition_scheme,
         sort_keys=sort_keys,

deltacat/{catalog → experimental/catalog}/iceberg/overrides.py

@@ -5,12 +5,11 @@ from typing import Iterator, List
 from pyarrow.fs import FileSystem
 
 from pyiceberg.io.pyarrow import (
-    fill_parquet_file_metadata,
+    data_file_statistics_from_parquet_metadata,
     compute_statistics_plan,
     parquet_path_to_id_mapping,
 )
-from pyiceberg.table import Table, _MergingSnapshotProducer
-from pyiceberg.table.snapshots import Operation
+from pyiceberg.table import Table
 from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
 from pyiceberg.types import StructType, NestedField, IntegerType
 from pyiceberg.typedef import Record
@@ -24,11 +23,10 @@ def append(table: Table, paths: List[str]) -> None:
     #    raise ValueError("Cannot write to tables with a sort-order")
 
     data_files = write_file(table, paths)
-    merge = _MergingSnapshotProducer(operation=Operation.APPEND, table=table)
-    for data_file in data_files:
-        merge.append_data_file(data_file)
-
-    merge.commit()
+    with table.transaction() as txn:
+        with txn.update_snapshot().fast_append() as snapshot_update:
+            for data_file in data_files:
+                snapshot_update.append_data_file(data_file)
 
 
 def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
@@ -41,6 +39,11 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
         fs_path = fs_tuple[1]
         with fs.open_input_file(fs_path) as native_file:
             parquet_metadata = pq.read_metadata(native_file)
+            statistics = data_file_statistics_from_parquet_metadata(
+                parquet_metadata=parquet_metadata,
+                stats_columns=compute_statistics_plan(table.schema(), table.properties),
+                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+            )
             data_file = DataFile(
                 content=DataFileContent.DATA,
                 file_path=file_path,
@@ -63,12 +66,7 @@ def write_file(table: Table, paths: Iterator[str]) -> Iterator[DataFile]:
                 spec_id=table.spec().spec_id,
                 equality_ids=None,
                 key_metadata=None,
-            )
-            fill_parquet_file_metadata(
-                data_file=data_file,
-                parquet_metadata=parquet_metadata,
-                stats_columns=compute_statistics_plan(table.schema(), table.properties),
-                parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
+                **statistics.to_serialized_dict(),
             )
             data_files.append(data_file)
     return data_files