deltacat 2.0.0b7__py3-none-any.whl → 2.0.0b10__py3-none-any.whl

deltacat/daft/daft_scan.py

@@ -9,10 +9,11 @@ from daft.daft import (
     FileFormatConfig,
     ParquetSourceConfig,
 )
-from daft.io.scan import ScanOperator
+from daft.io.scan import ScanOperator, ScanPushdowns
 
 from deltacat.catalog.model.table_definition import TableDefinition
 from deltacat.daft.model import DaftPartitionKeyMapper
+from deltacat.daft.translator import translate_pushdown
 
 
 class DeltaCatScanOperator(ScanOperator):
@@ -44,8 +45,11 @@ class DeltaCatScanOperator(ScanOperator):
         ]
 
     def to_scan_tasks(self, pushdowns: Pushdowns) -> Iterator[ScanTask]:
-        # TODO: implement pushdown predicate on DeltaCAT
-        dc_scan_plan = self.table.create_scan_plan()
+        daft_pushdowns = ScanPushdowns._from_pypushdowns(
+            pushdowns, schema=self.schema()
+        )
+        dc_pushdown = translate_pushdown(daft_pushdowns)
+        dc_scan_plan = self.table.create_scan_plan(pushdown=dc_pushdown)
         scan_tasks = []
         file_format_config = FileFormatConfig.from_parquet_config(
             # maybe this: ParquetSourceConfig(field_id_mapping=self._field_id_mapping)

deltacat/daft/translator.py

@@ -0,0 +1,126 @@
+from daft.io.scan import ScanPushdowns
+import pyarrow as pa
+from typing import Callable, Dict
+from daft.io.pushdowns import (
+    Expr as DaftExpr,
+    Literal as DaftLiteral,
+    Reference as DaftReference,
+    TermVisitor,
+)
+
+from deltacat.storage.model.expression import (
+    Expression,
+    Reference,
+    Literal,
+    Equal,
+    NotEqual,
+    GreaterThan,
+    LessThan,
+    GreaterThanEqual,
+    LessThanEqual,
+    And,
+    Or,
+    Not,
+    IsNull,
+)
+from deltacat.storage.model.scan.push_down import PartitionFilter, Pushdown
+
+
+def translate_pushdown(pushdown: ScanPushdowns) -> Pushdown:
+    """
+    Helper method to translate a Daft ScanPushdowns object into a Deltacat Pushdown.
+
+    Args:
+        pushdown: Daft ScanPushdowns object
+
+    Returns:
+        Pushdown: Deltacat Pushdown object with translated filters
+    """
+    translator = DaftToDeltacatExpressionTranslator()
+    partition_filter = None
+
+    if pushdown.predicate:
+        predicate = translator.visit(pushdown.predicate, None)
+        partition_filter = PartitionFilter.of(predicate)
+
+    # TODO: translate other pushdown filters
+    return Pushdown.of(
+        row_filter=None,
+        column_filter=None,
+        partition_filter=partition_filter,
+        limit=None,
+    )
+
+
+class DaftToDeltacatExpressionTranslator(TermVisitor[None, Expression]):
+    """
+    This visitor implementation traverses a Daft expression tree and produces
+    an equivalent Deltacat expression tree for use in Deltacat's query pushdown
+    system.
+    """
+
+    _PROCEDURES: Dict[str, Callable[..., Expression]] = {
+        # Comparison predicates
+        "=": Equal.of,
+        "!=": NotEqual.of,
+        "<": LessThan.of,
+        ">": GreaterThan.of,
+        "<=": LessThanEqual.of,
+        ">=": GreaterThanEqual.of,
+        # Logical predicates
+        "and": And.of,
+        "or": Or.of,
+        "not": Not.of,
+        # Special operations
+        "is_null": IsNull.of,
+    }
+
+    def visit_reference(self, term: DaftReference, context: None) -> Expression:
+        """
+        Convert Daft Reference to Deltacat Reference.
+
+        Args:
+            term: A Daft Reference expression representing a field or column.
+            context: Not used in this visitor implementation.
+
+        Returns:
+            DeltacatExpression: A Deltacat Reference expression for the same field.
+        """
+        return Reference(term.path)
+
+    def visit_literal(self, term: DaftLiteral, context: None) -> Expression:
+        """
+        Convert Daft Literal to Deltacat Literal.
+
+        Args:
+            term: A Daft Literal expression representing a constant value.
+            context: Not used in this visitor implementation.
+
+        Returns:
+            DeltacatExpression: A Deltacat Literal expression wrapping the same value as a PyArrow scalar.
+        """
+        return Literal(pa.scalar(term.value))
+
+    def visit_expr(self, term: DaftExpr, context: None) -> Expression:
+        """
+        This method handles the translation of procedure calls (operations) from
+        Daft to Deltacat, including special cases for IN, BETWEEN, and LIKE.
+
+        Args:
+            term: A Daft Expr expression representing an operation.
+            context: Not used in this visitor implementation.
+
+        Returns:
+            DeltacatExpression: An equivalent Deltacat expression.
+
+        Raises:
+            ValueError: If the operation has an invalid number of arguments or
+                if the operation is not supported by Deltacat.
+        """
+        proc = term.proc
+        args = [self.visit(arg.term, context) for arg in term.args]
+
+        if proc not in self._PROCEDURES:
+            raise ValueError(f"Deltacat does not support procedure '{proc}'.")
+
+        return self._PROCEDURES[proc](*args)

deltacat/examples/basic_logging.py

@@ -1,7 +1,9 @@
 import os
-import ray
+import deltacat
 import logging
 
+import ray
+
 from deltacat import logs
 from deltacat.constants import DELTACAT_APP_LOG_DIR, DELTACAT_SYS_LOG_DIR
 from deltacat.examples.common.fixtures import (
@@ -94,8 +96,8 @@ if __name__ == "__main__":
     # create any runtime environment required to run the example
     runtime_env = create_ray_runtime_environment()
 
-    # initialize ray
-    ray.init(runtime_env=runtime_env)
+    # initialize deltacat
+    deltacat.init(ray_init_args={"runtime_env": runtime_env})
 
     # run the example using os.environ as kwargs
     run(**os.environ)

deltacat/examples/hello_world.py

@@ -1,12 +1,10 @@
 import ray
 import deltacat
 import daft
-import pyiceberg
 
 
 def print_package_version_info():
     print(f"DeltaCAT Version: {deltacat.__version__}")
-    print(f"PyIceberg Version: {pyiceberg.__version__}")
     print(f"Ray Version: {ray.__version__}")
     print(f"Daft Version: {daft.__version__}")
 
@@ -24,4 +22,8 @@ def run():
 
 
 if __name__ == "__main__":
+    # initialize deltacat
+    deltacat.init()
+
+    # run the example
     run()

deltacat/examples/indexer/indexer.py

@@ -0,0 +1,163 @@
+import argparse
+
+from datetime import datetime
+
+import ray
+
+import deltacat
+import daft
+import pyarrow as pa
+import pandas as pd
+import polars as pl
+import numpy as np
+
+from deltacat import DeltaCatUrl
+
+
+def print_package_version_info() -> None:
+    print(f"DeltaCAT Version: {deltacat.__version__}")
+    print(f"Ray Version: {ray.__version__}")
+    print(f"Daft Version: {daft.__version__}")
+    print(f"NumPy Version: {np.__version__}")
+    print(f"PyArrow Version: {pa.__version__}")
+    print(f"Polars Version: {pl.__version__}")
+    print(f"Pandas Version: {pd.__version__}")
+
+
+def json_path_to_regex(path: str):
+    if not path:
+        raise ValueError("Path cannot be empty")
+    parts = path.split("/")
+    leaf_key = parts.pop()
+    regex = r""
+    for part in parts:
+        if part.strip():  # discard leading and/or redundant separators
+            regex += rf'"{part}"\s*:\s*[{{\[].*?'
+    regex += rf'"{leaf_key}"\s*:\s*"(?<{leaf_key}>.*?)"'
+    return regex
+
+
+def run(
+    source: str,
+    dest: str,
+) -> None:
+    # print package version info
+    print_package_version_info()
+
+    # run a synchronous copy from the source to the destination
+    deltacat.copy(
+        DeltaCatUrl(source),
+        DeltaCatUrl(dest),
+        # reader arguments to pass to the default reader (polars)
+        # for the given text-based datasource, it accepts the same
+        # arguments as polars.read_csv except for `source`, `n_threads`
+        # `new_columns`, `separator`, `has_header`, `quote_char`, and
+        # `infer_schema`.
+        reader_args={
+            "low_memory": True,  # try to use less memory (++stability, --perf)
+            "batch_size": 1024,  # text line count read into a buffer at once
+            "use_pyarrow": True,  # use the native pyarrow reader
+        },
+        # writer arguments to pass to the default writer (polars)
+        # for the given parquet-based datasink, it accepts the same
+        # arguments as polars.DataFrame.write_parquet except for `file`
+        writer_args={
+            "compression": "lz4",  # faster compression & decompression
+            # "compression": "zstd",  # better compression ratio
+            # "compression": "snappy",  # compatible w/ older Parquet readers
+        },
+        # Transforms to run against the default polars dataframe read.
+        # By default, each transform takes a polars dataframe `df` as input
+        # and produces a polars dataframe as output. All transforms listed
+        # are run in order (i.e., the dataframe output from transform[0]
+        # is the dataframe input to transform[1]).
+        #
+        # See:
+        # https://docs.pola.rs/api/python/stable/reference/dataframe/index.html
+        # https://docs.pola.rs/api/python/stable/reference/expressions/index.html
+        transforms=[
+            lambda df, src: df.rename(
+                {"text": "utf8_body"},
+            ),
+            lambda df, src: df.with_columns(
+                pl.col("utf8_body").hash().alias("utf8_body_hash"),
+                pl.lit(datetime.utcnow()).dt.datetime().alias("processing_time"),
+                pl.lit(src.url_path).alias("source_file_path"),
+            ),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    """
+    Example 1: Run this script locally using Ray:
+    $ python indexer.py \
+    $   --source 'text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31' \
+    $   --dest 'parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet'
+
+    Example 2: Submit this script as a local Ray job using a local job client:
+    >>> from deltacat import local_job_client
+    >>> client = local_job_client()
+    >>> # read the source file as line-delimited text
+    >>> src = "text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31"
+    >>> # write to the destination file using the default DeltaCAT Parquet writer (i.e., polars.DataFrame.write_parquet)
+    >>> dst = "parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet"
+    >>> try:
+    >>>   job_run_result = client.run_job(
+    >>>       # Entrypoint shell command to run the indexer job
+    >>>       entrypoint=f"python indexer.py --source '{src}' --dest '{dst}'",
+    >>>       # Path to the local directory that contains the indexer.py file
+    >>>       runtime_env={"working_dir": "./deltacat/examples/indexer.py"},
+    >>>   )
+    >>>   print(f"Job ID {job_run_result.job_id} terminal state: {job_run_result.job_status}")
+    >>>   print(f"Job ID {job_run_result.job_id} logs: ")
+    >>>   print(job_run_result.job_logs)
+    >>> except RuntimeError as e:
+    >>>     print(f"Job Run Failed: {e}")
+    >>> except TimeoutError as e:
+    >>>     print(f"Job Run Timed Out: {e}")
+
+    Example 3: Submit this script as a remote Ray job using a remote job client:
+    >>> from deltacat import job_client
+    >>> # use `deltacat.yaml` from the current working directory as the ray cluster launcher config file
+    >>> # automatically launches the cluster if it doesn't exist or has died
+    >>> # automatically forwards the ray cluster's dashboard for viewing in a web browser @ http://localhost:8265
+    >>> client = job_client()
+    >>> # ... follow the same steps as above to submit a synchronous indexer job ...
+    >>>
+    >>> # OR use an explicit cluster launcher config file path
+    >>> client = job_client("/Users/pdames/workspace/deltacat.yaml")
+    >>> # ... follow the same steps as above to submit a synchronous indexer job ...
+    """
+    script_args = [
+        (
+            [
+                "--source",
+            ],
+            {
+                "help": "Source DeltaCAT URL to index.",
+                "type": str,
+            },
+        ),
+        (
+            [
+                "--dest",
+            ],
+            {
+                "help": "Destination DeltaCAT URL to index.",
+                "type": str,
+            },
+        ),
+    ]
+    # parse CLI input arguments
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+
+    # initialize deltacat
+    deltacat.init()
+
+    # run the example using the parsed arguments
+    run(**vars(args))

deltacat/examples/indexer/job_runner.py

@@ -0,0 +1,199 @@
+import argparse
+import pathlib
+
+from deltacat.compute import (
+    job_client,
+    JobStatus,
+)
+
+
+def run_async(
+    source: str,
+    dest: str,
+    jobs_to_submit: int,
+    job_timeout: int,
+    cloud: str,
+    restart_ray: bool,
+):
+    # print package version info
+    working_dir = pathlib.Path(__file__).parent
+    cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
+    job_number = 0
+    client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
+    job_ids = []
+    while jobs_to_submit > 0:
+        jobs_to_submit -= 1
+        job_dest = dest + f".{job_number}"
+        job_id = client.submit_job(
+            # Entrypoint shell command to execute
+            entrypoint=f"python3 indexer.py --source '{source}' --dest '{job_dest}'",
+            # Path to the local directory that contains the indexer.py file
+            # This entire directory will be zipped into a job package, so keep
+            # it small.
+            runtime_env={"working_dir": working_dir},
+        )
+        job_ids.append(job_id)
+        job_number += 1
+
+    print("Waiting for all jobs to complete...")
+    job_number = 0
+    all_job_logs = ""
+    for job_id in job_ids:
+        job_status = client.await_job(job_id, timeout_seconds=job_timeout)
+        if job_status != JobStatus.SUCCEEDED:
+            print(f"Job `{job_id}` logs: ")
+            print(client.get_job_logs(job_id))
+            raise RuntimeError(f"Job `{job_id}` terminated with status: {job_status}")
+        all_job_logs += f"\nJob #{job_number} logs: \n"
+        all_job_logs += client.get_job_logs(job_id)
+        job_number += 1
+    print("All jobs completed!")
+    print("Job Logs: ")
+    print(all_job_logs)
+
+
+def run_sync(
+    source: str,
+    dest: str,
+    jobs_to_submit: int,
+    job_timeout: int,
+    cloud: str,
+    restart_ray: bool,
+):
+    working_dir = pathlib.Path(__file__).parent
+    cluster_cfg_file_path = working_dir.joinpath(cloud).joinpath("deltacat.yaml")
+    client = job_client(cluster_cfg_file_path, restart_ray=restart_ray)
+    job_number = 0
+    while jobs_to_submit > 0:
+        jobs_to_submit -= 1
+        job_dest = dest + f".{job_number}"
+        job_run_result = client.run_job(
+            # Entrypoint shell command to execute
+            entrypoint=f"python3 indexer.py --source '{source}' --dest '{job_dest}'",
+            # Path to the local directory that contains the indexer.py file
+            # This entire directory will be zipped into a job package, so keep
+            # it small.
+            runtime_env={"working_dir": working_dir},
+            timeout_seconds=job_timeout,
+        )
+        print(
+            f"Job ID {job_run_result.job_id} terminal state: {job_run_result.job_status}"
+        )
+        print(f"Job ID {job_run_result.job_id} logs: ")
+        print(job_run_result.job_logs)
+        job_number += 1
+
+
+def run(
+    source: str,
+    dest: str,
+    restart_ray: bool,
+    jobs_to_submit: int,
+    job_timeout: int,
+    asynchronous: bool,
+    cloud_provider: str,
+):
+    run_func = run_async if asynchronous else run_sync
+    run_func(
+        source=source,
+        dest=dest,
+        jobs_to_submit=jobs_to_submit,
+        job_timeout=job_timeout,
+        cloud=cloud_provider,
+        restart_ray=restart_ray,
+    )
+
+
+if __name__ == "__main__":
+    """
+    # Run this example through a command of the form:
+    $ python ./deltacat/examples/job_runner.py -- \
+    $ --source text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31 \
+    $ --dest parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet \
+    $ --asynchronous \
+    $ --jobs-to-submit 100 \
+    $ --job-timeout 90 \
+    $ --cloud-provider aws
+    """
+    script_args = [
+        (
+            [
+                "--source",
+            ],
+            {
+                "help": "Source DeltaCAT URL to index.",
+                "type": str,
+                "default": "text+s3://openalex-mag-format/data_dump_v1/2022-07-08/nlp/PaperAbstractsInvertedIndex.txt_part31",
+            },
+        ),
+        (
+            [
+                "--dest",
+            ],
+            {
+                "help": "Destination DeltaCAT URL to store the indexed file.",
+                "type": str,
+                "default": "parquet+s3://deltacat-example-output/openalex/PaperAbstractsInvertedIndex.part31.parquet",
+            },
+        ),
+        (
+            [
+                "--restart-ray",
+            ],
+            {
+                "help": "Restart Ray on an existing cluster.",
+                "action": "store_true",
+                "default": False,
+            },
+        ),
+        (
+            [
+                "--asynchronous",
+            ],
+            {
+                "help": "Run jobs asynchronously.",
+                "action": "store_true",
+                "default": False,
+            },
+        ),
+        (
+            [
+                "--jobs-to-submit",
+            ],
+            {
+                "help": "Number of indexer jobs to submit for execution.",
+                "type": int,
+                "default": 1,
+            },
+        ),
+        (
+            [
+                "--job-timeout",
+            ],
+            {
+                "help": "Job timeout in seconds.",
+                "type": int,
+                "default": 300,
+            },
+        ),
+        (
+            [
+                "--cloud-provider",
+            ],
+            {
+                "help": "Ray Cluster Cloud Provider ('aws' or 'gcp')",
+                "type": str,
+                "default": "aws",
+            },
+        ),
+    ]
+
+    # parse CLI input arguments
+    parser = argparse.ArgumentParser()
+    for args, kwargs in script_args:
+        parser.add_argument(*args, **kwargs)
+    args = parser.parse_args()
+    print(f"Command Line Arguments: {args}")
+
+    # run the example using os.environ as kwargs
+    run(**vars(args))

deltacat/io/__init__.py

@@ -0,0 +1,13 @@
+from deltacat.io.reader.deltacat_read_api import read_deltacat
+from deltacat.io.datasource.deltacat_datasource import DeltacatReadType
+from deltacat.io.datasource.deltacat_datasource import (
+    METAFILE_DATA_COLUMN_NAME,
+    METAFILE_TYPE_COLUMN_NAME,
+)
+
+__all__ = [
+    "read_deltacat",
+    "DeltacatReadType",
+    "METAFILE_DATA_COLUMN_NAME",
+    "METAFILE_TYPE_COLUMN_NAME",
+]

deltacat/io/dataset/deltacat_dataset.py

@@ -0,0 +1,91 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional, cast
+
+import pyarrow as pa
+from ray.data import Dataset
+
+from deltacat.utils.url import DeltaCatUrl
+from deltacat.io.datasink.deltacat_datasink import DeltaCatDatasink
+
+
+class DeltaCatDataset(Dataset):
+    @staticmethod
+    def from_dataset(dataset: Dataset) -> DeltaCatDataset:
+        # cast to DeltacatDataset in-place since it only adds new methods
+        dataset.__class__ = DeltaCatDataset
+        return cast(DeltaCatDataset, dataset)
+
+    def write_deltacat(
+        self,
+        url: DeltaCatUrl,
+        *,
+        # if the source dataset only contains DeltaCAT metadata, then only copy the metadata to the destination... if it contains external source file paths, then register them in a new Delta.
+        metadata_only: bool = False,
+        # merge all deltas as part of the write operation
+        copy_on_write: Optional[bool] = False,
+        filesystem: Optional[pa.fs.S3FileSystem] = None,
+        try_create_dir: bool = True,
+        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
+        arrow_parquet_args_fn: Callable[[], Dict[str, Any]] = lambda: {},
+        min_rows_per_file: Optional[int] = None,
+        ray_remote_args: Dict[str, Any] = None,
+        concurrency: Optional[int] = None,
+        **arrow_parquet_args,
+    ) -> None:
+        """Writes the dataset to files and commits DeltaCAT metadata indexing
+        the files written.
+
+        This is only supported for datasets convertible to Arrow records.
+        To control the number of files, use ``.repartition()``.
+
+        Unless a custom block path provider is given, the format of the output
+        files will be {uuid}_{block_idx}.{extension}, where ``uuid`` is a
+        unique id for the dataset.
+
+        The DeltaCAT manifest will be written to ``f"{path}/manifest``
+
+        Examples:
+            >>> ds.write_deltacat("s3://catalog/root/path")
+
+        Time complexity: O(dataset size / parallelism)
+
+        Args:
+            url: The path to the root directory where materialized files and
+                DeltaCAT manifest will be written.
+            filesystem: The filesystem implementation to write to. This should
+                be either a PyArrow S3FileSystem.
+            try_create_dir: Try to create all directories in destination path
+                if True. Does nothing if all directories already exist.
+            arrow_open_stream_args: kwargs passed to
+                pyarrow.fs.S3FileSystem.open_output_stream
+            filename_provider: FilenameProvider implementation
+                to write each dataset block to a custom output path.
+            arrow_parquet_args_fn: Callable that returns a dictionary of write
+                arguments to use when writing each block to a file. Overrides
+                any duplicate keys from arrow_parquet_args. This should be used
+                instead of arrow_parquet_args if any of your write arguments
+                cannot be pickled, or if you'd like to lazily resolve the write
+                arguments for each dataset block.
+            arrow_parquet_args: Options to pass to
+                pyarrow.parquet.write_table(), which is used to write out each
+                block to a file.
+        """
+        datasink = DeltaCatDatasink(
+            url,
+            metadata_only=metadata_only,
+            copy_on_write=copy_on_write,
+            arrow_parquet_args_fn=arrow_parquet_args_fn,
+            arrow_parquet_args=arrow_parquet_args,
+            min_rows_per_file=min_rows_per_file,
+            filesystem=filesystem,
+            try_create_dir=try_create_dir,
+            open_stream_args=arrow_open_stream_args,
+            dataset_uuid=self._uuid,
+        )
+        self.write_datasink(
+            datasink,
+            ray_remote_args=ray_remote_args,
+            concurrency=concurrency,
+        )