deltacat-io-core 0.1.0__tar.gz

deltacat_io_core-0.1.0/.gitignore

@@ -0,0 +1,222 @@
+# uv
+.venv/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.egg-info
+MANIFEST
+package-lock.json
+*.db
+pyvenv.cfg
+**/.deltacat
+.deltacat_memory/
+
+# Generated Files
+**/.riv-meta-*
+**/*.parquet
+**/*.feather
+**/*.json
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Vi
+*.swp
+
+##############################
+## JetBrains
+##############################
+out/
+.idea/
+.idea_modules/
+*.iml
+*.ipr
+*.iws
+
+##############################
+## Eclipse
+##############################
+.settings/
+bin/
+tmp/
+.metadata
+.classpath
+.project
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.loadpath
+.factorypath
+/eclipse-bin/
+/eclipse-bin/*
+
+##############################
+## NetBeans
+##############################
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+nbactions.xml
+nb-configuration.xml
+
+##############################
+## Visual Studio Code
+##############################
+.vscode/
+.code-workspace
+
+
+##############################
+## Cursor
+##############################
+.cursor/
+
+##############################
+## OS X
+##############################
+.DS_Store
+
+# Iceberg Examples
+iceberg_warehouse
+
+# Analysis reports (large CSVs, not code)
+reports/
+
+# Local coding transcripts/scratch notes
+claude-coding-session-*.txt

deltacat_io_core-0.1.0/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: deltacat-io-core
+Version: 0.1.0
+Summary: Shared local IO execution layer for DeltaCAT read/write clients.
+Author: Ray Team
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: <3.11,>=3.10
+Requires-Dist: fsspec
+Requires-Dist: msgpack~=1.0.7
+Requires-Dist: numpy==1.23.2
+Requires-Dist: pyarrow==17.0.0
+Provides-Extra: all
+Requires-Dist: fastavro; extra == 'all'
+Requires-Dist: pandas==2.2.3; extra == 'all'
+Requires-Dist: polars==1.28.1; extra == 'all'
+Requires-Dist: pylance>=0.37.0; extra == 'all'
+Provides-Extra: io
+Requires-Dist: fastavro; extra == 'io'
+Provides-Extra: lance
+Requires-Dist: pylance>=0.37.0; extra == 'lance'
+Provides-Extra: pandas
+Requires-Dist: pandas==2.2.3; extra == 'pandas'
+Provides-Extra: polars
+Requires-Dist: polars==1.28.1; extra == 'polars'
+Description-Content-Type: text/markdown
+
+# deltacat-io-core
+
+`deltacat-io-core` is the shared local execution layer for DeltaCAT reads and
+writes.
+
+It is used by both:
+
+- `deltacat-client` for direct thin-plan execution
+- `deltacat` for shared local execution and compatibility wrappers
+
+## Naming
+
+- distribution/package name: `deltacat-io-core`
+- Python import module: `deltacat_io_core`
+
+The distribution uses dashes for consistency with `deltacat-client`. The import
+module keeps underscores because Python module names cannot contain `-`.
+
+## Scope
+
+`deltacat-io-core` owns the code that should behave the same regardless of
+whether the caller is using the thin client or the thick DeltaCAT package.
+
+Today that includes:
+
+- direct execution of thin `Plan` objects
+- MOR execution for thin and thick paths
+- local file materialization and manifest building
+- schema alignment and table conversion helpers
+- sort-aware file ordering and manifest handling
+- shared compaction/MOR helper layers and model types
+- format-specific local readers/writers
+
+## Non-Goals
+
+`deltacat-io-core` does not own:
+
+- server routes or REST/MCP request handling
+- authoritative catalog/storage mutations
+- native Ray job orchestration surfaces
+- public end-user API shape for `deltacat` or `deltacat-client`
+
+It is a shared implementation layer, not the top-level user product.
+
+## Architecture
+
+The current read architecture is:
+
+1. The server resolves a thin `Plan`.
+2. `client.catalog.read(plan=...)` executes that plan directly through
+   `deltacat-io-core`.
+3. `dc.read_table(plan=...)` for thin plans also executes through the same
+   shared path.
+
+There is no longer a runtime bridge back into thick DeltaCAT for thin plan
+execution. The plan contract is expected to carry the metadata required for
+direct execution.
+
+The current write architecture is:
+
+1. The client stages local files or materializes local data through shared
+   helpers.
+2. The authoritative commit still happens through DeltaCAT server/native
+   boundaries.
+3. Shared write-preparation and manifest logic lives in `deltacat-io-core`.
+
+## Installation
+
+Base install:
+
+```bash
+uv pip install deltacat-io-core
+```
+
+Optional extras:
+
+- `deltacat-io-core[io]` for local file readers/writers (`pyarrow`, `fastavro`)
+- `deltacat-io-core[pandas]` for Pandas conversions
+- `deltacat-io-core[polars]` for Polars conversions and lazy scan helpers
+- `deltacat-io-core[lance]` for Lance dataset support
+- `deltacat-io-core[all]` for the full local IO stack
+
+## Read Capabilities
+
+The shared read executor currently handles:
+
+- schema-table reads
+- schemaless manifest-table reads
+- MOR reads
+- direct `pyarrow`, `pandas`, `polars`, `numpy`, `daft`, and `ray_dataset`
+  outputs where supported
+- lazy `pyarrow_parquet`
+- lazy `lance`
+
+It also enforces direct validation for unsupported combinations, for example:
+
+- schemaless + `pyarrow_parquet`
+- schemaless + `lance`
+- mixed-content lazy plans for format-specific readers
+- unknown content types in the shared path
+
+## Write Capabilities
+
+The shared write layer currently covers:
+
+- write input normalization
+- local data materialization
+- manifest construction for existing files and datasets
+- schema/read compatibility helpers
+- standard catalog write orchestration slices
+
+Authoritative catalog mutation, commit, retention, and compaction boundaries
+still remain on the native/server side where they belong.
+
+## Relationship To Other Packages
+
+Use `deltacat-client` when you want the public thin client.
+
+Use `deltacat` when you want the thick/native package.
+
+Use `deltacat-io-core` directly only if you are intentionally building against
+the shared execution layer itself.

deltacat_io_core-0.1.0/README.md

@@ -0,0 +1,122 @@
+# deltacat-io-core
+
+`deltacat-io-core` is the shared local execution layer for DeltaCAT reads and
+writes.
+
+It is used by both:
+
+- `deltacat-client` for direct thin-plan execution
+- `deltacat` for shared local execution and compatibility wrappers
+
+## Naming
+
+- distribution/package name: `deltacat-io-core`
+- Python import module: `deltacat_io_core`
+
+The distribution uses dashes for consistency with `deltacat-client`. The import
+module keeps underscores because Python module names cannot contain `-`.
+
+## Scope
+
+`deltacat-io-core` owns the code that should behave the same regardless of
+whether the caller is using the thin client or the thick DeltaCAT package.
+
+Today that includes:
+
+- direct execution of thin `Plan` objects
+- MOR execution for thin and thick paths
+- local file materialization and manifest building
+- schema alignment and table conversion helpers
+- sort-aware file ordering and manifest handling
+- shared compaction/MOR helper layers and model types
+- format-specific local readers/writers
+
+## Non-Goals
+
+`deltacat-io-core` does not own:
+
+- server routes or REST/MCP request handling
+- authoritative catalog/storage mutations
+- native Ray job orchestration surfaces
+- public end-user API shape for `deltacat` or `deltacat-client`
+
+It is a shared implementation layer, not the top-level user product.
+
+## Architecture
+
+The current read architecture is:
+
+1. The server resolves a thin `Plan`.
+2. `client.catalog.read(plan=...)` executes that plan directly through
+   `deltacat-io-core`.
+3. `dc.read_table(plan=...)` for thin plans also executes through the same
+   shared path.
+
+There is no longer a runtime bridge back into thick DeltaCAT for thin plan
+execution. The plan contract is expected to carry the metadata required for
+direct execution.
+
+The current write architecture is:
+
+1. The client stages local files or materializes local data through shared
+   helpers.
+2. The authoritative commit still happens through DeltaCAT server/native
+   boundaries.
+3. Shared write-preparation and manifest logic lives in `deltacat-io-core`.
+
+## Installation
+
+Base install:
+
+```bash
+uv pip install deltacat-io-core
+```
+
+Optional extras:
+
+- `deltacat-io-core[io]` for local file readers/writers (`pyarrow`, `fastavro`)
+- `deltacat-io-core[pandas]` for Pandas conversions
+- `deltacat-io-core[polars]` for Polars conversions and lazy scan helpers
+- `deltacat-io-core[lance]` for Lance dataset support
+- `deltacat-io-core[all]` for the full local IO stack
+
+## Read Capabilities
+
+The shared read executor currently handles:
+
+- schema-table reads
+- schemaless manifest-table reads
+- MOR reads
+- direct `pyarrow`, `pandas`, `polars`, `numpy`, `daft`, and `ray_dataset`
+  outputs where supported
+- lazy `pyarrow_parquet`
+- lazy `lance`
+
+It also enforces direct validation for unsupported combinations, for example:
+
+- schemaless + `pyarrow_parquet`
+- schemaless + `lance`
+- mixed-content lazy plans for format-specific readers
+- unknown content types in the shared path
+
+## Write Capabilities
+
+The shared write layer currently covers:
+
+- write input normalization
+- local data materialization
+- manifest construction for existing files and datasets
+- schema/read compatibility helpers
+- standard catalog write orchestration slices
+
+Authoritative catalog mutation, commit, retention, and compaction boundaries
+still remain on the native/server side where they belong.
+
+## Relationship To Other Packages
+
+Use `deltacat-client` when you want the public thin client.
+
+Use `deltacat` when you want the thick/native package.
+
+Use `deltacat-io-core` directly only if you are intentionally building against
+the shared execution layer itself.

deltacat_io_core-0.1.0/__init__.py

@@ -0,0 +1,40 @@
+"""Shared local IO execution helpers for DeltaCAT thin and thick read/write paths."""
+
+from __future__ import annotations
+
+import logging
+
+__version__ = "0.1.0"
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = [
+    "__version__",
+    "build_entries_for_local_paths",
+    "coerce_write_input",
+    "execute_read_plan",
+    "materialize_arrow_write",
+]
+
+
+def __getattr__(name: str):
+    if name in {
+        "build_entries_for_local_paths",
+        "coerce_write_input",
+        "execute_read_plan",
+        "materialize_arrow_write",
+    }:
+        from deltacat_io_core.execution import (
+            build_entries_for_local_paths,
+            coerce_write_input,
+            execute_read_plan,
+            materialize_arrow_write,
+        )
+
+        return {
+            "build_entries_for_local_paths": build_entries_for_local_paths,
+            "coerce_write_input": coerce_write_input,
+            "execute_read_plan": execute_read_plan,
+            "materialize_arrow_write": materialize_arrow_write,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

deltacat_io_core-0.1.0/bucket_task_support.py

@@ -0,0 +1,101 @@
+"""Shared helpers for hash/range bucket task execution."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+
+
+def execute_bucket_grouping(
+    *,
+    input: Any,
+    load_delta_file_envelope_groups_fn,
+    group_hash_bucket_indices_fn,
+    current_process_peak_memory_fn,
+    hash_bucket_result_factory,
+    wall_time_fn,
+    log_info_fn,
+    peak_memory_log_message: str,
+) -> Any:
+    """Execute the common bucket-task core shared by hash and range bucketing."""
+    (
+        delta_file_envelope_groups,
+        total_record_count,
+        total_size_bytes,
+    ) = load_delta_file_envelope_groups_fn()
+    hash_bucket_group_to_obj_id_tuple = group_hash_bucket_indices_fn(
+        hash_bucket_object_groups=delta_file_envelope_groups,
+        num_buckets=input.num_hash_buckets,
+        num_groups=input.num_hash_groups,
+        object_store=input.object_store,
+    )
+
+    peak_memory_usage_bytes = current_process_peak_memory_fn()
+    log_info_fn(
+        peak_memory_log_message.format(peak_memory_usage_bytes=peak_memory_usage_bytes)
+    )
+    return hash_bucket_result_factory(
+        hash_bucket_group_to_obj_id_tuple,
+        np.int64(total_size_bytes),
+        np.int64(total_record_count),
+        np.double(peak_memory_usage_bytes),
+        np.double(0.0),
+        np.double(wall_time_fn()),
+    )
+
+
+def execute_bucket_task(
+    *,
+    input: Any,
+    task_fn,
+    task_fn_kwargs: dict[str, Any],
+    process_utilization_context_factory,
+    timed_invocation_fn,
+    emit_timer_metrics_fn,
+    metrics_name: str,
+    hash_bucket_result_factory,
+    bytes_per_gibibyte: float,
+    log_info_fn,
+    log_debug_fn,
+    started_message: str,
+    finished_message: str,
+) -> Any:
+    """Execute the common outer wrapper shared by hash/range bucket tasks."""
+    with process_utilization_context_factory() as process_util:
+        log_info_fn(started_message)
+
+        def log_peak_memory():
+            log_debug_fn(
+                "Process peak memory utilization so far: %s bytes (%s GB)",
+                process_util.max_memory,
+                process_util.max_memory / bytes_per_gibibyte,
+            )
+
+        if input.memory_logs_enabled:
+            process_util.schedule_callback(log_peak_memory, 10)
+
+        bucket_result, duration = timed_invocation_fn(
+            func=task_fn,
+            **task_fn_kwargs,
+        )
+
+        emit_metrics_time = 0.0
+        if input.metrics_config:
+            _, latency = timed_invocation_fn(
+                func=emit_timer_metrics_fn,
+                metrics_name=metrics_name,
+                value=duration,
+                metrics_config=input.metrics_config,
+            )
+            emit_metrics_time = latency
+
+        log_info_fn(finished_message)
+        return hash_bucket_result_factory(
+            bucket_result[0],
+            bucket_result[1],
+            bucket_result[2],
+            bucket_result[3],
+            np.double(emit_metrics_time),
+            bucket_result[5],
+        )