deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/storage/model/transaction.py

@@ -0,0 +1,1733 @@
+from __future__ import annotations
+
+import os
+import copy
+import time
+import uuid
+import logging
+import posixpath
+from pathlib import PosixPath
+import threading
+import contextvars
+from collections import defaultdict
+
+from types import TracebackType
+from typing import Optional, List, Union, Tuple, Type, TYPE_CHECKING, Iterable
+
+if TYPE_CHECKING:
+    from deltacat.types.tables import Dataset
+
+import msgpack
+import pyarrow as pa
+import pyarrow.fs
+
+from deltacat.constants import (
+    TXN_DIR_NAME,
+    TXN_PART_SEPARATOR,
+    RUNNING_TXN_DIR_NAME,
+    FAILED_TXN_DIR_NAME,
+    PAUSED_TXN_DIR_NAME,
+    SUCCESS_TXN_DIR_NAME,
+    NANOS_PER_SEC,
+)
+from deltacat.storage.model.list_result import ListResult
+from deltacat.storage.model.types import (
+    TransactionOperationType,
+    TransactionState,
+    TransactionStatus,
+)
+from deltacat.storage.model.namespace import NamespaceLocator
+from deltacat.storage.model.table import TableLocator
+from deltacat.storage.model.table_version import TableVersionLocator
+from deltacat.storage.model.stream import StreamLocator
+from deltacat.storage.model.partition import PartitionLocator
+from deltacat.storage.model.delta import DeltaLocator
+from deltacat.storage.model.metafile import (
+    Metafile,
+    MetafileRevisionInfo,
+)
+from deltacat.types.tables import (
+    DatasetType,
+    from_pyarrow,
+)
+from deltacat.utils.filesystem import (
+    resolve_path_and_filesystem,
+    list_directory,
+    get_file_info,
+)
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+# Context variable to store the current transaction
+_current_transaction: contextvars.ContextVar[
+    Optional[Transaction]
+] = contextvars.ContextVar("current_transaction", default=None)
+
+
+def get_current_transaction() -> Optional[Transaction]:
+    """Get the currently active transaction from context."""
+    return _current_transaction.get()
+
+
+def set_current_transaction(transaction: Optional[Transaction]) -> contextvars.Token:
+    """Set the current transaction in context, returns token for restoration."""
+    return _current_transaction.set(transaction)
+
+
+def setup_transaction(
+    transaction: Optional[Transaction] = None,
+    **kwargs,
+) -> Tuple[Transaction, bool]:
+    """
+    Utility method to ensure a transaction exists and determine if it should be committed
+    within the caller's context. Creates a new transaction if none is provided.
+
+    Args:
+        transaction: Optional existing transaction to use
+        **kwargs: Additional arguments for catalog properties
+
+    Returns:
+        Tuple[Transaction, bool]: The transaction to use and whether to commit it
+    """
+    # Check for active transaction in context first
+    if transaction is None:
+        transaction = get_current_transaction()
+
+    commit_transaction = transaction is None
+    if commit_transaction:
+        from deltacat.catalog.model.properties import get_catalog_properties
+
+        catalog_properties = get_catalog_properties(**kwargs)
+        transaction = Transaction.of().start(
+            catalog_properties.root,
+            catalog_properties.filesystem,
+        )
+    return transaction, commit_transaction
+
+
+def transaction_log_dir_and_filesystem(
+    catalog_name: Optional[str] = None,
+) -> Tuple[str, pyarrow.fs.FileSystem]:
+    """
+    Get the transaction log directory and filesystem for the given catalog.
+
+    Args:
+        catalog_name: Name of the catalog to get the transaction log directory and filesystem for.
+            If None, uses the default catalog.
+
+    Returns:
+        Tuple[str, pyarrow.fs.FileSystem]: The transaction log directory and filesystem for the given catalog.
+    """
+    # Get the catalog and its properties
+    from deltacat.catalog.model.catalog import get_catalog
+
+    catalog = get_catalog(catalog_name)
+    catalog_properties = catalog.inner
+
+    # Get transaction directory paths
+    catalog_root_normalized, filesystem = resolve_path_and_filesystem(
+        catalog_properties.root,
+        catalog_properties.filesystem,
+    )
+
+    return posixpath.join(catalog_root_normalized, TXN_DIR_NAME), filesystem
+
+
+def transaction(
+    catalog_name: Optional[str] = None,
+    as_of: Optional[int] = None,
+    commit_message: Optional[str] = None,
+) -> Transaction:
+    """
+    Start a new interactive transaction for the given catalog.
+
+    Args:
+        catalog_name: Optional name of the catalog to run the transaction against.
+                     If None, uses the default catalog.
+        as_of: Optional historic timestamp in nanoseconds since epoch.
+                If provided, creates a read-only transaction that reads only transactions
+                with end times strictly less than the specified timestamp.
+        commit_message: Optional commit message to describe the transaction purpose.
+                Helps with time travel functionality by providing context
+                for each transaction when browsing transaction history.
+
+    Returns:
+        Transaction: A started interactive transaction ready for use with the given catalog.
+
+    Example:
+        # Read-write transaction with commit message
+        with dc.transaction(commit_message="Initial data load for Q4 analytics") as txn:
+            dc.write_to_table(data, "my_table")
+            dc.write_to_table(more_data, "my_other_table")
+
+        # Read-only historic transaction
+        import time
+        historic_time = time.time_ns() - 3600 * 1000000000  # 1 hour ago
+        with dc.transaction(as_of=historic_time) as txn:
+            # Only read operations allowed - provides snapshot as of historic_time
+            data = dc.read_table("my_table")
+    """
+    from deltacat.catalog.model.catalog import get_catalog
+
+    # Get the catalog and its properties
+    catalog = get_catalog(catalog_name)
+    catalog_properties = catalog.inner
+
+    # Create interactive transaction
+    if as_of is not None:
+        # Create read-only historic transaction
+        txn = Transaction.of(commit_message=commit_message).start(
+            catalog_properties.root,
+            catalog_properties.filesystem,
+            historic_timestamp=as_of,
+        )
+    else:
+        # Create regular read-write transaction
+        txn = Transaction.of(commit_message=commit_message).start(
+            catalog_properties.root, catalog_properties.filesystem
+        )
+        # Initialize the lazy transaction ID
+        logger.info(f"Created transaction with ID: {txn.id}")
+    return txn
+
+
+def _read_txn(
+    txn_log_dir: str,
+    txn_status: TransactionStatus,
+    transaction_id: str,
+    filesystem: pyarrow.fs.FileSystem,
+) -> Transaction:
+    """
+    Read a transaction ID with the expected status from the given root transaction log directory.
+
+    Args:
+        txn_log_dir: The directory containing the transaction log.
+        txn_status: The expected status of the transaction.
+        transaction_id: The ID of the transaction.
+        filesystem: The filesystem to use for reading the transaction.
+
+    Returns:
+        Transaction: The transaction.
+    """
+    # Transaction directories contain the actual transaction file
+    txn_dir_path = posixpath.join(
+        txn_log_dir, txn_status.dir_name(), posixpath.basename(transaction_id)
+    )
+
+    try:
+        file_info = get_file_info(txn_dir_path, filesystem)
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            f"Transaction with ID '{transaction_id}' and status '{txn_status}' not found."
+        )
+
+    # Only read transaction directories (skip any stray files)
+    if file_info.type != pyarrow.fs.FileType.Directory:
+        raise FileNotFoundError(
+            f"Transaction directory for transaction ID '{transaction_id}' with status '{txn_status}' not found."
+        )
+
+    # List files in the transaction directory
+    txn_files = list_directory(
+        path=txn_dir_path,
+        filesystem=filesystem,
+        ignore_missing_path=True,
+    )
+
+    if not txn_files:
+        raise FileNotFoundError(
+            f"No transaction file found for transaction ID '{transaction_id}' and status '{txn_status}'."
+        )
+
+    if len(txn_files) > 1:
+        raise RuntimeError(
+            f"Expected 1 transaction file in '{txn_dir_path}', but found {len(txn_files)}"
+        )
+
+    # Get the transaction file path
+    txn_file_path, _ = txn_files[0]
+
+    # Read the transaction from the file
+    return Transaction.read(txn_file_path, filesystem)
+
+
+def read_transaction(
+    transaction_id: str,
+    catalog_name: Optional[str] = None,
+    status: TransactionStatus = TransactionStatus.SUCCESS,
+) -> Transaction:
+    """
+    Read a transaction from the given catalog and transaction ID.
+    """
+    txn_log_dir, filesystem = transaction_log_dir_and_filesystem(catalog_name)
+    return _read_txn(txn_log_dir, status, transaction_id, filesystem)
+
+
+def transactions(
+    catalog_name: Optional[str] = None,
+    read_as: "DatasetType" = None,
+    start_time: Optional[int] = None,
+    end_time: Optional[int] = None,
+    limit: Optional[int] = None,
+    status_in: Iterable[TransactionStatus] = [TransactionStatus.SUCCESS],
+) -> Dataset:
+    """
+    Query transaction history for a catalog.
+
+    Args:
+        catalog_name: Optional name of the catalog to query. If None, uses the default catalog.
+        read_as: Dataset type to return results as. If None, defaults to DatasetType.PYARROW.
+        start_time: Optional start timestamp in nanoseconds since epoch to filter transactions.
+        end_time: Optional end timestamp in nanoseconds since epoch to filter transactions.
+        limit: Optional maximum number of transactions to return (most recent first).
+        status_in: Optional iterable of transaction status types to include. Defaults to [TransactionStatus.SUCCESS].
+
+    Returns:
+        Dataset: Transaction history as the specified dataset type with columns:
+                 - transaction_id: Unique transaction identifier
+                 - commit_message: Optional user-provided commit message
+                 - start_time: Transaction start timestamp (nanoseconds since epoch)
+                 - end_time: Transaction end timestamp (nanoseconds since epoch, None for running)
+                 - status: Transaction status (SUCCESS, RUNNING, FAILED, PAUSED)
+                 - operation_count: Number of operations in the transaction
+                 - operation_types: Comma-separated list of distinct operation types in the transaction
+                 - namespace_count: Number of distinct namespaces affected by the transaction
+                 - table_count: Number of distinct tables affected by the transaction
+                 - table_version_count: Number of distinct table versions affected by the transaction
+                 - stream_count: Number of distinct streams affected by the transaction
+                 - partition_count: Number of distinct partitions affected by the transaction
+                 - delta_count: Number of distinct deltas affected by the transaction
+
+    Example:
+        # Get recent successful transactions
+        recent = dc.transactions(limit=10)
+
+        # Get transactions for a specific time range
+        import time
+        hour_ago = time.time_ns() - 3600 * 1000000000
+        recent_hour = dc.transactions(start_time=hour_ago)
+
+        # Get transaction history as pandas DataFrame
+        df = dc.transactions(read_as=dc.DatasetType.PANDAS)
+    """
+    # Validate inputs
+    if limit is not None and limit <= 0:
+        raise ValueError("limit must be greater than 0")
+
+    # Set default read_as if not provided
+    if read_as is None:
+        read_as = DatasetType.PYARROW
+
+    if not status_in:
+        status_in = [TransactionStatus.SUCCESS]
+
+    # Get transaction directory path and filesystem
+    txn_log_dir, filesystem = transaction_log_dir_and_filesystem(catalog_name)
+
+    # Collect transaction data
+    transaction_records = {
+        "transaction_id": [],
+        "commit_message": [],
+        "start_time": [],
+        "end_time": [],
+        "status": [],
+        "operation_count": [],
+        "operation_types": [],
+        "namespace_count": [],
+        "table_count": [],
+        "table_version_count": [],
+        "stream_count": [],
+        "partition_count": [],
+        "delta_count": [],
+    }
+
+    # Helper function to process transactions in a directory
+    def process_transactions_in_directory(
+        directory: str, expected_status: TransactionStatus
+    ):
+        # TODO(pdames): Do a recursive listing to get the transaction files returned directly.
+        file_info_and_sizes = list_directory(
+            path=directory,
+            filesystem=filesystem,
+            ignore_missing_path=True,
+        )
+
+        for file_path, _ in file_info_and_sizes:
+            # Read the transaction from the file
+            # TODO(pdames): Do a recursive listing to get the transaction files returned directly.
+            try:
+                txn = _read_txn(
+                    txn_log_dir,
+                    expected_status,
+                    posixpath.basename(file_path),
+                    filesystem,
+                )
+            except FileNotFoundError:
+                # this may be a stray file or the transaction is being created - skip it
+                continue
+
+            # Apply time filters
+            # TODO(pdames): Parse start and end times from the transaction file path.
+            if (
+                start_time is not None
+                and txn.start_time
+                and txn.start_time < start_time
+            ):
+                continue
+            if end_time is not None and txn.end_time and txn.end_time > end_time:
+                continue
+
+            # Count operations and affected metadata objects by type.
+            operation_count = len(txn.operations)
+            operation_types = set()
+            affected_namespaces = set()
+            affected_tables = set()
+            affected_table_versions = set()
+            affected_streams = set()
+            affected_partitions = set()
+            affected_deltas = set()
+
+            for op in txn.operations:
+                operation_types.add(op.type)
+
+                # Determine locator type and cast to appropriate locator class
+                locator_dict = op.dest_metafile.get("locator", {})
+                if "tableName" in locator_dict and "namespaceLocator" in locator_dict:
+                    locator = TableLocator(locator_dict)
+                elif "namespace" in locator_dict:
+                    locator = NamespaceLocator(locator_dict)
+                elif "tableVersion" in locator_dict:
+                    locator = TableVersionLocator(locator_dict)
+                elif "streamId" in locator_dict:
+                    locator = StreamLocator(locator_dict)
+                elif "partitionId" in locator_dict:
+                    locator = PartitionLocator(locator_dict)
+                elif "streamPosition" in locator_dict:
+                    locator = DeltaLocator(locator_dict)
+                else:
+                    raise ValueError(
+                        f"Unknown locator type from structure: {locator_dict}"
+                    )
+
+                # Extract distinct metafiles updated by common/alias name (e.g., a table rename impacts 2 tables instead of 1)
+                if op.type in TransactionOperationType.write_operations():
+                    if locator.namespace is not None:
+                        affected_namespaces.add(locator.namespace)
+                    if isinstance(locator, TableLocator):
+                        affected_tables.add((locator.namespace, locator.table_name))
+                    elif isinstance(locator, TableVersionLocator):
+                        affected_table_versions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                            )
+                        )
+                    elif isinstance(locator, StreamLocator):
+                        affected_tables.add((locator.namespace, locator.table_name))
+                        affected_table_versions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                            )
+                        )
+                        affected_streams.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                            )
+                        )
+                    elif isinstance(locator, PartitionLocator):
+                        affected_tables.add((locator.namespace, locator.table_name))
+                        affected_table_versions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                            )
+                        )
+                        affected_streams.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                            )
+                        )
+                        affected_partitions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                                locator.partition_id,
+                            )
+                        )
+                    elif isinstance(locator, DeltaLocator):
+                        affected_tables.add((locator.namespace, locator.table_name))
+                        affected_table_versions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                            )
+                        )
+                        affected_streams.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                            )
+                        )
+                        affected_partitions.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                                locator.partition_id,
+                            )
+                        )
+                        affected_deltas.add(
+                            (
+                                locator.namespace,
+                                locator.table_name,
+                                locator.table_version,
+                                locator.stream_id,
+                                locator.partition_id,
+                                locator.stream_position,
+                            )
+                        )
+
+            # Create transaction record
+            transaction_records["transaction_id"].append(txn.id)
+            transaction_records["commit_message"].append(txn.commit_message)
+            transaction_records["start_time"].append(txn.start_time)
+            transaction_records["end_time"].append(txn.end_time)
+            transaction_records["status"].append(expected_status)
+            transaction_records["operation_count"].append(operation_count)
+            transaction_records["operation_types"].append(operation_types)
+            transaction_records["namespace_count"].append(len(affected_namespaces))
+            transaction_records["table_count"].append(len(affected_tables))
+            transaction_records["table_version_count"].append(
+                len(affected_table_versions)
+            )
+            transaction_records["stream_count"].append(len(affected_streams))
+            transaction_records["partition_count"].append(len(affected_partitions))
+            transaction_records["delta_count"].append(len(affected_deltas))
+
+    for status in status_in:
+        dir_path = posixpath.join(txn_log_dir, status.dir_name())
+        process_transactions_in_directory(dir_path, status)
+
+    # Sort by start_time descending (most recent first)
+    # Convert to list of records, sort, then convert back
+    if transaction_records["transaction_id"]:  # Only sort if we have records
+        # Create list of tuples (start_time, record_index)
+        sorted_indices = sorted(
+            range(len(transaction_records["start_time"])),
+            key=lambda i: transaction_records["start_time"][i] or 0,
+            reverse=True,
+        )
+
+        # Reorder all columns based on sorted indices
+        for key in transaction_records:
+            transaction_records[key] = [
+                transaction_records[key][i] for i in sorted_indices
+            ]
+
+    # Apply limit
+    # TODO(pdames): Apply limit during listing (pyarrow fs doesn't support limits natively).
+    if limit is not None and limit > 0:
+        for key in transaction_records:
+            transaction_records[key] = transaction_records[key][:limit]
+
+    # Convert to requested dataset type
+    return from_pyarrow(pa.Table.from_pydict(transaction_records), read_as)
+
+
+class TransactionTimeProvider:
+    """
+    Provider interface for transaction start and end times. An ideal
+    transaction time provider is externally consistent (e.g.,
+    https://cloud.google.com/spanner/docs/true-time-external-consistency),
+    such that:
+      1. A transaction start time is never less than a previously completed
+      transaction's end time.
+      2. A transaction end time is never less than an in-progress
+      transaction's start time.
+      3. Every transaction has a unique start and end time.
+      4. Start/end time assignment is non-blocking.
+    """
+
+    def start_time(self) -> int:
+        raise NotImplementedError("start_time not implemented")
+
+    def end_time(self) -> int:
+        raise NotImplementedError("end_time not implemented")
+
+
+class TransactionSystemTimeProvider(TransactionTimeProvider):
+    """
+    A local transaction time provider that returns the current system clock
+    epoch time in nanoseconds. Ensures that all local transaction start
+    times are greater than all last known end times, and that all known end
+    times are no less than all last known start time across all local threads
+    using this time provider.
+
+    Note that this time provider gives no external consistency guarantees due
+    to potential clock skew between distributed nodes writing to the same
+    catalog, and is only recommended for use with local catalogs.
+    """
+
+    last_known_start_times = defaultdict(int)
+    last_known_end_times = defaultdict(int)
+
+    # don't wait more than 60 seconds for the system clock to catch up
+    # between transactions (assumed to be indicative of a larger system
+    # clock change made between transactions)
+    max_sync_wait_time = 60 * NANOS_PER_SEC
+
+    def start_time(self) -> int:
+        """
+        Gets the current system time in nanoseconds since the epoch. Ensures
+        that the start time returned is greater than the last known end time
+        recorded at the time this method is invoked.
+        :return: Current epoch time in nanoseconds.
+        """
+        # ensure serial transactions in a single process have start times after
+        # the last known end time
+        last_known_end_times = self.last_known_end_times.values() or [0]
+        max_known_end_time = max(last_known_end_times)
+
+        elapsed_start_time = time.monotonic_ns()
+        current_time = time.time_ns()
+        while current_time <= max_known_end_time:
+            elapsed_time = time.monotonic_ns() - elapsed_start_time
+            if elapsed_time > self.max_sync_wait_time:
+                raise TimeoutError(
+                    f"Failed to sync cross-transaction system clock time after "
+                    f"{self.max_sync_wait_time / NANOS_PER_SEC} seconds, "
+                    f"aborting."
+                )
+            time.sleep(0.000001)
+            current_time = time.time_ns()
+
+        # update the current thread's last known end time
+        pid = os.getpid()
+        tid = threading.current_thread().ident
+        current_thread_time_key = (pid, tid)
+        self.last_known_end_times[current_thread_time_key] = current_time
+
+        return current_time
+
+    def end_time(self) -> int:
+        """
+        Gets the current system time in nanoseconds since the epoch. Ensures
+        that the end time returned is no less than the last known start time
+        recorded at the time this method is invoked.
+        :return: Current epoch time in nanoseconds.
+        """
+        # ensure serial transactions in a single process have end times no less
+        # than the last known start time
+        last_known_start_times = self.last_known_start_times.values() or [0]
+        last_start_time = max(last_known_start_times)
+
+        elapsed_start_time = time.monotonic_ns()
+        current_time = time.time_ns()
+        while current_time < last_start_time:
+            elapsed_time = time.monotonic_ns() - elapsed_start_time
+            if elapsed_time > self.max_sync_wait_time:
+                raise TimeoutError(
+                    f"Failed to sync cross-transaction system clock time after "
+                    f"{self.max_sync_wait_time / NANOS_PER_SEC} seconds, "
+                    f"aborting."
+                )
+            time.sleep(0.000001)
+            current_time = time.time_ns()
+
+        # update the current thread's last known end time
+        pid = os.getpid()
+        tid = threading.current_thread().ident
+        current_thread_time_key = (pid, tid)
+        self.last_known_start_times[current_thread_time_key] = current_time
+
+        return current_time
+
+
+class TransactionHistoricTimeProvider(TransactionTimeProvider):
+    """
+    A transaction time provider that returns a fixed historic timestamp
+    for read-only transactions. This enables MVCC snapshot isolation
+    as-of the specified timestamp.
+    """
+
+    def __init__(
+        self,
+        historic_timestamp: int,
+        base_time_provider: TransactionTimeProvider,
+    ):
+        """
+        Initialize with a fixed historic timestamp and a base time provider.
+
+        Args:
+            historic_timestamp: Timestamp in nanoseconds since epoch to use
+                              for both start and end times.
+            base_time_provider: Time provider to use for the end time.
+        """
+        # Validate that historic timestamp is not in the future
+        if historic_timestamp > base_time_provider.start_time():
+            raise ValueError(
+                f"Historic timestamp {historic_timestamp} cannot be set in the future."
+            )
+        self.base_time_provider = base_time_provider
+        self.historic_timestamp = historic_timestamp
+
+    def start_time(self) -> int:
+        """
+        Returns the fixed historic timestamp.
+        """
+        return self.historic_timestamp
+
+    def end_time(self) -> int:
+        """
+        Returns the end time of the base time provider.
+        """
+        return self.base_time_provider.end_time()
+
+
+class TransactionOperation(dict):
+    """
+    Base class for DeltaCAT transaction operations against individual metafiles.
+    """
+
+    @staticmethod
+    def of(
+        operation_type: Optional[TransactionOperationType],
+        dest_metafile: Metafile,
+        src_metafile: Optional[Metafile] = None,
+        read_limit: Optional[int] = None,
+    ) -> TransactionOperation:
+        if not dest_metafile:
+            raise ValueError("Transaction operations must have a destination metafile.")
+        if operation_type in [
+            TransactionOperationType.UPDATE,
+            TransactionOperationType.REPLACE,
+        ]:
+            if not src_metafile:
+                raise ValueError(
+                    f"{operation_type.value} transaction operations must have a source metafile."
+                )
+            elif type(dest_metafile) is not type(src_metafile):
+                raise ValueError(
+                    f"Source metafile type `{type(src_metafile)}` is not "
+                    f"equal to dest metafile type `{type(dest_metafile)}`."
+                )
+        elif src_metafile:
+            raise ValueError(
+                f"Only {TransactionOperationType.UPDATE.value} and {TransactionOperationType.REPLACE.value} transaction operations may have a source metafile."
+            )
+        if operation_type.is_write_operation() and read_limit:
+            raise ValueError(
+                f"Only {TransactionOperationType.READ.value} transaction operations may have a read limit."
+            )
+        txn_op = TransactionOperation()
+        txn_op.type = operation_type
+        txn_op.dest_metafile = dest_metafile
+        txn_op.src_metafile = src_metafile
+        txn_op.read_limit = read_limit
+        return txn_op
+
+    @property
+    def type(self) -> TransactionOperationType:
+        """
+        Returns the type of the transaction operation.
+        """
+        val = self["type"]
+        if val is not None and not isinstance(val, TransactionOperationType):
+            self["type"] = val = TransactionOperationType(val)
+        return val
+
+    @type.setter
+    def type(self, txn_op_type: TransactionOperationType):
+        self["type"] = txn_op_type
+
+    @property
+    def dest_metafile(self) -> Metafile:
+        """
+        Returns the metafile that is the target of this transaction operation.
+        """
+        val = self["dest_metafile"]
+        if val is not None and not isinstance(val, Metafile):
+            self["dest_metafile"] = val = Metafile(val)
+        return val
+
+    @dest_metafile.setter
+    def dest_metafile(self, metafile: Metafile):
+        self["dest_metafile"] = metafile
+
+    @property
+    def src_metafile(self) -> Optional[Metafile]:
+        """
+        Returns the metafile that is the source of this transaction operation.
+        """
+        val = self.get("src_metafile")
+        if val is not None and not isinstance(val, Metafile):
+            self["src_metafile"] = val = Metafile(val)
+        return val
+
+    @src_metafile.setter
+    def src_metafile(self, src_metafile: Optional[Metafile]):
+        self["src_metafile"] = src_metafile
+
+    @property
+    def read_limit(self) -> Optional[int]:
+        """
+        Returns the read limit for this transaction operation.
+        """
+        return self.get("read_limit")
+
+    @read_limit.setter
+    def read_limit(self, read_limit: Optional[int]):
+        self["read_limit"] = read_limit
+
+    @property
+    def metafile_write_paths(self) -> List[str]:
+        return self.get("metafile_write_paths") or []
+
+    @property
+    def locator_write_paths(self) -> List[str]:
+        return self.get("locator_write_paths") or []
+
+    def append_metafile_write_path(self, write_path: str):
+        metafile_write_paths = self.get("metafile_write_paths")
+        if not metafile_write_paths:
+            metafile_write_paths = self["metafile_write_paths"] = []
+        metafile_write_paths.append(write_path)
+
+    def append_locator_write_path(self, write_path: str):
+        locator_write_paths = self.get("locator_write_paths")
+        if not locator_write_paths:
+            locator_write_paths = self["locator_write_paths"] = []
+        locator_write_paths.append(write_path)
+
+    @metafile_write_paths.setter
+    def metafile_write_paths(self, write_paths: List[str]) -> None:
+        self["metafile_write_paths"] = write_paths
+
+    @locator_write_paths.setter
+    def locator_write_paths(self, write_paths: List[str]):
+        self["locator_write_paths"] = write_paths
+
+
+class TransactionOperationList(List[TransactionOperation]):
+    @staticmethod
+    def of(items: List[TransactionOperation]) -> TransactionOperationList:
+        typed_items = TransactionOperationList()
+        for item in items:
+            if item is not None and not isinstance(item, TransactionOperation):
+                item = TransactionOperation(item)
+            typed_items.append(item)
+        return typed_items
+
+    def __getitem__(self, item):
+        val = super().__getitem__(item)
+        if val is not None and not isinstance(val, TransactionOperation):
+            self[item] = val = TransactionOperation(val)
+        return val
+
+    def __iter__(self):
+        """Support enumeration by returning TransactionOperation objects."""
+        for i in range(len(self)):
+            yield self[i]  # This triggers __getitem__ conversion
+
+
+class Transaction(dict):
+    """
+    Base class for DeltaCAT transactions.
+    """
+
+    @staticmethod
+    def of(
+        txn_operations: Optional[TransactionOperationList] = None,
+        commit_message: Optional[str] = None,
+    ) -> Transaction:
+        if txn_operations is None:
+            txn_operations = []
+        transaction = Transaction()
+        transaction.operations = txn_operations
+        transaction.interactive = len(txn_operations) == 0
+        if commit_message:
+            transaction.commit_message = commit_message
+        return transaction
+
+    @staticmethod
+    def read_end_time(
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Optional[int]:
+        """
+        Returns the end time of the transaction, or None if the transaction
+        log file does not exist.
+        :param path: Transaction log path to read.
+        :param filesystem: File system to use for reading the Transaction file.
+        :return: Deserialized object from the Transaction file.
+        """
+        # TODO(pdames): Validate that input file path is a valid txn log.
+        if not filesystem:
+            path, filesystem = resolve_path_and_filesystem(path, filesystem)
+        file_info_and_sizes = list_directory(
+            path=path,
+            filesystem=filesystem,
+            ignore_missing_path=True,
+        )
+        end_time = None
+        if file_info_and_sizes:
+            if len(file_info_and_sizes) > 1:
+                raise ValueError(
+                    f"Expected to find only one transaction log at {path}, "
+                    f"but found {len(file_info_and_sizes)}"
+                )
+            end_time = Transaction._parse_end_time(file_info_and_sizes[0][0])
+        return end_time
+
+    @staticmethod
+    def _parse_end_time(txn_log_file_name_or_path: str) -> int:
+        return int(posixpath.basename(txn_log_file_name_or_path))
+
+    @classmethod
+    def read(
+        cls,
+        path: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Transaction:
+        """
+        Read a Transaction file and return the deserialized object.
+        :param path: Transaction file path to read.
+        :param filesystem: File system to use for reading the Transaction file.
+        :return: Deserialized object from the Transaction file.
+        """
+
+        if not filesystem:
+            path, filesystem = resolve_path_and_filesystem(path, filesystem)
+        with filesystem.open_input_stream(path) as file:
+            binary = file.readall()
+        obj = cls(**msgpack.loads(binary))
+        return obj
+
+    @staticmethod
+    def read_time_provider(provider_name: str):
+        """
+        Given the string name of a time provider class, return a new instance of it.
+        Raises ValueError if the provider name is unknown.
+        """
+        TIME_PROVIDER_CLASSES = {
+            "TransactionSystemTimeProvider": TransactionSystemTimeProvider,
+            # Add additional mappings as needed
+        }
+
+        provider_cls = TIME_PROVIDER_CLASSES.get(provider_name)
+        if provider_cls is None:
+            raise ValueError(f"Unknown time provider: {provider_name}")
+
+        return provider_cls()
+
+    @property
+    def id(self) -> Optional[str]:
+        """
+        Returns this transaction's unique ID assigned at commit start time, or
+        None if the unique ID has not yet been assigned.
+        """
+        _id = self.get("id")
+        if not _id and self.start_time:
+            _id = self["id"] = f"{self.start_time}{TXN_PART_SEPARATOR}{uuid.uuid4()}"
+        return _id
+
+    def state(self, catalog_root_dir: str, filesystem: pyarrow.fs.FileSystem = None):
+        """
+        Infer the transaction state based on its presence in different directories.
+        """
+
+        txn_name = self.id
+
+        catalog_root_normalized, filesystem = resolve_path_and_filesystem(
+            catalog_root_dir
+        )
+
+        txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+        running_txn_log_dir = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME)
+        filesystem.create_dir(running_txn_log_dir, recursive=True)
+        failed_txn_log_dir = posixpath.join(txn_log_dir, FAILED_TXN_DIR_NAME)
+        filesystem.create_dir(failed_txn_log_dir, recursive=False)
+        success_txn_log_dir = posixpath.join(txn_log_dir, SUCCESS_TXN_DIR_NAME)
+        filesystem.create_dir(success_txn_log_dir, recursive=False)
+        paused_txn_log_dir = posixpath.join(txn_log_dir, PAUSED_TXN_DIR_NAME)
+        filesystem.create_dir(paused_txn_log_dir, recursive=False)
+
+        # Check if the transaction file exists in the failed directory
+        in_failed = os.path.exists(os.path.join(failed_txn_log_dir, txn_name))
+
+        # Check if the transaction file exists in the running directory
+        in_running = os.path.exists(os.path.join(running_txn_log_dir, txn_name))
+
+        # Check if the transaction file exists in the success directory
+        in_success = os.path.exists(os.path.join(success_txn_log_dir, txn_name))
+
+        # Check if the transaction file exists in the paused directory
+        in_paused = os.path.exists(os.path.join(paused_txn_log_dir, txn_name))
+
+        if in_failed and in_running:
+            return TransactionState.FAILED
+        elif in_failed and not in_running:
+            return TransactionState.PURGED
+        elif in_success:
+            return TransactionState.SUCCESS
+        elif in_running:
+            return TransactionState.RUNNING
+        elif in_paused:
+            return TransactionState.PAUSED
+
+    @property
+    def operations(self) -> TransactionOperationList:
+        """
+        Returns the list of transaction operations.
+        """
+        return TransactionOperationList(self["operations"])
+
+    @operations.setter
+    def operations(self, operations: TransactionOperationList):
+        self["operations"] = operations
+
+    @property
+    def metafile_write_paths(self) -> List[str]:
+        return [path for op in self.operations for path in op.metafile_write_paths]
+
+    @property
+    def locator_write_paths(self) -> List[str]:
+        return [path for op in self.operations for path in op.locator_write_paths]
+
+    @property
+    def catalog_root_normalized(self) -> str:
+        """
+        Returns the catalog_root_normalized for this transaction.
+        """
+        return self.get("catalog_root_normalized")
+
+    @catalog_root_normalized.setter
+    def catalog_root_normalized(self, path: str):
+        self["catalog_root_normalized"] = path
+
+    @property
+    def _time_provider(self) -> TransactionSystemTimeProvider:
+        """
+        Returns the time_provider of the transaction.
+        """
+        return self.get("_time_provider")
+
+    @_time_provider.setter
+    def _time_provider(
+        self, tp: TransactionSystemTimeProvider
+    ) -> TransactionSystemTimeProvider:
+        self["_time_provider"] = tp
+
+    @property
+    def start_time(self) -> Optional[int]:
+        """
+        Returns the start time of the transaction.
+        """
+        return self.get("start_time")
+
+    @property
+    def pause_time(self) -> Optional[int]:
+        """
+        Returns the last pause time of the transaction.
+        """
+        return self.get("pause_time")
+
+    @property
+    def end_time(self) -> Optional[int]:
+        """
+        Returns the end time of the transaction.
+        """
+        return self.get("end_time")
+
+    @property
+    def commit_message(self) -> Optional[str]:
+        """
+        Returns the commit message for the transaction.
+        """
+        return self.get("commit_message")
+
+    @commit_message.setter
+    def commit_message(self, message: str):
+        """
+        Sets the commit message for the transaction.
+        """
+        self["commit_message"] = message
+
+    @property
+    def historic_timestamp(self) -> Optional[int]:
+        """
+        Returns the historic timestamp for the transaction.
+        """
+        return self.get("historic_timestamp")
+
+    @historic_timestamp.setter
+    def historic_timestamp(self, timestamp: int):
+        """
+        Sets the historic timestamp for the transaction.
+        """
+        self["historic_timestamp"] = timestamp
+
+    def _mark_start_time(self, time_provider: TransactionTimeProvider) -> int:
+        """
+        Sets the start time of the transaction using the given
+        TransactionTimeProvider. Raises a runtime error if the transaction
+        start time has already been set by a previous commit.
+        """
+        if self.get("start_time"):
+            raise RuntimeError("Cannot restart a previously started transaction.")
+        start_time = self["start_time"] = time_provider.start_time()
+        return start_time
+
+    def _mark_end_time(self, time_provider: TransactionTimeProvider) -> int:
+        """
+        Sets the end time of the transaction using the given
+        TransactionTimeProvider. Raises a runtime error if the transaction end
+        time has already been set by a previous commit, or if the transaction
+        start time has not been set.
+        """
+        if not self.get("start_time"):
+            raise RuntimeError("Cannot end an unstarted transaction.")
+        if self.get("end_time"):
+            raise RuntimeError("Cannot end a completed transaction.")
+        end_time = self["end_time"] = time_provider.end_time()
+        return end_time
+
+    def _mark_pause_time(self, time_provider: TransactionTimeProvider) -> int:
+        """
+        Sets the pause time of the transaction using the given
+        TransactionTimeProvider. Raises a runtime error if the transaction pause
+        time has already been set by a previous commit, or if the transaction
+        start time has not been set.
+        """
+        if not self.get("start_time"):
+            raise RuntimeError("Cannot pause an unstarted transaction.")
+        if self.get("end_time"):
+            raise RuntimeError("Cannot pause a completed transaction.")
+        pause_time = self["pause_time"] = time_provider.end_time()
+        return pause_time
+
+    @staticmethod
+    def _abs_txn_meta_path_to_relative(root: str, target: str) -> str:
+        """
+        Takes an absolute root directory path and target absolute path to
+        relativize with respect to the root directory. Returns the target
+        path relative to the root directory path. Raises an error if the
+        target path is not contained in the given root directory path, if
+        either path is not an absolute path, or if the target path is equal
+        to the root directory path.
+        """
+        root_path = PosixPath(root)
+        target_path = PosixPath(target)
+        # TODO (martinezdavid): Check why is_absolute() fails for certain Delta paths
+        # if not root_path.is_absolute() or not target_path.is_absolute():
+        #     raise ValueError("Both root and target must be absolute paths.")
+        if root_path == target_path:
+            raise ValueError(
+                "Target and root are identical, but expected target to be a child of root."
+            )
+        try:
+            relative_path = target_path.relative_to(root_path)
+        except ValueError:
+            raise ValueError("Expected target to be a child of root.")
+        return str(relative_path)
+
+    def relativize_operation_paths(
+        self, operation: TransactionOperation, catalog_root: str
+    ) -> None:
+        """
+        Converts all absolute paths in an operation to relative paths
+        with respect to the catalog root directory.
+        """
+        # handle metafile paths
+        if operation.metafile_write_paths:
+            metafile_write_paths = [
+                Transaction._abs_txn_meta_path_to_relative(catalog_root, path)
+                for path in operation.metafile_write_paths
+            ]
+            operation.metafile_write_paths = metafile_write_paths
+        # handle locator paths
+        if operation.locator_write_paths:
+            locator_write_paths = [
+                Transaction._abs_txn_meta_path_to_relative(catalog_root, path)
+                for path in operation.locator_write_paths
+            ]
+            operation.locator_write_paths = locator_write_paths
+
+    def to_serializable(self, catalog_root) -> Transaction:
+        """
+        Prepare the object for serialization by converting any non-serializable
+        types to serializable types. May also run any required pre-write
+        validations on the serialized or deserialized object.
+        :return: a serializable version of the object
+        """
+        # Only copy dictionary keys - all other members should not be serialized
+        serializable = Transaction({})
+        for key, value in self.items():
+            serializable[key] = copy.deepcopy(value)
+
+        # remove all src/dest metafile contents except IDs and locators to
+        # reduce file size (they can be reconstructed from their corresponding
+        # files as required).
+        for operation in serializable.operations:
+            # Sanity check that IDs exist on source and dest metafiles
+            if operation.dest_metafile and operation.dest_metafile.id is None:
+                raise ValueError(
+                    f"Transaction operation ${operation} dest metafile does "
+                    f"not have ID: ${operation.dest_metafile}"
+                )
+            if operation.src_metafile and operation.src_metafile.id is None:
+                raise ValueError(
+                    f"Transaction operation ${operation} src metafile does "
+                    f"not have ID: ${operation.src_metafile}"
+                )
+            # relativize after checking that dest and src metafiles are valid
+            self.relativize_operation_paths(operation, catalog_root)
+            operation.dest_metafile = {
+                "id": operation.dest_metafile.id,
+                "locator": operation.dest_metafile.locator,
+                "locator_alias": operation.dest_metafile.locator_alias,
+            }
+            if operation.src_metafile:
+                operation.src_metafile = {
+                    "id": operation.src_metafile.id,
+                    "locator": operation.src_metafile.locator,
+                    "locator_alias": operation.src_metafile.locator_alias,
+                }
+        # TODO(pdames): Ensure that all file paths recorded are relative to the
+        #  catalog root.
+
+        # TODO: check if we care about order or exact time stamps --> pickling time_provider?
+        # serializable.pop("_time_provider", None)
+
+        serializable["_time_provider"] = {
+            "type": type(self._time_provider).__name__,
+            "params": {},
+        }
+
+        serializable.catalog_root_normalized = self.catalog_root_normalized
+
+        return serializable
+
+    @staticmethod
+    def _validate_txn_log_file(success_txn_log_file: str) -> None:
+        txn_log_dir_name = posixpath.basename(posixpath.dirname(success_txn_log_file))
+        txn_log_parts = txn_log_dir_name.split(TXN_PART_SEPARATOR)
+        # ensure that the transaction start time is valid
+        try:
+            start_time = int(txn_log_parts[0])
+        except ValueError as e:
+            raise ValueError(
+                f"Transaction log file `{success_txn_log_file}` does not "
+                f"contain a valid start time."
+            ) from e
+        # ensure that the txn uuid is valid
+        txn_uuid_str = txn_log_parts[1]
+        try:
+            uuid.UUID(txn_uuid_str)
+        except ValueError as e:
+            raise OSError(
+                f"Transaction log file `{success_txn_log_file}` does not "
+                f"contain a valid UUID string."
+            ) from e
+        # ensure that the transaction end time is valid
+        try:
+            end_time = Transaction._parse_end_time(success_txn_log_file)
+        except ValueError as e:
+            raise ValueError(
+                f"Transaction log file `{success_txn_log_file}` does not "
+                f"contain a valid end time."
+            ) from e
+        # ensure transaction end time was not recorded before start time
+        if end_time < start_time:
+            raise OSError(
+                f"Transaction end time {end_time} is earlier than start "
+                f"time {start_time}! To preserve catalog integrity, the "
+                f"corresponding completed transaction log at "
+                f"`{success_txn_log_file}` has been removed."
+            )
+
+    def commit(
+        self,
+        catalog_root_dir: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+    ) -> Union[
+        List[ListResult[Metafile]],
+        Tuple[List[str], str],
+        Tuple[List["ListResult[Metafile]"], List[str], str],
+    ]:
+        """
+        Legacy wrapper that preserves the original `commit()` contract while
+        delegating the heavy lifting to the incremental helpers.
+
+        Returns
+        -------
+        - For READ transactions:  List[ListResult[Metafile]]
+        - For WRITE transactions: Tuple[List[str], str]
+            (list of successful write-paths, path to success-txn log file)
+        - For mixed READ/WRITE transactions: Tuple[List["ListResult[Metafile]"], List[str], str]
+        """
+
+        if hasattr(self, "interactive") and self.interactive:
+            raise RuntimeError(
+                "Cannot commit an interactive transaction. Use transaction.start(),transaction.step(), and transaction.seal() instead."
+            )
+
+        if self.operations and len(self.operations) > 0:
+            # Start a working copy (deep-copy, directory scaffolding, start-time, running/failed/success/paused dirs …)
+            txn_active = self.start(catalog_root_dir, filesystem)  # deep copy
+            # Sequentially execute every TransactionOperation
+            for op in txn_active.operations:
+                txn_active.step(op)
+        return txn_active._seal_steps()
+
+    def start(
+        self,
+        catalog_root_dir: str,
+        filesystem: Optional[pyarrow.fs.FileSystem] = None,
+        historic_timestamp: Optional[int] = None,
+    ) -> "Transaction":
+        """
+        Create directory scaffolding, timestamp the txn, and return a DEEP COPY
+        that the caller should use for all subsequent calls to step(), pause(),
+        and seal().  The original object remains read-only.
+
+        Args:
+            catalog_root_dir: Root directory for the catalog
+            filesystem: Optional filesystem to use
+            historic_timestamp: Optional timestamp in nanoseconds since epoch for snapshot isolation
+        """
+        # Create a deep copy
+        txn: "Transaction" = copy.deepcopy(self)
+
+        # Set up time provider based on transaction type
+        if historic_timestamp is not None:
+            # Use historic time provider for snapshot isolation
+            # TODO(pdames): Set base time provider to the catalog's configured time provider when more than one is supported.
+            txn._time_provider = TransactionHistoricTimeProvider(
+                historic_timestamp,
+                TransactionSystemTimeProvider(),
+            )
+            txn.historic_timestamp = historic_timestamp
+        else:
+            # Use system time provider for regular transactions
+            txn._time_provider = TransactionSystemTimeProvider()
+
+        txn._mark_start_time(txn._time_provider)  # start time on deep_copy
+
+        # Set up filesystem and directories
+        catalog_root_normalized, filesystem = resolve_path_and_filesystem(
+            catalog_root_dir,
+            filesystem,
+        )
+        txn.catalog_root_normalized = catalog_root_normalized
+        txn._filesystem = filesystem  # keep for pause/resume
+        txn.running_log_written = False  # internal flags
+        txn._list_results = []
+
+        # Make sure txn/ directories exist (idempotent)
+        txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+        filesystem.create_dir(
+            posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME),
+            recursive=True,
+        )
+        for subdir in (FAILED_TXN_DIR_NAME, SUCCESS_TXN_DIR_NAME, PAUSED_TXN_DIR_NAME):
+            try:
+                filesystem.create_dir(
+                    posixpath.join(txn_log_dir, subdir),
+                    recursive=False,
+                )
+            except FileExistsError:
+                pass  # allowed when catalog already initialised
+        return txn
+
+    def step(
+        self,
+        operation: "TransactionOperation",
+    ) -> Union[ListResult[Metafile], Tuple[List[str], List[str]]]:
+        """
+        Executes a single transaction operation.
+
+        Parameters
+        ----------
+        operation: TransactionOperation
+            The transaction operation to execute.
+
+        Returns
+        -------
+        - For READ transaction operation: ListResult[Metafile]
+        - For WRITE transaction operation: Tuple[List[str], List[str]]
+            (list of successful write-paths, list of successful locator write-paths)
+        """
+
+        catalog_root_normalized = self.catalog_root_normalized
+        filesystem = self._filesystem
+        txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+
+        running_txn_log_file_path = posixpath.join(
+            txn_log_dir, RUNNING_TXN_DIR_NAME, self.id
+        )
+
+        # Validate read-only transaction constraints
+        if self.historic_timestamp is not None:
+            if not operation.type.is_read_operation():
+                raise RuntimeError(
+                    f"Cannot perform {operation.type.value} operation in a read-only historic transaction."
+                )
+
+        # Add new operation to the transaction's list of operations
+        if self.interactive:
+            self.operations = self.operations + [operation]
+
+        # (a) READ txn op
+        if operation.type.is_read_operation():
+            list_result = operation.dest_metafile.read_txn(
+                catalog_root_dir=catalog_root_normalized,
+                success_txn_log_dir=posixpath.join(txn_log_dir, SUCCESS_TXN_DIR_NAME),
+                current_txn_op=operation,
+                current_txn_start_time=self.start_time,
+                current_txn_id=self.id,
+                filesystem=filesystem,
+            )
+            self._list_results.append(list_result)
+            return list_result
+
+        # (b) WRITE txn op
+        # First operation? -> create running log so an external janitor can
+        # see that a txn is in-flight.
+        if not self.running_log_written:
+            self._write_running_log(running_txn_log_file_path)
+
+        try:
+            (
+                metafile_write_paths,
+                locator_write_paths,
+            ) = operation.dest_metafile.write_txn(
+                catalog_root_dir=catalog_root_normalized,
+                success_txn_log_dir=posixpath.join(txn_log_dir, SUCCESS_TXN_DIR_NAME),
+                current_txn_op=operation,
+                current_txn_start_time=self.start_time,
+                current_txn_id=self.id,
+                filesystem=filesystem,
+            )
+            # Check for concurrent txn conflicts on the metafile and locator write paths just written
+            # TODO(pdames): Remove the fast-fail check here if it grows too expensive?
+            for path in metafile_write_paths + locator_write_paths:
+                MetafileRevisionInfo.check_for_concurrent_txn_conflict(
+                    success_txn_log_dir=posixpath.join(
+                        txn_log_dir,
+                        SUCCESS_TXN_DIR_NAME,
+                    ),
+                    current_txn_revision_file_path=path,
+                    filesystem=filesystem,
+                )
+            return metafile_write_paths, locator_write_paths
+        except Exception:
+            # convert in-flight txn → FAILED and clean up partial files
+            self._fail_and_cleanup(
+                failed_txn_log_dir=posixpath.join(txn_log_dir, FAILED_TXN_DIR_NAME),
+                running_log_path=running_txn_log_file_path,
+            )
+            raise  # surface original error
+
+    def pause(self) -> None:
+        fs = self._filesystem
+        root = self.catalog_root_normalized
+        txn_log_dir = posixpath.join(root, TXN_DIR_NAME)
+
+        running_path = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME, self.id)
+        paused_path = posixpath.join(txn_log_dir, PAUSED_TXN_DIR_NAME, self.id)
+
+        fs.create_dir(posixpath.dirname(paused_path), recursive=True)
+
+        # Record pause time (e.g., for time consistency guarantees)
+        self._mark_pause_time(self._time_provider)
+
+        # Serialize current transaction state into paused/txn_id
+        with fs.open_output_stream(paused_path) as f:
+            f.write(msgpack.dumps(self.to_serializable(root)))
+
+        # Clean up original running log
+        fs.delete_file(running_path)
+
+    def resume(self) -> None:
+        fs = self._filesystem
+        root = self.catalog_root_normalized
+        txn_log_dir = posixpath.join(root, TXN_DIR_NAME)
+
+        running_path = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME, self.id)
+        paused_path = posixpath.join(txn_log_dir, PAUSED_TXN_DIR_NAME, self.id)
+
+        # Load serialized transaction state
+        with fs.open_input_stream(paused_path) as f:
+            loaded_txn_data = msgpack.loads(f.readall())
+
+        # Restore relevant fields
+        restored_txn = Transaction(**loaded_txn_data)
+        self.__dict__.update(
+            restored_txn.__dict__
+        )  # make curr txn the same as restored (fill vars and stuff)
+
+        # To support restoring time provider state if we ever add non-ephemeral ones.
+        new_provider = Transaction.read_time_provider(
+            restored_txn["_time_provider"]["type"]
+        )
+
+        # evaluate system clock
+        now = new_provider.start_time()
+        self._time_provider = new_provider  # start time should be preserved
+        if now < self.pause_time:
+            raise RuntimeError(
+                f"System clock {now} is behind paused transaction time {self._pause_time}"
+            )
+            # TODO: set new start time or keep error if clock is off?
+
+        # Move back to running state
+        fs.create_dir(posixpath.dirname(running_path), recursive=True)
+        with fs.open_output_stream(running_path) as f:
+            f.write(msgpack.dumps(self.to_serializable(root)))
+        fs.delete_file(paused_path)
+
+    def seal(
+        self,
+    ) -> Union[
+        List["ListResult[Metafile]"],
+        Tuple[List[str], str],
+        Tuple[List["ListResult[Metafile]"], List[str], str],
+    ]:
+        """
+        For READ → returns list_results collected during step().
+        For WRITE → returns (written_paths, success_log_path).
+        """
+        if not self.interactive:
+            raise RuntimeError(
+                "Cannot seal a non-interactive transaction. Call transaction.commit() instead."
+            )
+
+        # Read-only transactions can only perform read operations
+        if self.historic_timestamp is not None:
+            if self._has_write_operations():
+                raise RuntimeError(
+                    "Cannot seal a read-only historic transaction that contains write operations."
+                )
+
+        return self._seal_steps()
+
+    def _has_write_operations(self) -> bool:
+        """
+        Check if the transaction contains any write operations.
+        Read-only transactions should only contain READ operations.
+        """
+        for operation in self.operations:
+            if not operation.type.is_read_operation():
+                return True
+        return False
+
+    def _seal_steps(
+        self,
+    ) -> Union[
+        List["ListResult[Metafile]"],
+        Tuple[List[str], str],
+        Tuple[List["ListResult[Metafile]"], List[str], str],
+    ]:
+        fs = self._filesystem
+        root = self.catalog_root_normalized
+        txn_log_dir = posixpath.join(root, TXN_DIR_NAME)
+        end_time = self._mark_end_time(self._time_provider)
+
+        # READ path: nothing persisted, so we are done
+        if all(op.type.is_read_operation() for op in self.operations):
+            return self._list_results
+
+        running_path = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME, self.id)
+        failed_dir = posixpath.join(txn_log_dir, FAILED_TXN_DIR_NAME)
+        success_dir = posixpath.join(txn_log_dir, SUCCESS_TXN_DIR_NAME)
+
+        # If no operations ever succeeded we still need a running log.
+        if not self.running_log_written:
+            self._write_running_log(running_path)
+        try:
+            # Check for concurrent txn conflicts on metafile and locator write paths
+            for path in self.metafile_write_paths + self.locator_write_paths:
+                MetafileRevisionInfo.check_for_concurrent_txn_conflict(
+                    success_txn_log_dir=posixpath.join(
+                        txn_log_dir, SUCCESS_TXN_DIR_NAME
+                    ),
+                    current_txn_revision_file_path=path,
+                    filesystem=fs,
+                )
+        except Exception:
+            self._fail_and_cleanup(
+                failed_txn_log_dir=failed_dir,
+                running_log_path=running_path,
+            )
+            # raise the original error
+            raise
+        success_log_path = None
+        try:
+            # write transaction log
+            success_txn_dir = posixpath.join(success_dir, self.id)
+            fs.create_dir(success_txn_dir, recursive=False)
+
+            success_log_path = posixpath.join(success_txn_dir, str(end_time))
+            with fs.open_output_stream(success_log_path) as f:
+                f.write(msgpack.dumps(self.to_serializable(root)))
+
+            Transaction._validate_txn_log_file(success_txn_log_file=success_log_path)
+
+        except Exception as e1:
+            self._fail_and_cleanup(
+                failed_txn_log_dir=failed_dir,
+                running_log_path=running_path,
+                success_log_path=success_log_path,
+            )
+            raise RuntimeError(
+                f"Transaction validation failed. To preserve catalog integrity, "
+                f"the corresponding completed transaction log at "
+                f"`{success_log_path}` has been removed."
+            ) from e1
+
+        else:
+            fs.delete_file(running_path)
+            if all(op.type.is_write_operation() for op in self.operations):
+                # pure write transaction - just return write paths and success log path
+                return self.metafile_write_paths, success_log_path
+            else:
+                # mixed read/write transaction - return read results, write paths, and success log path
+                return self._list_results, self.metafile_write_paths, success_log_path
+
+    #  Helper: write or overwrite the running/ID file exactly once
+    def _write_running_log(self, running_log_path: str) -> None:
+        with self._filesystem.open_output_stream(running_log_path) as f:
+            f.write(msgpack.dumps(self.to_serializable(self.catalog_root_normalized)))
+        self.running_log_written = True
+
+    #  Helper: mark txn FAILED and clean partial output
+    def _fail_and_cleanup(
+        self,
+        failed_txn_log_dir: str,
+        running_log_path: str,
+        success_log_path: Optional[str] = None,
+    ) -> None:
+        fs = self._filesystem
+
+        # 1. write failed/ID
+        failed_log_path = posixpath.join(failed_txn_log_dir, self.id)
+        with fs.open_output_stream(failed_log_path) as f:
+            f.write(msgpack.dumps(self.to_serializable(self.catalog_root_normalized)))
+
+        # 2. delete all provisional files
+        for path in self.metafile_write_paths:
+            try:
+                fs.delete_file(path)
+            except Exception:
+                pass  # best-effort; janitor job will catch leftovers
+        for path in self.locator_write_paths:
+            try:
+                fs.delete_file(path)
+            except Exception:
+                pass  # best-effort; janitor job will catch leftovers
+
+        # 3. tidy up bookkeeping logs
+        try:
+            fs.delete_file(running_log_path)
+        except Exception:
+            pass
+        if success_log_path:
+            try:
+                fs.delete_file(success_log_path)
+            except Exception:
+                pass
+
+    def __enter__(self) -> "Transaction":
+        """
+        Context manager entry point. Sets this transaction as the current context.
+        Supports nested transactions by preserving the context stack.
+        """
+        if not hasattr(self, "interactive") or not self.interactive:
+            raise RuntimeError(
+                "Transaction must be interactive to use with context manager. "
+                "Use dc.transaction() to create an interactive transaction."
+            )
+        if self.start_time is None:
+            raise RuntimeError(
+                "Transaction has not been started. "
+                "Use dc.transaction() to create a properly initialized transaction."
+            )
+
+        # Store the context token for restoration in __exit__
+        self._context_token = set_current_transaction(self)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        """
+        Context manager exit point. Restores previous transaction context and
+        automatically seals the transaction on successful completion or fails it
+        if an exception occurred.
+
+        Args:
+            exc_type: Exception type if an exception occurred, None otherwise
+            exc_value: Exception value if an exception occurred, None otherwise
+            traceback: Exception traceback if an exception occurred, None otherwise
+        """
+        try:
+            if exc_type is None and exc_value is None and traceback is None:
+                # No exception occurred - seal the transaction
+                self.seal()
+            else:
+                # Exception occurred during transaction - fail and cleanup
+                try:
+                    catalog_root_normalized = self.catalog_root_normalized
+                    txn_log_dir = posixpath.join(catalog_root_normalized, TXN_DIR_NAME)
+                    running_txn_log_file_path = posixpath.join(
+                        txn_log_dir, RUNNING_TXN_DIR_NAME, self.id
+                    )
+                    self._fail_and_cleanup(
+                        failed_txn_log_dir=posixpath.join(
+                            txn_log_dir, FAILED_TXN_DIR_NAME
+                        ),
+                        running_log_path=running_txn_log_file_path,
+                    )
+                except Exception:
+                    # If cleanup fails, still let the original exception propagate
+                    pass
+        finally:
+            # Always restore the previous transaction context using the token
+            if hasattr(self, "_context_token"):
+                try:
+                    # Get the previous value from the token
+                    old_value = self._context_token.old_value
+                    # Only set if the old value is a valid transaction or None
+                    if old_value is None or isinstance(old_value, Transaction):
+                        _current_transaction.set(old_value)
+                    else:
+                        # If old_value is not valid (e.g., Token.MISSING), set to None
+                        _current_transaction.set(None)
+                except (AttributeError, LookupError):
+                    # If token doesn't have old_value or context is corrupted, clear it
+                    try:
+                        _current_transaction.set(None)
+                    except LookupError:
+                        pass