pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/globals.py

@@ -3,15 +3,18 @@ from __future__ import annotations
 import logging
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, Literal, NamedTuple, Union
 
 import pandas as pd
+import pydantic
 from pandas.io.formats.style import Styler
 
-from pixeltable import DataFrame, catalog, exceptions as excs, exprs, func, share
+from pixeltable import DataFrame, catalog, exceptions as excs, exprs, func, share, type_system as ts
 from pixeltable.catalog import Catalog, TableVersionPath
 from pixeltable.catalog.insertable_table import OnErrorParameter
+from pixeltable.config import Config
 from pixeltable.env import Env
+from pixeltable.io.table_data_conduit import DFTableDataConduit, TableDataConduit
 from pixeltable.iterators import ComponentIterator
 
 if TYPE_CHECKING:
@@ -22,46 +25,62 @@ if TYPE_CHECKING:
         str,
         os.PathLike,
         Path,  # OS paths, filenames, URLs
-        Iterator[dict[str, Any]],  # iterator producing dictionaries of values
-        RowData,  # list of dictionaries
+        Iterable[dict[str, Any]],  # dictionaries of values
+        Iterable[pydantic.BaseModel],  # Pydantic model instances
         DataFrame,  # Pixeltable DataFrame
         pd.DataFrame,  # pandas DataFrame
-        'datasets.Dataset',
-        'datasets.DatasetDict',  # Huggingface datasets
+        datasets.Dataset,
+        datasets.DatasetDict,  # Huggingface datasets
     ]
 
 
 _logger = logging.getLogger('pixeltable')
 
 
-def init() -> None:
+def init(config_overrides: dict[str, Any] | None = None) -> None:
     """Initializes the Pixeltable environment."""
+    if config_overrides is None:
+        config_overrides = {}
+    Config.init(config_overrides)
     _ = Catalog.get()
 
 
 def create_table(
-    path_str: str,
-    schema: Optional[dict[str, Any]] = None,
+    path: str,
+    schema: dict[str, Any] | None = None,
     *,
-    source: Optional[TableDataSource] = None,
-    source_format: Optional[Literal['csv', 'excel', 'parquet', 'json']] = None,
-    schema_overrides: Optional[dict[str, Any]] = None,
+    source: TableDataSource | None = None,
+    source_format: Literal['csv', 'excel', 'parquet', 'json'] | None = None,
+    schema_overrides: dict[str, Any] | None = None,
+    create_default_idxs: bool = True,
     on_error: Literal['abort', 'ignore'] = 'abort',
-    primary_key: Optional[Union[str, list[str]]] = None,
+    primary_key: str | list[str] | None = None,
     num_retained_versions: int = 10,
     comment: str = '',
     media_validation: Literal['on_read', 'on_write'] = 'on_write',
     if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error',
-    extra_args: Optional[dict[str, Any]] = None,  # Additional arguments to data source provider
+    extra_args: dict[str, Any] | None = None,  # Additional arguments to data source provider
 ) -> catalog.Table:
-    """Create a new base table.
+    """Create a new base table. Exactly one of `schema` or `source` must be provided.
+
+    If a `schema` is provided, then an empty table will be created with the specified schema.
+
+    If a `source` is provided, then Pixeltable will attempt to infer a data source format and table schema from the
+    contents of the specified data, and the data will be imported from the specified source into the new table. The
+    source format and/or schema can be specified directly via the `source_format` and `schema_overrides` parameters.
 
     Args:
-        path_str: Path to the table.
-        schema: A dictionary that maps column names to column types
-        source: A data source from which a table schema can be inferred and data imported
-        source_format: A hint to the format of the source data
-        schema_overrides: If specified, then columns in `schema_overrides` will be given the specified types
+        path: Pixeltable path (qualified name) of the table, such as `'my_table'` or `'my_dir.my_subdir.my_table'`.
+        schema: Schema for the new table, mapping column names to Pixeltable types.
+        source: A data source (file, URL, DataFrame, or list of rows) to import from.
+        source_format: Must be used in conjunction with a `source`.
+            If specified, then the given format will be used to read the source data. (Otherwise,
+            Pixeltable will attempt to infer the format from the source data.)
+        schema_overrides: Must be used in conjunction with a `source`.
+            If specified, then columns in `schema_overrides` will be given the specified types.
+            (Pixeltable will attempt to infer the types of any columns not specified.)
+        create_default_idxs: If True, creates a B-tree index on every scalar and media column that is not computed,
+            except for boolean columns.
         on_error: Determines the behavior if an error occurs while evaluating a computed column or detecting an
             invalid media file (such as a corrupt image) for one of the inserted rows.
 
@@ -77,14 +96,15 @@ def create_table(
 
             - `'on_read'`: validate media files at query time
             - `'on_write'`: validate media files during insert/update operations
-        if_exists: Directive regarding how to handle if the path already exists.
-            Must be one of the following:
+        if_exists: Determines the behavior if a table already exists at the specified path location.
 
             - `'error'`: raise an error
             - `'ignore'`: do nothing and return the existing table handle
-            - `'replace'`: if the existing table has no views, drop and replace it with a new one
-            - `'replace_force'`: drop the existing table and all its views, and create a new one
-        extra_args: Additional arguments to pass to the source data provider
+            - `'replace'`: if the existing table has no views or snapshots, drop and replace it with a new one;
+                raise an error if the existing table has views or snapshots
+            - `'replace_force'`: drop the existing table and all its views and snapshots, and create a new one
+        extra_args: Must be used in conjunction with a `source`. If specified, then additional arguments will be
+            passed along to the source data provider.
 
     Returns:
         A handle to the newly created table, or to an already existing table at the path when `if_exists='ignore'`.
@@ -110,7 +130,7 @@ def create_table(
         >>> tbl1 = pxt.get_table('orig_table')
         ... tbl2 = pxt.create_table('new_table', tbl1.where(tbl1.col1 < 10).select(tbl1.col2))
 
-        Create a table if does not already exist, otherwise get the existing table:
+        Create a table if it does not already exist, otherwise get the existing table:
 
         >>> tbl = pxt.create_table('my_table', schema={'col1': pxt.Int, 'col2': pxt.String}, if_exists='ignore')
 
@@ -122,27 +142,39 @@ def create_table(
 
         >>> tbl = pxt.create_table('my_table', source='data.csv')
     """
-    from pixeltable.io.table_data_conduit import DFTableDataConduit, UnkTableDataConduit
+    from pixeltable.io.table_data_conduit import UnkTableDataConduit
     from pixeltable.io.utils import normalize_primary_key_parameter
 
     if (schema is None) == (source is None):
-        raise excs.Error('Must provide either a `schema` or a `source`')
+        raise excs.Error('Either a `schema` or a `source` must be provided (but not both)')
 
     if schema is not None and (len(schema) == 0 or not isinstance(schema, dict)):
         raise excs.Error('`schema` must be a non-empty dictionary')
 
-    path_obj = catalog.Path(path_str)
+    path_obj = catalog.Path.parse(path)
     if_exists_ = catalog.IfExistsParam.validated(if_exists, 'if_exists')
     media_validation_ = catalog.MediaValidation.validated(media_validation, 'media_validation')
-    primary_key: Optional[list[str]] = normalize_primary_key_parameter(primary_key)
-    table: catalog.Table = None
-    tds = None
-    data_source = None
+    primary_key: list[str] | None = normalize_primary_key_parameter(primary_key)
+    data_source: TableDataConduit | None = None
     if source is not None:
+        if isinstance(source, str) and source.strip().startswith('pxt://'):
+            raise excs.Error(
+                'create_table(): Creating a table directly from a cloud URI is not supported.'
+                ' Please replicate the table locally first using `pxt.replicate()`:\n'
+                "replica_tbl = pxt.replicate('pxt://path/to/remote_table', 'local_replica_name')\n"
+                "pxt.create_table('new_table_name', source=replica_tbl)"
+            )
         tds = UnkTableDataConduit(source, source_format=source_format, extra_fields=extra_args)
         tds.check_source_format()
         data_source = tds.specialize()
-        data_source.src_schema_overrides = schema_overrides
+        src_schema_overrides: dict[str, ts.ColumnType] = {}
+        if schema_overrides is not None:
+            for col_name, py_type in schema_overrides.items():
+                col_type = ts.ColumnType.normalize_type(py_type, nullable_default=True, allow_builtin_types=False)
+                if col_type is None:
+                    raise excs.Error(f'Invalid type for column {col_name!r} in `schema_overrides`: {py_type}')
+                src_schema_overrides[col_name] = col_type
+        data_source.src_schema_overrides = src_schema_overrides
         data_source.src_pk = primary_key
         data_source.infer_schema()
         schema = data_source.pxt_schema
@@ -156,35 +188,43 @@ def create_table(
             'Unable to create a proper schema from supplied `source`. Please use appropriate `schema_overrides`.'
         )
 
-    table = Catalog.get().create_table(
+    tbl, was_created = Catalog.get().create_table(
         path_obj,
         schema,
-        data_source.pxt_df if isinstance(data_source, DFTableDataConduit) else None,
         if_exists=if_exists_,
         primary_key=primary_key,
         comment=comment,
         media_validation=media_validation_,
         num_retained_versions=num_retained_versions,
+        create_default_idxs=create_default_idxs,
     )
-    if data_source is not None and not is_direct_df:
+
+    # TODO: combine data loading with table creation into a single transaction
+    if was_created:
         fail_on_exception = OnErrorParameter.fail_on_exception(on_error)
-        table.insert_table_data_source(data_source=data_source, fail_on_exception=fail_on_exception)
+        if isinstance(data_source, DFTableDataConduit):
+            df = data_source.pxt_df
+            with Catalog.get().begin_xact(tbl=tbl._tbl_version_path, for_write=True, lock_mutable_tree=True):
+                tbl._tbl_version.get().insert(None, df, fail_on_exception=fail_on_exception)
+        elif data_source is not None and not is_direct_df:
+            tbl.insert_table_data_source(data_source=data_source, fail_on_exception=fail_on_exception)
 
-    return table
+    return tbl
 
 
 def create_view(
     path: str,
-    base: Union[catalog.Table, DataFrame],
+    base: catalog.Table | DataFrame,
     *,
-    additional_columns: Optional[dict[str, Any]] = None,
+    additional_columns: dict[str, Any] | None = None,
     is_snapshot: bool = False,
-    iterator: Optional[tuple[type[ComponentIterator], dict[str, Any]]] = None,
+    create_default_idxs: bool = False,
+    iterator: tuple[type[ComponentIterator], dict[str, Any]] | None = None,
     num_retained_versions: int = 10,
     comment: str = '',
     media_validation: Literal['on_read', 'on_write'] = 'on_write',
     if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error',
-) -> Optional[catalog.Table]:
+) -> catalog.Table | None:
     """Create a view of an existing table object (which itself can be a view or a snapshot or a base table).
 
     Args:
@@ -197,6 +237,8 @@ def create_view(
             [`create_table`][pixeltable.create_table].
         is_snapshot: Whether the view is a snapshot. Setting this to `True` is equivalent to calling
             [`create_snapshot`][pixeltable.create_snapshot].
+        create_default_idxs: Whether to create default indexes on the view's columns (the base's columns are excluded).
+            Cannot be `True` for snapshots.
         iterator: The iterator to use for this view. If specified, then this view will be a one-to-many view of
             the base table.
         num_retained_versions: Number of versions of the view to retain.
@@ -244,16 +286,16 @@ def create_view(
         >>> tbl = pxt.get_table('my_table')
         ... view = pxt.create_view('my_view', tbl.where(tbl.col1 > 100), if_exists='replace_force')
     """
+    if is_snapshot and create_default_idxs is True:
+        raise excs.Error('Cannot create default indexes on a snapshot')
     tbl_version_path: TableVersionPath
-    select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]] = None
-    where: Optional[exprs.Expr] = None
+    select_list: list[tuple[exprs.Expr, str | None]] | None = None
+    where: exprs.Expr | None = None
     if isinstance(base, catalog.Table):
         tbl_version_path = base._tbl_version_path
         sample_clause = None
     elif isinstance(base, DataFrame):
-        base._validate_mutable('create_view', allow_select=True)
-        if len(base._from_clause.tbls) > 1:
-            raise excs.Error('Cannot create a view of a join')
+        base._validate_mutable_op_sequence('create_view', allow_select=True)
         tbl_version_path = base._from_clause.tbls[0]
         where = base.where_clause
         sample_clause = base.sample_clause
@@ -264,7 +306,7 @@ def create_view(
         raise excs.Error('`base` must be an instance of `Table` or `DataFrame`')
     assert isinstance(base, (catalog.Table, DataFrame))
 
-    path_obj = catalog.Path(path)
+    path_obj = catalog.Path.parse(path)
     if_exists_ = catalog.IfExistsParam.validated(if_exists, 'if_exists')
     media_validation_ = catalog.MediaValidation.validated(media_validation, 'media_validation')
 
@@ -276,7 +318,7 @@ def create_view(
             if col_name in [c.name for c in tbl_version_path.columns()]:
                 raise excs.Error(
                     f'Column {col_name!r} already exists in the base table '
-                    f'{tbl_version_path.get_column(col_name).tbl.name}.'
+                    f'{tbl_version_path.get_column(col_name).get_tbl().name}.'
                 )
 
     return Catalog.get().create_view(
@@ -287,6 +329,7 @@ def create_view(
         sample_clause=sample_clause,
         additional_columns=additional_columns,
         is_snapshot=is_snapshot,
+        create_default_idxs=create_default_idxs,
         iterator=iterator,
         num_retained_versions=num_retained_versions,
         comment=comment,
@@ -297,15 +340,15 @@ def create_view(
 
 def create_snapshot(
     path_str: str,
-    base: Union[catalog.Table, DataFrame],
+    base: catalog.Table | DataFrame,
     *,
-    additional_columns: Optional[dict[str, Any]] = None,
-    iterator: Optional[tuple[type[ComponentIterator], dict[str, Any]]] = None,
+    additional_columns: dict[str, Any] | None = None,
+    iterator: tuple[type[ComponentIterator], dict[str, Any]] | None = None,
     num_retained_versions: int = 10,
     comment: str = '',
     media_validation: Literal['on_read', 'on_write'] = 'on_write',
     if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error',
-) -> Optional[catalog.Table]:
+) -> catalog.Table | None:
     """Create a snapshot of an existing table object (which itself can be a view or a snapshot or a base table).
 
     Args:
@@ -376,36 +419,67 @@ def create_snapshot(
     )
 
 
-def create_replica(destination: str, source: Union[str, catalog.Table]) -> Optional[catalog.Table]:
+def publish(
+    source: str | catalog.Table,
+    destination_uri: str,
+    bucket_name: str | None = None,
+    access: Literal['public', 'private'] = 'private',
+) -> None:
     """
-    Create a replica of a table. Can be used either to create a remote replica of a local table, or to create a local
-    replica of a remote table. A given table can have at most one replica per Pixeltable instance.
+    Publishes a replica of a local Pixeltable table to Pixeltable cloud. A given table can be published to at most one
+    URI per Pixeltable cloud database.
 
     Args:
-        destination: Path where the replica will be created. Can be either a local path such as `'my_dir.my_table'`, or
-            a remote URI such as `'pxt://username/mydir.my_table'`.
-        source: Path to the source table, or (if the source table is a local table) a handle to the source table.
+        source: Path or table handle of the local table to be published.
+        destination_uri: Remote URI where the replica will be published, such as `'pxt://org_name/my_dir/my_table'`.
+        bucket_name: The name of the bucket to use to store replica's data. The bucket must be registered with
+            Pixeltable cloud. If no `bucket_name` is provided, the default storage bucket for the destination
+            database will be used.
+        access: Access control for the replica.
+
+            - `'public'`: Anyone can access this replica.
+            - `'private'`: Only the host organization can access.
     """
-    remote_dest = destination.startswith('pxt://')
-    remote_source = isinstance(source, str) and source.startswith('pxt://')
-    if remote_dest == remote_source:
-        raise excs.Error('Exactly one of `destination` or `source` must be a remote URI.')
-
-    if remote_dest:
-        if isinstance(source, str):
-            source = get_table(source)
-        share.push_replica(destination, source)
-        return None
-    else:
-        assert isinstance(source, str)
-        return share.pull_replica(destination, source)
+    if not destination_uri.startswith('pxt://'):
+        raise excs.Error("`destination_uri` must be a remote Pixeltable URI with the prefix 'pxt://'")
+
+    if isinstance(source, str):
+        source = get_table(source)
+
+    share.push_replica(destination_uri, source, bucket_name, access)
+
+
+def replicate(remote_uri: str, local_path: str) -> catalog.Table:
+    """
+    Retrieve a replica from Pixeltable cloud as a local table. This will create a full local copy of the replica in a
+    way that preserves the table structure of the original source data. Once replicated, the local table can be
+    queried offline just as any other Pixeltable table.
+
+    Args:
+        remote_uri: Remote URI of the table to be replicated, such as `'pxt://org_name/my_dir/my_table'` or
+            `'pxt://org_name/my_dir/my_table:5'` (with version 5).
+        local_path: Local table path where the replica will be created, such as `'my_new_dir.my_new_tbl'`. It can be
+            the same or different from the cloud table name.
+
+    Returns:
+        A handle to the newly created local replica table.
+    """
+    if not remote_uri.startswith('pxt://'):
+        raise excs.Error("`remote_uri` must be a remote Pixeltable URI with the prefix 'pxt://'")
+
+    return share.pull_replica(local_path, remote_uri)
 
 
-def get_table(path: str) -> catalog.Table:
+def get_table(path: str, if_not_exists: Literal['error', 'ignore'] = 'error') -> catalog.Table | None:
     """Get a handle to an existing table, view, or snapshot.
 
     Args:
         path: Path to the table.
+        if_not_exists: Directive regarding how to handle if the path does not exist.
+            Must be one of the following:
+
+            - `'error'`: raise an error
+            - `'ignore'`: do nothing and return `None`
 
     Returns:
         A handle to the [`Table`][pixeltable.Table].
@@ -425,20 +499,39 @@ def get_table(path: str) -> catalog.Table:
         Handles to views and snapshots are retrieved in the same way:
 
         >>> tbl = pxt.get_table('my_snapshot')
+
+        Get a handle to a specific version of a table:
+
+        >>> tbl = pxt.get_table('my_table:722')
     """
-    path_obj = catalog.Path(path)
-    tbl = Catalog.get().get_table(path_obj)
-    tv = tbl._tbl_version.get()
-    _logger.debug(f'get_table(): tbl={tv.id}:{tv.effective_version} sa_tbl={id(tv.store_tbl.sa_tbl):x} tv={id(tv):x}')
+    if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
+    path_obj = catalog.Path.parse(path, allow_versioned_path=True)
+    tbl = Catalog.get().get_table(path_obj, if_not_exists_)
     return tbl
 
 
-def move(path: str, new_path: str) -> None:
+def move(
+    path: str,
+    new_path: str,
+    *,
+    if_exists: Literal['error', 'ignore'] = 'error',
+    if_not_exists: Literal['error', 'ignore'] = 'error',
+) -> None:
     """Move a schema object to a new directory and/or rename a schema object.
 
     Args:
         path: absolute path to the existing schema object.
         new_path: absolute new path for the schema object.
+        if_exists: Directive regarding how to handle if a schema object already exists at the new path.
+            Must be one of the following:
+
+            - `'error'`: raise an error
+            - `'ignore'`: do nothing and return
+        if_not_exists: Directive regarding how to handle if the source path does not exist.
+            Must be one of the following:
+
+            - `'error'`: raise an error
+            - `'ignore'`: do nothing and return
 
     Raises:
         Error: If path does not exist or new_path already exists.
@@ -452,22 +545,26 @@ def move(path: str, new_path: str) -> None:
 
         >>>> pxt.move('dir1.my_table', 'dir1.new_name')
     """
+    if_exists_ = catalog.IfExistsParam.validated(if_exists, 'if_exists')
+    if if_exists_ not in (catalog.IfExistsParam.ERROR, catalog.IfExistsParam.IGNORE):
+        raise excs.Error("`if_exists` must be one of 'error' or 'ignore'")
+    if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
     if path == new_path:
         raise excs.Error('move(): source and destination cannot be identical')
-    path_obj, new_path_obj = catalog.Path(path), catalog.Path(new_path)
+    path_obj, new_path_obj = catalog.Path.parse(path), catalog.Path.parse(new_path)
     if path_obj.is_ancestor(new_path_obj):
         raise excs.Error(f'move(): cannot move {path!r} into its own subdirectory')
-    cat = Catalog.get()
-    cat.move(path_obj, new_path_obj)
+    Catalog.get().move(path_obj, new_path_obj, if_exists_, if_not_exists_)
 
 
 def drop_table(
-    table: Union[str, catalog.Table], force: bool = False, if_not_exists: Literal['error', 'ignore'] = 'error'
+    table: str | catalog.Table, force: bool = False, if_not_exists: Literal['error', 'ignore'] = 'error'
 ) -> None:
-    """Drop a table, view, or snapshot.
+    """Drop a table, view, snapshot, or replica.
 
     Args:
-        table: Fully qualified name, or handle, of the table to be dropped.
+        table: Fully qualified name or table handle of the table to be dropped; or a remote URI of a cloud replica to
+            be deleted.
         force: If `True`, will also drop all views and sub-views of this table.
         if_not_exists: Directive regarding how to handle if the path does not exist.
             Must be one of the following:
@@ -507,9 +604,69 @@ def drop_table(
         assert isinstance(table, str)
         tbl_path = table
 
-    path_obj = catalog.Path(tbl_path)
     if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
-    Catalog.get().drop_table(path_obj, force=force, if_not_exists=if_not_exists_)
+
+    if tbl_path.startswith('pxt://'):
+        # Remote table
+        if force:
+            raise excs.Error('Cannot use `force=True` with a cloud replica URI.')
+        # TODO: Handle if_not_exists properly
+        share.delete_replica(tbl_path)
+    else:
+        # Local table
+        path_obj = catalog.Path.parse(tbl_path)
+        Catalog.get().drop_table(path_obj, force=force, if_not_exists=if_not_exists_)
+
+
+def get_dir_contents(dir_path: str = '', recursive: bool = True) -> 'DirContents':
+    """Get the contents of a Pixeltable directory.
+
+    Args:
+        dir_path: Path to the directory. Defaults to the root directory.
+        recursive: If `False`, returns only those tables and directories that are directly contained in specified
+            directory; if `True`, returns all tables and directories that are descendants of the specified directory,
+            recursively.
+
+    Returns:
+        A [`DirContents`][pixeltable.DirContents] object representing the contents of the specified directory.
+
+    Raises:
+        Error: If the path does not exist or does not designate a directory.
+
+    Examples:
+        Get contents of top-level directory:
+
+        >>> pxt.get_dir_contents()
+
+        Get contents of 'dir1':
+
+        >>> pxt.get_dir_contents('dir1')
+    """
+    path_obj = catalog.Path.parse(dir_path, allow_empty_path=True)
+    catalog_entries = Catalog.get().get_dir_contents(path_obj, recursive=recursive)
+    dirs: list[str] = []
+    tables: list[str] = []
+    _assemble_dir_contents(dir_path, catalog_entries, dirs, tables)
+    dirs.sort()
+    tables.sort()
+    return DirContents(dirs, tables)
+
+
+def _assemble_dir_contents(
+    dir_path: str, catalog_entries: dict[str, Catalog.DirEntry], dirs: list[str], tables: list[str]
+) -> None:
+    for name, entry in catalog_entries.items():
+        if name.startswith('_'):
+            continue  # Skip system paths
+        path = f'{dir_path}.{name}' if len(dir_path) > 0 else name
+        if entry.dir is not None:
+            dirs.append(path)
+            if entry.dir_entries is not None:
+                _assemble_dir_contents(path, entry.dir_entries, dirs, tables)
+        else:
+            assert entry.table is not None
+            assert not entry.dir_entries
+            tables.append(path)
 
 
 def list_tables(dir_path: str = '', recursive: bool = True) -> list[str]:
@@ -535,15 +692,18 @@ def list_tables(dir_path: str = '', recursive: bool = True) -> list[str]:
 
         >>> pxt.list_tables('dir1')
     """
-    path_obj = catalog.Path(dir_path, empty_is_valid=True)  # validate format
-    cat = Catalog.get()
-    contents = cat.get_dir_contents(path_obj, recursive=recursive)
+    return _list_tables(dir_path, recursive=recursive, allow_system_paths=False)
+
+
+def _list_tables(dir_path: str = '', recursive: bool = True, allow_system_paths: bool = False) -> list[str]:
+    path_obj = catalog.Path.parse(dir_path, allow_empty_path=True, allow_system_path=allow_system_paths)
+    contents = Catalog.get().get_dir_contents(path_obj, recursive=recursive)
     return [str(p) for p in _extract_paths(contents, parent=path_obj, entry_type=catalog.Table)]
 
 
 def create_dir(
-    path: str, if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error', parents: bool = False
-) -> Optional[catalog.Dir]:
+    path: str, *, if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error', parents: bool = False
+) -> catalog.Dir | None:
     """Create a directory.
 
     Args:
@@ -588,7 +748,7 @@ def create_dir(
 
         >>> pxt.create_dir('parent1.parent2.sub_dir', parents=True)
     """
-    path_obj = catalog.Path(path)
+    path_obj = catalog.Path.parse(path)
     if_exists_ = catalog.IfExistsParam.validated(if_exists, 'if_exists')
     return Catalog.get().create_dir(path_obj, if_exists=if_exists_, parents=parents)
 
@@ -630,15 +790,75 @@ def drop_dir(path: str, force: bool = False, if_not_exists: Literal['error', 'ig
 
         >>> pxt.drop_dir('my_dir', force=True)
     """
-    path_obj = catalog.Path(path)  # validate format
+    path_obj = catalog.Path.parse(path)  # validate format
     if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
     Catalog.get().drop_dir(path_obj, if_not_exists=if_not_exists_, force=force)
 
 
+def ls(path: str = '') -> pd.DataFrame:
+    """
+    List the contents of a Pixeltable directory.
+
+    This function returns a Pandas DataFrame representing a human-readable listing of the specified directory,
+    including various attributes such as version and base table, as appropriate.
+
+    To get a programmatic list of the directory's contents, use [get_dir_contents()][pixeltable.get_dir_contents]
+    instead.
+    """
+    from pixeltable.catalog import retry_loop
+    from pixeltable.metadata import schema
+
+    cat = Catalog.get()
+    path_obj = catalog.Path.parse(path, allow_empty_path=True)
+    dir_entries = cat.get_dir_contents(path_obj)
+
+    @retry_loop(for_write=False)
+    def op() -> list[list[str]]:
+        rows: list[list[str]] = []
+        for name, entry in dir_entries.items():
+            if name.startswith('_'):
+                continue
+            if entry.dir is not None:
+                kind = 'dir'
+                version = ''
+                base = ''
+            else:
+                assert entry.table is not None
+                assert isinstance(entry.table, schema.Table)
+                tbl = cat.get_table_by_id(entry.table.id)
+                md = tbl.get_metadata()
+                base = md['base'] or ''
+                if base.startswith('_'):
+                    base = '<anonymous base table>'
+                if md['is_replica']:
+                    kind = 'replica'
+                elif md['is_snapshot']:
+                    kind = 'snapshot'
+                elif md['is_view']:
+                    kind = 'view'
+                else:
+                    kind = 'table'
+                version = '' if kind == 'snapshot' else str(md['version'])
+            rows.append([name, kind, version, base])
+        return rows
+
+    rows = op()
+
+    rows = sorted(rows, key=lambda x: x[0])
+    df = pd.DataFrame(
+        {
+            'Name': [row[0] for row in rows],
+            'Kind': [row[1] for row in rows],
+            'Version': [row[2] for row in rows],
+            'Base': [row[3] for row in rows],
+        },
+        index=([''] * len(rows)),
+    )
+    return df
+
+
 def _extract_paths(
-    dir_entries: dict[str, Catalog.DirEntry],
-    parent: catalog.Path,
-    entry_type: Optional[type[catalog.SchemaObject]] = None,
+    dir_entries: dict[str, Catalog.DirEntry], parent: catalog.Path, entry_type: type[catalog.SchemaObject] | None = None
 ) -> list[catalog.Path]:
     """Convert nested dir_entries structure to a flattened list of paths."""
     matches: list[str]
@@ -676,7 +896,7 @@ def list_dirs(path: str = '', recursive: bool = True) -> list[str]:
         >>> cl.list_dirs('my_dir', recursive=True)
         ['my_dir', 'my_dir.sub_dir1']
     """
-    path_obj = catalog.Path(path, empty_is_valid=True)  # validate format
+    path_obj = catalog.Path.parse(path, allow_empty_path=True)  # validate format
     cat = Catalog.get()
     contents = cat.get_dir_contents(path_obj, recursive=recursive)
     return [str(p) for p in _extract_paths(contents, parent=path_obj, entry_type=catalog.Dir)]
@@ -711,7 +931,7 @@ def list_functions() -> Styler:
     return pd_df.hide(axis='index')
 
 
-def tools(*args: Union[func.Function, func.tools.Tool]) -> func.tools.Tools:
+def tools(*args: func.Function | func.tools.Tool) -> func.tools.Tools:
     """
     Specifies a collection of UDFs to be used as LLM tools. Pixeltable allows any UDF to be used as an input into an
     LLM tool-calling API. To use one or more UDFs as tools, wrap them in a `pxt.tools` call and pass the return value
@@ -748,7 +968,7 @@ def tools(*args: Union[func.Function, func.tools.Tool]) -> func.tools.Tools:
     return func.tools.Tools(tools=[arg if isinstance(arg, func.tools.Tool) else tool(arg) for arg in args])
 
 
-def tool(fn: func.Function, name: Optional[str] = None, description: Optional[str] = None) -> func.tools.Tool:
+def tool(fn: func.Function, name: str | None = None, description: str | None = None) -> func.tools.Tool:
     """
     Specifies a Pixeltable UDF to be used as an LLM tool with customizable metadata. See the documentation for
     [pxt.tools()][pixeltable.tools] for more details.
@@ -769,11 +989,7 @@ def tool(fn: func.Function, name: Optional[str] = None, description: Optional[st
 
 
 def configure_logging(
-    *,
-    to_stdout: Optional[bool] = None,
-    level: Optional[int] = None,
-    add: Optional[str] = None,
-    remove: Optional[str] = None,
+    *, to_stdout: bool | None = None, level: int | None = None, add: str | None = None, remove: str | None = None
 ) -> None:
     """Configure logging.
 
@@ -788,3 +1004,14 @@ def configure_logging(
 
 def array(elements: Iterable) -> exprs.Expr:
     return exprs.Expr.from_array(elements)
+
+
+class DirContents(NamedTuple):
+    """
+    Represents the contents of a Pixeltable directory.
+    """
+
+    dirs: list[str]
+    """List of directory paths contained in this directory."""
+    tables: list[str]
+    """List of table paths contained in this directory."""