pixeltable 0.1.2__tar.gz → 0.2.1__tar.gz

pixeltable-0.2.1/LICENSE

@@ -0,0 +1,18 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+
+   Copyright 2023 Marcel Kornacker
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pixeltable-0.2.1/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.1
+Name: pixeltable
+Version: 0.2.1
+Summary: Pixeltable: The Multimodal AI Data Plane
+Author: Marcel Kornacker
+Author-email: marcelk@gmail.com
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: av (>=10.0.0)
+Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
+Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
+Requires-Dist: jmespath (>=1.0.1,<2.0.0)
+Requires-Dist: numpy (>=1.24.1,<2.0.0)
+Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
+Requires-Dist: pandas (>=1.5.3,<2.0.0)
+Requires-Dist: pgserver (==0.0.5)
+Requires-Dist: pgvector (>=0.2.1,<0.3.0)
+Requires-Dist: pillow (>=9.4.0,<10.0.0)
+Requires-Dist: psutil (>=5.9.5,<6.0.0)
+Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: regex (>=2022.10.31,<2023.0.0)
+Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
+Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
+Requires-Dist: tqdm (>=4.64.1,<5.0.0)
+Description-Content-Type: text/markdown
+
+<img src="docs/pixeltable-banner.png" width="45%"/>
+
+# Pixeltable: The Multimodal AI Data Plane
+
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+&nbsp;&nbsp;
+![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
+
+Pixeltable is a Python library that lets AI engineers and data scientists focus on
+exploration, modeling, and app development without having to deal with the customary
+data plumbing.
+
+**Pixeltable redefines data infrastructure and workflow orchestration for AI development.**
+It brings together data storage, versioning, and indexing with orchestration and model
+versioning under a declarative table interface, with transformations, model inference,
+and custom logic represented as computed columns.
+<!--
+## Quick Start
+
+If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
+the Pixeltable Basics tutorial in colab:
+
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/docs/tutorials/pixeltable-basics.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+-->
+## Installation
+
+Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
+
+```
+pip install pixeltable
+```
+
+To verify that it's working:
+
+```
+import pixeltable as pxt
+cl = pxt.Client()
+```
+
+For more detailed installation instructions, see the
+[Getting Started with Pixeltable](https://pixeltable.github.io/pixeltable/getting-started/)
+guide. Then, check out the
+[Pixeltable Basics](https://pixeltable.github.io/pixeltable/tutorials/pixeltable-basics/)
+tutorial for a tour of its most important features.
+
+## What problems does Pixeltable solve?
+
+Today’s solutions for AI app development require extensive custom coding and infrastructure
+plumbing. Tracking lineage and versions between and across data transformations, models, and
+deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
+a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
+building and productionizing AI applications.
+
+## Why should you use Pixeltable?
+
+- It gives you transparency and reproducibility
+    - All generated data is automatically recorded and versioned
+    - You will never need to re-run a workload because you lost track of the input data
+- It saves you money
+    - All data changes are automatically incremental
+    - You never need to re-run pipelines from scratch because you’re adding data
+- It integrates with any existing Python code or libraries
+    - Bring your ever-changing code and workloads
+    - You choose the models, tools, and AI practices (e.g., your embedding model for a vector index); Pixeltable orchestrates the data
+
+## Example Use Cases
+
+* Interact with video data at the frame level without having to think about frame extraction,
+intermediate file storage, or storage space explosion.
+* Augment your data incrementally and interactively with built-in functions and UDFs, such as
+image transformations, model inference, and visualizations, without having to think about data pipelines,
+incremental updates, or capturing function output.
+* Interact with all the data relevant to your AI application (video, images, documents, audio, structured data, JSON) through
+a simple dataframe-style API directly in Python. This includes:
+    * similarity search on embeddings, supported by high-dimensional vector indexing
+    * path expressions and transformations on JSON data
+    * PIL and OpenCV image operations
+    * assembling frames into videos
+* Perform keyword and image similarity search at the video frame level without having to worry about frame
+storage.
+* Access all Pixeltable-resident data directly as a PyTorch dataset in your training scripts.
+* Understand the compute and storage costs of your data at the granularity of individual augmentations and
+get cost projections before adding new data and new augmentations.
+* Rely on Pixeltable's automatic versioning and snapshot functionality to protect against regressions
+and to ensure reproducibility.
+

pixeltable-0.2.1/README.md

@@ -0,0 +1,87 @@
+<img src="docs/pixeltable-banner.png" width="45%"/>
+
+# Pixeltable: The Multimodal AI Data Plane
+
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+&nbsp;&nbsp;
+![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
+
+Pixeltable is a Python library that lets AI engineers and data scientists focus on
+exploration, modeling, and app development without having to deal with the customary
+data plumbing.
+
+**Pixeltable redefines data infrastructure and workflow orchestration for AI development.**
+It brings together data storage, versioning, and indexing with orchestration and model
+versioning under a declarative table interface, with transformations, model inference,
+and custom logic represented as computed columns.
+<!--
+## Quick Start
+
+If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
+the Pixeltable Basics tutorial in colab:
+
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/docs/tutorials/pixeltable-basics.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+-->
+## Installation
+
+Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
+
+```
+pip install pixeltable
+```
+
+To verify that it's working:
+
+```
+import pixeltable as pxt
+cl = pxt.Client()
+```
+
+For more detailed installation instructions, see the
+[Getting Started with Pixeltable](https://pixeltable.github.io/pixeltable/getting-started/)
+guide. Then, check out the
+[Pixeltable Basics](https://pixeltable.github.io/pixeltable/tutorials/pixeltable-basics/)
+tutorial for a tour of its most important features.
+
+## What problems does Pixeltable solve?
+
+Today’s solutions for AI app development require extensive custom coding and infrastructure
+plumbing. Tracking lineage and versions between and across data transformations, models, and
+deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
+a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
+building and productionizing AI applications.
+
+## Why should you use Pixeltable?
+
+- It gives you transparency and reproducibility
+    - All generated data is automatically recorded and versioned
+    - You will never need to re-run a workload because you lost track of the input data
+- It saves you money
+    - All data changes are automatically incremental
+    - You never need to re-run pipelines from scratch because you’re adding data
+- It integrates with any existing Python code or libraries
+    - Bring your ever-changing code and workloads
+    - You choose the models, tools, and AI practices (e.g., your embedding model for a vector index); Pixeltable orchestrates the data
+
+## Example Use Cases
+
+* Interact with video data at the frame level without having to think about frame extraction,
+intermediate file storage, or storage space explosion.
+* Augment your data incrementally and interactively with built-in functions and UDFs, such as
+image transformations, model inference, and visualizations, without having to think about data pipelines,
+incremental updates, or capturing function output.
+* Interact with all the data relevant to your AI application (video, images, documents, audio, structured data, JSON) through
+a simple dataframe-style API directly in Python. This includes:
+    * similarity search on embeddings, supported by high-dimensional vector indexing
+    * path expressions and transformations on JSON data
+    * PIL and OpenCV image operations
+    * assembling frames into videos
+* Perform keyword and image similarity search at the video frame level without having to worry about frame
+storage.
+* Access all Pixeltable-resident data directly as a PyTorch dataset in your training scripts.
+* Understand the compute and storage costs of your data at the granularity of individual augmentations and
+get cost projections before adding new data and new augmentations.
+* Rely on Pixeltable's automatic versioning and snapshot functionality to protect against regressions
+and to ensure reproducibility.

pixeltable-0.2.1/pixeltable/__init__.py

@@ -0,0 +1,44 @@
+from .catalog import Column, Table, InsertableTable, View
+from .client import Client
+from .dataframe import DataFrame
+from .exceptions import Error, Error
+from .exprs import RELATIVE_PATH_ROOT
+from .func import Function, udf, uda, Aggregator, expr_udf
+from .type_system import \
+    ColumnType, StringType, IntType, FloatType, BoolType, TimestampType, JsonType, ArrayType, ImageType, VideoType, \
+    AudioType, DocumentType
+from .utils.help import help
+# noinspection PyUnresolvedReferences
+from . import functions
+
+__all__ = [
+    'Client',
+    'DataFrame',
+    'Column',
+    'Table',
+    'InsertableTable',
+    'View',
+    'Error',
+    'ColumnType',
+    'StringType',
+    'IntType',
+    'FloatType',
+    'BoolType',
+    'TimestampType',
+    'JsonType',
+    'RELATIVE_PATH_ROOT',
+    'ArrayType',
+    'ImageType',
+    'VideoType',
+    'AudioType',
+    'DocumentType',
+    'Function',
+    'help',
+    'udf',
+    'Aggregator',
+    'uda',
+    'expr_udf',
+]
+
+
+

pixeltable-0.2.1/pixeltable/catalog/__init__.py

@@ -0,0 +1,13 @@
+from .catalog import Catalog
+from .column import Column
+from .table_version_path import TableVersionPath
+from .table_version import TableVersion
+from .schema_object import SchemaObject
+from .named_function import NamedFunction
+from .dir import Dir
+from .table import Table
+from .insertable_table import InsertableTable
+from .view import View
+from .path import Path
+from .path_dict import PathDict
+from .globals import is_valid_identifier, is_valid_path

pixeltable-0.2.1/pixeltable/catalog/catalog.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+from typing import Optional, List, Any, Dict, Tuple
+from uuid import UUID
+import dataclasses
+import logging
+
+import sqlalchemy as sql
+import sqlalchemy.orm as orm
+
+from .table_version import TableVersion
+from .table_version_path import TableVersionPath
+from .table import Table
+from .named_function import NamedFunction
+from .path_dict import PathDict
+import pixeltable.env as env
+import pixeltable.metadata.schema as schema
+
+_logger = logging.getLogger('pixeltable')
+
+class Catalog:
+    """A repository of catalog objects"""
+    _instance: Optional[Catalog] = None
+
+    @classmethod
+    def get(cls) -> Catalog:
+        if cls._instance is None:
+            cls._instance = cls()
+            with orm.Session(env.Env.get().engine, future=True) as session:
+                cls._instance._load_table_versions(session)
+                #cls._instance._load_functions(session)
+        return cls._instance
+
+    @classmethod
+    def clear(cls) -> None:
+        """Remove the instance. Used for testing."""
+        cls._instance = None
+
+    def __init__(self) -> None:
+        # key: [id, version]
+        # - mutable version of a table: version == None (even though TableVersion.version is set correctly)
+        # - snapshot versions: records the version of the snapshot
+        self.tbl_versions: Dict[Tuple[UUID, int], TableVersion] = {}
+
+        self.tbls: Dict[UUID, Table] = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
+        self.tbl_dependents: Dict[UUID, List[Table]] = {}
+
+        self._init_store()
+        self.paths = PathDict()  # do this after _init_catalog()
+
+    def _init_store(self) -> None:
+        """One-time initialization of the stored catalog. Idempotent."""
+        with orm.Session(env.Env.get().engine, future=True) as session:
+            if session.query(sql.func.count(schema.Dir.id)).scalar() > 0:
+                return
+            # create a top-level directory, so that every schema object has a directory
+            dir_md = schema.DirMd(name='')
+            dir_record = schema.Dir(parent_id=None, md=dataclasses.asdict(dir_md))
+            session.add(dir_record)
+            session.flush()
+            session.commit()
+            _logger.info(f'Initialized catalog')
+
+    def _load_snapshot_version(
+            self, tbl_id: UUID, version: int, base: Optional[TableVersion], session: orm.Session
+    ) -> TableVersion:
+        q = session.query(schema.Table, schema.TableSchemaVersion) \
+            .select_from(schema.Table) \
+            .join(schema.TableVersion) \
+            .join(schema.TableSchemaVersion) \
+            .where(schema.Table.id == tbl_id) \
+            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {version}")) \
+            .where(sql.text((
+                f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
+                f"{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}")))
+        tbl_record, schema_version_record = q.one()
+        tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+        schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
+        # we ignore tbl_record.base_tbl_id/base_snapshot_id and use 'base' instead: if the base is a snapshot
+        # we'd have to look that up first
+        return TableVersion(tbl_record.id, tbl_md, version, schema_version_md, is_snapshot=True, base=base)
+
+    def _load_table_versions(self, session: orm.Session) -> None:
+        from .insertable_table import InsertableTable
+        from .view import View
+
+        # load tables/views;
+        # do this in ascending order of creation ts so that we can resolve base references in one pass
+        q = session.query(schema.Table, schema.TableSchemaVersion) \
+            .select_from(schema.Table) \
+            .join(schema.TableVersion) \
+            .join(schema.TableSchemaVersion) \
+            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = 0")) \
+            .where(sql.text((
+                f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
+                f"{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}"))) \
+            .order_by(sql.text(f"({schema.TableVersion.__table__}.md->>'created_at')::float"))
+
+        for tbl_record, schema_version_record in q.all():
+            tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+            schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
+            view_md = tbl_md.view_md
+
+            if view_md is not None:
+                assert len(view_md.base_versions) > 0
+                # construct a TableVersionPath for the view
+                refd_versions = [(UUID(tbl_id), version) for tbl_id, version in view_md.base_versions]
+                base_path: Optional[TableVersionPath] = None
+                base: Optional[TableVersion] = None
+                # go through the versions in reverse order, so we can construct TableVersionPaths
+                for base_id, version in refd_versions[::-1]:
+                    base_version = self.tbl_versions.get((base_id, version), None)
+                    if base_version is None:
+                        if version is None:
+                            # debugging
+                            pass
+                        # if this is a reference to a mutable table, we should have loaded it already
+                        assert version is not None
+                        base_version = self._load_snapshot_version(base_id, version, base, session)
+                    base_path = TableVersionPath(base_version, base=base_path)
+                    base = base_version
+                assert base_path is not None
+
+                base_tbl = self.tbls[base_path.tbl_version.id]
+                is_snapshot = view_md is not None and view_md.is_snapshot
+                snapshot_only = is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
+                if snapshot_only:
+                    # this is a pure snapshot, without a physical table backing it
+                    view_path = base_path
+                else:
+                    tbl_version = TableVersion(
+                        tbl_record.id, tbl_md, tbl_md.current_version, schema_version_md, is_snapshot=is_snapshot,
+                        base=base_path.tbl_version if is_snapshot else None,
+                        base_path=base_path if not is_snapshot else None)
+                    view_path = TableVersionPath(tbl_version, base=base_path)
+
+                tbl = View(
+                    tbl_record.id, tbl_record.dir_id, tbl_md.name, view_path, base_tbl,
+                    snapshot_only=snapshot_only)
+                self.tbl_dependents[base_tbl._id].append(tbl)
+
+            else:
+                tbl_version = TableVersion(tbl_record.id, tbl_md, tbl_md.current_version, schema_version_md)
+                tbl = InsertableTable(tbl_record.dir_id, tbl_version)
+
+            self.tbls[tbl._id] = tbl
+            self.tbl_dependents[tbl._id] = []
+            self.paths.add_schema_obj(tbl._dir_id, tbl_md.name, tbl)
+
+    # def _load_functions(self, session: orm.Session) -> None:
+    #     # load Function metadata; doesn't load the actual callable, which can be large and is only done on-demand by the
+    #     # FunctionRegistry
+    #     q = session.query(schema.Function.id, schema.Function.dir_id, schema.Function.md) \
+    #         .where(sql.text(f"({schema.Function.__table__}.md->>'name')::text IS NOT NULL"))
+    #     for id, dir_id, md in q.all():
+    #         assert 'name' in md
+    #         name = md['name']
+    #         assert name is not None
+    #         named_fn = NamedFunction(id, dir_id, name)
+    #         self.paths.add_schema_obj(dir_id, name, named_fn)

pixeltable-0.2.1/pixeltable/catalog/column.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional, Union, Callable, Set
+
+import sqlalchemy as sql
+from pgvector.sqlalchemy import Vector
+
+from pixeltable import exceptions as excs
+from pixeltable.metadata import schema
+from pixeltable.type_system import ColumnType, StringType
+from .globals import is_valid_identifier
+
+_logger = logging.getLogger('pixeltable')
+
+class Column:
+    """Representation of a column in the schema of a Table/DataFrame.
+
+    A Column contains all the metadata necessary for executing queries and updates against a particular version of a
+    table/view.
+    """
+    def __init__(
+            self, name: str, col_type: Optional[ColumnType] = None,
+            computed_with: Optional[Union['Expr', Callable]] = None,
+            primary_key: bool = False, stored: Optional[bool] = None,
+            indexed: bool = False,
+            # these parameters aren't set by users
+            col_id: Optional[int] = None):
+        """Column constructor.
+
+        Args:
+            name: column name
+            col_type: column type; can be None if the type can be derived from ``computed_with``
+            computed_with: a callable or an Expr object that computes the column value
+            primary_key: if True, this column is part of the primary key
+            stored: determines whether a computed column is present in the stored table or recomputed on demand
+            indexed: if True, this column has a nearest neighbor index (only valid for image columns)
+            col_id: column ID (only used internally)
+
+        Computed columns: those have a non-None ``computed_with`` argument
+
+        - when constructed by the user: ``computed_with`` was constructed explicitly and is passed in;
+          col_type is None
+        - when loaded from md store: ``computed_with`` is set and col_type is set
+
+        ``computed_with`` is a Callable:
+
+        - the callable's parameter names must correspond to existing columns in the table for which this Column
+          is being used
+        - ``col_type`` needs to be set to the callable's return type
+
+        ``stored`` (only valid for computed image columns):
+
+        - if True: the column is present in the stored table
+        - if False: the column is not present in the stored table and recomputed during a query
+        - if None: the system chooses for you (at present, this is always False, but this may change in the future)
+
+        indexed: only valid for image columns; if true, maintains an NN index for this column
+        """
+        if not is_valid_identifier(name):
+            raise excs.Error(f"Invalid column name: '{name}'")
+        self.name = name
+        if col_type is None and computed_with is None:
+            raise excs.Error(f'Column {name}: col_type is required if computed_with is not specified')
+
+        self.value_expr: Optional['Expr'] = None
+        self.compute_func: Optional[Callable] = None
+        from pixeltable import exprs
+        if computed_with is not None:
+            value_expr = exprs.Expr.from_object(computed_with)
+            if value_expr is None:
+                # computed_with needs to be a Callable
+                if not isinstance(computed_with, Callable):
+                    raise excs.Error(
+                        f'Column {name}: computed_with needs to be either a Pixeltable expression or a Callable, '
+                        f'but it is a {type(computed_with)}')
+                if col_type is None:
+                    raise excs.Error(f'Column {name}: col_type is required if computed_with is a Callable')
+                # we need to turn the computed_with function into an Expr, but this requires resolving
+                # column name references and for that we need to wait until we're assigned to a Table
+                self.compute_func = computed_with
+            else:
+                self.value_expr = value_expr.copy()
+                self.col_type = self.value_expr.col_type
+
+        if col_type is not None:
+            self.col_type = col_type
+        assert self.col_type is not None
+
+        self.stored = stored
+        self.dependent_cols: Set[Column] = set()  # cols with value_exprs that reference us; set by TableVersion
+        self.id = col_id
+        self.primary_key = primary_key
+
+        # column in the stored table for the values of this Column
+        self.sa_col: Optional[sql.schema.Column] = None
+
+        # computed cols also have storage columns for the exception string and type
+        self.sa_errormsg_col: Optional[sql.schema.Column] = None
+        self.sa_errortype_col: Optional[sql.schema.Column] = None
+        # indexed columns also have a column for the embeddings
+        self.sa_idx_col: Optional[sql.schema.Column] = None
+        from .table_version import TableVersion
+        self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
+
+        if indexed and not self.col_type.is_image_type():
+            raise excs.Error(f'Column {name}: indexed=True requires ImageType')
+        self.is_indexed = indexed
+
+    @classmethod
+    def from_md(cls, col_id: int, md: schema.SchemaColumn, tbl: 'TableVersion') -> Column:
+        """Construct a Column from metadata.
+
+        Leaves out value_expr, because that requires TableVersion.cols to be complete.
+        """
+        col = cls(
+            md.name, col_type=ColumnType.from_dict(md.col_type), primary_key=md.is_pk,
+            stored=md.stored, indexed=md.is_indexed, col_id=col_id)
+        col.tbl = tbl
+        return col
+
+    def __hash__(self) -> int:
+        assert self.tbl is not None
+        return hash((self.tbl.id, self.id))
+
+    def check_value_expr(self) -> None:
+        assert self.value_expr is not None
+        if self.stored == False and self.is_computed and self.has_window_fn_call():
+            raise excs.Error(
+                f'Column {self.name}: stored={self.stored} not supported for columns computed with window functions:'
+                f'\n{self.value_expr}')
+
+    def has_window_fn_call(self) -> bool:
+        if self.value_expr is None:
+            return False
+        from pixeltable import exprs
+        l = list(self.value_expr.subexprs(filter=lambda e: isinstance(e, exprs.FunctionCall) and e.is_window_fn_call))
+        return len(l) > 0
+
+    @property
+    def is_computed(self) -> bool:
+        return self.compute_func is not None or self.value_expr is not None
+
+    @property
+    def is_stored(self) -> bool:
+        """Returns True if column is materialized in the stored table."""
+        assert self.stored is not None
+        return self.stored
+
+    @property
+    def records_errors(self) -> bool:
+        """True if this column also stores error information."""
+        return self.is_stored and (self.is_computed or self.col_type.is_media_type())
+
+    def source(self) -> None:
+        """
+        If this is a computed col and the top-level expr is a function call, print the source, if possible.
+        """
+        from pixeltable import exprs
+        if self.value_expr is None or not isinstance(self.value_expr, exprs.FunctionCall):
+            return
+        self.value_expr.fn.source()
+
+    def create_sa_cols(self) -> None:
+        """
+        These need to be recreated for every new table schema version.
+        """
+        assert self.is_stored
+        # all storage columns are nullable (we deal with null errors in Pixeltable directly)
+        self.sa_col = sql.Column(self.storage_name(), self.col_type.to_sa_type(), nullable=True)
+        if self.is_computed or self.col_type.is_media_type():
+            self.sa_errormsg_col = sql.Column(self.errormsg_storage_name(), StringType().to_sa_type(), nullable=True)
+            self.sa_errortype_col = sql.Column(self.errortype_storage_name(), StringType().to_sa_type(), nullable=True)
+        if self.is_indexed:
+            self.sa_idx_col = sql.Column(self.index_storage_name(), Vector(512), nullable=True)
+
+    def storage_name(self) -> str:
+        assert self.id is not None
+        assert self.is_stored
+        return f'col_{self.id}'
+
+    def errormsg_storage_name(self) -> str:
+        return f'{self.storage_name()}_errormsg'
+
+    def errortype_storage_name(self) -> str:
+        return f'{self.storage_name()}_errortype'
+
+    def index_storage_name(self) -> str:
+        return f'{self.storage_name()}_idx_0'
+
+    def __str__(self) -> str:
+        return f'{self.name}: {self.col_type}'
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Column):
+            return False
+        assert self.tbl is not None
+        assert other.tbl is not None
+        return self.tbl.id == other.tbl.id and self.id == other.id
+

pixeltable-0.2.1/pixeltable/catalog/dir.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import dataclasses
+import logging
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from .schema_object import SchemaObject
+from pixeltable.env import Env
+from pixeltable.metadata import schema
+
+
+_logger = logging.getLogger('pixeltable')
+
+class Dir(SchemaObject):
+    def __init__(self, id: UUID, parent_id: UUID, name: str):
+        super().__init__(id, name, parent_id)
+
+    @classmethod
+    def display_name(cls) -> str:
+        return 'directory'
+
+    def move(self, new_name: str, new_dir_id: UUID) -> None:
+        super().move(new_name, new_dir_id)
+        with Env.get().engine.begin() as conn:
+            dir_md = schema.DirMd(name=new_name)
+            conn.execute(
+                sql.update(schema.Dir.__table__)
+                .values({schema.Dir.parent_id: self._dir_id, schema.Dir.md: dataclasses.asdict(dir_md)})
+                .where(schema.Dir.id == self._id))
+