pixeltable 0.2.4__tar.gz → 0.2.6__tar.gz

{pixeltable-0.2.4 → pixeltable-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.4
+Version: 0.2.6
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -13,19 +13,22 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: av (>=10.0.0)
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
+Requires-Dist: ftfy (>=6.2.0,<7.0.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: numpy (>=1.26)
+Requires-Dist: mistune (>=3.0.2,<4.0.0)
+Requires-Dist: numpy (>=1.25)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.1.0)
+Requires-Dist: pgserver (==0.1.2)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
-Requires-Dist: pillow (>=10.0)
+Requires-Dist: pillow (>=9.3.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
+Requires-Dist: pymupdf (>=1.24.1,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: regex (>=2022.10.31,<2023.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: setuptools (==69.1.1)
 Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
 Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
 Requires-Dist: tenacity (>=8.2,<9.0)
@@ -33,13 +36,13 @@ Requires-Dist: tqdm (>=4.64.1,<5.0.0)
 Description-Content-Type: text/markdown
 
 <div align="center">
-<img src="docs/pixeltable-banner.png" width="45%"/>
+<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/master/docs/pixeltable-banner.png" alt="Pixeltable" width="45%" />
 
 # Unifying Data, Models, and Orchestration for AI Products
 
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 &nbsp;&nbsp;
-![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
+[![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)](https://github.com/pixeltable/pixeltable/actions)
 
 [Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.github.io/pixeltable/)
 </div>
@@ -60,11 +63,12 @@ Learn the basics of Pixeltable through interactive examples. View the notebooks
 ### Pixeltable Basics
 In this tutorial, we'll survey how to create tables, populate them with data, and enhance them with built-in and user-defined transformations and AI operations.
 
-[![Open in Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://www.kaggle.com/code/brunep/pixeltable-basics) <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
+[![Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://kaggle.com/kernels/welcome?src=https://github.com/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb)&nbsp;&nbsp;
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
 
 
 ## 💾 Installation
-Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
+Pixeltable works with Python 3.9, 3.10, 3.11, or 3.12 running on Linux, MacOS, or Windows.
 
 ```
 pip install pixeltable
@@ -74,7 +78,7 @@ To verify that it's working:
 
 ```
 import pixeltable as pxt
-cl = pxt.Client()
+pxt.init()
 ```
 
 For more detailed installation instructions, see the

{pixeltable-0.2.4 → pixeltable-0.2.6}/README.md

@@ -1,11 +1,11 @@
 <div align="center">
-<img src="docs/pixeltable-banner.png" width="45%"/>
+<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/master/docs/pixeltable-banner.png" alt="Pixeltable" width="45%" />
 
 # Unifying Data, Models, and Orchestration for AI Products
 
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 &nbsp;&nbsp;
-![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
+[![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)](https://github.com/pixeltable/pixeltable/actions)
 
 [Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.github.io/pixeltable/)
 </div>
@@ -26,11 +26,12 @@ Learn the basics of Pixeltable through interactive examples. View the notebooks
 ### Pixeltable Basics
 In this tutorial, we'll survey how to create tables, populate them with data, and enhance them with built-in and user-defined transformations and AI operations.
 
-[![Open in Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://www.kaggle.com/code/brunep/pixeltable-basics) <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
+[![Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://kaggle.com/kernels/welcome?src=https://github.com/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb)&nbsp;&nbsp;
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
 
 
 ## 💾 Installation
-Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
+Pixeltable works with Python 3.9, 3.10, 3.11, or 3.12 running on Linux, MacOS, or Windows.
 
 ```
 pip install pixeltable
@@ -40,7 +41,7 @@ To verify that it's working:
 
 ```
 import pixeltable as pxt
-cl = pxt.Client()
+pxt.init()
 ```
 
 For more detailed installation instructions, see the

{pixeltable-0.2.4 → pixeltable-0.2.6}/pixeltable/__init__.py

@@ -1,18 +1,30 @@
 from .catalog import Column, Table, InsertableTable, View
-from .client import Client
 from .dataframe import DataFrame
 from .exceptions import Error, Error
 from .exprs import RELATIVE_PATH_ROOT
 from .func import Function, udf, uda, Aggregator, expr_udf
-from .type_system import \
-    ColumnType, StringType, IntType, FloatType, BoolType, TimestampType, JsonType, ArrayType, ImageType, VideoType, \
-    AudioType, DocumentType
+from .globals import *
+from .type_system import (
+    ColumnType,
+    StringType,
+    IntType,
+    FloatType,
+    BoolType,
+    TimestampType,
+    JsonType,
+    ArrayType,
+    ImageType,
+    VideoType,
+    AudioType,
+    DocumentType,
+)
 from .utils.help import help
+
 # noinspection PyUnresolvedReferences
-from . import functions
+from . import functions, io
+from .__version__ import __version__, __version_tuple__
 
 __all__ = [
-    'Client',
     'DataFrame',
     'Column',
     'Table',
@@ -39,6 +51,3 @@ __all__ = [
     'uda',
     'expr_udf',
 ]
-
-
-

pixeltable-0.2.6/pixeltable/__version__.py

@@ -0,0 +1,3 @@
+# These version placeholders will be replaced during build.
+__version__ = "0.2.6"
+__version_tuple__ = (0, 2, 6)

{pixeltable-0.2.4 → pixeltable-0.2.6}/pixeltable/catalog/column.py

@@ -4,11 +4,9 @@ import logging
 from typing import Optional, Union, Callable, Set
 
 import sqlalchemy as sql
-from pgvector.sqlalchemy import Vector
 
-from pixeltable import exceptions as excs
-from pixeltable.metadata import schema
-from pixeltable.type_system import ColumnType, StringType
+import pixeltable.exceptions as excs
+import pixeltable.type_system as ts
 from .globals import is_valid_identifier
 
 _logger = logging.getLogger('pixeltable')
@@ -20,44 +18,38 @@ class Column:
     table/view.
     """
     def __init__(
-            self, name: str, col_type: Optional[ColumnType] = None,
+            self, name: Optional[str], col_type: Optional[ts.ColumnType] = None,
             computed_with: Optional[Union['Expr', Callable]] = None,
-            primary_key: bool = False, stored: Optional[bool] = None,
-            indexed: bool = False,
-            # these parameters aren't set by users
-            col_id: Optional[int] = None):
+            is_pk: bool = False, stored: Optional[bool] = None,
+            col_id: Optional[int] = None, schema_version_add: Optional[int] = None,
+            schema_version_drop: Optional[int] = None, sa_col_type: Optional[sql.sqltypes.TypeEngine] = None
+    ):
         """Column constructor.
 
         Args:
-            name: column name
+            name: column name; None for system columns (eg, index columns)
             col_type: column type; can be None if the type can be derived from ``computed_with``
             computed_with: a callable or an Expr object that computes the column value
-            primary_key: if True, this column is part of the primary key
+            is_pk: if True, this column is part of the primary key
             stored: determines whether a computed column is present in the stored table or recomputed on demand
-            indexed: if True, this column has a nearest neighbor index (only valid for image columns)
             col_id: column ID (only used internally)
 
         Computed columns: those have a non-None ``computed_with`` argument
-
         - when constructed by the user: ``computed_with`` was constructed explicitly and is passed in;
           col_type is None
         - when loaded from md store: ``computed_with`` is set and col_type is set
 
         ``computed_with`` is a Callable:
-
         - the callable's parameter names must correspond to existing columns in the table for which this Column
           is being used
         - ``col_type`` needs to be set to the callable's return type
 
         ``stored`` (only valid for computed image columns):
-
         - if True: the column is present in the stored table
         - if False: the column is not present in the stored table and recomputed during a query
         - if None: the system chooses for you (at present, this is always False, but this may change in the future)
-
-        indexed: only valid for image columns; if true, maintains an NN index for this column
         """
-        if not is_valid_identifier(name):
+        if name is not None and not is_valid_identifier(name):
             raise excs.Error(f"Invalid column name: '{name}'")
         self.name = name
         if col_type is None and computed_with is None:
@@ -90,35 +82,20 @@ class Column:
         self.stored = stored
         self.dependent_cols: Set[Column] = set()  # cols with value_exprs that reference us; set by TableVersion
         self.id = col_id
-        self.primary_key = primary_key
+        self.is_pk = is_pk
+        self.schema_version_add = schema_version_add
+        self.schema_version_drop = schema_version_drop
 
         # column in the stored table for the values of this Column
         self.sa_col: Optional[sql.schema.Column] = None
+        self.sa_col_type = sa_col_type
 
         # computed cols also have storage columns for the exception string and type
         self.sa_errormsg_col: Optional[sql.schema.Column] = None
         self.sa_errortype_col: Optional[sql.schema.Column] = None
-        # indexed columns also have a column for the embeddings
-        self.sa_idx_col: Optional[sql.schema.Column] = None
         from .table_version import TableVersion
         self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
 
-        if indexed and not self.col_type.is_image_type():
-            raise excs.Error(f'Column {name}: indexed=True requires ImageType')
-        self.is_indexed = indexed
-
-    @classmethod
-    def from_md(cls, col_id: int, md: schema.SchemaColumn, tbl: 'TableVersion') -> Column:
-        """Construct a Column from metadata.
-
-        Leaves out value_expr, because that requires TableVersion.cols to be complete.
-        """
-        col = cls(
-            md.name, col_type=ColumnType.from_dict(md.col_type), primary_key=md.is_pk,
-            stored=md.stored, indexed=md.is_indexed, col_id=col_id)
-        col.tbl = tbl
-        return col
-
     def __hash__(self) -> int:
         assert self.tbl is not None
         return hash((self.tbl.id, self.id))
@@ -137,6 +114,10 @@ class Column:
         l = list(self.value_expr.subexprs(filter=lambda e: isinstance(e, exprs.FunctionCall) and e.is_window_fn_call))
         return len(l) > 0
 
+    def get_idx_info(self) -> dict[str, 'pixeltable.catalog.TableVersion.IndexInfo']:
+        assert self.tbl is not None
+        return {name: info for name, info in self.tbl.idxs_by_name.items() if info.col == self}
+
     @property
     def is_computed(self) -> bool:
         return self.compute_func is not None or self.value_expr is not None
@@ -167,26 +148,26 @@ class Column:
         """
         assert self.is_stored
         # all storage columns are nullable (we deal with null errors in Pixeltable directly)
-        self.sa_col = sql.Column(self.storage_name(), self.col_type.to_sa_type(), nullable=True)
+        self.sa_col = sql.Column(
+            self.store_name(), self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type,
+            nullable=True)
         if self.is_computed or self.col_type.is_media_type():
-            self.sa_errormsg_col = sql.Column(self.errormsg_storage_name(), StringType().to_sa_type(), nullable=True)
-            self.sa_errortype_col = sql.Column(self.errortype_storage_name(), StringType().to_sa_type(), nullable=True)
-        if self.is_indexed:
-            self.sa_idx_col = sql.Column(self.index_storage_name(), Vector(512), nullable=True)
+            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), ts.StringType().to_sa_type(), nullable=True)
+            self.sa_errortype_col = sql.Column(self.errortype_store_name(), ts.StringType().to_sa_type(), nullable=True)
+
+    def get_sa_col_type(self) -> sql.sqltypes.TypeEngine:
+        return self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type
 
-    def storage_name(self) -> str:
+    def store_name(self) -> str:
         assert self.id is not None
         assert self.is_stored
         return f'col_{self.id}'
 
-    def errormsg_storage_name(self) -> str:
-        return f'{self.storage_name()}_errormsg'
-
-    def errortype_storage_name(self) -> str:
-        return f'{self.storage_name()}_errortype'
+    def errormsg_store_name(self) -> str:
+        return f'{self.store_name()}_errormsg'
 
-    def index_storage_name(self) -> str:
-        return f'{self.storage_name()}_idx_0'
+    def errortype_store_name(self) -> str:
+        return f'{self.store_name()}_errortype'
 
     def __str__(self) -> str:
         return f'{self.name}: {self.col_type}'

{pixeltable-0.2.4 → pixeltable-0.2.6}/pixeltable/catalog/insertable_table.py

@@ -11,14 +11,17 @@ import pixeltable.type_system as ts
 from pixeltable import exceptions as excs
 from pixeltable.env import Env
 from .catalog import Catalog
+from .globals import UpdateStatus
 from .table import Table
 from .table_version import TableVersion
 from .table_version_path import TableVersionPath
 
 _logger = logging.getLogger('pixeltable')
 
+
 class InsertableTable(Table):
     """A `Table` that allows inserting and deleting rows."""
+
     def __init__(self, dir_id: UUID, tbl_version: TableVersion):
         tbl_version_path = TableVersionPath(tbl_version)
         super().__init__(tbl_version.id, dir_id, tbl_version.name, tbl_version_path)
@@ -42,7 +45,7 @@ class InsertableTable(Table):
             col = columns[column_names.index(pk_col)]
             if col.col_type.nullable:
                 raise excs.Error(f'Primary key column {pk_col} cannot be nullable')
-            col.primary_key = True
+            col.is_pk = True
 
         with orm.Session(Env.get().engine, future=True) as session:
             _, tbl_version = TableVersion.create(session, dir_id, name, columns, num_retained_versions, comment)
@@ -62,7 +65,7 @@ class InsertableTable(Table):
     @overload
     def insert(self, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any): ...
 
-    def insert(self, *args, **kwargs) -> Table.UpdateStatus:
+    def insert(self, *args, **kwargs) -> UpdateStatus:
         """Insert rows into table.
 
         To insert multiple rows at a time:
@@ -161,7 +164,7 @@ class InsertableTable(Table):
                     msg = str(e)
                     raise excs.Error(f'Error in column {col.name}: {msg[0].lower() + msg[1:]}\nRow: {row}')
 
-    def delete(self, where: Optional['pixeltable.exprs.Predicate'] = None) -> Table.UpdateStatus:
+    def delete(self, where: Optional['pixeltable.exprs.Predicate'] = None) -> UpdateStatus:
         """Delete rows in this table.
 
         Args:
@@ -181,9 +184,7 @@ class InsertableTable(Table):
         if where is not None:
             if not isinstance(where, Predicate):
                 raise excs.Error(f"'where' argument must be a Predicate, got {type(where)}")
-            analysis_info = Planner.analyze(self.tbl_version, where)
-            if analysis_info.similarity_clause is not None:
-                raise excs.Error('nearest() cannot be used with delete()')
+            analysis_info = Planner.analyze(self.tbl_version_path, where)
             # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
             if analysis_info.filter is not None:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')