pixeltable 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

pixeltable/tests/test_table.py

@@ -18,7 +18,7 @@ from pixeltable import exceptions as excs
 from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import \
     make_tbl, create_table_data, read_data_file, get_video_files, get_audio_files, get_image_files, get_documents, \
-    assert_resultset_eq, assert_hf_dataset_equal, make_test_arrow_table
+    assert_resultset_eq, assert_hf_dataset_equal, make_test_arrow_table, validate_update_status
 from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import \
     StringType, IntType, FloatType, TimestampType, ImageType, VideoType, JsonType, BoolType, ArrayType, AudioType, \
@@ -41,6 +41,21 @@ class TestTable:
     def add1(a: int) -> int:
         return a + 1
 
+    @pxt.uda(
+        update_types=[IntType()], value_type=IntType(), requires_order_by=True,
+        allows_window=True)
+    class window_fn:
+        def __init__(self):
+            pass
+        def update(self, i: int) -> None:
+            pass
+        def value(self) -> int:
+            return 1
+
+    @pxt.expr_udf(param_types=[IntType(nullable=False)])
+    def add1(a: int) -> int:
+        return a + 1
+
     def test_create(self, test_client: pxt.Client) -> None:
         cl = test_client
         cl.create_dir('dir1')
@@ -56,7 +71,7 @@ class TestTable:
         with pytest.raises(excs.Error):
             _ = cl.create_table('1test', schema)
         with pytest.raises(excs.Error):
-            _ = catalog.Column('1c', StringType())
+            _ = cl.create_table('bad name', schema={'c1': StringType()})
         with pytest.raises(excs.Error):
             _ = cl.create_table('test', schema)
         with pytest.raises(excs.Error):
@@ -299,10 +314,6 @@ class TestTable:
             cl.create_table('test', {'c1': {'type': StringType(), 'stored': 'true'}})
         assert '"stored" must be a bool' in str(exc_info.value)
 
-        with pytest.raises(excs.Error) as exc_info:
-            cl.create_table('test', {'c1': {'type': StringType(), 'indexed': 'true'}})
-        assert '"indexed" must be a bool' in str(exc_info.value)
-
         with pytest.raises(excs.Error) as exc_info:
             cl.create_table('test', {'c1': StringType()}, primary_key='c2')
         assert 'primary key column c2 not found' in str(exc_info.value).lower()
@@ -501,18 +512,8 @@ class TestTable:
         # a non-materialized column that refers to another non-materialized column
         view.add_column(c4=view.c2.rotate(60), stored=False)
 
-        @pxt.uda(
-            name='window_fn', update_types=[IntType()], value_type=IntType(), requires_order_by = True,
-            allows_window = True)
-        class WindowFnAggregator:
-            def __init__(self):
-                pass
-            def update(self, i: int) -> None:
-                pass
-            def value(self) -> int:
-                return 1
         # cols computed with window functions are stored by default
-        view.add_column(c5=window_fn(view.frame_idx, 1, group_by=view.video))
+        view.add_column(c5=self.window_fn(view.frame_idx, 1, group_by=view.video))
 
         # reload to make sure that metadata gets restored correctly
         cl = pxt.Client(reload=True)
@@ -553,6 +554,23 @@ class TestTable:
         cl.drop_table('test_tbl')
         assert MediaStore.count(view.get_id()) == 0
 
+    def test_insert_nulls(self, test_client: pxt.Client) -> None:
+        cl = test_client
+        schema = {
+            'c1': StringType(nullable=True),
+            'c2': IntType(nullable=True),
+            'c3': FloatType(nullable=True),
+            'c4': BoolType(nullable=True),
+            'c5': ArrayType((2, 3), dtype=IntType(), nullable=True),
+            'c6': JsonType(nullable=True),
+            'c7': ImageType(nullable=True),
+            'c8': VideoType(nullable=True),
+        }
+        t = cl.create_table('test1', schema)
+        status = t.insert(c1='abc')
+        assert status.num_rows == 1
+        assert status.num_excs == 0
+
     def test_insert(self, test_client: pxt.Client) -> None:
         cl = test_client
         schema = {
@@ -650,7 +668,63 @@ class TestTable:
         t2 = cl.get_table('test')
         _  = t2.show(n=0)
 
-    def test_update(self, test_tbl: pxt.Table, indexed_img_tbl: pxt.Table) -> None:
+    def test_batch_update(self, test_tbl: pxt.Table) -> None:
+        t = test_tbl
+        validate_update_status(
+            t.batch_update([{'c1': '1', 'c2': 1}, {'c1': '2', 'c2': 2}]),
+            expected_rows=2)
+        assert t.where(t.c2 == 1).collect()[0]['c1'] == '1'
+        assert t.where(t.c2 == 2).collect()[0]['c1'] == '2'
+        validate_update_status(
+            t.batch_update([{'c1': 'one', '_rowid': (1,)}, {'c1': 'two', '_rowid': (2,)}]),
+            expected_rows=2)
+        assert t.where(t.c2 == 1).collect()[0]['c1'] == 'one'
+        assert t.where(t.c2 == 2).collect()[0]['c1'] == 'two'
+
+        cl = pxt.Client()
+        # test composite primary key
+        schema = {'c1': StringType(), 'c2': IntType(), 'c3': FloatType()}
+        t = cl.create_table('composite', schema=schema, primary_key=['c1', 'c2'])
+        rows = [{'c1': str(i), 'c2': i, 'c3': float(i)} for i in range(10)]
+        validate_update_status(t.insert(rows), expected_rows=10)
+
+        validate_update_status(
+            t.batch_update([{'c1': '1', 'c2': 1, 'c3': 2.0}, {'c1': '2', 'c2': 2, 'c3': 3.0}]),
+            expected_rows=2)
+
+        with pytest.raises(excs.Error) as exc_info:
+            # can't mix _rowid with primary key
+            _ = t.batch_update([{'c1': '1', 'c2': 1, 'c3': 2.0, '_rowid': (1,)}])
+        assert 'c1 is a primary key column' in str(exc_info.value).lower()
+
+        with pytest.raises(excs.Error) as exc_info:
+            # bad literal
+            _ = t.batch_update([{'c2': 1, 'c3': 'a'}])
+        assert "'a' is not a valid literal" in str(exc_info.value).lower()
+
+        with pytest.raises(excs.Error) as exc_info:
+            # missing primary key column
+            t.batch_update([{'c1': '1', 'c3': 2.0}])
+        assert 'primary key columns (c2) missing' in str(exc_info.value).lower()
+
+        # table without primary key
+        t2 = cl.create_table('no_pk', schema=schema)
+        validate_update_status(t2.insert(rows), expected_rows=10)
+        with pytest.raises(excs.Error) as exc_info:
+            _ = t2.batch_update([{'c1': '1', 'c2': 1, 'c3': 2.0}])
+        assert 'must have primary key for batch update' in str(exc_info.value).lower()
+
+        # updating with _rowid still works
+        validate_update_status(
+            t2.batch_update([{'c1': 'one', '_rowid': (1,)}, {'c1': 'two', '_rowid': (2,)}]),
+            expected_rows=2)
+        assert t2.where(t2.c2 == 1).collect()[0]['c1'] == 'one'
+        assert t2.where(t2.c2 == 2).collect()[0]['c1'] == 'two'
+        with pytest.raises(AssertionError):
+            # some rows are missing rowids
+            _ = t2.batch_update([{'c1': 'one', '_rowid': (1,)}, {'c1': 'two'}])
+
+    def test_update(self, test_tbl: pxt.Table, small_img_tbl) -> None:
         t = test_tbl
         # update every type with a literal
         test_cases = [
@@ -755,7 +829,7 @@ class TestTable:
             t.update({'c3': 1.0}, where=lambda c2: c2 == 10)
         assert 'Predicate' in str(excinfo.value)
 
-        img_t = indexed_img_tbl
+        img_t = small_img_tbl
 
         # can't update image col
         with pytest.raises(excs.Error) as excinfo:
@@ -785,7 +859,7 @@ class TestTable:
         r2 = t.where(t.c2 < 5).select(t.c3, t.c10, t.d1, t.d2).order_by(t.c2).show(0)
         assert_resultset_eq(r1, r2)
 
-    def test_delete(self, test_tbl: pxt.Table, indexed_img_tbl: pxt.Table) -> None:
+    def test_delete(self, test_tbl: pxt.Table, small_img_tbl) -> None:
         t = test_tbl
 
         cnt = t.where(t.c3 < 10.0).count()
@@ -813,7 +887,7 @@ class TestTable:
             t.delete(where=lambda c2: c2 == 10)
         assert 'Predicate' in str(excinfo.value)
 
-        img_t = indexed_img_tbl
+        img_t = small_img_tbl
         # similarity search is not supported
         with pytest.raises(excs.Error) as excinfo:
             img_t.delete(where=img_t.img.nearest('car'))
@@ -979,6 +1053,10 @@ class TestTable:
         t2.revert()
         assert MediaStore.count(t2.get_id()) == t2.count() * stores_img_col
 
+    @pxt.udf(return_type=ImageType(), param_types=[ImageType()])
+    def img_fn_with_exc(img: PIL.Image.Image) -> PIL.Image.Image:
+        raise RuntimeError
+
     def test_computed_img_cols(self, test_client: pxt.Client) -> None:
         cl = test_client
         schema = {'img': ImageType(nullable=False)}
@@ -996,10 +1074,7 @@ class TestTable:
 
         # computed img col with exceptions
         t = cl.create_table('test3', schema)
-        @pxt.udf(return_type=ImageType(), param_types=[ImageType()])
-        def f(img: PIL.Image.Image) -> PIL.Image.Image:
-            raise RuntimeError
-        t.add_column(c3=f(t.img), stored=True)
+        t.add_column(c3=self.img_fn_with_exc(t.img), stored=True)
         rows = read_data_file('imagenette2-160', 'manifest.csv', ['img'])
         rows = [{'img': r['img']} for r in rows[:20]]
         t.insert(rows, fail_on_exception=False)
@@ -1211,6 +1286,7 @@ class TestTable:
         check_rename(t, 'c1_renamed', 'c1')
 
         # revert() works
+        _ = t.select(t.c1_renamed).collect()
         t.revert()
         _ = t.select(t.c1).collect()
         #check_rename(t, 'c1', 'c1_renamed')

pixeltable/tests/test_types.py

@@ -1,3 +1,7 @@
+import datetime
+from copy import copy
+from typing import List, Dict, Optional
+
 from pixeltable.type_system import \
     ColumnType, StringType, IntType, BoolType, ImageType, InvalidType, FloatType, TimestampType, JsonType, ArrayType
 
@@ -20,3 +24,29 @@ class TestTypes:
             t_serialized = t.serialize()
             t_deserialized = ColumnType.deserialize(t_serialized)
             assert t == t_deserialized
+
+    def test_from_python_type(self) -> None:
+        test_cases = {
+            str: StringType(),
+            int: IntType(),
+            float: FloatType(),
+            bool: BoolType(),
+            datetime.date: TimestampType(),
+            datetime.datetime: TimestampType(),
+            list: JsonType(),
+            dict: JsonType(),
+            list[int]: JsonType(),
+            list[dict[str, int]]: JsonType(),
+            dict[int, str]: JsonType(),
+            dict[dict[str, int], list[int]]: JsonType(),
+            List: JsonType(),
+            Dict: JsonType(),
+            List[int]: JsonType(),
+            List[Dict[str, int]]: JsonType(),
+            Dict[int, str]: JsonType()
+        }
+        for py_type, pxt_type in test_cases.items():
+            assert ColumnType.from_python_type(py_type) == pxt_type
+            opt_pxt_type = copy(pxt_type)
+            opt_pxt_type.nullable = True
+            assert ColumnType.from_python_type(Optional[py_type]) == opt_pxt_type

pixeltable/tests/test_video.py

@@ -107,6 +107,18 @@ class TestVideo:
         base_t.insert({'video': p} for p in video_filepaths)
         _ = view_t[view_t.c1, view_t.c2, view_t.c3, view_t.c4].show(0)
 
+    # window function that simply passes through the frame
+    @pxt.uda(
+        update_types=[ImageType()], value_type=ImageType(),
+        requires_order_by=True, allows_std_agg=False, allows_window=True)
+    class agg_fn:
+        def __init__(self):
+            self.img = None
+        def update(self, frame: PIL.Image.Image) -> None:
+            self.img = frame
+        def value(self) -> PIL.Image.Image:
+            return self.img
+
     def test_make_video(self, test_client: pxt.Client) -> None:
         video_filepaths = get_video_files()
         cl = test_client
@@ -131,29 +143,17 @@ class TestVideo:
                 make_video(view_t.pos, view_t.frame),
                 make_video(view_t.pos - 1, view_t.transformed)).group_by(base_t).show()
 
-        # window function that simply passes through the frame
-        @pxt.uda(
-            update_types=[ImageType()], value_type=ImageType(), name='agg_fn',
-            requires_order_by=True, allows_std_agg=False, allows_window=True)
-        class WindowAgg:
-            def __init__(self):
-                self.img = None
-            def update(self, frame: PIL.Image.Image) -> None:
-                self.img = frame
-            def value(self) -> PIL.Image.Image:
-                return self.img
-
         # make sure it works
-        _ = view_t.select(agg_fn(view_t.pos, view_t.frame, group_by=base_t)).show()
-        status = view_t.add_column(agg=agg_fn(view_t.pos, view_t.frame, group_by=base_t))
+        _ = view_t.select(self.agg_fn(view_t.pos, view_t.frame, group_by=base_t)).show()
+        status = view_t.add_column(agg=self.agg_fn(view_t.pos, view_t.frame, group_by=base_t))
         assert status.num_excs == 0
         _ = view_t.select(make_video(view_t.pos, view_t.agg)).group_by(base_t).show()
 
         # image cols computed with a window function currently need to be stored
         with pytest.raises(excs.Error):
-            view_t.add_column(agg2=agg_fn(view_t.pos, view_t.frame, group_by=base_t), stored=False)
+            view_t.add_column(agg2=self.agg_fn(view_t.pos, view_t.frame, group_by=base_t), stored=False)
 
         # reload from store
         cl = pxt.Client(reload=True)
         base_t, view_t = cl.get_table(base_t.get_name()), cl.get_table(view_t.get_name())
-        _ = view_t.select(agg_fn(view_t.pos, view_t.frame, group_by=base_t)).show()
+        _ = view_t.select(self.agg_fn(view_t.pos, view_t.frame, group_by=base_t)).show()

pixeltable/tests/test_view.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 
 import PIL
@@ -414,6 +415,10 @@ class TestView:
             v.order_by(v.c2).show(0),
             t.where(t.c2 < 10).order_by(t.c2).show(0))
 
+        # create views with filters containing date and datetime
+        _ = cl.create_view('test_view_2', t, filter=t.c5 >= datetime.date.today())
+        _ = cl.create_view('test_view_3', t, filter=t.c5 < datetime.datetime.now())
+
     def test_view_of_snapshot(self, test_client: pxt.Client) -> None:
         """Test view over a snapshot"""
         cl = test_client

pixeltable/tests/utils.py

@@ -6,6 +6,7 @@ from collections import namedtuple
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set
 
+import PIL.Image
 import numpy as np
 import pandas as pd
 import pytest
@@ -16,6 +17,7 @@ from pixeltable import catalog
 from pixeltable.catalog.globals import UpdateStatus
 from pixeltable.dataframe import DataFrameResultSet
 from pixeltable.env import Env
+from pixeltable.functions.huggingface import clip_image, clip_text
 from pixeltable.type_system import (
     ArrayType,
     BoolType,
@@ -43,6 +45,7 @@ def make_default_type(t: ColumnType.Type) -> ColumnType:
         return TimestampType()
     assert False
 
+
 def make_tbl(cl: pxt.Client, name: str = 'test', col_names: Optional[List[str]] = None) -> catalog.InsertableTable:
     if col_names is None:
         col_names = ['c1']
@@ -51,7 +54,9 @@ def make_tbl(cl: pxt.Client, name: str = 'test', col_names: Optional[List[str]]
         schema[f'{col_name}'] = make_default_type(ColumnType.Type(i % 5))
     return cl.create_table(name, schema)
 
-def create_table_data(t: catalog.Table, col_names: Optional[List[str]] = None, num_rows: int = 10) -> List[Dict[str, Any]]:
+
+def create_table_data(t: catalog.Table, col_names: Optional[List[str]] = None, num_rows: int = 10) -> List[
+    Dict[str, Any]]:
     if col_names is None:
         col_names = []
     data: Dict[str, Any] = {}
@@ -124,6 +129,7 @@ def create_table_data(t: catalog.Table, col_names: Optional[List[str]] = None, n
     rows = [{col_name: data[col_name][i] for col_name in col_names} for i in range(num_rows)]
     return rows
 
+
 def create_test_tbl(client: pxt.Client, name: str = 'test_tbl') -> catalog.Table:
     schema = {
         'c1': StringType(nullable=False),
@@ -189,12 +195,25 @@ def create_test_tbl(client: pxt.Client, name: str = 'test_tbl') -> catalog.Table
     t.insert(rows)
     return t
 
+
+def create_img_tbl(cl: pxt.Client, name: str = 'test_img_tbl') -> catalog.Table:
+    schema = {
+        'img': ImageType(nullable=False),
+        'category': StringType(nullable=False),
+        'split': StringType(nullable=False),
+    }
+    tbl = cl.create_table(name, schema)
+    rows = read_data_file('imagenette2-160', 'manifest.csv', ['img'])
+    tbl.insert(rows)
+    return tbl
+
+
 def create_all_datatypes_tbl(test_client: pxt.Client) -> catalog.Table:
     """ Creates a table with all supported datatypes.
     """
     schema = {
-        'row_id': IntType(nullable=False), # used for row selection
-        'c_array': ArrayType(shape=(10,),  dtype=FloatType(), nullable=True),
+        'row_id': IntType(nullable=False),  # used for row selection
+        'c_array': ArrayType(shape=(10,), dtype=FloatType(), nullable=True),
         'c_bool': BoolType(nullable=True),
         'c_float': FloatType(nullable=True),
         'c_image': ImageType(nullable=True),
@@ -207,12 +226,13 @@ def create_all_datatypes_tbl(test_client: pxt.Client) -> catalog.Table:
     tbl = test_client.create_table('all_datatype_tbl', schema)
     example_rows = create_table_data(tbl, num_rows=11)
 
-    for i,r in enumerate(example_rows):
-        r['row_id'] = i # row_id
+    for i, r in enumerate(example_rows):
+        r['row_id'] = i  # row_id
 
     tbl.insert(example_rows)
     return tbl
 
+
 def read_data_file(dir_name: str, file_name: str, path_col_names: Optional[List[str]] = None) -> List[Dict[str, Any]]:
     """
     Locate dir_name, create df out of file_name.
@@ -223,7 +243,7 @@ def read_data_file(dir_name: str, file_name: str, path_col_names: Optional[List[
     """
     if path_col_names is None:
         path_col_names = []
-    tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
+    tests_dir = os.path.dirname(__file__)  # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/{dir_name}', recursive=True)
     assert len(glob_result) == 1, f'Could not find {dir_name}'
     abs_path = Path(glob_result[0])
@@ -235,8 +255,9 @@ def read_data_file(dir_name: str, file_name: str, path_col_names: Optional[List[
         df[col_name] = df.apply(lambda r: str(abs_path / r[col_name]), axis=1)
     return df.to_dict(orient='records')
 
+
 def get_video_files(include_bad_video: bool = False) -> List[str]:
-    tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
+    tests_dir = os.path.dirname(__file__)  # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/videos/*', recursive=True)
     if not include_bad_video:
         glob_result = [f for f in glob_result if 'bad_video' not in f]
@@ -244,18 +265,21 @@ def get_video_files(include_bad_video: bool = False) -> List[str]:
     half_res = [f for f in glob_result if 'half_res' in f or 'bad_video' in f]
     return half_res
 
+
 def get_test_video_files() -> List[str]:
-    tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
+    tests_dir = os.path.dirname(__file__)  # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/test_videos/*', recursive=True)
     return glob_result
 
+
 def get_image_files(include_bad_image: bool = False) -> List[str]:
-    tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
+    tests_dir = os.path.dirname(__file__)  # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/imagenette2-160/*', recursive=True)
     if not include_bad_image:
         glob_result = [f for f in glob_result if 'bad_image' not in f]
     return glob_result
 
+
 def get_audio_files(include_bad_audio: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     glob_result = glob.glob(f'{tests_dir}/**/audio/*', recursive=True)
@@ -263,11 +287,13 @@ def get_audio_files(include_bad_audio: bool = False) -> List[str]:
         glob_result = [f for f in glob_result if 'bad_audio' not in f]
     return glob_result
 
+
 def get_documents() -> List[str]:
     tests_dir = os.path.dirname(__file__)
     # for now, we can only handle .html and .md
     return [p for p in glob.glob(f'{tests_dir}/**/documents/*', recursive=True) if not p.endswith('.pdf')]
 
+
 def get_sentences(n: int = 100) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     path = glob.glob(f'{tests_dir}/**/jeopardy.json', recursive=True)[0]
@@ -403,6 +429,14 @@ def assert_hf_dataset_equal(hf_dataset: 'datasets.Dataset', df: pxt.DataFrame, s
         check_tup = DatasetTuple(**encoded_tup)
         assert check_tup in acc_dataset
 
+@pxt.expr_udf
+def img_embed(img: PIL.Image.Image) -> np.ndarray:
+    return clip_image(img, model_id='openai/clip-vit-base-patch32')
+
+@pxt.expr_udf
+def text_embed(txt: str) -> np.ndarray:
+    return clip_text(txt, model_id='openai/clip-vit-base-patch32')
 
 SAMPLE_IMAGE_URL = \
     'https://raw.githubusercontent.com/pixeltable/pixeltable/master/docs/source/data/images/000000000009.jpg'
+

pixeltable/tool/create_test_db_dump.py

@@ -136,6 +136,22 @@ class Dumper:
             for i in range(num_rows)
         ]
         t.insert(rows)
+        self.cl.create_dir('views')
+        v = self.cl.create_view('views.sample_view', t, filter=(t.c2 < 50))
+        _ = self.cl.create_view('views.sample_snapshot', t, filter=(t.c2 >= 75), is_snapshot=True)
+        # Computed column using a library function
+        v['str_format'] = pxt.functions.string.str_format('{0} {key}', t.c1, key=t.c1)
+        # Computed column using a bespoke udf
+        v['test_udf'] = test_udf(t.c2)
+        # astype
+        v['astype'] = t.c1.astype(pxt.FloatType())
+        # computed column using a stored function
+        v['stored'] = t.c1.apply(lambda x: f'Hello, {x}', col_type=pxt.StringType())
+
+
+@pxt.udf
+def test_udf(n: int) -> int:
+    return n + 1
 
 
 def main() -> None: