pixeltable 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

pixeltable/tests/test_function.py

@@ -1,332 +0,0 @@
-from typing import Optional
-
-import numpy as np
-import pytest
-
-import pixeltable as pxt
-import pixeltable.exceptions as excs
-from pixeltable import catalog
-from pixeltable.func import Function, FunctionRegistry, Batch
-from pixeltable.type_system import IntType, FloatType
-from pixeltable.tests.utils import assert_resultset_eq
-import pixeltable.func as func
-
-
-def dummy_fn(i: int) -> int:
-    return i
-
-class TestFunction:
-    @pxt.udf(return_type=IntType(), param_types=[IntType()])
-    def func(x: int) -> int:
-        return x + 1
-
-    @pxt.uda(value_type=IntType(), update_types=[IntType()])
-    class agg:
-        def __init__(self):
-            self.sum = 0
-        def update(self, val: int) -> None:
-            if val is not None:
-                self.sum += val
-        def value(self) -> int:
-            return self.sum
-
-    def test_serialize_anonymous(self, init_env) -> None:
-        d = self.func.as_dict()
-        FunctionRegistry.get().clear_cache()
-        deserialized = Function.from_dict(d)
-        # TODO: add Function.exec() and then use that
-        assert deserialized.py_fn(1) == 2
-
-    @pytest.mark.skip(reason='deprecated')
-    def test_create(self, test_client: pxt.Client) -> None:
-        cl = test_client
-        cl.create_function('test_fn', self.func)
-        assert self.func.md.fqn == 'test_fn'
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        _ = cl.list_functions()
-        fn2 = cl.get_function('test_fn')
-        assert fn2.md.fqn == 'test_fn'
-        assert fn2.py_fn(1) == 2
-
-        with pytest.raises(excs.Error):
-            cl.create_function('test_fn', self.func)
-        with pytest.raises(excs.Error):
-            cl.create_function('dir1.test_fn', self.func)
-        with pytest.raises(excs.Error):
-            library_fn = make_library_function(IntType(), [IntType()], __name__, 'dummy_fn')
-            cl.create_function('library_fn', library_fn)
-
-    @pytest.mark.skip(reason='deprecated')
-    def test_update(self, test_client: pxt.Client, test_tbl: catalog.Table) -> None:
-        cl = test_client
-        t = test_tbl
-        cl.create_function('test_fn', self.func)
-        res1 = t[self.func(t.c2)].show(0).to_pandas()
-
-        # load function from db and make sure it computes the same thing as before
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        fn = cl.get_function('test_fn')
-        res2 = t[fn(t.c2)].show(0).to_pandas()
-        assert res1.col_0.equals(res2.col_0)
-        fn.py_fn = lambda x: x + 2
-        cl.update_function('test_fn', fn)
-        assert self.func.md.fqn == fn.md.fqn  # fqn doesn't change
-
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        fn = cl.get_function('test_fn')
-        assert self.func.md.fqn == fn.md.fqn  # fqn doesn't change
-        res3 = t[fn(t.c2)].show(0).to_pandas()
-        assert (res2.col_0 + 1).equals(res3.col_0)
-
-        # signature changes
-        with pytest.raises(excs.Error):
-            cl.update_function('test_fn', make_function(FloatType(), [IntType()], fn.py_fn))
-        with pytest.raises(excs.Error):
-            cl.update_function('test_fn', make_function(IntType(), [FloatType()], fn.py_fn))
-        with pytest.raises(excs.Error):
-            cl.update_function('test_fn', self.agg)
-
-    @pytest.mark.skip(reason='deprecated')
-    def test_move(self, test_client: pxt.Client) -> None:
-        cl = test_client
-        cl.create_function('test_fn', self.func)
-
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        with pytest.raises(excs.Error):
-            cl.move('test_fn2', 'test_fn')
-        cl.move('test_fn', 'test_fn2')
-        func = cl.get_function('test_fn2')
-        assert func.py_fn(1) == 2
-        assert func.md.fqn == 'test_fn2'
-
-        with pytest.raises(excs.Error):
-            _ = cl.get_function('test_fn')
-
-        # move function between directories
-        cl.create_dir('functions')
-        cl.create_dir('functions2')
-        cl.create_function('functions.func1', self.func)
-        with pytest.raises(excs.Error):
-            cl.move('functions2.func1', 'functions.func1')
-        cl.move('functions.func1', 'functions2.func1')
-        func = cl.get_function('functions2.func1')
-        assert func.md.fqn == 'functions2.func1'
-
-
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        func = cl.get_function('functions2.func1')
-        assert func.py_fn(1) == 2
-        assert func.md.fqn == 'functions2.func1'
-        with pytest.raises(excs.Error):
-            _ = cl.get_function('functions.func1')
-
-    @pytest.mark.skip(reason='deprecated')
-    def test_drop(self, test_client: pxt.Client) -> None:
-        cl = test_client
-        cl.create_function('test_fn', self.func)
-        FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        cl.drop_function('test_fn')
-
-        with pytest.raises(excs.Error):
-            _ = cl.get_function('test_fn')
-
-    def test_list(self, test_client: pxt.Client) -> None:
-        _ = FunctionRegistry.get().list_functions()
-        print(_)
-
-    def test_stored_udf(self, test_client: pxt.Client) -> None:
-        cl = test_client
-        t = cl.create_table('test', {'c1': pxt.IntType(), 'c2': pxt.FloatType()})
-        rows = [{'c1': i, 'c2': i + 0.5} for i in range(100)]
-        status = t.insert(rows)
-        assert status.num_rows == len(rows)
-        assert status.num_excs == 0
-
-        @pxt.udf(_force_stored=True)
-        def f1(a: int, b: float) -> float:
-            return a + b
-        t['f1'] = f1(t.c1, t.c2)
-
-        func.FunctionRegistry.get().clear_cache()
-        cl = pxt.Client(reload=True)
-        t = cl.get_table('test')
-        status = t.insert(rows)
-        assert status.num_rows == len(rows)
-        assert status.num_excs == 0
-
-    @pxt.udf(return_type=IntType(), param_types=[IntType(), FloatType(), FloatType(), FloatType()])
-    def f1(a: int, b: float, c: float = 0.0, d: float = 1.0) -> float:
-        return a + b + c + d
-
-    @pxt.udf(
-        return_type=IntType(),
-        param_types=[IntType(nullable=True), FloatType(nullable=False), FloatType(nullable=True)])
-    def f2(a: int, b: float = 0.0, c: float = 1.0) -> float:
-        return (0.0 if a is None else a) + b + (0.0 if c is None else c)
-
-    def test_call(self, test_tbl: catalog.Table) -> None:
-        t = test_tbl
-
-        r0 = t[t.c2, t.c3].show(0).to_pandas()
-        # positional params with default args
-        r1 = t[self.f1(t.c2, t.c3)].show(0).to_pandas()['col_0']
-        assert np.all(r1 == r0.c2 + r0.c3 + 1.0)
-        # kw args only
-        r2 = t[self.f1(c=0.0, b=t.c3, a=t.c2)].show(0).to_pandas()['col_0']
-        assert np.all(r1 == r2)
-        # overriding default args
-        r3 = t[self.f1(d=0.0, c=1.0, b=t.c3, a=t.c2)].show(0).to_pandas()['col_0']
-        assert np.all(r2 == r3)
-        # overriding default with positional arg
-        r4 = t[self.f1(t.c2, t.c3, 0.0)].show(0).to_pandas()['col_0']
-        assert np.all(r3 == r4)
-        # overriding default with positional arg and kw arg
-        r5 = t[self.f1(t.c2, t.c3, 1.0, d=0.0)].show(0).to_pandas()['col_0']
-        assert np.all(r4 == r5)
-        # d is kwarg
-        r6 = t[self.f1(t.c2, d=1.0, b=t.c3)].show(0).to_pandas()['col_0']
-        assert np.all(r5 == r6)
-        # d is Expr kwarg
-        r6 = t[self.f1(1, d=t.c3, b=t.c3)].show(0).to_pandas()['col_0']
-        assert np.all(r5 == r6)
-
-        # test handling of Nones
-        r0 = t[self.f2(1, t.c3)].show(0).to_pandas()['col_0']
-        r1 = t[self.f2(None, t.c3, 2.0)].show(0).to_pandas()['col_0']
-        assert np.all(r0 == r1)
-        r2 = t[self.f2(2, t.c3, None)].show(0).to_pandas()['col_0']
-        assert np.all(r1 == r2)
-        # kwarg with None
-        r3 = t[self.f2(c=None, a=t.c2)].show(0).to_pandas()['col_0']
-        # kwarg with Expr
-        r4 = t[self.f2(c=t.c3, a=None)].show(0).to_pandas()['col_0']
-        assert np.all(r3 == r4)
-
-        with pytest.raises(TypeError) as exc_info:
-            _ = t[self.f1(t.c2, c=0.0)].show(0)
-        assert "'b'" in str(exc_info.value)
-        with pytest.raises(TypeError) as exc_info:
-            _ = t[self.f1(t.c2)].show(0)
-        assert "'b'" in str(exc_info.value)
-        with pytest.raises(TypeError) as exc_info:
-            _ = t[self.f1(c=1.0, a=t.c2)].show(0)
-        assert "'b'" in str(exc_info.value)
-
-        # bad default value
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf(return_type=IntType(), param_types=[IntType(), FloatType(), FloatType()])
-            def f1(a: int, b: float, c: str = '') -> float:
-                return a + b + c
-        assert 'default value' in str(exc_info.value).lower()
-        # missing param type
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf(return_type=IntType(), param_types=[IntType(), FloatType()])
-            def f1(a: int, b: float, c: str = '') -> float:
-                return a + b + c
-        assert 'missing type for parameter c' in str(exc_info.value).lower()
-        # bad parameter name
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf(return_type=IntType(), param_types=[IntType()])
-            def f1(group_by: int) -> int:
-                return group_by
-        assert 'reserved' in str(exc_info.value)
-        # bad parameter name
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf(return_type=IntType(), param_types=[IntType()])
-            def f1(order_by: int) -> int:
-                return order_by
-        assert 'reserved' in str(exc_info.value)
-
-    @pxt.expr_udf
-    def add1(x: int) -> int:
-        return x + 1
-
-    @pxt.expr_udf
-    def add2(x: int, y: int):
-        return x + y
-
-    @pxt.expr_udf
-    def add2_with_default(x: int, y: int = 1) -> int:
-        return x + y
-
-    def test_expr_udf(self, test_tbl: catalog.Table) -> None:
-        t = test_tbl
-
-        res1 = t.select(out=self.add1(t.c2)).order_by(t.c2).collect()
-        res2 = t.select(t.c2 + 1).order_by(t.c2).collect()
-        assert_resultset_eq(res1, res2)
-
-        # return type inferred from expression
-        res1 = t.select(out=self.add2(t.c2, t.c2)).order_by(t.c2).collect()
-        res2 = t.select(t.c2 * 2).order_by(t.c2).collect()
-        assert_resultset_eq(res1, res2)
-
-        with pytest.raises(TypeError) as exc_info:
-            _ = t.select(self.add1(y=t.c2)).collect()
-        assert 'missing a required argument' in str(exc_info.value).lower()
-
-        with pytest.raises(excs.Error) as exc_info:
-            # parameter types cannot be inferred
-            @pxt.expr_udf
-            def add1(x, y) -> int:
-                return x + y
-        assert 'cannot infer pixeltable type' in str(exc_info.value).lower()
-
-        with pytest.raises(excs.Error) as exc_info:
-            # missing param types
-            @pxt.expr_udf(param_types=[IntType()])
-            def add1(x, y) -> int:
-                return x + y
-        assert 'missing type for parameter y' in str(exc_info.value).lower()
-
-        with pytest.raises(TypeError) as exc_info:
-            # signature has correct parameter kind
-            @pxt.expr_udf
-            def add1(*, x: int) -> int:
-                return x + y
-            _ = t.select(add1(t.c2)).collect()
-        assert 'takes 0 positional arguments' in str(exc_info.value).lower()
-
-        res1 = t.select(out=self.add2_with_default(t.c2)).order_by(t.c2).collect()
-        res2 = t.select(out=self.add2(t.c2, 1)).order_by(t.c2).collect()
-        assert_resultset_eq(res1, res2)
-
-    # Test that various invalid udf definitions generate
-    # correct error messages.
-    def test_invalid_udfs(self):
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf
-            def udf1(name: Batch[str]) -> str:
-                return ''
-        assert 'batched parameters in udf, but no `batch_size` given' in str(exc_info.value).lower()
-
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf(batch_size=32)
-            def udf2(name: Batch[str]) -> str:
-                return ''
-        assert 'batch_size is specified; Python return type must be a `Batch`' in str(exc_info.value)
-
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf
-            def udf3(name: str) -> Optional[np.ndarray]:
-                return None
-        assert 'cannot infer pixeltable return type' in str(exc_info.value).lower()
-
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf
-            def udf4(array: np.ndarray) -> str:
-                return ''
-        assert 'cannot infer pixeltable type for parameter array' in str(exc_info.value).lower()
-
-        with pytest.raises(excs.Error) as exc_info:
-            @pxt.udf
-            def udf5(name: str, untyped) -> str:
-                return ''
-        assert 'cannot infer pixeltable type for parameter untyped' in str(exc_info.value).lower()
-

pixeltable/tests/test_index.py

@@ -1,138 +0,0 @@
-import PIL.Image
-import numpy as np
-import pytest
-
-import pixeltable as pxt
-from pixeltable.functions.huggingface import clip_image, clip_text
-from pixeltable.tests.utils import text_embed, img_embed, skip_test_if_not_installed
-
-
-class TestIndex:
-
-    # wrong signature
-    @pxt.udf
-    def bad_embed(x: str) -> str:
-        return x
-
-    def test_embedding_basic(self, img_tbl: pxt.Table, test_tbl: pxt.Table) -> None:
-        skip_test_if_not_installed('transformers')
-        img_t = img_tbl
-        rows = list(img_t.select(img=img_t.img.fileurl, category=img_t.category, split=img_t.split).collect())
-        # create table with fewer rows to speed up testing
-        cl = pxt.Client()
-        schema = {
-            'img': pxt.ImageType(nullable=False),
-            'category': pxt.StringType(nullable=False),
-            'split': pxt.StringType(nullable=False),
-        }
-        tbl_name = 'index_test'
-        img_t = cl.create_table(tbl_name, schema=schema)
-        img_t.insert(rows[:30])
-
-        img_t.add_embedding_index('img', img_embed=img_embed, text_embed=text_embed)
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # duplicate name
-            img_t.add_embedding_index('img', idx_name='idx0', img_embed=img_embed)
-        assert 'duplicate index name' in str(exc_info.value).lower()
-
-        img_t.add_embedding_index('category', text_embed=text_embed)
-        # revert() removes the index
-        img_t.revert()
-        with pytest.raises(pxt.Error) as exc_info:
-            img_t.drop_index(column_name='category')
-        assert 'does not have an index' in str(exc_info.value).lower()
-
-        rows = list(img_t.collect())
-        status = img_t.update({'split': 'other'}, where=img_t.split == 'test')
-        assert status.num_excs == 0
-
-        status = img_t.delete()
-        assert status.num_excs == 0
-
-        # revert delete()
-        img_t.revert()
-        # revert update()
-        img_t.revert()
-
-        # make sure we can still do DML after reloading the metadata
-        cl = pxt.Client(reload=True)
-        img_t = cl.get_table(tbl_name)
-        status = img_t.insert(rows)
-        assert status.num_excs == 0
-
-        status = img_t.update({'split': 'other'}, where=img_t.split == 'test')
-        assert status.num_excs == 0
-
-        status = img_t.delete()
-        assert status.num_excs == 0
-
-        # revert delete()
-        img_t.revert()
-        # revert update()
-        img_t.revert()
-
-        img_t.drop_index(idx_name='idx0')
-        with pytest.raises(pxt.Error) as exc_info:
-            img_t.drop_index(column_name='img')
-        assert 'does not have an index' in str(exc_info.value).lower()
-
-        # revert() makes the index reappear
-        img_t.revert()
-        with pytest.raises(pxt.Error) as exc_info:
-            img_t.add_embedding_index('img', idx_name='idx0', img_embed=img_embed)
-        assert 'duplicate index name' in str(exc_info.value).lower()
-
-        # dropping the indexed column also drops indices
-        img_t.drop_column('img')
-        with pytest.raises(pxt.Error) as exc_info:
-            img_t.drop_index(idx_name='idx0')
-        assert 'does not exist' in str(exc_info.value).lower()
-
-    def test_errors(self, img_tbl: pxt.Table, test_tbl: pxt.Table) -> None:
-        img_t = img_tbl
-        rows = list(img_t.select(img=img_t.img.fileurl, category=img_t.category, split=img_t.split).collect())
-        # create table with fewer rows to speed up testing
-        cl = pxt.Client()
-        schema = {
-            'img': pxt.ImageType(nullable=False),
-            'category': pxt.StringType(nullable=False),
-            'split': pxt.StringType(nullable=False),
-        }
-        tbl_name = 'index_test'
-        img_t = cl.create_table(tbl_name, schema=schema)
-        img_t.insert(rows[:30])
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # unknown column
-            img_t.add_embedding_index('does_not_exist', idx_name='idx0', img_embed=img_embed)
-        assert 'column does_not_exist unknown' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # wrong column type
-            test_tbl.add_embedding_index('c2', img_embed=img_embed)
-        assert 'requires string or image column' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # missing embedding function
-            img_tbl.add_embedding_index('img', text_embed=text_embed)
-        assert 'image embedding function is required' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # wrong signature
-            img_tbl.add_embedding_index('img', img_embed=clip_image)
-        assert 'but has signature' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # missing embedding function
-            img_tbl.add_embedding_index('category', img_embed=img_embed)
-        assert 'text embedding function is required' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            # wrong signature
-            img_tbl.add_embedding_index('category', text_embed=clip_text)
-        assert 'but has signature' in str(exc_info.value).lower()
-
-        with pytest.raises(pxt.Error) as exc_info:
-            img_tbl.add_embedding_index('category', text_embed=self.bad_embed)
-        assert 'must return an array' in str(exc_info.value).lower()

pixeltable/tests/test_migration.py

@@ -1,44 +0,0 @@
-import glob
-import logging
-import os
-import platform
-import subprocess
-
-import pgserver
-import pytest
-
-import pixeltable as pxt
-from pixeltable.env import Env
-from pixeltable.tests.conftest import clean_db
-
-_logger = logging.getLogger('pixeltable')
-
-
-class TestMigration:
-
-    @pytest.mark.skipif(platform.system() == 'Windows', reason='Does not run on Windows')
-    def test_db_migration(self, init_env) -> None:
-        env = Env.get()
-        pg_package_dir = os.path.dirname(pgserver.__file__)
-        pg_restore_binary = f'{pg_package_dir}/pginstall/bin/pg_restore'
-        _logger.info(f'Using pg_restore binary at: {pg_restore_binary}')
-        dump_files = glob.glob('pixeltable/tests/data/dbdumps/*.dump.gz')
-        dump_files.sort()
-        for dump_file in dump_files:
-            _logger.info(f'Testing migration from DB dump {dump_file}.')
-            _logger.info(f'DB URL: {env.db_url}')
-            clean_db(restore_tables=False)
-            with open(dump_file, 'rb') as dump:
-                gunzip_process = subprocess.Popen(
-                    ["gunzip", "-c"],
-                    stdin=dump,
-                    stdout=subprocess.PIPE
-                )
-                subprocess.run(
-                    [pg_restore_binary, '-d', env.db_url, '-U', 'postgres'],
-                    stdin=gunzip_process.stdout,
-                    check=True
-                )
-            # TODO(aaron-siegel) This will test that the migration succeeds without raising any exceptions.
-            # We should also add some assertions to sanity-check the outcome.
-            _ = pxt.Client()

pixeltable/tests/test_nos.py

@@ -1,54 +0,0 @@
-import pytest
-
-import pixeltable as pxt
-from pixeltable.iterators import FrameIterator
-from pixeltable.tests.utils import get_video_files, skip_test_if_not_installed
-from pixeltable.type_system import ImageType, VideoType
-
-
-class TestNOS:
-    def test_basic(self, test_client: pxt.Client) -> None:
-        skip_test_if_not_installed('nos')
-        cl = test_client
-        video_t = cl.create_table('video_tbl', {'video': VideoType()})
-        # create frame view
-        args = {'video': video_t.video, 'fps': 1}
-        v = cl.create_view('test_view', video_t, iterator_class=FrameIterator, iterator_args=args)
-        v.add_column(transform1=v.frame.rotate(30), stored=False)
-        from pixeltable.functions.nos.object_detection_2d import \
-            torchvision_fasterrcnn_mobilenet_v3_large_320_fpn as fasterrcnn
-        v.add_column(detections=fasterrcnn(v.transform1))
-        from pixeltable.functions.nos.image_embedding import openai_clip
-        v.add_column(embed=openai_clip(v.transform1.resize([224, 224])))
-        # add a stored column that isn't referenced in nos calls
-        v.add_column(transform2=v.frame.rotate(60), stored=True)
-
-        status = video_t.insert(video=get_video_files()[0])
-        pass
-
-    def test_exceptions(self, test_client: pxt.Client) -> None:
-        skip_test_if_not_installed('nos')
-        cl = test_client
-        video_t = cl.create_table('video_tbl', {'video': VideoType()})
-        # create frame view
-        args = {'video': video_t.video, 'fps': 1}
-        v = cl.create_view('test_view', video_t, iterator_class=FrameIterator, iterator_args=args)
-        video_t.insert(video=get_video_files()[0])
-
-        v.add_column(frame_s=v.frame.resize([640, 480]))
-        # 'rotated' has exceptions
-        v.add_column(rotated=lambda frame_s, frame_idx: frame_s.rotate(int(360 / frame_idx)), type=ImageType())
-        from pixeltable.functions.nos.object_detection_2d import yolox_medium
-        v.add_column(detections=yolox_medium(v.rotated), stored=True)
-        assert v.where(v.detections.errortype != None).count() == 1
-
-    @pytest.mark.skip(reason='too slow')
-    def test_sd(self, test_client: pxt.Client) -> None:
-        skip_test_if_not_installed('nos')
-        """Test model that mixes batched with scalar parameters"""
-        t = test_client.create_table('sd_test', {'prompt': pxt.StringType()})
-        t.insert(prompt='cat on a sofa')
-        from pixeltable.functions.nos.image_generation import stabilityai_stable_diffusion_2 as sd2
-        t.add_column(img=sd2(t.prompt, 1, 512, 512), stored=True)
-        img = t[t.img].show(1)[0, 0]
-        assert img.size == (512, 512)