pixeltable 0.2.1__tar.gz → 0.2.3__tar.gz

{pixeltable-0.2.1 → pixeltable-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.1
+Version: 0.2.3
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -11,19 +11,21 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: av (>=10.0.0)
+Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: numpy (>=1.24.1,<2.0.0)
+Requires-Dist: numpy (>=1.26,<2.0)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
-Requires-Dist: pandas (>=1.5.3,<2.0.0)
-Requires-Dist: pgserver (==0.0.5)
+Requires-Dist: pandas (>=2.0,<3.0)
+Requires-Dist: pgserver (==0.0.9)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
 Requires-Dist: pillow (>=9.4.0,<10.0.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: regex (>=2022.10.31,<2023.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
 Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
 Requires-Dist: tqdm (>=4.64.1,<5.0.0)
@@ -45,16 +47,16 @@ data plumbing.
 It brings together data storage, versioning, and indexing with orchestration and model
 versioning under a declarative table interface, with transformations, model inference,
 and custom logic represented as computed columns.
-<!--
+
 ## Quick Start
 
 If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
 the Pixeltable Basics tutorial in colab:
 
-<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/docs/tutorials/pixeltable-basics.ipynb">
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb">
   <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
 </a>
--->
+
 ## Installation
 
 Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.

{pixeltable-0.2.1 → pixeltable-0.2.3}/README.md

@@ -14,16 +14,16 @@ data plumbing.
 It brings together data storage, versioning, and indexing with orchestration and model
 versioning under a declarative table interface, with transformations, model inference,
 and custom logic represented as computed columns.
-<!--
+
 ## Quick Start
 
 If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
 the Pixeltable Basics tutorial in colab:
 
-<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/docs/tutorials/pixeltable-basics.ipynb">
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb">
   <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
 </a>
--->
+
 ## Installation
 
 Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/dataframe.py

@@ -39,7 +39,7 @@ def _format_img(img: object) -> str:
     with io.BytesIO() as buffer:
         img.save(buffer, 'jpeg')
         img_base64 = base64.b64encode(buffer.getvalue()).decode()
-        return f'<div style=\'width:200px;\'><img src="data:image/jpeg;base64,{img_base64}"></div>'
+        return f'<div style="width:200px;"><img src="data:image/jpeg;base64,{img_base64}" width="200" /></div>'
 
 def _create_source_tag(file_path: str) -> str:
     abs_path = Path(file_path)

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/env.py

@@ -92,7 +92,7 @@ class Env:
     def db_url(self) -> str:
         assert self._db_url is not None
         return self._db_url
-    
+
     @property
     def http_address(self) -> str:
         assert self._http_address is not None
@@ -142,7 +142,7 @@ class Env:
     def set_up(self, echo: bool = False, reinit_db: bool = False) -> None:
         if self._initialized:
             return
-        
+
         self._initialized = True
         home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
         assert self._home is None or self._home == home
@@ -153,7 +153,6 @@ class Env:
         self._dataset_cache_dir = self._home / 'dataset_cache'
         self._log_dir = self._home / 'logs'
         self._tmp_dir = self._home / 'tmp'
-        self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(self._home / 'pgdata')))
 
         # Read in the config
         if os.path.isfile(self._config_file):
@@ -204,8 +203,9 @@ class Env:
             os.remove(path)
 
         self._db_name = os.environ.get('PIXELTABLE_DB', 'pixeltable')
+        self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(self._home / 'pgdata')))
 
-        # cleanup_mode=None will leave db on for debugging purposes
+        # in pgserver.get_server(): cleanup_mode=None will leave db on for debugging purposes
         self._db_server = pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
         self._db_url = self._db_server.get_uri(database=self._db_name)
 
@@ -257,16 +257,17 @@ class Env:
         _ = create_nos_modules()
 
     def _create_openai_client(self) -> None:
+        if not self.is_installed_package('openai'):
+            raise excs.Error('OpenAI client not initialized (cannot find package `openai`: `pip install openai`?)')
+        import openai
         if 'openai' in self._config and 'api_key' in self._config['openai']:
             api_key = self._config['openai']['api_key']
         else:
             api_key = os.environ.get('OPENAI_API_KEY')
         if api_key is None or api_key == '':
-            self._logger.info("OpenAI client not initialized (no API key configured).")
-            return
-        import openai
-        self._logger.info('Initializing OpenAI client.')
+            raise excs.Error('OpenAI client not initialized (no API key configured).')
         self._openai_client = openai.OpenAI(api_key=api_key)
+        self._logger.info('Initialized OpenAI client.')
 
     def _create_together_client(self) -> None:
         if 'together' in self._config and 'api_key' in self._config['together']:
@@ -285,15 +286,15 @@ class Env:
         """
         The http server root is the file system root.
         eg: /home/media/foo.mp4 is located at http://127.0.0.1:{port}/home/media/foo.mp4
-        This arrangement enables serving media hosted within _home, 
+        This arrangement enables serving media hosted within _home,
         as well as external media inserted into pixeltable or produced by pixeltable.
         The port is chosen dynamically to prevent conflicts.
-        """        
+        """
         # Port 0 means OS picks one for us.
         address = ("127.0.0.1", 0)
         class FixedRootHandler(http.server.SimpleHTTPRequestHandler):
             def __init__(self, *args, **kwargs):
-                super().__init__(*args, directory='/', **kwargs)        
+                super().__init__(*args, directory='/', **kwargs)
         self._httpd = socketserver.TCPServer(address, FixedRootHandler)
         port = self._httpd.server_address[1]
         self._http_address = f'http://127.0.0.1:{port}'
@@ -330,8 +331,6 @@ class Env:
             self._spacy_nlp = spacy.load('en_core_web_sm')
         check('tiktoken')
         check('openai')
-        if self.is_installed_package('openai'):
-            self._create_openai_client()
         check('together')
         if self.is_installed_package('together'):
             self._create_together_client()
@@ -401,7 +400,10 @@ class Env:
         return self._nos_client
 
     @property
-    def openai_client(self) -> Optional['openai.OpenAI']:
+    def openai_client(self) -> 'openai.OpenAI':
+        if self._openai_client is None:
+            self._create_openai_client()
+        assert self._openai_client is not None
         return self._openai_client
 
     @property

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/exec/cache_prefetch_node.py

@@ -1,19 +1,21 @@
 from __future__ import annotations
-from typing import List, Optional, Any, Tuple, Dict
+
+import concurrent.futures
+import logging
 import threading
+import urllib.parse
+import urllib.request
 from collections import defaultdict
-from uuid import UUID
-import concurrent
-import logging
-import urllib
 from pathlib import Path
+from typing import List, Optional, Any, Tuple, Dict
+from uuid import UUID
 
-from .data_row_batch import DataRowBatch
-from .exec_node import ExecNode
-import pixeltable.exprs as exprs
-from pixeltable.utils.filecache import FileCache
 import pixeltable.env as env
 import pixeltable.exceptions as excs
+import pixeltable.exprs as exprs
+from pixeltable.utils.filecache import FileCache
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
 
 _logger = logging.getLogger('pixeltable')
 
@@ -81,7 +83,9 @@ class CachePrefetchNode(ExecNode):
         """Fetches a remote URL into Env.tmp_dir and returns its path"""
         url = row.file_urls[slot_idx]
         parsed = urllib.parse.urlparse(url)
-        assert parsed.scheme != '' and parsed.scheme != 'file'
+        # Use len(parsed.scheme) > 1 here to ensure we're not being passed
+        # a Windows filename
+        assert len(parsed.scheme) > 1 and parsed.scheme != 'file'
         # preserve the file extension, if there is one
         extension = ''
         if parsed.path != '':
@@ -95,7 +99,6 @@ class CachePrefetchNode(ExecNode):
                     if self.boto_client is None:
                         self.boto_client = get_client()
                     self.boto_client.download_file(parsed.netloc, parsed.path.lstrip('/'), str(tmp_path))
-                    return tmp_path
             elif parsed.scheme == 'http' or parsed.scheme == 'https':
                 with urllib.request.urlopen(url) as resp, open(tmp_path, 'wb') as f:
                     data = resp.read()

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/exprs/data_row.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
-from typing import Optional, List, Any, Tuple
+
 import io
-import urllib
+import urllib.parse
+import urllib.request
+from typing import Optional, List, Any, Tuple
 
 import PIL
 import numpy as np
@@ -104,6 +106,7 @@ class DataRow:
             assert self.file_paths[index] is not None
             if self.vals[index] is None:
                 self.vals[index] = PIL.Image.open(self.file_paths[index])
+                self.vals[index].load()
 
         return self.vals[index]
 
@@ -137,14 +140,19 @@ class DataRow:
         if (idx in self.img_slot_idxs or idx in self.media_slot_idxs) and isinstance(val, str):
             # this is either a local file path or a URL
             parsed = urllib.parse.urlparse(val)
-            if parsed.scheme == '' or parsed.scheme == 'file':
+            # Determine if this is a local file or a remote URL. If the scheme length is <= 1,
+            # we assume it's a local file. (This is because a Windows path will be interpreted
+            # by urllib as a URL with scheme equal to the drive letter.)
+            if len(parsed.scheme) <= 1 or parsed.scheme == 'file':
                 # local file path
                 assert self.file_urls[idx] is None and self.file_paths[idx] is None
-                if parsed.scheme == '':
-                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(parsed.path))
+                if len(parsed.scheme) <= 1:
+                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(val))
+                    self.file_paths[idx] = val
                 else:
                     self.file_urls[idx] = val
-                self.file_paths[idx] = urllib.parse.unquote(parsed.path)
+                    # Wrap the path in a url2pathname() call to ensure proper handling on Windows.
+                    self.file_paths[idx] = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))
             else:
                 # URL
                 assert self.file_urls[idx] is None

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/iterators/document.py

@@ -61,7 +61,7 @@ class DocumentSplitter(ComponentIterator):
         import bs4
         if html_skip_tags is None:
             html_skip_tags = ['nav']
-        with open(document, 'r') as fh:
+        with open(document, 'r', encoding='utf8') as fh:
             s = fh.read()
             self._doc_handle = get_document_handle(s)
             assert self._doc_handle is not None

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/store.py

@@ -1,24 +1,26 @@
 from __future__ import annotations
 
+import abc
+import logging
 import os
 import sys
+import urllib.parse
+import urllib.request
 import warnings
 from typing import Optional, Dict, Any, List, Tuple, Set
-import logging
-import urllib
+
 import sqlalchemy as sql
 from tqdm import tqdm, TqdmWarning
-import abc
 
 import pixeltable.catalog as catalog
+import pixeltable.env as env
+from pixeltable import exprs
+import pixeltable.exceptions as excs
+from pixeltable.exec import ExecNode
 from pixeltable.metadata import schema
 from pixeltable.type_system import StringType
-from pixeltable.exec import ExecNode
-from pixeltable import exprs
-from pixeltable.utils.sql import log_stmt, log_explain
-import pixeltable.env as env
 from pixeltable.utils.media_store import MediaStore
-
+from pixeltable.utils.sql import log_stmt, log_explain
 
 _logger = logging.getLogger('pixeltable')
 
@@ -121,10 +123,13 @@ class StoreBase:
         if file_url is None:
             return None
         parsed = urllib.parse.urlparse(file_url)
-        if parsed.scheme != '' and parsed.scheme != 'file':
+        # We should never be passed a local file path here. The "len > 1" ensures that Windows
+        # file paths aren't mistaken for URLs with a single-character scheme.
+        assert len(parsed.scheme) > 1
+        if parsed.scheme != 'file':
             # remote url
             return file_url
-        file_path = urllib.parse.unquote(parsed.path)
+        file_path = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))
         if not file_path.startswith(pxt_tmp_dir):
             # not a tmp file
             return file_url

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/conftest.py

@@ -23,7 +23,7 @@ def init_env(tmp_path_factory) -> None:
     from pixeltable.env import Env
     # set the relevant env vars for Client() to connect to the test db
 
-    shared_home = pathlib.Path(os.environ.get('PIXELTABLE_HOME', '~/.pixeltable')).expanduser()
+    shared_home = pathlib.Path(os.environ.get('PIXELTABLE_HOME', str(pathlib.Path.home() / '.pixeltable')))
     home_dir = str(tmp_path_factory.mktemp('base') / '.pixeltable')
     os.environ['PIXELTABLE_HOME'] = home_dir
     os.environ['PIXELTABLE_CONFIG'] = str(shared_home / 'config.yaml')
@@ -31,6 +31,8 @@ def init_env(tmp_path_factory) -> None:
     os.environ['PIXELTABLE_DB'] = test_db
     os.environ['PIXELTABLE_PGDATA'] = str(shared_home / 'pgdata')
 
+    # ensure this home dir exits
+    shared_home.mkdir(parents=True, exist_ok=True)
     # this also runs create_all()
     Env.get().set_up(echo=True)
     yield

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_component_view.py

@@ -9,10 +9,9 @@ import pixeltable as pxt
 from pixeltable import exceptions as excs
 from pixeltable.iterators import ComponentIterator
 from pixeltable.iterators.video import FrameIterator
-from pixeltable.tests.utils import assert_resultset_eq, get_video_files
+from pixeltable.tests.utils import assert_resultset_eq, get_test_video_files
 from pixeltable.type_system import IntType, VideoType, JsonType
 
-
 class ConstantImgIterator(ComponentIterator):
     """Component iterator that generates a fixed number of all-black 1280x720 images."""
     def __init__(self, video: str, num_frames: int = 10):
@@ -59,14 +58,13 @@ class ConstantImgIterator(ComponentIterator):
             return
         self.next_frame_idx = pos
 
-
 class TestComponentView:
     def test_basic(self, test_client: pxt.Client) -> None:
         cl = test_client
         # create video table
         schema = {'video': VideoType(), 'angle': IntType(), 'other_angle': IntType()}
         video_t = cl.create_table('video_tbl', schema)
-        video_filepaths = get_video_files()
+        video_filepaths = get_test_video_files()
 
         # cannot add 'pos' column
         with pytest.raises(excs.Error) as excinfo:
@@ -124,7 +122,7 @@ class TestComponentView:
         cl = test_client
         # create video table
         video_t = cl.create_table('video_tbl', {'video': VideoType()})
-        video_filepaths = get_video_files()
+        video_filepaths = get_test_video_files()
         # create frame view
         args = {'video': video_t.video, 'fps': 1}
         view_t = cl.create_view('test_view', video_t, iterator_class=FrameIterator, iterator_args=args)
@@ -153,7 +151,7 @@ class TestComponentView:
             'test_view', video_t, schema={'annotation': JsonType(nullable=True)},
             iterator_class=FrameIterator, iterator_args=args)
 
-        video_filepaths = get_video_files()
+        video_filepaths = get_test_video_files()
         rows = [{'video': p} for p in video_filepaths]
         status = video_t.insert(rows)
         assert status.num_excs == 0
@@ -206,7 +204,7 @@ class TestComponentView:
 
         # create video table
         video_t = cl.create_table(base_path, {'video': VideoType(), 'margin': IntType()})
-        video_filepaths = get_video_files()
+        video_filepaths = get_test_video_files()
         rows = [{'video': path, 'margin': i * 10} for i, path in enumerate(video_filepaths)]
         status = video_t.insert(rows)
         assert status.num_rows == len(rows)
@@ -277,7 +275,7 @@ class TestComponentView:
         # create video table
         schema = {'video': VideoType(), 'int1': IntType(), 'int2': IntType()}
         video_t = cl.create_table('video_tbl', schema)
-        video_filepaths = get_video_files()
+        video_filepaths = get_test_video_files()
 
         # create first view
         args = {'video': video_t.video}

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_dataframe.py

@@ -7,7 +7,6 @@ import bs4
 import numpy as np
 import pytest
 import requests
-from pycocotools.coco import COCO
 
 import pixeltable as pxt
 from pixeltable import catalog
@@ -184,6 +183,8 @@ class TestDataFrame:
         res = t.select(1.0).where(t.c2 < 10).collect()
         assert res[res.column_names()[0]] == [1.0] * 10
 
+    # TODO This test doesn't work on Windows due to reliance on the structure of file URLs
+    @pytest.mark.skip('Test is not portable')
     def test_html_media_url(self, test_client: pxt.Client) -> None:
         tab = test_client.create_table('test_html_repr', {'video': pxt.VideoType(), 'audio': pxt.AudioType()})
         status = tab.insert(video=get_video_files()[0], audio=get_audio_files()[0])
@@ -208,6 +209,7 @@ class TestDataFrame:
     def test_to_pytorch_dataset(self, all_datatypes_tbl: catalog.Table):
         """ tests all types are handled correctly in this conversion
         """
+        skip_test_if_not_installed('torch')
         import torch
 
         t = all_datatypes_tbl
@@ -238,6 +240,7 @@ class TestDataFrame:
     def test_to_pytorch_image_format(self, all_datatypes_tbl: catalog.Table) -> None:
         """ tests the image_format parameter is honored
         """
+        skip_test_if_not_installed('torch')
         import torch
         import torchvision.transforms as T
 
@@ -295,6 +298,7 @@ class TestDataFrame:
             1. compatibility with multiprocessing
             2. compatibility of all types with default collate_fn
         """
+        skip_test_if_not_installed('torch')
         import torch.utils.data
         @pxt.udf(param_types=[pxt.JsonType()], return_type=pxt.JsonType())
         def restrict_json_for_default_collate(obj):
@@ -352,6 +356,7 @@ class TestDataFrame:
             2. adding a row to the table invalidates the cached version
             3. changing the select list invalidates the cached version
         """
+        skip_test_if_not_installed('torch')
         t = all_datatypes_tbl
 
         t.drop_column('c_video') # null value video column triggers internal assertions in DataRow
@@ -383,6 +388,7 @@ class TestDataFrame:
 
     def test_to_coco(self, test_client: pxt.Client) -> None:
         skip_test_if_not_installed('nos')
+        from pycocotools.coco import COCO
         cl = test_client
         base_t = cl.create_table('videos', {'video': pxt.VideoType()})
         args = {'video': base_t.video, 'fps': 1}

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_document.py

@@ -8,6 +8,7 @@ import pytest
 import pixeltable as pxt
 from pixeltable.iterators.document import DocumentSplitter
 from pixeltable.tests.utils import get_documents, get_video_files, get_audio_files, get_image_files
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import DocumentType
 
 
@@ -34,6 +35,7 @@ class TestDocument:
         assert status.num_excs == len(file_paths)
 
     def test_doc_splitter(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('tiktoken')
         file_paths = self.valid_doc_paths()
         cl = test_client
         doc_t = cl.create_table('docs', {'doc': DocumentType()})
@@ -88,6 +90,7 @@ class TestDocument:
                     cl.drop_table('chunks')
 
     def test_doc_splitter_headings(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('spacy')
         file_paths = self.valid_doc_paths()
         cl = test_client
         doc_t = cl.create_table('docs', {'doc': DocumentType()})

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_exprs.py

@@ -1,5 +1,6 @@
 import json
 import urllib.parse
+import urllib.request
 from typing import List, Dict
 
 import pytest
@@ -174,7 +175,7 @@ class TestExprs:
         res = img_t.select(img_t.img.fileurl).show(0).to_pandas()
         stored_urls = set(res.iloc[:, 0])
         assert len(stored_urls) == len(res)
-        all_urls  = set([urllib.parse.urljoin('file:', path) for path in get_image_files()])
+        all_urls = set(urllib.parse.urljoin('file:', urllib.request.pathname2url(path)) for path in get_image_files())
         assert stored_urls <= all_urls
 
         # localpath

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_functions.py

@@ -5,6 +5,7 @@ import pytest
 import pixeltable as pxt
 from pixeltable import catalog
 from pixeltable.env import Env
+import pixeltable.exceptions as excs
 from pixeltable.functions.pil.image import blend
 from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import get_video_files, skip_test_if_not_installed, get_sentences, get_image_files
@@ -67,8 +68,6 @@ class TestFunctions:
     def test_openai(self, test_client: pxt.Client) -> None:
         skip_test_if_not_installed('openai')
         TestFunctions.skip_test_if_no_openai_client()
-        if Env.get().openai_client is None:
-            pytest.skip(f'OpenAI client does not exist (missing API key?).')
         cl = test_client
         t = cl.create_table('test_tbl', {'input': StringType()})
         from pixeltable.functions.openai import chat_completions, embeddings, moderations
@@ -110,8 +109,10 @@ class TestFunctions:
 
     @staticmethod
     def skip_test_if_no_openai_client() -> None:
-        if Env.get().openai_client is None:
-            pytest.skip(f'OpenAI client does not exist (missing API key?)')
+        try:
+            _ = Env.get().openai_client
+        except excs.Error as exc:
+            pytest.skip(str(exc))
 
     def test_together(self, test_client: pxt.Client) -> None:
         skip_test_if_not_installed('together')

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_table.py

@@ -18,6 +18,7 @@ from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import \
     make_tbl, create_table_data, read_data_file, get_video_files, get_audio_files, get_image_files, get_documents, \
     assert_resultset_eq
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import \
     StringType, IntType, FloatType, TimestampType, ImageType, VideoType, JsonType, BoolType, ArrayType, AudioType, \
     DocumentType
@@ -296,6 +297,7 @@ class TestTable:
         self.check_bad_media(test_client, rows, DocumentType(nullable=True))
 
     def test_validate_external_url(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         rows = [
             {'media': 's3://open-images-dataset/validation/doesnotexist.jpg', 'is_bad_media': True},
             {'media': 'https://archive.random.org/download?file=2024-01-28.bin', 'is_bad_media': True},  # 403 error
@@ -315,6 +317,7 @@ class TestTable:
         self.check_bad_media(test_client, rows, VideoType(nullable=True))
 
     def test_create_s3_image_table(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         tbl = cl.create_table('test', {'img': ImageType(nullable=False)})
         # this is needed because Client.reset_catalog() doesn't call TableVersion.drop(), which would
@@ -371,6 +374,7 @@ class TestTable:
         assert cache_stats.total_size == 0
 
     def test_video_url(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         schema = {
             'payload': IntType(nullable=False),
@@ -390,6 +394,7 @@ class TestTable:
         cap.release()
 
     def test_create_video_table(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         tbl = cl.create_table(
             'test_tbl',
@@ -529,6 +534,7 @@ class TestTable:
         assert 'expected ndarray((2, 3)' in str(exc_info.value)
 
     def test_query(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         col_names = ['c1', 'c2', 'c3', 'c4', 'c5']
         t = make_tbl(cl, 'test', col_names)

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/test_video.py

@@ -8,6 +8,7 @@ from pixeltable import catalog
 from pixeltable import exceptions as excs
 from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import get_video_files
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import VideoType, ImageType
 from pixeltable.utils.media_store import MediaStore
 
@@ -61,6 +62,7 @@ class TestVideo:
         assert MediaStore.count(view.get_id()) == view.count()
 
     def test_query(self, test_client: pxt.client) -> None:
+        skip_test_if_not_installed('boto3')
         video_filepaths = get_video_files()
         cl = test_client
         base_t, view_t = self.create_tbls(cl)

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/tests/utils.py

@@ -18,6 +18,8 @@ from pixeltable.type_system import \
     ColumnType, StringType, IntType, FloatType, ArrayType, BoolType, TimestampType, JsonType, ImageType, VideoType
 
 
+
+
 def make_default_type(t: ColumnType.Type) -> ColumnType:
     if t == ColumnType.Type.STRING:
         return StringType()
@@ -223,19 +225,28 @@ def read_data_file(dir_name: str, file_name: str, path_col_names: Optional[List[
         df[col_name] = df.apply(lambda r: str(abs_path / r[col_name]), axis=1)
     return df.to_dict(orient='records')
 
-def get_video_files(include_bad_video=False) -> List[str]:
+def get_video_files(include_bad_video: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/videos/*', recursive=True)
     if not include_bad_video:
         glob_result = [f for f in glob_result if 'bad_video' not in f]
+
+    half_res = [f for f in glob_result if 'half_res' in f or 'bad_video' in f]
+    return half_res
+
+def get_test_video_files() -> List[str]:
+    tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
+    glob_result = glob.glob(f'{tests_dir}/**/test_videos/*', recursive=True)
     return glob_result
 
-def get_image_files() -> List[str]:
+def get_image_files(include_bad_image: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/imagenette2-160/*', recursive=True)
+    if not include_bad_image:
+        glob_result = [f for f in glob_result if 'bad_image' not in f]
     return glob_result
 
-def get_audio_files(include_bad_audio=False) -> List[str]:
+def get_audio_files(include_bad_audio: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     glob_result = glob.glob(f'{tests_dir}/**/audio/*', recursive=True)
     if not include_bad_audio:
@@ -250,7 +261,7 @@ def get_documents() -> List[str]:
 def get_sentences(n: int = 100) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     path = glob.glob(f'{tests_dir}/**/jeopardy.json', recursive=True)[0]
-    with open(path, 'r') as f:
+    with open(path, 'r', encoding='utf8') as f:
         questions_list = json.load(f)
     # this dataset contains \' around the questions
     return [q['question'].replace("'", '') for q in questions_list[:n]]

pixeltable-0.2.3/pixeltable/tool/create_test_video.py

@@ -0,0 +1,81 @@
+import av
+import PIL.Image
+import PIL.ImageDraw
+import PIL.ImageFont
+
+from pathlib import Path
+from typing import Optional
+import tempfile
+import math
+
+def create_test_video(
+    frame_count: int,
+    frame_rate: float = 1.0,
+    frame_width: int = 224,
+    aspect_ratio: str = '16:9',
+    frame_height: Optional[int] = None,
+    output_path: Optional[Path] = None,
+    font_file: str = '/Library/Fonts/Arial Unicode.ttf',
+) -> Path:
+    """
+        Creates an .mp4 video file such as the ones in /tests/data/test_videos
+        The video contains a frame number in each frame (for visual sanity check).
+
+        Args:
+            frame_count: Number of frames to create
+            frame_rate: Frame rate of the video
+            frame_width (int): Width in pixels of the video frame. Note: cost of decoding increases dramatically
+                with frame width * frame height.
+            aspect_ratio: Aspect ratio (width/height) of the video frames string of form 'width:height'
+            frame_height: Height of the video frame, if given, aspect_ratio is ignored
+            output_path: Path to save the video file
+            font_file: Path to the font file used for text.
+    """
+
+    if output_path is None:
+        output_path = Path(tempfile.NamedTemporaryFile(suffix='.mp4', delete=False).name)
+
+    parts = [int(p) for p in aspect_ratio.split(':')]
+    assert len(parts) == 2
+    aspect_ratio = parts[0] / parts[1]
+
+    if frame_height is None:
+        frame_height = math.ceil(frame_width / aspect_ratio)
+
+    frame_size = (frame_width, frame_height)
+
+    font_size = min(frame_height, frame_width) // 4
+    font = PIL.ImageFont.truetype(font=font_file, size=font_size)
+    font_fill = 0xFFFFFF  # white
+    frame_color = 0xFFFFFF - font_fill  # black
+    # Create a video container
+    container = av.open(str(output_path), mode='w')
+
+    # Add a video stream
+    stream = container.add_stream('h264', rate=frame_rate)
+    stream.width, stream.height = frame_size
+    stream.pix_fmt = 'yuv420p'
+
+    for frame_number in range(frame_count):
+        # Create an image with a number in it
+        image = PIL.Image.new('RGB', frame_size, color=frame_color)
+        draw = PIL.ImageDraw.Draw(image)
+        # Optionally, add a font here if you have one
+        text = str(frame_number)
+        _, _, text_width, text_height = draw.textbbox((0, 0), text, font=font)
+        text_position = ((frame_size[0] - text_width) // 2, (frame_size[1] - text_height) // 2)
+        draw.text(text_position, text, font=font, fill=font_fill)
+
+        # Convert the PIL image to an AVFrame
+        frame = av.VideoFrame.from_image(image)
+
+        # Encode and write the frame
+        for packet in stream.encode(frame):
+            container.mux(packet)
+
+    # Flush and close the stream
+    for packet in stream.encode():
+        container.mux(packet)
+
+    container.close()
+    return output_path

{pixeltable-0.2.1 → pixeltable-0.2.3}/pixeltable/type_system.py

@@ -911,7 +911,7 @@ class DocumentType(ColumnType):
     def validate_media(self, val: Any) -> None:
         assert isinstance(val, str)
         from pixeltable.utils.documents import get_document_handle
-        with open(val, 'r') as fh:
+        with open(val, 'r', encoding='utf8') as fh:
             try:
                 s = fh.read()
                 dh = get_document_handle(s)

{pixeltable-0.2.1 → pixeltable-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pixeltable"
-version = "0.2.1"
+version = "0.2.3"
 description = "Pixeltable: The Multimodal AI Data Plane"
 authors = ["Marcel Kornacker <marcelk@gmail.com>"]
 readme = "README.md"
@@ -17,8 +17,8 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
-numpy = "^1.24.1"
-pandas = "^1.5.3"
+numpy = "^1.26"
+pandas = ">=2.0,<3.0"
 pillow = "^9.4.0"
 opencv-python-headless = "^4.7.0.68"
 tqdm = "^4.64.1"
@@ -31,9 +31,11 @@ sqlalchemy = {extras = ["mypy"], version = "^2.0.23"}
 sqlalchemy-utils = "^0.41.1"
 pgvector = "^0.2.1"
 av = ">=10.0.0"
+beautifulsoup4 = "^4.0.0"
+requests = "^2.31.0"
 pyyaml = "^6.0.1"
 jinja2 = "^3.1.3"
-pgserver = "0.0.5"
+pgserver = "0.0.9"
 
 [tool.poetry.group.dev]
 optional = true
@@ -61,8 +63,6 @@ mkdocs-jupyter = "^0.24"
 pycocotools = "^2.0.7"
 ipykernel = "^6.27.1"
 nbmake = "^1.4.6"
-bs4 = "^0.0.2"
-requests = "^2.31.0"
 # packages required by various optional pieces of the codebase
 torch = "^2.2"
 torchvision = "^0.17"
@@ -74,7 +74,6 @@ boto3 = "^1.17"
 spacy = "^3.0"
 en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl"}
 tiktoken = ">=0.3"
-beautifulsoup4 = "^4.0.0"
 sentence-transformers = "^2.0.0"
 transformers = "^4.20"