pixeltable 0.2.2__tar.gz → 0.2.3__tar.gz

{pixeltable-0.2.2 → pixeltable-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.2
+Version: 0.2.3
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -11,19 +11,21 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: av (>=10.0.0)
+Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: numpy (>=1.24.1,<2.0.0)
+Requires-Dist: numpy (>=1.26,<2.0)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.0.7)
+Requires-Dist: pgserver (==0.0.9)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
 Requires-Dist: pillow (>=9.4.0,<10.0.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: regex (>=2022.10.31,<2023.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
 Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
 Requires-Dist: tqdm (>=4.64.1,<5.0.0)

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/exec/cache_prefetch_node.py

@@ -1,19 +1,21 @@
 from __future__ import annotations
-from typing import List, Optional, Any, Tuple, Dict
+
+import concurrent.futures
+import logging
 import threading
+import urllib.parse
+import urllib.request
 from collections import defaultdict
-from uuid import UUID
-import concurrent
-import logging
-import urllib
 from pathlib import Path
+from typing import List, Optional, Any, Tuple, Dict
+from uuid import UUID
 
-from .data_row_batch import DataRowBatch
-from .exec_node import ExecNode
-import pixeltable.exprs as exprs
-from pixeltable.utils.filecache import FileCache
 import pixeltable.env as env
 import pixeltable.exceptions as excs
+import pixeltable.exprs as exprs
+from pixeltable.utils.filecache import FileCache
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
 
 _logger = logging.getLogger('pixeltable')
 
@@ -81,7 +83,9 @@ class CachePrefetchNode(ExecNode):
         """Fetches a remote URL into Env.tmp_dir and returns its path"""
         url = row.file_urls[slot_idx]
         parsed = urllib.parse.urlparse(url)
-        assert parsed.scheme != '' and parsed.scheme != 'file'
+        # Use len(parsed.scheme) > 1 here to ensure we're not being passed
+        # a Windows filename
+        assert len(parsed.scheme) > 1 and parsed.scheme != 'file'
         # preserve the file extension, if there is one
         extension = ''
         if parsed.path != '':
@@ -95,7 +99,6 @@ class CachePrefetchNode(ExecNode):
                     if self.boto_client is None:
                         self.boto_client = get_client()
                     self.boto_client.download_file(parsed.netloc, parsed.path.lstrip('/'), str(tmp_path))
-                    return tmp_path
             elif parsed.scheme == 'http' or parsed.scheme == 'https':
                 with urllib.request.urlopen(url) as resp, open(tmp_path, 'wb') as f:
                     data = resp.read()

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/exprs/data_row.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
-from typing import Optional, List, Any, Tuple
+
 import io
-import urllib
+import urllib.parse
+import urllib.request
+from typing import Optional, List, Any, Tuple
 
 import PIL
 import numpy as np
@@ -104,6 +106,7 @@ class DataRow:
             assert self.file_paths[index] is not None
             if self.vals[index] is None:
                 self.vals[index] = PIL.Image.open(self.file_paths[index])
+                self.vals[index].load()
 
         return self.vals[index]
 
@@ -137,14 +140,19 @@ class DataRow:
         if (idx in self.img_slot_idxs or idx in self.media_slot_idxs) and isinstance(val, str):
             # this is either a local file path or a URL
             parsed = urllib.parse.urlparse(val)
-            if parsed.scheme == '' or parsed.scheme == 'file':
+            # Determine if this is a local file or a remote URL. If the scheme length is <= 1,
+            # we assume it's a local file. (This is because a Windows path will be interpreted
+            # by urllib as a URL with scheme equal to the drive letter.)
+            if len(parsed.scheme) <= 1 or parsed.scheme == 'file':
                 # local file path
                 assert self.file_urls[idx] is None and self.file_paths[idx] is None
-                if parsed.scheme == '':
-                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(parsed.path))
+                if len(parsed.scheme) <= 1:
+                    self.file_urls[idx] = urllib.parse.urljoin('file:', urllib.request.pathname2url(val))
+                    self.file_paths[idx] = val
                 else:
                     self.file_urls[idx] = val
-                self.file_paths[idx] = urllib.parse.unquote(parsed.path)
+                    # Wrap the path in a url2pathname() call to ensure proper handling on Windows.
+                    self.file_paths[idx] = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))
             else:
                 # URL
                 assert self.file_urls[idx] is None

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/iterators/document.py

@@ -61,7 +61,7 @@ class DocumentSplitter(ComponentIterator):
         import bs4
         if html_skip_tags is None:
             html_skip_tags = ['nav']
-        with open(document, 'r') as fh:
+        with open(document, 'r', encoding='utf8') as fh:
             s = fh.read()
             self._doc_handle = get_document_handle(s)
             assert self._doc_handle is not None

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/store.py

@@ -1,24 +1,26 @@
 from __future__ import annotations
 
+import abc
+import logging
 import os
 import sys
+import urllib.parse
+import urllib.request
 import warnings
 from typing import Optional, Dict, Any, List, Tuple, Set
-import logging
-import urllib
+
 import sqlalchemy as sql
 from tqdm import tqdm, TqdmWarning
-import abc
 
 import pixeltable.catalog as catalog
+import pixeltable.env as env
+from pixeltable import exprs
+import pixeltable.exceptions as excs
+from pixeltable.exec import ExecNode
 from pixeltable.metadata import schema
 from pixeltable.type_system import StringType
-from pixeltable.exec import ExecNode
-from pixeltable import exprs
-from pixeltable.utils.sql import log_stmt, log_explain
-import pixeltable.env as env
 from pixeltable.utils.media_store import MediaStore
-
+from pixeltable.utils.sql import log_stmt, log_explain
 
 _logger = logging.getLogger('pixeltable')
 
@@ -121,10 +123,13 @@ class StoreBase:
         if file_url is None:
             return None
         parsed = urllib.parse.urlparse(file_url)
-        if parsed.scheme != '' and parsed.scheme != 'file':
+        # We should never be passed a local file path here. The "len > 1" ensures that Windows
+        # file paths aren't mistaken for URLs with a single-character scheme.
+        assert len(parsed.scheme) > 1
+        if parsed.scheme != 'file':
             # remote url
             return file_url
-        file_path = urllib.parse.unquote(parsed.path)
+        file_path = urllib.parse.unquote(urllib.request.url2pathname(parsed.path))
         if not file_path.startswith(pxt_tmp_dir):
             # not a tmp file
             return file_url

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/test_dataframe.py

@@ -7,7 +7,6 @@ import bs4
 import numpy as np
 import pytest
 import requests
-from pycocotools.coco import COCO
 
 import pixeltable as pxt
 from pixeltable import catalog
@@ -184,6 +183,8 @@ class TestDataFrame:
         res = t.select(1.0).where(t.c2 < 10).collect()
         assert res[res.column_names()[0]] == [1.0] * 10
 
+    # TODO This test doesn't work on Windows due to reliance on the structure of file URLs
+    @pytest.mark.skip('Test is not portable')
     def test_html_media_url(self, test_client: pxt.Client) -> None:
         tab = test_client.create_table('test_html_repr', {'video': pxt.VideoType(), 'audio': pxt.AudioType()})
         status = tab.insert(video=get_video_files()[0], audio=get_audio_files()[0])
@@ -208,6 +209,7 @@ class TestDataFrame:
     def test_to_pytorch_dataset(self, all_datatypes_tbl: catalog.Table):
         """ tests all types are handled correctly in this conversion
         """
+        skip_test_if_not_installed('torch')
         import torch
 
         t = all_datatypes_tbl
@@ -238,6 +240,7 @@ class TestDataFrame:
     def test_to_pytorch_image_format(self, all_datatypes_tbl: catalog.Table) -> None:
         """ tests the image_format parameter is honored
         """
+        skip_test_if_not_installed('torch')
         import torch
         import torchvision.transforms as T
 
@@ -295,6 +298,7 @@ class TestDataFrame:
             1. compatibility with multiprocessing
             2. compatibility of all types with default collate_fn
         """
+        skip_test_if_not_installed('torch')
         import torch.utils.data
         @pxt.udf(param_types=[pxt.JsonType()], return_type=pxt.JsonType())
         def restrict_json_for_default_collate(obj):
@@ -352,6 +356,7 @@ class TestDataFrame:
             2. adding a row to the table invalidates the cached version
             3. changing the select list invalidates the cached version
         """
+        skip_test_if_not_installed('torch')
         t = all_datatypes_tbl
 
         t.drop_column('c_video') # null value video column triggers internal assertions in DataRow
@@ -383,6 +388,7 @@ class TestDataFrame:
 
     def test_to_coco(self, test_client: pxt.Client) -> None:
         skip_test_if_not_installed('nos')
+        from pycocotools.coco import COCO
         cl = test_client
         base_t = cl.create_table('videos', {'video': pxt.VideoType()})
         args = {'video': base_t.video, 'fps': 1}

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/test_document.py

@@ -8,6 +8,7 @@ import pytest
 import pixeltable as pxt
 from pixeltable.iterators.document import DocumentSplitter
 from pixeltable.tests.utils import get_documents, get_video_files, get_audio_files, get_image_files
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import DocumentType
 
 
@@ -34,6 +35,7 @@ class TestDocument:
         assert status.num_excs == len(file_paths)
 
     def test_doc_splitter(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('tiktoken')
         file_paths = self.valid_doc_paths()
         cl = test_client
         doc_t = cl.create_table('docs', {'doc': DocumentType()})
@@ -88,6 +90,7 @@ class TestDocument:
                     cl.drop_table('chunks')
 
     def test_doc_splitter_headings(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('spacy')
         file_paths = self.valid_doc_paths()
         cl = test_client
         doc_t = cl.create_table('docs', {'doc': DocumentType()})

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/test_exprs.py

@@ -1,5 +1,6 @@
 import json
 import urllib.parse
+import urllib.request
 from typing import List, Dict
 
 import pytest
@@ -174,7 +175,7 @@ class TestExprs:
         res = img_t.select(img_t.img.fileurl).show(0).to_pandas()
         stored_urls = set(res.iloc[:, 0])
         assert len(stored_urls) == len(res)
-        all_urls  = set([urllib.parse.urljoin('file:', path) for path in get_image_files()])
+        all_urls = set(urllib.parse.urljoin('file:', urllib.request.pathname2url(path)) for path in get_image_files())
         assert stored_urls <= all_urls
 
         # localpath

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/test_table.py

@@ -18,6 +18,7 @@ from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import \
     make_tbl, create_table_data, read_data_file, get_video_files, get_audio_files, get_image_files, get_documents, \
     assert_resultset_eq
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import \
     StringType, IntType, FloatType, TimestampType, ImageType, VideoType, JsonType, BoolType, ArrayType, AudioType, \
     DocumentType
@@ -296,6 +297,7 @@ class TestTable:
         self.check_bad_media(test_client, rows, DocumentType(nullable=True))
 
     def test_validate_external_url(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         rows = [
             {'media': 's3://open-images-dataset/validation/doesnotexist.jpg', 'is_bad_media': True},
             {'media': 'https://archive.random.org/download?file=2024-01-28.bin', 'is_bad_media': True},  # 403 error
@@ -315,6 +317,7 @@ class TestTable:
         self.check_bad_media(test_client, rows, VideoType(nullable=True))
 
     def test_create_s3_image_table(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         tbl = cl.create_table('test', {'img': ImageType(nullable=False)})
         # this is needed because Client.reset_catalog() doesn't call TableVersion.drop(), which would
@@ -371,6 +374,7 @@ class TestTable:
         assert cache_stats.total_size == 0
 
     def test_video_url(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         schema = {
             'payload': IntType(nullable=False),
@@ -390,6 +394,7 @@ class TestTable:
         cap.release()
 
     def test_create_video_table(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         tbl = cl.create_table(
             'test_tbl',
@@ -529,6 +534,7 @@ class TestTable:
         assert 'expected ndarray((2, 3)' in str(exc_info.value)
 
     def test_query(self, test_client: pxt.Client) -> None:
+        skip_test_if_not_installed('boto3')
         cl = test_client
         col_names = ['c1', 'c2', 'c3', 'c4', 'c5']
         t = make_tbl(cl, 'test', col_names)

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/test_video.py

@@ -8,6 +8,7 @@ from pixeltable import catalog
 from pixeltable import exceptions as excs
 from pixeltable.iterators import FrameIterator
 from pixeltable.tests.utils import get_video_files
+from pixeltable.tests.utils import skip_test_if_not_installed
 from pixeltable.type_system import VideoType, ImageType
 from pixeltable.utils.media_store import MediaStore
 
@@ -61,6 +62,7 @@ class TestVideo:
         assert MediaStore.count(view.get_id()) == view.count()
 
     def test_query(self, test_client: pxt.client) -> None:
+        skip_test_if_not_installed('boto3')
         video_filepaths = get_video_files()
         cl = test_client
         base_t, view_t = self.create_tbls(cl)

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/tests/utils.py

@@ -225,7 +225,7 @@ def read_data_file(dir_name: str, file_name: str, path_col_names: Optional[List[
         df[col_name] = df.apply(lambda r: str(abs_path / r[col_name]), axis=1)
     return df.to_dict(orient='records')
 
-def get_video_files(include_bad_video=False) -> List[str]:
+def get_video_files(include_bad_video: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/videos/*', recursive=True)
     if not include_bad_video:
@@ -239,12 +239,14 @@ def get_test_video_files() -> List[str]:
     glob_result = glob.glob(f'{tests_dir}/**/test_videos/*', recursive=True)
     return glob_result
 
-def get_image_files() -> List[str]:
+def get_image_files(include_bad_image: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__) # search with respect to tests/ dir
     glob_result = glob.glob(f'{tests_dir}/**/imagenette2-160/*', recursive=True)
+    if not include_bad_image:
+        glob_result = [f for f in glob_result if 'bad_image' not in f]
     return glob_result
 
-def get_audio_files(include_bad_audio=False) -> List[str]:
+def get_audio_files(include_bad_audio: bool = False) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     glob_result = glob.glob(f'{tests_dir}/**/audio/*', recursive=True)
     if not include_bad_audio:
@@ -259,7 +261,7 @@ def get_documents() -> List[str]:
 def get_sentences(n: int = 100) -> List[str]:
     tests_dir = os.path.dirname(__file__)
     path = glob.glob(f'{tests_dir}/**/jeopardy.json', recursive=True)[0]
-    with open(path, 'r') as f:
+    with open(path, 'r', encoding='utf8') as f:
         questions_list = json.load(f)
     # this dataset contains \' around the questions
     return [q['question'].replace("'", '') for q in questions_list[:n]]

{pixeltable-0.2.2 → pixeltable-0.2.3}/pixeltable/type_system.py

@@ -911,7 +911,7 @@ class DocumentType(ColumnType):
     def validate_media(self, val: Any) -> None:
         assert isinstance(val, str)
         from pixeltable.utils.documents import get_document_handle
-        with open(val, 'r') as fh:
+        with open(val, 'r', encoding='utf8') as fh:
             try:
                 s = fh.read()
                 dh = get_document_handle(s)

{pixeltable-0.2.2 → pixeltable-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pixeltable"
-version = "0.2.2"
+version = "0.2.3"
 description = "Pixeltable: The Multimodal AI Data Plane"
 authors = ["Marcel Kornacker <marcelk@gmail.com>"]
 readme = "README.md"
@@ -17,7 +17,7 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
-numpy = "^1.24.1"
+numpy = "^1.26"
 pandas = ">=2.0,<3.0"
 pillow = "^9.4.0"
 opencv-python-headless = "^4.7.0.68"
@@ -31,9 +31,11 @@ sqlalchemy = {extras = ["mypy"], version = "^2.0.23"}
 sqlalchemy-utils = "^0.41.1"
 pgvector = "^0.2.1"
 av = ">=10.0.0"
+beautifulsoup4 = "^4.0.0"
+requests = "^2.31.0"
 pyyaml = "^6.0.1"
 jinja2 = "^3.1.3"
-pgserver = "0.0.7"
+pgserver = "0.0.9"
 
 [tool.poetry.group.dev]
 optional = true
@@ -61,8 +63,6 @@ mkdocs-jupyter = "^0.24"
 pycocotools = "^2.0.7"
 ipykernel = "^6.27.1"
 nbmake = "^1.4.6"
-bs4 = "^0.0.2"
-requests = "^2.31.0"
 # packages required by various optional pieces of the codebase
 torch = "^2.2"
 torchvision = "^0.17"
@@ -74,7 +74,6 @@ boto3 = "^1.17"
 spacy = "^3.0"
 en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl"}
 tiktoken = ">=0.3"
-beautifulsoup4 = "^4.0.0"
 sentence-transformers = "^2.0.0"
 transformers = "^4.20"