pixeltable 0.2.26__tar.gz → 0.2.27__tar.gz

{pixeltable-0.2.26 → pixeltable-0.2.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.26
+Version: 0.2.27
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 Home-page: https://pixeltable.com/
 License: Apache-2.0

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.26"
-__version_tuple__ = (0, 2, 26)
+__version__ = "0.2.27"
+__version_tuple__ = (0, 2, 27)

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/env.py

@@ -496,6 +496,7 @@ class Env:
         self.__register_package('datasets')
         self.__register_package('fiftyone')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('google.generativeai', library_name='google-generativeai')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
@@ -520,8 +521,14 @@ class Env:
         self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
 
     def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+        is_installed: bool
+        try:
+            is_installed = importlib.util.find_spec(package_name) is not None
+        except ModuleNotFoundError:
+            # This can happen if the parent of `package_name` is not installed.
+            is_installed = False
         self.__optional_packages[package_name] = PackageInfo(
-            is_installed=importlib.util.find_spec(package_name) is not None,
+            is_installed=is_installed,
             library_name=library_name or package_name  # defaults to package_name unless specified otherwise
         )
 

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/functions/__init__.py

@@ -1,7 +1,7 @@
 from pixeltable.utils.code import local_public_names
 
-from . import (anthropic, audio, fireworks, huggingface, image, json, llama_cpp, mistralai, ollama, openai, string,
-               timestamp, together, video, vision, whisper)
+from . import (anthropic, audio, fireworks, gemini, huggingface, image, json, llama_cpp, mistralai, ollama, openai,
+               string, timestamp, together, video, vision, whisper)
 from .globals import *
 
 __all__ = local_public_names(__name__, exclude=['globals']) + local_public_names(globals.__name__)

pixeltable-0.2.27/pixeltable/functions/gemini.py

@@ -0,0 +1,85 @@
+"""
+Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
+that wrap various endpoints from the Google Gemini API. In order to use them, you must
+first `pip install google-generativeai` and configure your Gemini credentials, as described in
+the [Working with Gemini](https://pixeltable.readme.io/docs/working-with-gemini) tutorial.
+"""
+
+from typing import Optional
+
+import pixeltable as pxt
+from pixeltable import env
+
+
+@env.register_client('gemini')
+def _(api_key: str) -> None:
+    import google.generativeai as genai  # type: ignore[import-untyped]
+    genai.configure(api_key=api_key)
+
+
+def _ensure_loaded() -> None:
+    env.Env.get().get_client('gemini')
+
+
+@pxt.udf
+def generate_content(
+    contents: str,
+    *,
+    model_name: str,
+    candidate_count: Optional[int] = None,
+    stop_sequences: Optional[list[str]] = None,
+    max_output_tokens: Optional[int] = None,
+    temperature: Optional[float] = None,
+    top_p: Optional[float] = None,
+    top_k: Optional[int] = None,
+    response_mime_type: Optional[str] = None,
+    response_schema: Optional[dict] = None,
+    presence_penalty: Optional[float] = None,
+    frequency_penalty: Optional[float] = None,
+    response_logprobs: Optional[bool] = None,
+    logprobs: Optional[int] = None,
+) -> dict:
+    """
+    Generate content from the specified model. For additional details, see:
+    <https://ai.google.dev/gemini-api/docs>
+
+    __Requirements:__
+
+    - `pip install google-generativeai`
+
+    Args:
+        contents: The input content to generate from.
+        model_name: The name of the model to use.
+
+    For details on the other parameters, see: <https://ai.google.dev/gemini-api/docs>
+
+    Returns:
+        A dictionary containing the response and other metadata.
+
+    Examples:
+        Add a computed column that applies the model `gemini-1.5-flash`
+        to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
+
+        >>> tbl['response'] = generate_content(tbl.prompt, model_name='gemini-1.5-flash')
+    """
+    env.Env.get().require_package('google.generativeai')
+    _ensure_loaded()
+    import google.generativeai as genai
+
+    model = genai.GenerativeModel(model_name=model_name)
+    gc = genai.GenerationConfig(
+        candidate_count=candidate_count,
+        stop_sequences=stop_sequences,
+        max_output_tokens=max_output_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        response_mime_type=response_mime_type,
+        response_schema=response_schema,
+        presence_penalty=presence_penalty,
+        frequency_penalty=frequency_penalty,
+        response_logprobs=response_logprobs,
+        logprobs=logprobs,
+    )
+    response = model.generate_content(contents, generation_config=gc)
+    return response.to_dict()

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/iterators/document.py

@@ -151,6 +151,9 @@ class DocumentSplitter(ComponentIterator):
         elif self._doc_handle.format == DocumentType.DocumentFormat.PDF:
             assert self._doc_handle.pdf_doc is not None
             self._sections = self._pdf_sections()
+        elif self._doc_handle.format == DocumentType.DocumentFormat.TXT:
+            assert self._doc_handle.txt_doc is not None
+            self._sections = self._txt_sections()
         else:
             assert False, f'Unsupported document format: {self._doc_handle.format}'
 
@@ -389,6 +392,15 @@ class DocumentSplitter(ComponentIterator):
         if accumulated_text and not emit_on_page:
             yield DocumentSection(text=_emit_text(), metadata=DocumentSectionMetadata())
 
+    def _txt_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections for text files.
+
+        Currently, it returns the entire text as a single section.
+        TODO: Add support for paragraphs.
+        """
+        assert self._doc_handle.txt_doc is not None
+        yield DocumentSection(text=ftfy.fix_text(self._doc_handle.txt_doc), metadata=DocumentSectionMetadata())
+
     def _sentence_sections(self, input_sections: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
         """Split the input sections into sentences"""
         for section in input_sections:

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/type_system.py

@@ -940,6 +940,7 @@ class DocumentType(ColumnType):
         MD = 1
         PDF = 2
         XML = 3
+        TXT = 4
 
     def __init__(self, nullable: bool = False, doc_formats: Optional[str] = None):
         super().__init__(self.Type.DOCUMENT, nullable=nullable)

{pixeltable-0.2.26 → pixeltable-0.2.27}/pixeltable/utils/documents.py

@@ -15,6 +15,7 @@ class DocumentHandle:
     bs_doc: Optional[bs4.BeautifulSoup] = None
     md_ast: Optional[dict] = None
     pdf_doc: Optional[fitz.Document] = None
+    txt_doc: Optional[str] = None
 
 
 def get_document_handle(path: str) -> Optional[DocumentHandle]:
@@ -40,6 +41,11 @@ def get_document_handle(path: str) -> Optional[DocumentHandle]:
         if bs_doc is not None:
             return DocumentHandle(format=ts.DocumentType.DocumentFormat.XML, bs_doc=bs_doc)
 
+    if doc_format == '.txt':
+        txt_doc = get_txt(path)
+        if txt_doc is not None:
+            return DocumentHandle(format=ts.DocumentType.DocumentFormat.TXT, txt_doc=txt_doc)
+
     return None
 
 
@@ -84,3 +90,11 @@ def get_markdown_handle(path: str) -> Optional[dict]:
         return md_ast(text)
     except Exception:
         return None
+
+def get_txt(path: str) -> Optional[str]:
+    try:
+        with open(path, "r") as f:
+            doc = f.read()
+        return doc if doc != '' else None
+    except Exception:
+        return None

{pixeltable-0.2.26 → pixeltable-0.2.27}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry_dynamic_versioning.backend"
 
 [tool.poetry]
 name = "pixeltable"
-version = "0.2.26"
+version = "0.2.27"
 description = "AI Data Infrastructure: Declarative, Multimodal, and Incremental"
 license = "Apache-2.0"
 authors = ["Pixeltable, Inc. <contact@pixeltable.com>"]
@@ -114,6 +114,7 @@ together = "^1.3.1"
 fireworks-ai = "^0.13.0"
 mistralai = "^1.2.1"
 replicate = "^1.0.2"
+google-generativeai = "^0.8.3"
 boto3 = "==1.35.5"  # Locking a specific version of boto3 dramatically improves `poetry lock` runtimes
 spacy = ">=3.7"
 pydantic = ">=2.7.4"  # Required minimum version to work with spacy + python 3.12