pixeltable 0.2.26__py3-none-any.whl → 0.2.27__py3-none-any.whl

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.26"
-__version_tuple__ = (0, 2, 26)
+__version__ = "0.2.27"
+__version_tuple__ = (0, 2, 27)

pixeltable/env.py

@@ -496,6 +496,7 @@ class Env:
         self.__register_package('datasets')
         self.__register_package('fiftyone')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('google.generativeai', library_name='google-generativeai')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
@@ -520,8 +521,14 @@ class Env:
         self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
 
     def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+        is_installed: bool
+        try:
+            is_installed = importlib.util.find_spec(package_name) is not None
+        except ModuleNotFoundError:
+            # This can happen if the parent of `package_name` is not installed.
+            is_installed = False
         self.__optional_packages[package_name] = PackageInfo(
-            is_installed=importlib.util.find_spec(package_name) is not None,
+            is_installed=is_installed,
             library_name=library_name or package_name  # defaults to package_name unless specified otherwise
         )
 

pixeltable/functions/__init__.py

@@ -1,7 +1,7 @@
 from pixeltable.utils.code import local_public_names
 
-from . import (anthropic, audio, fireworks, huggingface, image, json, llama_cpp, mistralai, ollama, openai, string,
-               timestamp, together, video, vision, whisper)
+from . import (anthropic, audio, fireworks, gemini, huggingface, image, json, llama_cpp, mistralai, ollama, openai,
+               string, timestamp, together, video, vision, whisper)
 from .globals import *
 
 __all__ = local_public_names(__name__, exclude=['globals']) + local_public_names(globals.__name__)

pixeltable/functions/gemini.py

@@ -0,0 +1,85 @@
+"""
+Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
+that wrap various endpoints from the Google Gemini API. In order to use them, you must
+first `pip install google-generativeai` and configure your Gemini credentials, as described in
+the [Working with Gemini](https://pixeltable.readme.io/docs/working-with-gemini) tutorial.
+"""
+
+from typing import Optional
+
+import pixeltable as pxt
+from pixeltable import env
+
+
+@env.register_client('gemini')
+def _(api_key: str) -> None:
+    import google.generativeai as genai  # type: ignore[import-untyped]
+    genai.configure(api_key=api_key)
+
+
+def _ensure_loaded() -> None:
+    env.Env.get().get_client('gemini')
+
+
+@pxt.udf
+def generate_content(
+    contents: str,
+    *,
+    model_name: str,
+    candidate_count: Optional[int] = None,
+    stop_sequences: Optional[list[str]] = None,
+    max_output_tokens: Optional[int] = None,
+    temperature: Optional[float] = None,
+    top_p: Optional[float] = None,
+    top_k: Optional[int] = None,
+    response_mime_type: Optional[str] = None,
+    response_schema: Optional[dict] = None,
+    presence_penalty: Optional[float] = None,
+    frequency_penalty: Optional[float] = None,
+    response_logprobs: Optional[bool] = None,
+    logprobs: Optional[int] = None,
+) -> dict:
+    """
+    Generate content from the specified model. For additional details, see:
+    <https://ai.google.dev/gemini-api/docs>
+
+    __Requirements:__
+
+    - `pip install google-generativeai`
+
+    Args:
+        contents: The input content to generate from.
+        model_name: The name of the model to use.
+
+    For details on the other parameters, see: <https://ai.google.dev/gemini-api/docs>
+
+    Returns:
+        A dictionary containing the response and other metadata.
+
+    Examples:
+        Add a computed column that applies the model `gemini-1.5-flash`
+        to an existing Pixeltable column `tbl.prompt` of the table `tbl`:
+
+        >>> tbl['response'] = generate_content(tbl.prompt, model_name='gemini-1.5-flash')
+    """
+    env.Env.get().require_package('google.generativeai')
+    _ensure_loaded()
+    import google.generativeai as genai
+
+    model = genai.GenerativeModel(model_name=model_name)
+    gc = genai.GenerationConfig(
+        candidate_count=candidate_count,
+        stop_sequences=stop_sequences,
+        max_output_tokens=max_output_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        response_mime_type=response_mime_type,
+        response_schema=response_schema,
+        presence_penalty=presence_penalty,
+        frequency_penalty=frequency_penalty,
+        response_logprobs=response_logprobs,
+        logprobs=logprobs,
+    )
+    response = model.generate_content(contents, generation_config=gc)
+    return response.to_dict()

pixeltable/iterators/document.py

@@ -151,6 +151,9 @@ class DocumentSplitter(ComponentIterator):
         elif self._doc_handle.format == DocumentType.DocumentFormat.PDF:
             assert self._doc_handle.pdf_doc is not None
             self._sections = self._pdf_sections()
+        elif self._doc_handle.format == DocumentType.DocumentFormat.TXT:
+            assert self._doc_handle.txt_doc is not None
+            self._sections = self._txt_sections()
         else:
             assert False, f'Unsupported document format: {self._doc_handle.format}'
 
@@ -389,6 +392,15 @@ class DocumentSplitter(ComponentIterator):
         if accumulated_text and not emit_on_page:
             yield DocumentSection(text=_emit_text(), metadata=DocumentSectionMetadata())
 
+    def _txt_sections(self) -> Iterator[DocumentSection]:
+        """Create DocumentSections for text files.
+
+        Currently, it returns the entire text as a single section.
+        TODO: Add support for paragraphs.
+        """
+        assert self._doc_handle.txt_doc is not None
+        yield DocumentSection(text=ftfy.fix_text(self._doc_handle.txt_doc), metadata=DocumentSectionMetadata())
+
     def _sentence_sections(self, input_sections: Iterable[DocumentSection]) -> Iterator[DocumentSection]:
         """Split the input sections into sentences"""
         for section in input_sections:

pixeltable/type_system.py

@@ -940,6 +940,7 @@ class DocumentType(ColumnType):
         MD = 1
         PDF = 2
         XML = 3
+        TXT = 4
 
     def __init__(self, nullable: bool = False, doc_formats: Optional[str] = None):
         super().__init__(self.Type.DOCUMENT, nullable=nullable)

pixeltable/utils/documents.py

@@ -15,6 +15,7 @@ class DocumentHandle:
     bs_doc: Optional[bs4.BeautifulSoup] = None
     md_ast: Optional[dict] = None
     pdf_doc: Optional[fitz.Document] = None
+    txt_doc: Optional[str] = None
 
 
 def get_document_handle(path: str) -> Optional[DocumentHandle]:
@@ -40,6 +41,11 @@ def get_document_handle(path: str) -> Optional[DocumentHandle]:
         if bs_doc is not None:
             return DocumentHandle(format=ts.DocumentType.DocumentFormat.XML, bs_doc=bs_doc)
 
+    if doc_format == '.txt':
+        txt_doc = get_txt(path)
+        if txt_doc is not None:
+            return DocumentHandle(format=ts.DocumentType.DocumentFormat.TXT, txt_doc=txt_doc)
+
     return None
 
 
@@ -84,3 +90,11 @@ def get_markdown_handle(path: str) -> Optional[dict]:
         return md_ast(text)
     except Exception:
         return None
+
+def get_txt(path: str) -> Optional[str]:
+    try:
+        with open(path, "r") as f:
+            doc = f.read()
+        return doc if doc != '' else None
+    except Exception:
+        return None

{pixeltable-0.2.26.dist-info → pixeltable-0.2.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.26
+Version: 0.2.27
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 Home-page: https://pixeltable.com/
 License: Apache-2.0

{pixeltable-0.2.26.dist-info → pixeltable-0.2.27.dist-info}/RECORD

@@ -1,5 +1,5 @@
 pixeltable/__init__.py,sha256=gv2jvZ7H5tEjLear10E7hSH9jF5Mw6iSeryvRp88bOE,1391
-pixeltable/__version__.py,sha256=5OkFhhUIXyKPz2LjEv1DUzoaDvZXzsOKyF_UesfvD_U,114
+pixeltable/__version__.py,sha256=QUwAvaVSNmDc6Bk6QhVpLDxZX1U9OQRfWmmw1ZoKK_A,114
 pixeltable/catalog/__init__.py,sha256=Ar6_F_6C7tkznIlCPBHVHDop5YssBDjKQr2NPQ21QCI,484
 pixeltable/catalog/catalog.py,sha256=tyDyI5wQw7vV6_FChrp9qgGCRClcjiSdW3eygYT0p9s,7849
 pixeltable/catalog/column.py,sha256=ezeKoGl6aBTzSZBihDA6vdETcvyCguAD4OsmrxWs73A,9595
@@ -15,7 +15,7 @@ pixeltable/catalog/table_version.py,sha256=W4yrvn-mF63CQ8cQGmjbjYTCf1xYuV6k6rI__
 pixeltable/catalog/table_version_path.py,sha256=CczGbcz5ESq663arreri_p4chMZHozgG6k3y-ajkJN4,5787
 pixeltable/catalog/view.py,sha256=mJbaVE63GaBDy7EcQkQ2q5NKlrHlGyENkRyQVzU-ak8,10854
 pixeltable/dataframe.py,sha256=RnRpOWLOqKNHGK2cO-Zbbc63yLUqrX7eSCYW11K4n7A,41320
-pixeltable/env.py,sha256=oIFeLJVDQuQgW8REAoNkY45BF2o02NxtysgAlJNEBc8,30520
+pixeltable/env.py,sha256=zmEdcMj_nKIsPJ-sKB1Avi3AnacGQOuk-uP1tQhh1tU,30840
 pixeltable/exceptions.py,sha256=NuFY2WtkQpLfLHT_J70kOw9Tr0kEDkkgo-u7As4Gaq4,410
 pixeltable/exec/__init__.py,sha256=bWbIA7U7y1hC40VOSv_p9doaMuOB0i3r1O2UFNw0VEs,494
 pixeltable/exec/aggregation_node.py,sha256=0LdoPp_CR_UbcS60XkDw66SqlrQnw6Dy6KDWqi4PJ6k,4005
@@ -69,10 +69,11 @@ pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,14
 pixeltable/func/query_template_function.py,sha256=pGqwtWiPsEmo7phVoJJODiuD1Sh0gZoW4BpKnZV9cRE,3537
 pixeltable/func/signature.py,sha256=MzOHayuf2S3EWIl4IRtdNWnaKj5tphnw1z7P6xNU6Xg,8691
 pixeltable/func/udf.py,sha256=_883xbGujeIhYOz-lWH5SvLn22F5JVpP6O_MRvpU4hU,7362
-pixeltable/functions/__init__.py,sha256=EtR9M3ewYpmtHeshNULqZVBd87bNeKSFAdpOuWCMl6o,389
+pixeltable/functions/__init__.py,sha256=2lsAds20ZUEJETkGmGXjWSqzmGhPABcuGWOe1IZjqxs,397
 pixeltable/functions/anthropic.py,sha256=P1E5o4-8QP1LTIUsWVgo_wMJ4WOnxtXUUXuFWUagChU,3032
 pixeltable/functions/audio.py,sha256=7213nTnqKJ6vM9kalaoJ283OwX5SGEJN10vDhaRNZ6E,644
 pixeltable/functions/fireworks.py,sha256=qwFC_eIaDs-glxyJ_IVXaNGkpgPzeRsQ_SdpzueBxq0,2605
+pixeltable/functions/gemini.py,sha256=RQ3I25nXSXUXdF0rmhnv5XOgZXb_6SIgMc_hEyF83SI,2783
 pixeltable/functions/globals.py,sha256=pCFX2a_N87SwG9GxyPjSOC3TVMowMB6XIHSWKfFOuGE,3917
 pixeltable/functions/huggingface.py,sha256=s5KmOfi9-TOYyrL1Wv-voKP7ykkUN7LlLAA_uo01UQc,21210
 pixeltable/functions/image.py,sha256=3Qm4ybAT_o4YUl3bzhEXy8dKOwgZ7RCUV-ky-dbL_jc,13836
@@ -104,7 +105,7 @@ pixeltable/io/pandas.py,sha256=7eHg7wnAfRA9eBk4iC0iSSVTKOM59Ne4pXokKWdt3dY,9793
 pixeltable/io/parquet.py,sha256=bvwTRIKhS99M3bAcAP63jnh5-R6Qww1SrYBGWjZvN1g,8800
 pixeltable/iterators/__init__.py,sha256=qA9cJTwPO3gk-7y8mUXw2anWSbnUAVGwzl52SxiUjNU,398
 pixeltable/iterators/base.py,sha256=ZC0ZvXL4iw6AmT8cu-Mdx-T2UG9nmJYV1C6LK4efAfw,1669
-pixeltable/iterators/document.py,sha256=AsvEmZ5RGRi3AFGCrH2_-UNx5rTCFA-0WmMYQBTQI20,19679
+pixeltable/iterators/document.py,sha256=e5EMUAW4f5SqfC5mRN7Oo2E3gw_vcMSzq2707NVLByU,20254
 pixeltable/iterators/image.py,sha256=7eJupbyLEl_q4bemEozL8WgTgkxjl-DYnSXyHTQ7Rck,3628
 pixeltable/iterators/string.py,sha256=NG_fWc_GAITDfzl6MvrDOMrSoMcZdMZf6hPQztCSatE,1305
 pixeltable/iterators/video.py,sha256=anedQpSOnjYhHLa8UiEx-6ROagb14rpXh9am8oTriUg,9382
@@ -134,13 +135,13 @@ pixeltable/tool/doc_plugins/mkdocstrings.py,sha256=afq7XOaSC5WRmugkh-FMFMK8PqOgI
 pixeltable/tool/doc_plugins/templates/material/udf.html.jinja,sha256=R-7Q57nmDd5BUea-F1-MjwjK3pq7uBHXNoSo8_QjZG4,4890
 pixeltable/tool/embed_udf.py,sha256=EXvfvuzZm0uTgH-aAATSrKV8ixCU8OMwpzXlJMg845Y,299
 pixeltable/tool/mypy_plugin.py,sha256=__oTFElirrK25jCX1z_asD_gxGnGxtD2TaU6r1if-Ic,1784
-pixeltable/type_system.py,sha256=QPQ2Q065WQJY7uz1U3Y5dlgUPYwnWxBYfPmiZZXb-JE,41936
+pixeltable/type_system.py,sha256=Q6W4Vp3SZOgJwjzt2Ugwo-YA4NEe6sJ2xhIXOP_HoM0,41952
 pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
 pixeltable/utils/arrow.py,sha256=4AyCAkkNfNChVTaPAVS5j1ZO7BtTxn11duonolfktV8,3931
 pixeltable/utils/coco.py,sha256=WCUyoj0dTyJFbPB7frEQUvY92SlEPxQ068r-3QAwROE,7285
 pixeltable/utils/code.py,sha256=AOw1u2r8_DQXpX-lxJhyHWARGrCRDXOJHFVgKOi54Uc,1231
 pixeltable/utils/description_helper.py,sha256=P-8EE2pRFP8s3coe73frgV68yt4Dp3saErCUehMxKUw,3759
-pixeltable/utils/documents.py,sha256=B984nVigJgHZ5Rm-zX4LLuHuMnfmz-zr24bbAsc_y3w,2511
+pixeltable/utils/documents.py,sha256=pytTYY167wYc1rGaDd9HPK6GOrz1ML78eAD3hIHrG68,2930
 pixeltable/utils/filecache.py,sha256=6HKQdItqSSTQvj2HkSJulyhfBedi4PgC7umwxXGOVG8,10637
 pixeltable/utils/formatter.py,sha256=5E_gDg11ClFI-5SthwkiqyE3hAok3JHDj4OSK9cJklM,9257
 pixeltable/utils/http_server.py,sha256=xYPTvmYrkUpKfOaLDq08D-eHswkcgDf4qAt76ZFH6lM,2411
@@ -149,8 +150,8 @@ pixeltable/utils/pytorch.py,sha256=6RvOCjy_QV4gc-aht-3d0zoASkuv-warfpl87vgmuKw,3
 pixeltable/utils/s3.py,sha256=huA5hxDGkPIu18zWet76o0FsO7Vbtp-iPmnOzCe-MvA,586
 pixeltable/utils/sql.py,sha256=j_tj0h4ffm-DhUIJbvGphxrVyBKlNTwDKqWGhRQ5_PY,795
 pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
-pixeltable-0.2.26.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pixeltable-0.2.26.dist-info/METADATA,sha256=qgtP333OuS_5GC1MBPPuS7Eu3V_Kb1hXxcnEs3l0QLA,19476
-pixeltable-0.2.26.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-pixeltable-0.2.26.dist-info/entry_points.txt,sha256=TNI1Gb5vPwFrTdw6TimSYjO8FeK8c_HuPr28vcf7o_I,108
-pixeltable-0.2.26.dist-info/RECORD,,
+pixeltable-0.2.27.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pixeltable-0.2.27.dist-info/METADATA,sha256=GaJYBIIbc-7XgT5z4Ni7ADUvaxF2dPoAnuYzJZlZIgw,19476
+pixeltable-0.2.27.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+pixeltable-0.2.27.dist-info/entry_points.txt,sha256=TNI1Gb5vPwFrTdw6TimSYjO8FeK8c_HuPr28vcf7o_I,108
+pixeltable-0.2.27.dist-info/RECORD,,