palimpzest 1.2.0__tar.gz → 1.3.1__tar.gz

{palimpzest-1.2.0/src/palimpzest.egg-info → palimpzest-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: palimpzest
-Version: 1.2.0
+Version: 1.3.1
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
 Project-URL: homepage, https://palimpzest.org
@@ -34,6 +34,7 @@ Requires-Dist: PyLD>=2.0.4
 Requires-Dist: pyarrow>=20.0.0
 Requires-Dist: pypdf>=5.1.0
 Requires-Dist: pytest-mock>=3.14.0
+Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: requests>=2.25
 Requires-Dist: ruff>=0.9.0

{palimpzest-1.2.0 → palimpzest-1.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "palimpzest"
-version = "1.2.0"
+version = "1.3.1"
 description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,6 +28,7 @@ dependencies = [
     "pyarrow>=20.0.0",
     "pypdf>=5.1.0",
     "pytest-mock>=3.14.0",
+    "python-dotenv>=1.2.1",
     "pyyaml>=6.0.1",
     "requests>=2.25",
     "ruff>=0.9.0",

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/constants.py

@@ -31,9 +31,9 @@ class Model(str, Enum):
     GEMINI_2_0_FLASH = "vertex_ai/gemini-2.0-flash"
     GEMINI_2_5_FLASH = "vertex_ai/gemini-2.5-flash"
     GEMINI_2_5_PRO = "vertex_ai/gemini-2.5-pro"
-    GOOGLE_GEMINI_2_5_FLASH = "google/gemini-2.5-flash"
-    GOOGLE_GEMINI_2_5_FLASH_LITE = "google/gemini-2.5-flash-lite"
-    GOOGLE_GEMINI_2_5_PRO = "google/gemini-2.5-pro"
+    GOOGLE_GEMINI_2_5_FLASH = "gemini/gemini-2.5-flash"
+    GOOGLE_GEMINI_2_5_FLASH_LITE = "gemini/gemini-2.5-flash-lite"
+    GOOGLE_GEMINI_2_5_PRO = "gemini/gemini-2.5-pro"
     LLAMA_4_MAVERICK = "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas"
     GPT_4o_AUDIO_PREVIEW = "openai/gpt-4o-audio-preview"
     GPT_4o_MINI_AUDIO_PREVIEW = "openai/gpt-4o-mini-audio-preview"
@@ -72,8 +72,8 @@ class Model(str, Enum):
     def is_vertex_model(self):
         return "vertex_ai" in self.value.lower()
 
-    def is_google_model(self):
-        return "google" in self.value.lower()
+    def is_google_ai_studio_model(self):
+        return "gemini/" in self.value.lower()
 
     def is_vllm_model(self):
         return "hosted_vllm" in self.value.lower()

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/query/generators/generators.py

@@ -108,7 +108,7 @@ class Generator(Generic[ContextType, InputType]):
         self,
         model: Model,
         prompt_strategy: PromptStrategy,
-        reasoning_effort: str | None = None,
+        reasoning_effort: str | None,
         api_base: str | None = None,
         cardinality: Cardinality = Cardinality.ONE_TO_ONE,
         desc: str | None = None,
@@ -325,18 +325,7 @@ class Generator(Generic[ContextType, InputType]):
             if is_audio_op:
                 completion_kwargs = {"modalities": ["text"], **completion_kwargs}
             if self.model.is_reasoning_model():
-                if self.model.is_vertex_model():
-                    reasoning_effort = self.reasoning_effort
-                    if self.reasoning_effort is None and self.model == Model.GEMINI_2_5_PRO:
-                        reasoning_effort = "low"
-                    elif self.reasoning_effort is None:
-                        reasoning_effort = "disable"
-                    completion_kwargs = {"reasoning_effort": reasoning_effort, **completion_kwargs}
-                elif self.model.is_anthropic_model() and self.reasoning_effort is not None:
-                    completion_kwargs = {"reasoning_effort": self.reasoning_effort, **completion_kwargs}
-                elif self.model.is_openai_model():
-                    reasoning_effort = "minimal" if self.reasoning_effort is None else self.reasoning_effort
-                    completion_kwargs = {"reasoning_effort": reasoning_effort, **completion_kwargs}
+                completion_kwargs = {"reasoning_effort": self.reasoning_effort, **completion_kwargs}
             if self.model.is_vllm_model():
                 completion_kwargs = {"api_base": self.api_base, "api_key": os.environ.get("VLLM_API_KEY", "fake-api-key"), **completion_kwargs}
             completion = litellm.completion(model=self.model_name, messages=messages, **completion_kwargs)

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/query/optimizer/optimizer.py

@@ -75,7 +75,7 @@ class Optimizer:
         cost_model: BaseCostModel,
         available_models: list[Model],
         join_parallelism: int = 64,
-        reasoning_effort: str | None = None,
+        reasoning_effort: str | None = "default",
         api_base: str | None = None,
         verbose: bool = False,
         allow_bonded_query: bool = True,

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/query/optimizer/rules.py

@@ -54,6 +54,7 @@ from palimpzest.query.operators.search import (
 from palimpzest.query.operators.split import SplitConvert, SplitFilter
 from palimpzest.query.operators.topk import TopKOp
 from palimpzest.query.optimizer.primitives import Expression, Group, LogicalExpression, PhysicalExpression
+from palimpzest.utils.model_helpers import resolve_reasoning_settings
 
 logger = logging.getLogger(__name__)
 
@@ -628,15 +629,17 @@ class LLMConvertBondedRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        no_reasoning = runtime_kwargs["reasoning_effort"] in [None, "minimal", "low"]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.MAP_NO_REASONING if model.is_reasoning_model() and no_reasoning else PromptStrategy.MAP,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
-            }
-            for model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = PromptStrategy.MAP if use_reasoning_prompt else PromptStrategy.MAP_NO_REASONING
+            variable_op_kwargs.append(
+                {
+                    "model": model,
+                    "prompt_strategy": prompt_strategy,
+                    "reasoning_effort": reasoning_effort,
+                }
+            )
 
         return cls._perform_substitution(logical_expression, LLMConvertBonded, runtime_kwargs, variable_op_kwargs)
 
@@ -665,18 +668,27 @@ class RAGRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.MAP if phys_op_cls is RAGConvert else PromptStrategy.FILTER,
-                "num_chunks_per_field": num_chunks_per_field,
-                "chunk_size": chunk_size,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
-            }
-            for model in models
-            for num_chunks_per_field in cls.num_chunks_per_fields
-            for chunk_size in cls.chunk_sizes
-        ]
+        variable_op_kwargs = []
+        for model in models:
+             use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+             prompt_strategy = (
+                 PromptStrategy.MAP if use_reasoning_prompt else PromptStrategy.MAP_NO_REASONING
+                 if phys_op_cls is RAGConvert
+                 else PromptStrategy.FILTER if use_reasoning_prompt else PromptStrategy.FILTER_NO_REASONING
+             )
+             variable_op_kwargs.extend(
+                 [
+                    {
+                        "model": model,
+                        "prompt_strategy": prompt_strategy,
+                        "num_chunks_per_field": num_chunks_per_field,
+                        "chunk_size": chunk_size,
+                        "reasoning_effort": reasoning_effort,
+                    }
+                    for num_chunks_per_field in cls.num_chunks_per_fields
+                    for chunk_size in cls.chunk_sizes
+                 ]
+             )
 
         return cls._perform_substitution(logical_expression, phys_op_cls, runtime_kwargs, variable_op_kwargs)
 
@@ -704,6 +716,7 @@ class MixtureOfAgentsRule(ImplementationRule):
         phys_op_cls = MixtureOfAgentsConvert if isinstance(logical_expression.operator, ConvertScan) else MixtureOfAgentsFilter
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
+        _, reasoning_effort = resolve_reasoning_settings(None, runtime_kwargs["reasoning_effort"])
         proposer_model_set = {model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)}
         aggregator_model_set = {model for model in runtime_kwargs["available_models"] if model.is_text_model()}
         variable_op_kwargs = [
@@ -711,7 +724,7 @@ class MixtureOfAgentsRule(ImplementationRule):
                 "proposer_models": list(proposer_models),
                 "temperatures": [temp] * len(proposer_models),
                 "aggregator_model": aggregator_model,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
+                "reasoning_effort": reasoning_effort,
             }
             for k in cls.num_proposer_models
             for temp in cls.temperatures
@@ -743,18 +756,27 @@ class CritiqueAndRefineRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "critic_model": critic_model,
-                "refine_model": refine_model,
-                "prompt_strategy": PromptStrategy.MAP if phys_op_cls is CritiqueAndRefineConvert else PromptStrategy.FILTER,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
-            }
-            for model in models
-            for critic_model in models
-            for refine_model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = (
+                PromptStrategy.MAP if use_reasoning_prompt else PromptStrategy.MAP_NO_REASONING
+                if phys_op_cls is CritiqueAndRefineConvert
+                else PromptStrategy.FILTER if use_reasoning_prompt else PromptStrategy.FILTER_NO_REASONING
+            )
+            variable_op_kwargs.extend(
+                [
+                    {
+                        "model": model,
+                        "critic_model": critic_model,
+                        "refine_model": refine_model,
+                        "prompt_strategy": prompt_strategy,
+                        "reasoning_effort": reasoning_effort,
+                    }
+                    for critic_model in models
+                    for refine_model in models
+                ]
+            )
 
         return cls._perform_substitution(logical_expression, phys_op_cls, runtime_kwargs, variable_op_kwargs)
 
@@ -782,12 +804,13 @@ class SplitRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
+        _, reasoning_effort = resolve_reasoning_settings(None, runtime_kwargs["reasoning_effort"])
         variable_op_kwargs = [
             {
                 "model": model,
                 "min_size_to_chunk": min_size_to_chunk,
                 "num_chunks": num_chunks,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
+                "reasoning_effort": reasoning_effort,
             }
             for model in models
             for min_size_to_chunk in cls.min_size_to_chunk
@@ -855,15 +878,17 @@ class LLMFilterRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        no_reasoning = runtime_kwargs["reasoning_effort"] in [None, "minimal", "low"]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.FILTER_NO_REASONING if model.is_reasoning_model() and no_reasoning else PromptStrategy.FILTER,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"]
-            }
-            for model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = PromptStrategy.FILTER if use_reasoning_prompt else PromptStrategy.FILTER_NO_REASONING
+            variable_op_kwargs.append(
+                {
+                    "model": model,
+                    "prompt_strategy": prompt_strategy,
+                    "reasoning_effort": reasoning_effort,
+                }
+            )
 
         return cls._perform_substitution(logical_expression, LLMFilter, runtime_kwargs, variable_op_kwargs)
 
@@ -902,17 +927,19 @@ class NestedLoopsJoinRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        no_reasoning = runtime_kwargs["reasoning_effort"] in [None, "minimal", "low"]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.JOIN_NO_REASONING if model.is_reasoning_model() and no_reasoning else PromptStrategy.JOIN,
-                "join_parallelism": runtime_kwargs["join_parallelism"],
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
-                "retain_inputs": not runtime_kwargs["is_validation"],
-            }
-            for model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = PromptStrategy.JOIN if use_reasoning_prompt else PromptStrategy.JOIN_NO_REASONING
+            variable_op_kwargs.append(
+                {
+                    "model": model,
+                    "prompt_strategy": prompt_strategy,
+                    "join_parallelism": runtime_kwargs["join_parallelism"],
+                    "reasoning_effort": reasoning_effort,
+                    "retain_inputs": not runtime_kwargs["is_validation"],
+                }
+            )
 
         return cls._perform_substitution(logical_expression, NestedLoopsJoin, runtime_kwargs, variable_op_kwargs)
 
@@ -934,18 +961,20 @@ class EmbeddingJoinRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression)]
-        no_reasoning = runtime_kwargs["reasoning_effort"] in [None, "minimal", "low"]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.JOIN_NO_REASONING if model.is_reasoning_model() and no_reasoning else PromptStrategy.JOIN,
-                "join_parallelism": runtime_kwargs["join_parallelism"],
-                "reasoning_effort": runtime_kwargs["reasoning_effort"],
-                "retain_inputs": not runtime_kwargs["is_validation"],
-                "num_samples": 10, # TODO: iterate over different choices of num_samples
-            }
-            for model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = PromptStrategy.JOIN if use_reasoning_prompt else PromptStrategy.JOIN_NO_REASONING
+            variable_op_kwargs.append(
+                {
+                    "model": model,
+                    "prompt_strategy": prompt_strategy,
+                    "join_parallelism": runtime_kwargs["join_parallelism"],
+                    "reasoning_effort": reasoning_effort,
+                    "retain_inputs": not runtime_kwargs["is_validation"],
+                    "num_samples": 10, # TODO: iterate over different choices of num_samples
+                }
+            )
 
         return cls._perform_substitution(logical_expression, EmbeddingJoin, runtime_kwargs, variable_op_kwargs)
 
@@ -966,15 +995,17 @@ class SemanticAggregateRule(ImplementationRule):
 
         # create variable physical operator kwargs for each model which can implement this logical_expression
         models = [model for model in runtime_kwargs["available_models"] if cls._model_matches_input(model, logical_expression) and not model.is_llama_model()]
-        no_reasoning = runtime_kwargs["reasoning_effort"] in [None, "minimal", "low"]
-        variable_op_kwargs = [
-            {
-                "model": model,
-                "prompt_strategy": PromptStrategy.AGG_NO_REASONING if model.is_reasoning_model() and no_reasoning else PromptStrategy.AGG,
-                "reasoning_effort": runtime_kwargs["reasoning_effort"]
-            }
-            for model in models
-        ]
+        variable_op_kwargs = []
+        for model in models:
+            use_reasoning_prompt, reasoning_effort = resolve_reasoning_settings(model, runtime_kwargs["reasoning_effort"])
+            prompt_strategy = PromptStrategy.AGG if use_reasoning_prompt else PromptStrategy.AGG_NO_REASONING
+            variable_op_kwargs.append(
+                {
+                    "model": model,
+                    "prompt_strategy": prompt_strategy,
+                    "reasoning_effort": reasoning_effort,
+                }
+            )
 
         return cls._perform_substitution(logical_expression, SemanticAggregate, runtime_kwargs, variable_op_kwargs)
 

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/query/processor/config.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pydantic import BaseModel, ConfigDict, Field
 
 from palimpzest.constants import Model
@@ -18,7 +20,7 @@ class QueryProcessorConfig(BaseModel):
     policy: Policy = Field(default_factory=MaxQuality)
     enforce_types: bool = Field(default=False)
     scan_start_idx: int = Field(default=0)
-    num_samples: int = Field(default=None)
+    num_samples: int | None = Field(default=None)
     verbose: bool = Field(default=False)
     progress: bool = Field(default=True)
     available_models: list[Model] | None = Field(default=None)
@@ -26,8 +28,8 @@ class QueryProcessorConfig(BaseModel):
     max_workers: int | None = Field(default=64)
     join_parallelism: int = Field(default=64)
     batch_size: int | None = Field(default=None)
-    reasoning_effort: str | None = Field(default=None)  # Gemini: "disable", "low", "medium", "high"
-    use_vertex: bool = Field(default=True)  # Whether to use Vertex models for Gemini or Google models
+    reasoning_effort: str | None = Field(default="default")  # Gemini: "disable", "low", "medium", "high"
+    use_vertex: bool = Field(default=False)  # Whether to use Vertex models for Gemini or Google models
     gemini_credentials_path: str | None = Field(default=None)  # Path to Gemini credentials file
     api_base: str | None = Field(default=None)  # API base URL for vLLM
 
@@ -53,3 +55,7 @@ class QueryProcessorConfig(BaseModel):
     def to_dict(self) -> dict:
         """Convert the config to a dict representation."""
         return self.model_dump()
+
+    def copy(self) -> QueryProcessorConfig:
+        """Create a copy of the config."""
+        return QueryProcessorConfig(**self.to_dict())

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest/query/processor/query_processor_factory.py

@@ -2,6 +2,8 @@ import logging
 import os
 from enum import Enum
 
+from dotenv import load_dotenv
+
 from palimpzest.core.data.dataset import Dataset
 from palimpzest.core.elements.records import DataRecordCollection
 from palimpzest.query.execution.execution_strategy import ExecutionStrategy, SentinelExecutionStrategy
@@ -108,7 +110,7 @@ class QueryProcessorFactory:
                 raise ValueError("ANTHROPIC_API_KEY must be set to use Anthropic models.")
             if model.is_together_model() and not together_key:
                 raise ValueError("TOGETHER_API_KEY must be set to use Together models.")
-            if model.is_google_model() and not (gemini_key or google_key or config.gemini_credentials_path):
+            if model.is_google_ai_studio_model() and not (gemini_key or google_key or config.gemini_credentials_path):
                 raise ValueError("GEMINI_API_KEY, GOOGLE_API_KEY, or gemini_credentials path must be set to use Google Gemini models.")
             if model.is_vllm_model() and config.api_base is None:
                 raise ValueError("api_base must be set to use vLLM models.")
@@ -168,6 +170,9 @@ class QueryProcessorFactory:
         if config is None:
             config = QueryProcessorConfig()
 
+        # make a copy of the config to avoid modifying the original
+        config = config.copy()
+
         # apply any additional keyword arguments to the config and validate its contents
         config, validator = cls._config_validation_and_normalization(config, train_dataset, validator)
 
@@ -194,6 +199,7 @@ class QueryProcessorFactory:
         train_dataset: dict[str, Dataset] | None = None,
         validator: Validator | None = None,
     ) -> DataRecordCollection:
+        load_dotenv(override=True)
         logger.info(f"Creating processor for dataset: {dataset}")
         processor = cls.create_processor(dataset, config, train_dataset, validator)
         logger.info(f"Created processor: {processor}")

palimpzest-1.3.1/src/palimpzest/utils/model_helpers.py

@@ -0,0 +1,88 @@
+import os
+
+from palimpzest.constants import Model
+
+
+def get_models(include_embedding: bool = False, use_vertex: bool = False, gemini_credentials_path: str | None = None, api_base: str | None = None) -> list[Model]:
+    """
+    Return the set of models which the system has access to based on the set environment variables.
+    """
+    models = []
+    if os.getenv("OPENAI_API_KEY") not in [None, ""]:
+        openai_models = [model for model in Model if model.is_openai_model()]
+        if not include_embedding:
+            openai_models = [
+                model for model in openai_models if not model.is_embedding_model()
+            ]
+        models.extend(openai_models)
+
+    if os.getenv("TOGETHER_API_KEY") not in [None, ""]:
+        together_models = [model for model in Model if model.is_together_model()]
+        if not include_embedding:
+            together_models = [
+                model for model in together_models if not model.is_embedding_model()
+            ]
+        models.extend(together_models)
+
+    if os.getenv("ANTHROPIC_API_KEY") not in [None, ""]:
+        anthropic_models = [model for model in Model if model.is_anthropic_model()]
+        if not include_embedding:
+            anthropic_models = [
+                model for model in anthropic_models if not model.is_embedding_model()
+            ]
+        models.extend(anthropic_models)
+
+    gemini_credentials_path = (
+        os.path.join(os.path.expanduser("~"), ".config", "gcloud", "application_default_credentials.json")
+        if gemini_credentials_path is None
+        else gemini_credentials_path
+    )
+    if os.getenv("GEMINI_API_KEY") not in [None, ""] or (use_vertex and os.path.exists(gemini_credentials_path)):
+        vertex_models = [model for model in Model if model.is_vertex_model()]
+        google_ai_studio_models = [model for model in Model if model.is_google_ai_studio_model()]
+        if not include_embedding:
+            vertex_models = [
+                model for model in vertex_models if not model.is_embedding_model()
+            ]
+        if use_vertex:
+            models.extend(vertex_models)
+        else:
+            models.extend(google_ai_studio_models)
+
+    if api_base is not None:
+        vllm_models = [model for model in Model if model.is_vllm_model()]
+        if not include_embedding:
+            vllm_models = [
+                model for model in vllm_models if not model.is_embedding_model()
+            ]
+        models.extend(vllm_models)
+
+    return models
+
+
+def resolve_reasoning_settings(model: Model | None, reasoning_effort: str | None) -> tuple[bool, str]:
+    """
+    Resolve the reasoning settings based on the model and provided reasoning effort.
+    Returns a tuple indicating whether reasoning prompt should be used and the reasoning effort level.
+    By default, we use the reasoning prompt everywhere while setting the model reasoning effort to None (or minimal).
+    If a user explicitly provides a reasoning_effort, we pass that through to the model.
+    If the user explicitly disables reasoning_effort, we disable the reasoning prompt as well.
+    """
+    # turn off reasoning prompt if reasoning_effort is in [None, "disable", "minimal", "low"]
+    use_reasoning_prompt = reasoning_effort not in [None, "disable", "minimal", "low"]
+
+    # if reasoning_effort is set to "default", set it to None to use model defaults
+    if reasoning_effort == "default":
+        reasoning_effort = None
+
+    # translate reasoning_effort into model-specific settings
+    if model is not None and model.is_reasoning_model():
+        if model.is_vertex_model() or model.is_google_ai_studio_model():
+            if reasoning_effort is None and model in [Model.GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_PRO]:
+                reasoning_effort = "low"
+            elif reasoning_effort is None:
+                reasoning_effort = "disable"
+        elif model.is_openai_model():
+            reasoning_effort = "minimal" if reasoning_effort in [None, "disable", "minimal", "low"] else reasoning_effort
+
+    return use_reasoning_prompt, reasoning_effort

{palimpzest-1.2.0 → palimpzest-1.3.1/src/palimpzest.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: palimpzest
-Version: 1.2.0
+Version: 1.3.1
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
 Project-URL: homepage, https://palimpzest.org
@@ -34,6 +34,7 @@ Requires-Dist: PyLD>=2.0.4
 Requires-Dist: pyarrow>=20.0.0
 Requires-Dist: pypdf>=5.1.0
 Requires-Dist: pytest-mock>=3.14.0
+Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: requests>=2.25
 Requires-Dist: ruff>=0.9.0

{palimpzest-1.2.0 → palimpzest-1.3.1}/src/palimpzest.egg-info/requires.txt

@@ -17,6 +17,7 @@ PyLD>=2.0.4
 pyarrow>=20.0.0
 pypdf>=5.1.0
 pytest-mock>=3.14.0
+python-dotenv>=1.2.1
 pyyaml>=6.0.1
 requests>=2.25
 ruff>=0.9.0

palimpzest-1.2.0/src/palimpzest/utils/model_helpers.py

@@ -1,61 +0,0 @@
-import os
-
-from palimpzest.constants import Model
-
-
-# TODO: better handle vertex vs. google for gemini models
-def get_models(include_embedding: bool = False, use_vertex: bool = True, gemini_credentials_path: str | None = None, api_base: str | None = None) -> list[Model]:
-    """
-    Return the set of models which the system has access to based on the set environment variables.
-    """
-    models = []
-    if os.getenv("OPENAI_API_KEY") is not None:
-        openai_models = [model for model in Model if model.is_openai_model()]
-        if not include_embedding:
-            openai_models = [
-                model for model in openai_models if not model.is_embedding_model()
-            ]
-        models.extend(openai_models)
-
-    if os.getenv("TOGETHER_API_KEY") is not None:
-        together_models = [model for model in Model if model.is_together_model()]
-        if not include_embedding:
-            together_models = [
-                model for model in together_models if not model.is_embedding_model()
-            ]
-        models.extend(together_models)
-
-    if os.getenv("ANTHROPIC_API_KEY") is not None:
-        anthropic_models = [model for model in Model if model.is_anthropic_model()]
-        if not include_embedding:
-            anthropic_models = [
-                model for model in anthropic_models if not model.is_embedding_model()
-            ]
-        models.extend(anthropic_models)
-
-    gemini_credentials_path = (
-        os.path.join(os.path.expanduser("~"), ".config", "gcloud", "application_default_credentials.json")
-        if gemini_credentials_path is None
-        else gemini_credentials_path
-    )
-    if os.getenv("GEMINI_API_KEY") is not None or os.path.exists(gemini_credentials_path):
-        vertex_models = [model for model in Model if model.is_vertex_model()]
-        google_models = [model for model in Model if model.is_google_model()]
-        if not include_embedding:
-            vertex_models = [
-                model for model in vertex_models if not model.is_embedding_model()
-            ]
-        if use_vertex:
-            models.extend(vertex_models)
-        else:
-            models.extend(google_models)
-
-    if api_base is not None:
-        vllm_models = [model for model in Model if model.is_vllm_model()]
-        if not include_embedding:
-            vllm_models = [
-                model for model in vllm_models if not model.is_embedding_model()
-            ]
-        models.extend(vllm_models)
-
-    return models