palimpzest 0.7.21__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {palimpzest-0.7.21/src/palimpzest.egg-info → palimpzest-0.8.0}/PKG-INFO +6 -1
- {palimpzest-0.7.21 → palimpzest-0.8.0}/pyproject.toml +7 -1
- palimpzest-0.8.0/src/palimpzest/__init__.py +72 -0
- palimpzest-0.8.0/src/palimpzest/agents/search_agents.py +637 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/constants.py +259 -197
- palimpzest-0.8.0/src/palimpzest/core/data/context.py +393 -0
- palimpzest-0.8.0/src/palimpzest/core/data/context_manager.py +163 -0
- palimpzest-0.8.0/src/palimpzest/core/data/dataset.py +634 -0
- palimpzest-0.7.21/src/palimpzest/core/data/datareaders.py → palimpzest-0.8.0/src/palimpzest/core/data/iter_dataset.py +202 -126
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/core/elements/groupbysig.py +16 -13
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/core/elements/records.py +166 -75
- palimpzest-0.8.0/src/palimpzest/core/lib/schemas.py +205 -0
- palimpzest-0.7.21/src/palimpzest/core/data/dataclasses.py → palimpzest-0.8.0/src/palimpzest/core/models.py +306 -170
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/policy.py +2 -27
- palimpzest-0.8.0/src/palimpzest/prompts/__init__.py +52 -0
- palimpzest-0.8.0/src/palimpzest/prompts/agent_prompts.py +357 -0
- palimpzest-0.8.0/src/palimpzest/prompts/context_search.py +9 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/convert_prompts.py +61 -5
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/filter_prompts.py +50 -5
- palimpzest-0.8.0/src/palimpzest/prompts/join_prompts.py +163 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/moa_proposer_convert_prompts.py +5 -5
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/prompt_factory.py +358 -46
- palimpzest-0.8.0/src/palimpzest/prompts/validator.py +239 -0
- palimpzest-0.8.0/src/palimpzest/query/execution/all_sample_execution_strategy.py +274 -0
- palimpzest-0.8.0/src/palimpzest/query/execution/execution_strategy.py +346 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/execution/execution_strategy_type.py +5 -7
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/execution/mab_execution_strategy.py +249 -136
- palimpzest-0.8.0/src/palimpzest/query/execution/parallel_execution_strategy.py +261 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/execution/single_threaded_execution_strategy.py +107 -64
- palimpzest-0.8.0/src/palimpzest/query/generators/generators.py +436 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/__init__.py +15 -5
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/aggregate.py +50 -33
- palimpzest-0.8.0/src/palimpzest/query/operators/compute.py +201 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/convert.py +27 -21
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/critique_and_refine_convert.py +7 -5
- palimpzest-0.8.0/src/palimpzest/query/operators/distinct.py +62 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/filter.py +22 -13
- palimpzest-0.8.0/src/palimpzest/query/operators/join.py +402 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/limit.py +3 -3
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/logical.py +198 -80
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/mixture_of_agents_convert.py +10 -8
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/physical.py +27 -21
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/project.py +3 -3
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/rag_convert.py +7 -7
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/retrieve.py +9 -9
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/scan.py +81 -42
- palimpzest-0.8.0/src/palimpzest/query/operators/search.py +524 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/operators/split_convert.py +10 -8
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/__init__.py +7 -9
- palimpzest-0.8.0/src/palimpzest/query/optimizer/cost_model.py +262 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/optimizer.py +123 -181
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/optimizer_strategy.py +66 -61
- palimpzest-0.8.0/src/palimpzest/query/optimizer/plan.py +453 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/primitives.py +43 -19
- palimpzest-0.8.0/src/palimpzest/query/optimizer/rules.py +901 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/tasks.py +127 -58
- palimpzest-0.8.0/src/palimpzest/query/processor/config.py +51 -0
- palimpzest-0.8.0/src/palimpzest/query/processor/query_processor.py +152 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/processor/query_processor_factory.py +46 -38
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/schemabuilder/schema_builder.py +15 -28
- palimpzest-0.8.0/src/palimpzest/tools/__init__.py +0 -0
- palimpzest-0.8.0/src/palimpzest/utils/__init__.py +0 -0
- palimpzest-0.8.0/src/palimpzest/utils/model_helpers.py +56 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/utils/progress.py +114 -102
- palimpzest-0.8.0/src/palimpzest/validator/__init__.py +0 -0
- palimpzest-0.8.0/src/palimpzest/validator/validator.py +306 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0/src/palimpzest.egg-info}/PKG-INFO +6 -1
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest.egg-info/SOURCES.txt +20 -20
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest.egg-info/requires.txt +6 -0
- palimpzest-0.7.21/src/palimpzest/__init__.py +0 -41
- palimpzest-0.7.21/src/palimpzest/core/lib/fields.py +0 -141
- palimpzest-0.7.21/src/palimpzest/core/lib/schemas.py +0 -443
- palimpzest-0.7.21/src/palimpzest/prompts/__init__.py +0 -22
- palimpzest-0.7.21/src/palimpzest/prompts/code_synthesis_prompts.py +0 -28
- palimpzest-0.7.21/src/palimpzest/query/execution/all_sample_execution_strategy.py +0 -216
- palimpzest-0.7.21/src/palimpzest/query/execution/execution_strategy.py +0 -453
- palimpzest-0.7.21/src/palimpzest/query/execution/parallel_execution_strategy.py +0 -352
- palimpzest-0.7.21/src/palimpzest/query/execution/random_sampling_execution_strategy.py +0 -240
- palimpzest-0.7.21/src/palimpzest/query/generators/api_client_factory.py +0 -30
- palimpzest-0.7.21/src/palimpzest/query/generators/generators.py +0 -609
- palimpzest-0.7.21/src/palimpzest/query/operators/code_synthesis_convert.py +0 -488
- palimpzest-0.7.21/src/palimpzest/query/operators/map.py +0 -130
- palimpzest-0.7.21/src/palimpzest/query/optimizer/cost_model.py +0 -595
- palimpzest-0.7.21/src/palimpzest/query/optimizer/plan.py +0 -168
- palimpzest-0.7.21/src/palimpzest/query/optimizer/rules.py +0 -1063
- palimpzest-0.7.21/src/palimpzest/query/processor/config.py +0 -86
- palimpzest-0.7.21/src/palimpzest/query/processor/nosentinel_processor.py +0 -33
- palimpzest-0.7.21/src/palimpzest/query/processor/processing_strategy_type.py +0 -28
- palimpzest-0.7.21/src/palimpzest/query/processor/query_processor.py +0 -97
- palimpzest-0.7.21/src/palimpzest/query/processor/sentinel_processor.py +0 -88
- palimpzest-0.7.21/src/palimpzest/query/processor/streaming_processor.py +0 -149
- palimpzest-0.7.21/src/palimpzest/sets.py +0 -405
- palimpzest-0.7.21/src/palimpzest/utils/datareader_helpers.py +0 -61
- palimpzest-0.7.21/src/palimpzest/utils/demo_helpers.py +0 -75
- palimpzest-0.7.21/src/palimpzest/utils/field_helpers.py +0 -69
- palimpzest-0.7.21/src/palimpzest/utils/generation_helpers.py +0 -69
- palimpzest-0.7.21/src/palimpzest/utils/model_helpers.py +0 -106
- palimpzest-0.7.21/src/palimpzest/utils/sandbox.py +0 -183
- {palimpzest-0.7.21 → palimpzest-0.8.0}/LICENSE +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/README.md +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/setup.cfg +0 -0
- {palimpzest-0.7.21/src/palimpzest/core → palimpzest-0.8.0/src/palimpzest/agents}/__init__.py +0 -0
- /palimpzest-0.7.21/src/palimpzest/core/data/__init__.py → /palimpzest-0.8.0/src/palimpzest/agents/compute_agents.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/core/elements → palimpzest-0.8.0/src/palimpzest/core}/__init__.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/core/lib → palimpzest-0.8.0/src/palimpzest/core/data}/__init__.py +0 -0
- /palimpzest-0.7.21/src/palimpzest/core/elements/index.py → /palimpzest-0.8.0/src/palimpzest/core/data/index_dataset.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/query → palimpzest-0.8.0/src/palimpzest/core/elements}/__init__.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/core/elements/filters.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/query/execution → palimpzest-0.8.0/src/palimpzest/core/lib}/__init__.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/critique_and_refine_convert_prompts.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/moa_aggregator_convert_prompts.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/split_merge_prompts.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/split_proposer_prompts.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/prompts/util_phrases.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/query/generators → palimpzest-0.8.0/src/palimpzest/query}/__init__.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/query/processor → palimpzest-0.8.0/src/palimpzest/query/execution}/__init__.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/schemabuilder → palimpzest-0.8.0/src/palimpzest/query/generators}/__init__.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/query/optimizer/optimizer_strategy_type.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/tools → palimpzest-0.8.0/src/palimpzest/query/processor}/__init__.py +0 -0
- {palimpzest-0.7.21/src/palimpzest/utils → palimpzest-0.8.0/src/palimpzest/schemabuilder}/__init__.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/tools/README.md +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/tools/allenpdf.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/tools/pdfparser.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/tools/skema_tools.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/utils/env_helpers.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/utils/hash_helpers.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest/utils/udfs.py +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest.egg-info/dependency_links.txt +0 -0
- {palimpzest-0.7.21 → palimpzest-0.8.0}/src/palimpzest.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: palimpzest
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
|
|
5
5
|
Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
|
|
6
6
|
Project-URL: homepage, https://palimpzest.org
|
|
@@ -15,12 +15,14 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
15
15
|
Requires-Python: >=3.8
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE
|
|
18
|
+
Requires-Dist: anthropic>=0.55.0
|
|
18
19
|
Requires-Dist: beautifulsoup4>=4.13.4
|
|
19
20
|
Requires-Dist: chromadb>=1.0.15
|
|
20
21
|
Requires-Dist: colorama>=0.4.6
|
|
21
22
|
Requires-Dist: datasets>=4.0.0
|
|
22
23
|
Requires-Dist: fastapi~=0.115.0
|
|
23
24
|
Requires-Dist: gradio>=5.26.0
|
|
25
|
+
Requires-Dist: litellm>=1.73.1
|
|
24
26
|
Requires-Dist: numpy==2.0.2
|
|
25
27
|
Requires-Dist: openai>=1.0
|
|
26
28
|
Requires-Dist: pandas>=2.1.1
|
|
@@ -37,6 +39,7 @@ Requires-Dist: requests>=2.25
|
|
|
37
39
|
Requires-Dist: ruff>=0.9.0
|
|
38
40
|
Requires-Dist: sentence-transformers==5.0.0
|
|
39
41
|
Requires-Dist: setuptools>=70.1.1
|
|
42
|
+
Requires-Dist: smolagents[toolkit]
|
|
40
43
|
Requires-Dist: tabulate>=0.9.0
|
|
41
44
|
Requires-Dist: together>=1.5.5
|
|
42
45
|
Requires-Dist: tqdm~=4.66.1
|
|
@@ -46,6 +49,8 @@ Requires-Dist: mkdocs>=1.6.1; extra == "docs"
|
|
|
46
49
|
Requires-Dist: mkdocs-material>=9.6.3; extra == "docs"
|
|
47
50
|
Requires-Dist: mkdocstrings-python>=1.15.0; extra == "docs"
|
|
48
51
|
Requires-Dist: mkdocs-material[imaging]; extra == "docs"
|
|
52
|
+
Provides-Extra: vllm
|
|
53
|
+
Requires-Dist: vllm>=0.10.1.1; extra == "vllm"
|
|
49
54
|
Dynamic: license-file
|
|
50
55
|
|
|
51
56
|

|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "palimpzest"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.8.0"
|
|
4
4
|
description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.8"
|
|
@@ -9,12 +9,14 @@ authors = [
|
|
|
9
9
|
{name="MIT DSG Semantic Management Lab", email="michjc@csail.mit.edu"},
|
|
10
10
|
]
|
|
11
11
|
dependencies = [
|
|
12
|
+
"anthropic>=0.55.0",
|
|
12
13
|
"beautifulsoup4>=4.13.4",
|
|
13
14
|
"chromadb>=1.0.15",
|
|
14
15
|
"colorama>=0.4.6",
|
|
15
16
|
"datasets>=4.0.0",
|
|
16
17
|
"fastapi~=0.115.0",
|
|
17
18
|
"gradio>=5.26.0",
|
|
19
|
+
"litellm>=1.73.1",
|
|
18
20
|
"numpy==2.0.2",
|
|
19
21
|
"openai>=1.0",
|
|
20
22
|
"pandas>=2.1.1",
|
|
@@ -31,6 +33,7 @@ dependencies = [
|
|
|
31
33
|
"ruff>=0.9.0",
|
|
32
34
|
"sentence-transformers==5.0.0",
|
|
33
35
|
"setuptools>=70.1.1",
|
|
36
|
+
"smolagents[toolkit]",
|
|
34
37
|
"tabulate>=0.9.0",
|
|
35
38
|
"together>=1.5.5",
|
|
36
39
|
"tqdm~=4.66.1",
|
|
@@ -52,6 +55,9 @@ docs = [
|
|
|
52
55
|
"mkdocstrings-python>=1.15.0",
|
|
53
56
|
"mkdocs-material[imaging]",
|
|
54
57
|
]
|
|
58
|
+
vllm = [
|
|
59
|
+
"vllm>=0.10.1.1",
|
|
60
|
+
]
|
|
55
61
|
|
|
56
62
|
[tool.setuptools]
|
|
57
63
|
package-dir = {"" = "src"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from palimpzest.constants import Cardinality, Model
|
|
4
|
+
from palimpzest.core.data.context import Context, TextFileContext
|
|
5
|
+
from palimpzest.core.data.dataset import Dataset
|
|
6
|
+
from palimpzest.core.data.iter_dataset import (
|
|
7
|
+
AudioFileDataset,
|
|
8
|
+
HTMLFileDataset,
|
|
9
|
+
ImageFileDataset,
|
|
10
|
+
IterDataset,
|
|
11
|
+
MemoryDataset,
|
|
12
|
+
PDFFileDataset,
|
|
13
|
+
TextFileDataset,
|
|
14
|
+
XLSFileDataset,
|
|
15
|
+
)
|
|
16
|
+
from palimpzest.core.elements.groupbysig import GroupBySig
|
|
17
|
+
from palimpzest.core.lib.schemas import AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL
|
|
18
|
+
from palimpzest.policy import (
|
|
19
|
+
MaxQuality,
|
|
20
|
+
MaxQualityAtFixedCost,
|
|
21
|
+
MaxQualityAtFixedTime,
|
|
22
|
+
MinCost,
|
|
23
|
+
MinCostAtFixedQuality,
|
|
24
|
+
MinTime,
|
|
25
|
+
MinTimeAtFixedQuality,
|
|
26
|
+
PlanCost,
|
|
27
|
+
Policy,
|
|
28
|
+
)
|
|
29
|
+
from palimpzest.query.processor.config import QueryProcessorConfig
|
|
30
|
+
from palimpzest.validator.validator import Validator
|
|
31
|
+
|
|
32
|
+
# Initialize the root logger
|
|
33
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
# constants
|
|
37
|
+
"Cardinality",
|
|
38
|
+
"Model",
|
|
39
|
+
# core
|
|
40
|
+
"GroupBySig",
|
|
41
|
+
"Context",
|
|
42
|
+
"TextFileContext",
|
|
43
|
+
"Dataset",
|
|
44
|
+
"IterDataset",
|
|
45
|
+
"AudioFileDataset",
|
|
46
|
+
"MemoryDataset",
|
|
47
|
+
"HTMLFileDataset",
|
|
48
|
+
"ImageFileDataset",
|
|
49
|
+
"PDFFileDataset",
|
|
50
|
+
"TextFileDataset",
|
|
51
|
+
"XLSFileDataset",
|
|
52
|
+
# schemas
|
|
53
|
+
"AudioBase64",
|
|
54
|
+
"AudioFilepath",
|
|
55
|
+
"ImageBase64",
|
|
56
|
+
"ImageFilepath",
|
|
57
|
+
"ImageURL",
|
|
58
|
+
# policy
|
|
59
|
+
"MaxQuality",
|
|
60
|
+
"MaxQualityAtFixedCost",
|
|
61
|
+
"MaxQualityAtFixedTime",
|
|
62
|
+
"MinCost",
|
|
63
|
+
"MinCostAtFixedQuality",
|
|
64
|
+
"MinTime",
|
|
65
|
+
"MinTimeAtFixedQuality",
|
|
66
|
+
"PlanCost",
|
|
67
|
+
"Policy",
|
|
68
|
+
# query
|
|
69
|
+
"QueryProcessorConfig",
|
|
70
|
+
# validator
|
|
71
|
+
"Validator",
|
|
72
|
+
]
|