palimpzest 0.7.21__tar.gz → 0.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {palimpzest-0.7.21/src/palimpzest.egg-info → palimpzest-0.8.1}/PKG-INFO +6 -1
  2. {palimpzest-0.7.21 → palimpzest-0.8.1}/pyproject.toml +7 -1
  3. palimpzest-0.8.1/src/palimpzest/__init__.py +72 -0
  4. palimpzest-0.8.1/src/palimpzest/agents/search_agents.py +637 -0
  5. palimpzest-0.8.1/src/palimpzest/constants.py +650 -0
  6. palimpzest-0.8.1/src/palimpzest/core/data/context.py +393 -0
  7. palimpzest-0.8.1/src/palimpzest/core/data/context_manager.py +163 -0
  8. palimpzest-0.8.1/src/palimpzest/core/data/dataset.py +639 -0
  9. palimpzest-0.7.21/src/palimpzest/core/data/datareaders.py → palimpzest-0.8.1/src/palimpzest/core/data/iter_dataset.py +202 -126
  10. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/core/elements/groupbysig.py +16 -13
  11. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/core/elements/records.py +166 -75
  12. palimpzest-0.8.1/src/palimpzest/core/lib/schemas.py +205 -0
  13. palimpzest-0.7.21/src/palimpzest/core/data/dataclasses.py → palimpzest-0.8.1/src/palimpzest/core/models.py +306 -170
  14. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/policy.py +2 -27
  15. palimpzest-0.8.1/src/palimpzest/prompts/__init__.py +52 -0
  16. palimpzest-0.8.1/src/palimpzest/prompts/agent_prompts.py +357 -0
  17. palimpzest-0.8.1/src/palimpzest/prompts/context_search.py +9 -0
  18. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/convert_prompts.py +62 -6
  19. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/filter_prompts.py +51 -6
  20. palimpzest-0.8.1/src/palimpzest/prompts/join_prompts.py +163 -0
  21. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/moa_proposer_convert_prompts.py +6 -6
  22. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/prompt_factory.py +375 -47
  23. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/split_proposer_prompts.py +1 -1
  24. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/util_phrases.py +5 -0
  25. palimpzest-0.8.1/src/palimpzest/prompts/validator.py +239 -0
  26. palimpzest-0.8.1/src/palimpzest/query/execution/all_sample_execution_strategy.py +274 -0
  27. palimpzest-0.8.1/src/palimpzest/query/execution/execution_strategy.py +346 -0
  28. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/execution/execution_strategy_type.py +5 -7
  29. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/execution/mab_execution_strategy.py +249 -136
  30. palimpzest-0.8.1/src/palimpzest/query/execution/parallel_execution_strategy.py +261 -0
  31. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/execution/single_threaded_execution_strategy.py +107 -64
  32. palimpzest-0.8.1/src/palimpzest/query/generators/generators.py +438 -0
  33. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/__init__.py +15 -5
  34. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/aggregate.py +50 -33
  35. palimpzest-0.8.1/src/palimpzest/query/operators/compute.py +201 -0
  36. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/convert.py +33 -19
  37. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/critique_and_refine_convert.py +7 -5
  38. palimpzest-0.8.1/src/palimpzest/query/operators/distinct.py +62 -0
  39. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/filter.py +26 -16
  40. palimpzest-0.8.1/src/palimpzest/query/operators/join.py +403 -0
  41. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/limit.py +3 -3
  42. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/logical.py +205 -77
  43. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/mixture_of_agents_convert.py +10 -8
  44. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/physical.py +27 -21
  45. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/project.py +3 -3
  46. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/rag_convert.py +7 -7
  47. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/retrieve.py +9 -9
  48. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/scan.py +81 -42
  49. palimpzest-0.8.1/src/palimpzest/query/operators/search.py +524 -0
  50. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/operators/split_convert.py +10 -8
  51. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/__init__.py +7 -9
  52. palimpzest-0.8.1/src/palimpzest/query/optimizer/cost_model.py +262 -0
  53. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/optimizer.py +123 -181
  54. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/optimizer_strategy.py +66 -61
  55. palimpzest-0.8.1/src/palimpzest/query/optimizer/plan.py +453 -0
  56. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/primitives.py +43 -19
  57. palimpzest-0.8.1/src/palimpzest/query/optimizer/rules.py +901 -0
  58. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/tasks.py +127 -58
  59. palimpzest-0.8.1/src/palimpzest/query/processor/config.py +52 -0
  60. palimpzest-0.8.1/src/palimpzest/query/processor/query_processor.py +152 -0
  61. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/processor/query_processor_factory.py +46 -38
  62. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/schemabuilder/schema_builder.py +15 -28
  63. palimpzest-0.8.1/src/palimpzest/tools/__init__.py +0 -0
  64. palimpzest-0.8.1/src/palimpzest/utils/__init__.py +0 -0
  65. palimpzest-0.8.1/src/palimpzest/utils/model_helpers.py +61 -0
  66. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/utils/progress.py +114 -102
  67. palimpzest-0.8.1/src/palimpzest/validator/__init__.py +0 -0
  68. palimpzest-0.8.1/src/palimpzest/validator/validator.py +306 -0
  69. {palimpzest-0.7.21 → palimpzest-0.8.1/src/palimpzest.egg-info}/PKG-INFO +6 -1
  70. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest.egg-info/SOURCES.txt +20 -20
  71. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest.egg-info/requires.txt +6 -0
  72. palimpzest-0.7.21/src/palimpzest/__init__.py +0 -41
  73. palimpzest-0.7.21/src/palimpzest/constants.py +0 -516
  74. palimpzest-0.7.21/src/palimpzest/core/lib/fields.py +0 -141
  75. palimpzest-0.7.21/src/palimpzest/core/lib/schemas.py +0 -443
  76. palimpzest-0.7.21/src/palimpzest/prompts/__init__.py +0 -22
  77. palimpzest-0.7.21/src/palimpzest/prompts/code_synthesis_prompts.py +0 -28
  78. palimpzest-0.7.21/src/palimpzest/query/execution/all_sample_execution_strategy.py +0 -216
  79. palimpzest-0.7.21/src/palimpzest/query/execution/execution_strategy.py +0 -453
  80. palimpzest-0.7.21/src/palimpzest/query/execution/parallel_execution_strategy.py +0 -352
  81. palimpzest-0.7.21/src/palimpzest/query/execution/random_sampling_execution_strategy.py +0 -240
  82. palimpzest-0.7.21/src/palimpzest/query/generators/api_client_factory.py +0 -30
  83. palimpzest-0.7.21/src/palimpzest/query/generators/generators.py +0 -609
  84. palimpzest-0.7.21/src/palimpzest/query/operators/code_synthesis_convert.py +0 -488
  85. palimpzest-0.7.21/src/palimpzest/query/operators/map.py +0 -130
  86. palimpzest-0.7.21/src/palimpzest/query/optimizer/cost_model.py +0 -595
  87. palimpzest-0.7.21/src/palimpzest/query/optimizer/plan.py +0 -168
  88. palimpzest-0.7.21/src/palimpzest/query/optimizer/rules.py +0 -1063
  89. palimpzest-0.7.21/src/palimpzest/query/processor/config.py +0 -86
  90. palimpzest-0.7.21/src/palimpzest/query/processor/nosentinel_processor.py +0 -33
  91. palimpzest-0.7.21/src/palimpzest/query/processor/processing_strategy_type.py +0 -28
  92. palimpzest-0.7.21/src/palimpzest/query/processor/query_processor.py +0 -97
  93. palimpzest-0.7.21/src/palimpzest/query/processor/sentinel_processor.py +0 -88
  94. palimpzest-0.7.21/src/palimpzest/query/processor/streaming_processor.py +0 -149
  95. palimpzest-0.7.21/src/palimpzest/sets.py +0 -405
  96. palimpzest-0.7.21/src/palimpzest/utils/datareader_helpers.py +0 -61
  97. palimpzest-0.7.21/src/palimpzest/utils/demo_helpers.py +0 -75
  98. palimpzest-0.7.21/src/palimpzest/utils/field_helpers.py +0 -69
  99. palimpzest-0.7.21/src/palimpzest/utils/generation_helpers.py +0 -69
  100. palimpzest-0.7.21/src/palimpzest/utils/model_helpers.py +0 -106
  101. palimpzest-0.7.21/src/palimpzest/utils/sandbox.py +0 -183
  102. {palimpzest-0.7.21 → palimpzest-0.8.1}/LICENSE +0 -0
  103. {palimpzest-0.7.21 → palimpzest-0.8.1}/README.md +0 -0
  104. {palimpzest-0.7.21 → palimpzest-0.8.1}/setup.cfg +0 -0
  105. {palimpzest-0.7.21/src/palimpzest/core → palimpzest-0.8.1/src/palimpzest/agents}/__init__.py +0 -0
  106. /palimpzest-0.7.21/src/palimpzest/core/data/__init__.py → /palimpzest-0.8.1/src/palimpzest/agents/compute_agents.py +0 -0
  107. {palimpzest-0.7.21/src/palimpzest/core/elements → palimpzest-0.8.1/src/palimpzest/core}/__init__.py +0 -0
  108. {palimpzest-0.7.21/src/palimpzest/core/lib → palimpzest-0.8.1/src/palimpzest/core/data}/__init__.py +0 -0
  109. /palimpzest-0.7.21/src/palimpzest/core/elements/index.py → /palimpzest-0.8.1/src/palimpzest/core/data/index_dataset.py +0 -0
  110. {palimpzest-0.7.21/src/palimpzest/query → palimpzest-0.8.1/src/palimpzest/core/elements}/__init__.py +0 -0
  111. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/core/elements/filters.py +0 -0
  112. {palimpzest-0.7.21/src/palimpzest/query/execution → palimpzest-0.8.1/src/palimpzest/core/lib}/__init__.py +0 -0
  113. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/critique_and_refine_convert_prompts.py +0 -0
  114. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/moa_aggregator_convert_prompts.py +0 -0
  115. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/prompts/split_merge_prompts.py +0 -0
  116. {palimpzest-0.7.21/src/palimpzest/query/generators → palimpzest-0.8.1/src/palimpzest/query}/__init__.py +0 -0
  117. {palimpzest-0.7.21/src/palimpzest/query/processor → palimpzest-0.8.1/src/palimpzest/query/execution}/__init__.py +0 -0
  118. {palimpzest-0.7.21/src/palimpzest/schemabuilder → palimpzest-0.8.1/src/palimpzest/query/generators}/__init__.py +0 -0
  119. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/query/optimizer/optimizer_strategy_type.py +0 -0
  120. {palimpzest-0.7.21/src/palimpzest/tools → palimpzest-0.8.1/src/palimpzest/query/processor}/__init__.py +0 -0
  121. {palimpzest-0.7.21/src/palimpzest/utils → palimpzest-0.8.1/src/palimpzest/schemabuilder}/__init__.py +0 -0
  122. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/tools/README.md +0 -0
  123. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/tools/allenpdf.py +0 -0
  124. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/tools/pdfparser.py +0 -0
  125. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/tools/skema_tools.py +0 -0
  126. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/utils/env_helpers.py +0 -0
  127. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/utils/hash_helpers.py +0 -0
  128. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest/utils/udfs.py +0 -0
  129. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest.egg-info/dependency_links.txt +0 -0
  130. {palimpzest-0.7.21 → palimpzest-0.8.1}/src/palimpzest.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: palimpzest
3
- Version: 0.7.21
3
+ Version: 0.8.1
4
4
  Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
5
5
  Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
6
6
  Project-URL: homepage, https://palimpzest.org
@@ -15,12 +15,14 @@ Classifier: Programming Language :: Python :: 3.8
15
15
  Requires-Python: >=3.8
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE
18
+ Requires-Dist: anthropic>=0.55.0
18
19
  Requires-Dist: beautifulsoup4>=4.13.4
19
20
  Requires-Dist: chromadb>=1.0.15
20
21
  Requires-Dist: colorama>=0.4.6
21
22
  Requires-Dist: datasets>=4.0.0
22
23
  Requires-Dist: fastapi~=0.115.0
23
24
  Requires-Dist: gradio>=5.26.0
25
+ Requires-Dist: litellm>=1.73.1
24
26
  Requires-Dist: numpy==2.0.2
25
27
  Requires-Dist: openai>=1.0
26
28
  Requires-Dist: pandas>=2.1.1
@@ -37,6 +39,7 @@ Requires-Dist: requests>=2.25
37
39
  Requires-Dist: ruff>=0.9.0
38
40
  Requires-Dist: sentence-transformers==5.0.0
39
41
  Requires-Dist: setuptools>=70.1.1
42
+ Requires-Dist: smolagents[toolkit]
40
43
  Requires-Dist: tabulate>=0.9.0
41
44
  Requires-Dist: together>=1.5.5
42
45
  Requires-Dist: tqdm~=4.66.1
@@ -46,6 +49,8 @@ Requires-Dist: mkdocs>=1.6.1; extra == "docs"
46
49
  Requires-Dist: mkdocs-material>=9.6.3; extra == "docs"
47
50
  Requires-Dist: mkdocstrings-python>=1.15.0; extra == "docs"
48
51
  Requires-Dist: mkdocs-material[imaging]; extra == "docs"
52
+ Provides-Extra: vllm
53
+ Requires-Dist: vllm>=0.10.1.1; extra == "vllm"
49
54
  Dynamic: license-file
50
55
 
51
56
  ![pz-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "palimpzest"
3
- version = "0.7.21"
3
+ version = "0.8.1"
4
4
  description = "Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.8"
@@ -9,12 +9,14 @@ authors = [
9
9
  {name="MIT DSG Semantic Management Lab", email="michjc@csail.mit.edu"},
10
10
  ]
11
11
  dependencies = [
12
+ "anthropic>=0.55.0",
12
13
  "beautifulsoup4>=4.13.4",
13
14
  "chromadb>=1.0.15",
14
15
  "colorama>=0.4.6",
15
16
  "datasets>=4.0.0",
16
17
  "fastapi~=0.115.0",
17
18
  "gradio>=5.26.0",
19
+ "litellm>=1.73.1",
18
20
  "numpy==2.0.2",
19
21
  "openai>=1.0",
20
22
  "pandas>=2.1.1",
@@ -31,6 +33,7 @@ dependencies = [
31
33
  "ruff>=0.9.0",
32
34
  "sentence-transformers==5.0.0",
33
35
  "setuptools>=70.1.1",
36
+ "smolagents[toolkit]",
34
37
  "tabulate>=0.9.0",
35
38
  "together>=1.5.5",
36
39
  "tqdm~=4.66.1",
@@ -52,6 +55,9 @@ docs = [
52
55
  "mkdocstrings-python>=1.15.0",
53
56
  "mkdocs-material[imaging]",
54
57
  ]
58
+ vllm = [
59
+ "vllm>=0.10.1.1",
60
+ ]
55
61
 
56
62
  [tool.setuptools]
57
63
  package-dir = {"" = "src"}
@@ -0,0 +1,72 @@
1
+ import logging
2
+
3
+ from palimpzest.constants import Cardinality, Model
4
+ from palimpzest.core.data.context import Context, TextFileContext
5
+ from palimpzest.core.data.dataset import Dataset
6
+ from palimpzest.core.data.iter_dataset import (
7
+ AudioFileDataset,
8
+ HTMLFileDataset,
9
+ ImageFileDataset,
10
+ IterDataset,
11
+ MemoryDataset,
12
+ PDFFileDataset,
13
+ TextFileDataset,
14
+ XLSFileDataset,
15
+ )
16
+ from palimpzest.core.elements.groupbysig import GroupBySig
17
+ from palimpzest.core.lib.schemas import AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL
18
+ from palimpzest.policy import (
19
+ MaxQuality,
20
+ MaxQualityAtFixedCost,
21
+ MaxQualityAtFixedTime,
22
+ MinCost,
23
+ MinCostAtFixedQuality,
24
+ MinTime,
25
+ MinTimeAtFixedQuality,
26
+ PlanCost,
27
+ Policy,
28
+ )
29
+ from palimpzest.query.processor.config import QueryProcessorConfig
30
+ from palimpzest.validator.validator import Validator
31
+
32
+ # Initialize the root logger
33
+ logging.getLogger(__name__).addHandler(logging.NullHandler())
34
+
35
+ __all__ = [
36
+ # constants
37
+ "Cardinality",
38
+ "Model",
39
+ # core
40
+ "GroupBySig",
41
+ "Context",
42
+ "TextFileContext",
43
+ "Dataset",
44
+ "IterDataset",
45
+ "AudioFileDataset",
46
+ "MemoryDataset",
47
+ "HTMLFileDataset",
48
+ "ImageFileDataset",
49
+ "PDFFileDataset",
50
+ "TextFileDataset",
51
+ "XLSFileDataset",
52
+ # schemas
53
+ "AudioBase64",
54
+ "AudioFilepath",
55
+ "ImageBase64",
56
+ "ImageFilepath",
57
+ "ImageURL",
58
+ # policy
59
+ "MaxQuality",
60
+ "MaxQualityAtFixedCost",
61
+ "MaxQualityAtFixedTime",
62
+ "MinCost",
63
+ "MinCostAtFixedQuality",
64
+ "MinTime",
65
+ "MinTimeAtFixedQuality",
66
+ "PlanCost",
67
+ "Policy",
68
+ # query
69
+ "QueryProcessorConfig",
70
+ # validator
71
+ "Validator",
72
+ ]