palimpzest 0.5.3__py3-none-any.whl → 0.6.0__py3-none-any.whl

palimpzest/__init__.py

@@ -1,7 +1,5 @@
-from palimpzest.constants import MAX_ROWS, Cardinality
-
-# data management
-from palimpzest.datamanager.datamanager import DataDirectory
+from palimpzest.constants import Cardinality
+from palimpzest.core.data.datareaders import DataReader
 from palimpzest.policy import (
     MaxQuality,
     MaxQualityAtFixedCost,
@@ -13,16 +11,14 @@ from palimpzest.policy import (
     PlanCost,
     Policy,
 )
-
-# dataset functionality
+from palimpzest.query.processor.config import QueryProcessorConfig
 from palimpzest.sets import Dataset
 
 __all__ = [
     # constants
-    "MAX_ROWS",
     "Cardinality",
-    # datamanager
-    "DataDirectory",
+    # core
+    "DataReader",
     # policy
     "MaxQuality",
     "MaxQualityAtFixedCost",
@@ -33,6 +29,8 @@ __all__ = [
     "MinTimeAtFixedQuality",
     "PlanCost",
     "Policy",
+    # query
+    "QueryProcessorConfig",
     # sets
     "Dataset",
 ]

palimpzest/constants.py

@@ -27,14 +27,52 @@ class PromptStrategy(str, Enum):
     PromptStrategy describes the prompting technique to be used by a Generator when
     performing some task with a specified Model.
     """
+    # Chain-of-Thought Boolean Prompt Strategies
     COT_BOOL = "chain-of-thought-bool"
+    # COT_BOOL_CRITIC = "chain-of-thought-bool-critic"
+    # COT_BOOL_REFINE = "chain-of-thought-bool-refine"
+
+    # Chain-of-Thought Boolean with Image Prompt Strategies
     COT_BOOL_IMAGE = "chain-of-thought-bool-image"
+    # COT_BOOL_IMAGE_CRITIC = "chain-of-thought-bool-image-critic"
+    # COT_BOOL_IMAGE_REFINE = "chain-of-thought-bool-image-refine"
+
+    # Chain-of-Thought Question Answering Prompt Strategies
     COT_QA = "chain-of-thought-question"
+    COT_QA_CRITIC = "chain-of-thought-question-critic"
+    COT_QA_REFINE = "chain-of-thought-question-refine"
+
+    # Chain-of-Thought Question with Image Prompt Strategies
     COT_QA_IMAGE = "chain-of-thought-question-image"
+    COT_QA_IMAGE_CRITIC = "chain-of-thought-question-critic-image"
+    COT_QA_IMAGE_REFINE = "chain-of-thought-question-refine-image"
+
+    # Mixture-of-Agents Prompt Strategies
     COT_MOA_PROPOSER = "chain-of-thought-mixture-of-agents-proposer"
     COT_MOA_PROPOSER_IMAGE = "chain-of-thought-mixture-of-agents-proposer-image"
     COT_MOA_AGG = "chain-of-thought-mixture-of-agents-aggregation"
 
+    def is_image_prompt(self):
+        return "image" in self.value
+
+    def is_cot_bool_prompt(self):
+        return "chain-of-thought-bool" in self.value
+
+    def is_cot_qa_prompt(self):
+        return "chain-of-thought-question" in self.value
+
+    def is_critic_prompt(self):
+        return "critic" in self.value
+
+    def is_refine_prompt(self):
+        return "refine" in self.value
+
+    def is_moa_proposer_prompt(self):
+        return "mixture-of-agents-proposer" in self.value
+
+    def is_moa_aggregator_prompt(self):
+        return "mixture-of-agents-aggregation" in self.value
+
 
 class AggFunc(str, Enum):
     COUNT = "count"
@@ -67,10 +105,11 @@ HTML_EXTENSIONS = [".html", ".htm"]
 # the number of seconds the parallel execution will sleep for while waiting for futures to complete
 PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS = 0.3
 
+# default PDF parser
+DEFAULT_PDF_PROCESSOR = "pypdf"
+
 # character limit for various IDs
 MAX_ID_CHARS = 10
-DEFAULT_DATASET_ID_CHARS = 16
-MAX_DATASET_ID_CHARS = 100
 
 # retry LLM executions 2^x * (multiplier) for up to 10 seconds and at most 4 times
 RETRY_MULTIPLIER = 2
@@ -98,9 +137,15 @@ LOCAL_SCAN_TIME_PER_KB = 1 / (float(500) * 1024)
 # Assume 30 GB/sec for sequential access of memory
 MEMORY_SCAN_TIME_PER_KB = 1 / (float(30) * 1024 * 1024)
 
+# Assume 1 KB per record
+NAIVE_BYTES_PER_RECORD = 1024
+
 # Rough conversion from # of characters --> # of tokens; assumes 1 token ~= 4 chars
 TOKENS_PER_CHARACTER = 0.25
 
+# Rough estimate of the number of tokens the context is allowed to take up for MIXTRAL and LLAMA3 models
+MIXTRAL_LLAMA_CONTEXT_TOKENS_LIMIT = 6000
+
 # a naive estimate for the input record size
 NAIVE_EST_SOURCE_RECORD_SIZE_IN_BYTES = 1_000_000
 
@@ -128,11 +173,6 @@ NAIVE_PDF_PROCESSOR_TIME_PER_RECORD = 10.0
 # Whether or not to log LLM outputs
 LOG_LLM_OUTPUT = False
 
-# Derived schema prefix
-DERIVED_SCHEMA_PREFIX = "DerivedSchema_"
-
-# Derived source_id for records created from a DataFrame
-FROM_DF_PREFIX = "FROM_DF_"
 
 #### MODEL PERFORMANCE & COST METRICS ####
 # I've looked across models and grouped knowledge into commonly used categories:

palimpzest/core/__init__.py

@@ -1,15 +1,13 @@
-from palimpzest.core.data.datasources import (
-    DataSource,
-    DirectorySource,
-    FileSource,
-    HTMLFileDirectorySource,
-    ImageFileDirectorySource,
-    MemorySource,
-    PDFFileDirectorySource,
-    TextFileDirectorySource,
-    UserSource,
-    ValidationDataSource,
-    XLSFileDirectorySource,
+from palimpzest.core.data.datareaders import (
+    DataReader,
+    DirectoryReader,
+    FileReader,
+    HTMLFileDirectoryReader,
+    ImageFileDirectoryReader,
+    MemoryReader,
+    PDFFileDirectoryReader,
+    TextFileDirectoryReader,
+    XLSFileDirectoryReader,
 )
 from palimpzest.core.elements.records import DataRecord, DataRecordSet
 from palimpzest.core.lib.fields import (
@@ -33,7 +31,6 @@ from palimpzest.core.lib.schemas import (
     PlotImage,
     RawJSONObject,
     Schema,
-    SourceRecord,
     Table,
     TextFile,
     WebPage,
@@ -61,23 +58,20 @@ __all__ = [
     "PlotImage",
     "RawJSONObject",
     "Schema",
-    "SourceRecord",
     "Table",
     "TextFile",
     "WebPage",
     "XLSFile",
-    # datasources
-    "DataSource",
-    "DirectorySource",
-    "FileSource",
-    "HTMLFileDirectorySource",
-    "ImageFileDirectorySource",
-    "MemorySource",
-    "PDFFileDirectorySource",
-    "TextFileDirectorySource",
-    "UserSource",
-    "ValidationDataSource",
-    "XLSFileDirectorySource",
+    # datareaders
+    "DataReader",
+    "DirectoryReader",
+    "FileReader",
+    "HTMLFileDirectoryReader",
+    "ImageFileDirectoryReader",
+    "MemoryReader",
+    "PDFFileDirectoryReader",
+    "TextFileDirectoryReader",
+    "XLSFileDirectoryReader",
     # records
     "DataRecord",
     "DataRecordSet",

palimpzest/core/data/dataclasses.py

@@ -124,8 +124,8 @@ class RecordOpStats:
     # identifier for the parent of this record
     record_parent_id: str
 
-    # idenifier for the source of this record
-    record_source_id: str
+    # idenifier for the source idx of this record
+    record_source_idx: str
 
     # a dictionary with the record state after being processed by the operator
     record_state: dict[str, Any]
@@ -406,6 +406,13 @@ class OperatorCostEstimates:
     # upper bound on quality
     quality_upper_bound: float | None = None
 
+    def __rmul__(self, multiplier: float) -> OperatorCostEstimates:
+        """
+        Multiply all fields by a scalar.
+        """
+        dct = {field.name: getattr(self, field.name) * multiplier for field in fields(self)}
+        return OperatorCostEstimates(**dct)
+
     def __post_init__(self):
         if self.cardinality_lower_bound is None and self.cardinality_upper_bound is None:
             self.cardinality_lower_bound = self.cardinality