palimpzest 0.8.2__py3-none-any.whl → 0.8.3__py3-none-any.whl

palimpzest/query/operators/split_convert.py

@@ -1,170 +0,0 @@
-from __future__ import annotations
-
-import math
-
-from pydantic.fields import FieldInfo
-
-from palimpzest.constants import (
-    MODEL_CARDS,
-    NAIVE_EST_NUM_INPUT_TOKENS,
-    NAIVE_EST_NUM_OUTPUT_TOKENS,
-    PromptStrategy,
-)
-from palimpzest.core.elements.records import DataRecord
-from palimpzest.core.models import GenerationStats, OperatorCostEstimates
-from palimpzest.query.generators.generators import Generator
-from palimpzest.query.operators.convert import LLMConvert
-
-
-class SplitConvert(LLMConvert):
-    def __init__(self, num_chunks: int = 2, min_size_to_chunk: int = 1000, *args, **kwargs):
-        kwargs["prompt_strategy"] = None
-        super().__init__(*args, **kwargs)
-        self.num_chunks = num_chunks
-        self.min_size_to_chunk = min_size_to_chunk
-        self.split_generator = Generator(self.model, PromptStrategy.SPLIT_PROPOSER, self.reasoning_effort, self.api_base, self.cardinality, self.desc, self.verbose)
-        self.split_merge_generator = Generator(self.model, PromptStrategy.SPLIT_MERGER, self.reasoning_effort, self.api_base, self.cardinality, self.desc, self.verbose)
-
-        # crude adjustment factor for naive estimation in no-sentinel setting
-        self.naive_quality_adjustment = 0.6
-
-    def __str__(self):
-        op = super().__str__()
-        op += f"    Chunk Size: {str(self.num_chunks)}\n"
-        op += f"    Min Size to Chunk: {str(self.min_size_to_chunk)}\n"
-        return op
-
-    def get_id_params(self):
-        id_params = super().get_id_params()
-        id_params = {"num_chunks": self.num_chunks, "min_size_to_chunk": self.min_size_to_chunk, **id_params}
-
-        return id_params
-
-    def get_op_params(self):
-        op_params = super().get_op_params()
-        return {"num_chunks": self.num_chunks, "min_size_to_chunk": self.min_size_to_chunk, **op_params}
-
-    def naive_cost_estimates(self, source_op_cost_estimates: OperatorCostEstimates) -> OperatorCostEstimates:
-        """
-        Update the cost per record and quality estimates produced by LLMConvert's naive estimates.
-        We adjust the cost per record to account for the reduced number of input tokens following
-        the retrieval of relevant chunks, and we make a crude estimate of the quality degradation
-        that results from using a downsized input (although this may in fact improve quality in
-        some cases).
-        """
-        # get naive cost estimates from LLMConvert
-        naive_op_cost_estimates = super().naive_cost_estimates(source_op_cost_estimates)
-
-        # re-compute cost per record assuming we use fewer input tokens; naively assume a single input field
-        est_num_input_tokens = NAIVE_EST_NUM_INPUT_TOKENS
-        est_num_output_tokens = NAIVE_EST_NUM_OUTPUT_TOKENS
-        model_conversion_usd_per_record = (
-            MODEL_CARDS[self.model.value]["usd_per_input_token"] * est_num_input_tokens
-            + MODEL_CARDS[self.model.value]["usd_per_output_token"] * est_num_output_tokens
-        )
-
-        # set refined estimate of cost per record
-        naive_op_cost_estimates.cost_per_record = model_conversion_usd_per_record
-        naive_op_cost_estimates.cost_per_record_lower_bound = naive_op_cost_estimates.cost_per_record
-        naive_op_cost_estimates.cost_per_record_upper_bound = naive_op_cost_estimates.cost_per_record
-        naive_op_cost_estimates.quality = (naive_op_cost_estimates.quality) * self.naive_quality_adjustment
-        naive_op_cost_estimates.quality_lower_bound = naive_op_cost_estimates.quality
-        naive_op_cost_estimates.quality_upper_bound = naive_op_cost_estimates.quality
-
-        return naive_op_cost_estimates
-
-    def is_image_conversion(self) -> bool:
-        """SplitConvert is currently disallowed on image conversions, so this must be False."""
-        return False
-
-    def get_text_chunks(self, text: str, num_chunks: int) -> list[str]:
-        """
-        Given a text string, chunk it into num_chunks substrings of roughly equal size.
-        """
-        chunks = []
-
-        idx, chunk_size = 0, math.ceil(len(text) / num_chunks)
-        while idx + chunk_size < len(text):
-            chunks.append(text[idx : idx + chunk_size])
-            idx += chunk_size
-
-        if idx < len(text):
-            chunks.append(text[idx:])
-
-        return chunks
-
-    def get_chunked_candidate(self, candidate: DataRecord, input_fields: list[str]) -> list[DataRecord]:
-        """
-        For each text field, chunk the content. If a field is smaller than the chunk size,
-        simply include the full field.
-        """
-        # compute mapping from each field to its chunked content
-        field_name_to_chunked_content = {}
-        for field_name in input_fields:
-            field = candidate.get_field_type(field_name)
-            content = candidate[field_name]
-
-            # do not chunk this field if it is not a string or a list of strings
-            is_string_field = field.annotation in [str, str | None]
-            is_list_string_field = field.annotation in [list[str], list[str] | None]
-            if not (is_string_field or is_list_string_field):
-                field_name_to_chunked_content[field_name] = [content]
-                continue
-
-            # if this is a list of strings, join the strings
-            if is_list_string_field:
-                content = "[" + ", ".join(content) + "]"
-
-            # skip this field if its length is less than the min size to chunk
-            if len(content) < self.min_size_to_chunk:
-                field_name_to_chunked_content[field_name] = [content]
-                continue
-
-            # chunk the content
-            field_name_to_chunked_content[field_name] = self.get_text_chunks(content, self.num_chunks)
-
-        # compute the true number of chunks (may be 1 if all fields are not chunked)
-        num_chunks = max(len(chunks) for chunks in field_name_to_chunked_content.values())
-
-        # create the chunked canidates
-        candidates = []
-        for chunk_idx in range(num_chunks):
-            candidate_copy = candidate.copy()
-            for field_name in input_fields:
-                field_chunks = field_name_to_chunked_content[field_name]
-                candidate_copy[field_name] = field_chunks[chunk_idx] if len(field_chunks) > 1 else field_chunks[0]
-
-            candidates.append(candidate_copy)
-
-        return candidates
-
-    def convert(self, candidate: DataRecord, fields: dict[str, FieldInfo]) -> tuple[dict[str, list], GenerationStats]:
-        # get the set of input fields to use for the convert operation
-        input_fields = self.get_input_fields()
-
-        # lookup most relevant chunks for each field using embedding search
-        candidate_copy = candidate.copy()
-        chunked_candidates = self.get_chunked_candidate(candidate_copy, input_fields)
-
-        # construct kwargs for generation
-        gen_kwargs = {"project_cols": input_fields, "output_schema": self.output_schema}
-
-        # generate outputs for each chunk separately
-        chunk_outputs, chunk_generation_stats_lst = [], []
-        for candidate in chunked_candidates:
-            _, reasoning, chunk_generation_stats, _ = self.split_generator(candidate, fields, json_output=False, **gen_kwargs)
-            chunk_outputs.append(reasoning)
-            chunk_generation_stats_lst.append(chunk_generation_stats)
-
-        # call the merger
-        gen_kwargs = {
-            "project_cols": input_fields,
-            "output_schema": self.output_schema,
-            "chunk_outputs": chunk_outputs,
-        }
-        field_answers, _, merger_gen_stats, _ = self.split_merge_generator(candidate, fields, **gen_kwargs)
-
-        # compute the total generation stats
-        generation_stats = sum(chunk_generation_stats_lst) + merger_gen_stats
-
-        return field_answers, generation_stats

palimpzest-0.8.2.dist-info/RECORD

@@ -1,95 +0,0 @@
-palimpzest/__init__.py,sha256=1PzadDDOVMQJKNEYUH0_tw8tQKUYTT31M0vuzTr2Rqk,1694
-palimpzest/constants.py,sha256=GagsbJl1xCAjgt6Biw27KnHSZgiramxhnerhmYe3P_k,24690
-palimpzest/policy.py,sha256=lIvw_C_rmwCH4LZaeNkAuixl8zw9RAW_JcSWSHPjKyc,11628
-palimpzest/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/agents/compute_agents.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/agents/search_agents.py,sha256=t2QMreB5Ph71aoNk5bBtV-0l8im79z-pMAR3JDAySDw,29418
-palimpzest/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/models.py,sha256=fLO4T7x0njNeEbUpbhJm9cdnBva0y0Zw5WGBGdzdS_I,42442
-palimpzest/core/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/data/context.py,sha256=x1xYyu9qW65dvtK_XayIfv_CgsCEPW6Qe0DTiSf9sjU,16207
-palimpzest/core/data/context_manager.py,sha256=8hAKWD2jhFZgghTu7AYgjkvKDsJUPVxq8g4nG0HWvfo,6150
-palimpzest/core/data/dataset.py,sha256=M7SxPXzHsfj-ljy_P3ckcJNqGf4RwNxtZI02q_tmL2M,28178
-palimpzest/core/data/index_dataset.py,sha256=adO67DgzHhA4lBME0-h4SjXfdz9UcNMSDGXTpUdKbgE,1929
-palimpzest/core/data/iter_dataset.py,sha256=u7eZNWWT84rH_D8LNIuq0NAnm2roX81ifKTYp-hwY7g,20512
-palimpzest/core/elements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/elements/filters.py,sha256=fU2x0eWDwfP52_5fUmqJXTuhs4H0vvHtPZLdA3IIw8I,1642
-palimpzest/core/elements/groupbysig.py,sha256=Fcbt1GSAkxIILS5me0sWaNMLiptLJ6NIY6BIC0S-g3k,2318
-palimpzest/core/elements/records.py,sha256=Su4d8GRNp5-Q1nSWANm-jehyw53tHUl20iGluyEk8NI,19053
-palimpzest/core/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/lib/schemas.py,sha256=0qauaG3uW5tCJXNAo1i0G0UgbTaQLSLT6GoNDX8494Q,8376
-palimpzest/prompts/__init__.py,sha256=sdZbC8RWi_IGjFuzKQMdRjS2Ih4zQnkyzFoJ6Q3Ce70,1764
-palimpzest/prompts/agent_prompts.py,sha256=CUzBVLBiPSw8OShtKp4VTpQwtrNMtcMglo-IZHMvuDM,17459
-palimpzest/prompts/context_search.py,sha256=s3pti4XNRiIyiWzjVNL_NqmqEc31jzSKMF2SlN0Aaf8,357
-palimpzest/prompts/convert_prompts.py,sha256=quoIcdIrP4FoPBXlHKeOPjH5lVn8GH30f1tPiu3Xsyw,6037
-palimpzest/prompts/critique_and_refine_convert_prompts.py,sha256=WoXExBxQ7twswd9VCCST26c-2ehZtpD2iQoBi7sqDnQ,7814
-palimpzest/prompts/filter_prompts.py,sha256=drTivlA_WnWkAIzY9GjqO_hfwdy432nMtsV-OYa-mlE,4260
-palimpzest/prompts/join_prompts.py,sha256=fEGZY_zn_dvOJCeUFYrHdg5P3h_H6Fo3FMmdnXwp2l4,5957
-palimpzest/prompts/moa_aggregator_convert_prompts.py,sha256=BQRrtGdr53PTqvXzmFh8kfQ_w9KoKw-zTtmdo-8RFjo,2887
-palimpzest/prompts/moa_proposer_convert_prompts.py,sha256=8vhq0bnikbCzS4CDV5IskFPWF0TC7VZGjeGvpOyIBV8,3476
-palimpzest/prompts/prompt_factory.py,sha256=MpEYoyPXY3gfFGG60O9rlw5A5UejC3CTRHcO6KRhyww,51733
-palimpzest/prompts/split_merge_prompts.py,sha256=0mTZeJhxtvlmv-ro0KwQpxlGgSTwyUhGRHJ-uHk2Zlw,3146
-palimpzest/prompts/split_proposer_prompts.py,sha256=X3hufHPAiQyytZ_TFe2wJkVPgJtClZ9fVgz2zNk2Z5Q,2638
-palimpzest/prompts/util_phrases.py,sha256=ajxzj-B2gE56IENKVKElqw1xKWOF5IahOOqq026Pr00,876
-palimpzest/prompts/validator.py,sha256=pJTZjlt_OiFM3IFOgsJ0jQdayra8iRVrpqENlXI9tQQ,10532
-palimpzest/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/query/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/query/execution/all_sample_execution_strategy.py,sha256=3n2hl8m-WFWIu-a8DiSVsGkz4ej3yB7mSdFR0jsiwAU,14366
-palimpzest/query/execution/execution_strategy.py,sha256=KwBJbWOBOOPBiWRm3ypHcAQiWbCsvtW6UnVU4tHkYz8,18905
-palimpzest/query/execution/execution_strategy_type.py,sha256=vRQBPCQN5_aoyD3TLIeW3VPo15mqF-5RBvEXkENz9FE,987
-palimpzest/query/execution/mab_execution_strategy.py,sha256=paVfB8lqNyUuISqfhkTd6RqOZqpyVty1EAN1sZz7erA,43554
-palimpzest/query/execution/parallel_execution_strategy.py,sha256=Gn5hB5XddX2jCkxx6d7O-DmitK6fbuwBFnnyKhnGYEw,15706
-palimpzest/query/execution/single_threaded_execution_strategy.py,sha256=1eo-Z9G3u92_PjoSX8HmO3D3phYgA8f0Actbgd1-oKY,16247
-palimpzest/query/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/query/generators/generators.py,sha256=WYMcff7axgDEmYqXvy1A-C5FA4s6lI393CcCx-TKTgM,20941
-palimpzest/query/operators/__init__.py,sha256=j-yh0P5tzXGa0JU_g8aNn54wCJDXPCMbmtOmazXXEts,3459
-palimpzest/query/operators/aggregate.py,sha256=QvWr4C1arFSZWVqPSF5F5On6Ise5OF3VVWNGSq6Gfjk,11230
-palimpzest/query/operators/compute.py,sha256=bxMKLRU_o7v603daKeR0FayDZ_V6NLI1fGzgu6E-sac,8473
-palimpzest/query/operators/convert.py,sha256=mpXXYdgcH6zPqypzh3SjTqHjq7PDEbkfdZ1_XEml-nw,17334
-palimpzest/query/operators/critique_and_refine_convert.py,sha256=PbtKva6e3fh3yeUMGlkcpacPD003bFBzgBsw_yy-8fw,5293
-palimpzest/query/operators/distinct.py,sha256=MuF3NlC0QMTSGs0_fe2oly0I5Ow0hfOa7h8BFGhHiCs,2594
-palimpzest/query/operators/filter.py,sha256=jmSGV7xZ8uxXzH-Oko7l8ZPZxNf_qJNkYVAYgiSHl9g,10802
-palimpzest/query/operators/join.py,sha256=rs9_Y59082dlnSJu9rpRDEuv7jDPItKSpYsC8FCMFDM,17837
-palimpzest/query/operators/limit.py,sha256=upJ775cGkxjFHRJm8GpSvtJN1cspg2FVYLN_MrIfUo4,2113
-palimpzest/query/operators/logical.py,sha256=K_dRlNKkda35kQ7gYGsrW9PoFuDPzexpjtDq_FYdhVw,20223
-palimpzest/query/operators/mixture_of_agents_convert.py,sha256=4v2V612NqdVD0RmcJ5VSgTiVliObku-t-A79SXVnpk0,6753
-palimpzest/query/operators/physical.py,sha256=buPZjtP4HKNVfOCNWdBtDnRS217dSsIG74gqZ1jmoyo,8320
-palimpzest/query/operators/project.py,sha256=RX5SbHFRwHcMfiQRofIPQr-AHgIDYm68ifiFZAPu7Fo,2094
-palimpzest/query/operators/rag_convert.py,sha256=1QQGrE22-Ec3-MNbnaU3k4TGHdpi2qZqZR9MHUniEM4,10691
-palimpzest/query/operators/retrieve.py,sha256=v1FTFsSctqH4B37aWgBXYIxgOMJwRWQ2kwwXu1huwaQ,13106
-palimpzest/query/operators/scan.py,sha256=Da_EZUrArzlAameHYCmtqo-xbPOFvbTYSktrUcUEUSc,7398
-palimpzest/query/operators/search.py,sha256=xydO5Kni0RArpvLSn2ajzD4TcH442VjpP2x9NakjzaA,22842
-palimpzest/query/operators/split_convert.py,sha256=acCPlkrUfqHhGD7bU2AXQAhIEeAEIh0itamuCOm4KBk,7787
-palimpzest/query/optimizer/__init__.py,sha256=L2E1rOA-8O9oH6JL56wLI1qUVxXBLubJEG1IHMH-HU4,2384
-palimpzest/query/optimizer/cost_model.py,sha256=OldPy-TJdfsQbYRoKlb3yWeKbi15jcldTIUS6BTi9T8,12678
-palimpzest/query/optimizer/optimizer.py,sha256=mgM6c0d_voGNun2hMzqjfumJVieACtcHsNnBP4LyXAA,19626
-palimpzest/query/optimizer/optimizer_strategy.py,sha256=9YlNGkqwgX0WaV6y8tKOOHVN8kC8GjDI3DttvGW5SYY,10206
-palimpzest/query/optimizer/optimizer_strategy_type.py,sha256=V-MMHvJdnfZKoUX1xxxwh66q1RjN2FL35IsiT1C62c8,1084
-palimpzest/query/optimizer/plan.py,sha256=VIhN7tWT7EoRE9BKYa1qvvOhX7dEaM-aiobByX0qjzg,22900
-palimpzest/query/optimizer/primitives.py,sha256=jMMVq37y1tWiPU1lSSKQP9OP-mzkpSxSmUeDajRYYOQ,5445
-palimpzest/query/optimizer/rules.py,sha256=9AsuVjhiZUc0snQPNhIqeyKpmqFsSv7e-v6BEbp9CDw,43315
-palimpzest/query/optimizer/tasks.py,sha256=DJcKDNbVJox61rnTW0HgT1PtxGx2P_NiLvNroXie-Lg,29509
-palimpzest/query/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/query/processor/config.py,sha256=vHVsgeBnKigacO0QA7bLf5q8pJhFWA2j9-p_no2bmYo,2366
-palimpzest/query/processor/query_processor.py,sha256=T4ffPbnOX23G8FDITzmM7Iw7DUEDWIHnwl8XLYllgjg,6240
-palimpzest/query/processor/query_processor_factory.py,sha256=6w9R1Y8AOV22X8MUf7g2G5Qb15BGEZAXQKbCQJafWJ0,8048
-palimpzest/schemabuilder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/schemabuilder/schema_builder.py,sha256=QraGp66dcD-ej6Y2mER40o86G9JqlBkL7swkJzjUAIY,7968
-palimpzest/tools/README.md,sha256=56_6LPG80uc0CLVhTBP6I1wgIffNv9cyTr0TmVZqmrM,483
-palimpzest/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/tools/allenpdf.py,sha256=fXMOmSDdSSLXDKAPYYJ8k4egtWEBf_Me9Lq9tM3iyoA,1690
-palimpzest/tools/pdfparser.py,sha256=0DOVUZLxYfqjxM8WNEfYcyiXb1qW9BWVIHEB_B_YhWA,9570
-palimpzest/tools/skema_tools.py,sha256=HXUFpjMhbVxZwKKkATeK-FwtlTCawaCbeP-uHntI1Kg,669
-palimpzest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/utils/env_helpers.py,sha256=n81KzoJ459pRxo7QmJA7duazwWsfoMGTHc71D2LatFk,334
-palimpzest/utils/hash_helpers.py,sha256=3A8dA7SbXTwnnvZvPVNqqMLlVRhCKyKF_bjNNAu3Exk,334
-palimpzest/utils/model_helpers.py,sha256=X6SlMgD5I5Aj_cxaFaoGaaNvOOqTNZVmjj6zbfn63Yk,2476
-palimpzest/utils/progress.py,sha256=7gucyZr82udMDZitrrkAOSKHZVljE3R2wv9nf5gA5TM,20807
-palimpzest/utils/udfs.py,sha256=LjHic54B1az-rKgNLur0wOpaz2ko_UodjLEJrazkxvY,1854
-palimpzest/validator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/validator/validator.py,sha256=J2tGvJqfg6v5lOQDYYaqAa9d37uVHBrqkNs-a8d1Ic0,16365
-palimpzest-0.8.2.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
-palimpzest-0.8.2.dist-info/METADATA,sha256=bDa2zFfJr_v4Ef6fzq3SCALSoXoXc0uPnefnmVbAzTA,7286
-palimpzest-0.8.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-palimpzest-0.8.2.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
-palimpzest-0.8.2.dist-info/RECORD,,