themefinder 0.5.4__py3-none-any.whl → 0.6.2__py3-none-any.whl

themefinder/__init__.py

@@ -1,10 +1,10 @@
 from .core import (
     find_themes,
     sentiment_analysis,
-    theme_generation,
     theme_condensation,
-    theme_refinement,
+    theme_generation,
     theme_mapping,
+    theme_refinement,
 )
 
 __all__ = [

themefinder/core.py

@@ -6,6 +6,7 @@ from langchain_core.prompts import PromptTemplate
 from langchain_core.runnables import Runnable
 
 from .llm_batch_processor import batch_and_run, load_prompt_from_file
+from .models import SentimentAnalysisOutput, ThemeMappingOutput
 from .themefinder_logging import logger
 
 CONSULTATION_SYSTEM_PROMPT = load_prompt_from_file("consultation_system_prompt")
@@ -18,7 +19,7 @@ async def find_themes(
     target_n_themes: int | None = None,
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
     verbose: bool = True,
-) -> dict[str, pd.DataFrame]:
+) -> dict[str, str | pd.DataFrame]:
     """Process survey responses through a multi-stage theme analysis pipeline.
 
     This pipeline performs sequential analysis steps:
@@ -41,47 +42,46 @@ async def find_themes(
             Defaults to True.
 
     Returns:
-        dict[str, pd.DataFrame]: Dictionary containing results from each pipeline stage:
-            - question: The survey question
+        dict[str, str | pd.DataFrame]: Dictionary containing results from each pipeline stage:
+            - question: The survey question string
             - sentiment: DataFrame with sentiment analysis results
-            - topics: DataFrame with initial generated themes
-            - condensed_topics: DataFrame with combined similar themes
-            - refined_topics: DataFrame with refined theme definitions
+            - themes: DataFrame with the final themes output
             - mapping: DataFrame mapping responses to final themes
+            - unprocessables: Dataframe containing the inputs that could not be processed by the LLM
     """
     logger.setLevel(logging.INFO if verbose else logging.CRITICAL)
 
-    sentiment_df = await sentiment_analysis(
+    sentiment_df, sentiment_unprocessables = await sentiment_analysis(
         responses_df,
         llm,
         question=question,
         system_prompt=system_prompt,
     )
-    theme_df = await theme_generation(
+    theme_df, _ = await theme_generation(
         sentiment_df,
         llm,
         question=question,
         system_prompt=system_prompt,
     )
-    condensed_theme_df = await theme_condensation(
+    condensed_theme_df, _ = await theme_condensation(
         theme_df, llm, question=question, system_prompt=system_prompt
     )
-    refined_theme_df = await theme_refinement(
+    refined_theme_df, _ = await theme_refinement(
         condensed_theme_df,
         llm,
         question=question,
         system_prompt=system_prompt,
     )
     if target_n_themes is not None:
-        refined_theme_df = await theme_target_alignment(
+        refined_theme_df, _ = await theme_target_alignment(
             refined_theme_df,
             llm,
             question=question,
             target_n_themes=target_n_themes,
             system_prompt=system_prompt,
         )
-    mapping_df = await theme_mapping(
-        sentiment_df,
+    mapping_df, mapping_unprocessables = await theme_mapping(
+        sentiment_df[["response_id", "response"]],
         llm,
         question=question,
         refined_themes_df=refined_theme_df,
@@ -95,10 +95,9 @@ async def find_themes(
     return {
         "question": question,
         "sentiment": sentiment_df,
-        "themes": theme_df,
-        "condensed_themes": condensed_theme_df,
-        "refined_themes": refined_theme_df,
+        "themes": refined_theme_df,
         "mapping": mapping_df,
+        "unprocessables": pd.concat([sentiment_unprocessables, mapping_unprocessables]),
     }
 
 
@@ -109,7 +108,7 @@ async def sentiment_analysis(
     batch_size: int = 20,
     prompt_template: str | Path | PromptTemplate = "sentiment_analysis",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Perform sentiment analysis on survey responses using an LLM.
 
     This function processes survey responses in batches to analyze their sentiment
@@ -129,24 +128,29 @@ async def sentiment_analysis(
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: DataFrame containing the original responses enriched with
-            sentiment analysis results.
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
 
     Note:
-        The function uses response_id_integrity_check to ensure responses maintain
+        The function uses validation_check to ensure responses maintain
         their original order and association after processing.
     """
     logger.info(f"Running sentiment analysis on {len(responses_df)} responses")
-    return await batch_and_run(
+    processed_rows, unprocessable_rows = await batch_and_run(
         responses_df,
         prompt_template,
         llm,
         batch_size=batch_size,
         question=question,
-        response_id_integrity_check=True,
+        validation_check=True,
+        task_validation_model=SentimentAnalysisOutput,
         system_prompt=system_prompt,
     )
 
+    return processed_rows, unprocessable_rows
+
 
 async def theme_generation(
     responses_df: pd.DataFrame,
@@ -156,7 +160,7 @@ async def theme_generation(
     partition_key: str | None = "position",
     prompt_template: str | Path | PromptTemplate = "theme_generation",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Generate themes from survey responses using an LLM.
 
     This function processes batches of survey responses to identify common themes or topics.
@@ -179,10 +183,14 @@ async def theme_generation(
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: DataFrame containing identified themes and their associated metadata.
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
+
     """
     logger.info(f"Running theme generation on {len(responses_df)} responses")
-    return await batch_and_run(
+    generated_themes, _ = await batch_and_run(
         responses_df,
         prompt_template,
         llm,
@@ -191,6 +199,7 @@ async def theme_generation(
         question=question,
         system_prompt=system_prompt,
     )
+    return generated_themes, _
 
 
 async def theme_condensation(
@@ -201,7 +210,7 @@ async def theme_condensation(
     prompt_template: str | Path | PromptTemplate = "theme_condensation",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
     **kwargs,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Condense and combine similar themes identified from survey responses.
 
     This function processes the initially identified themes to combine similar or
@@ -221,18 +230,21 @@ async def theme_condensation(
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: DataFrame containing the condensed themes, where similar topics
-            have been combined into broader categories.
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
+
     """
     logger.info(f"Running theme condensation on {len(themes_df)} themes")
-    themes_df["response_id"] = range(len(themes_df))
+    themes_df["response_id"] = themes_df.index + 1
 
     n_themes = themes_df.shape[0]
     while n_themes > batch_size:
         logger.info(
             f"{n_themes} larger than batch size, using recursive theme condensation"
         )
-        themes_df = await batch_and_run(
+        themes_df, _ = await batch_and_run(
             themes_df,
             prompt_template,
             llm,
@@ -242,13 +254,13 @@ async def theme_condensation(
             **kwargs,
         )
         themes_df = themes_df.sample(frac=1).reset_index(drop=True)
-        themes_df["response_id"] = range(len(themes_df))
+        themes_df["response_id"] = themes_df.index + 1
         if len(themes_df) == n_themes:
             logger.info("Themes no longer being condensed")
             break
         n_themes = themes_df.shape[0]
 
-    themes_df = await batch_and_run(
+    themes_df, _ = await batch_and_run(
         themes_df,
         prompt_template,
         llm,
@@ -259,7 +271,7 @@ async def theme_condensation(
     )
 
     logger.info(f"Final number of condensed themes: {themes_df.shape[0]}")
-    return themes_df
+    return themes_df, _
 
 
 async def theme_refinement(
@@ -269,7 +281,7 @@ async def theme_refinement(
     batch_size: int = 10000,
     prompt_template: str | Path | PromptTemplate = "theme_refinement",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Refine and standardize condensed themes using an LLM.
 
     This function processes previously condensed themes to create clear, standardized
@@ -286,15 +298,15 @@ async def theme_refinement(
             Defaults to 10000.
         prompt_template (str | Path | PromptTemplate, optional): Template for structuring
             the prompt to the LLM. Can be a string identifier, path to template file,
-            or PromptTemplate instance. Defaults to "topic_refinement".
+            or PromptTemplate instance. Defaults to "theme_refinement".
         system_prompt (str): System prompt to guide the LLM's behavior.
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: A single-row DataFrame where:
-            - Each column represents a unique theme (identified by topic_id)
-            - The values contain the refined theme descriptions
-            - The format is optimized for subsequent theme mapping operations
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
 
     Note:
         The function adds sequential response_ids to the input DataFrame and
@@ -302,16 +314,9 @@ async def theme_refinement(
         processing.
     """
     logger.info(f"Running theme refinement on {len(condensed_themes_df)} responses")
-    condensed_themes_df["response_id"] = range(len(condensed_themes_df))
+    condensed_themes_df["response_id"] = condensed_themes_df.index + 1
 
-    def transpose_refined_themes(refined_themes: pd.DataFrame):
-        """Transpose topics for increased legibility."""
-        transposed_df = pd.DataFrame(
-            [refined_themes["topic"].to_numpy()], columns=refined_themes["topic_id"]
-        )
-        return transposed_df
-
-    refined_themes = await batch_and_run(
+    refined_themes, _ = await batch_and_run(
         condensed_themes_df,
         prompt_template,
         llm,
@@ -319,7 +324,7 @@ async def theme_refinement(
         question=question,
         system_prompt=system_prompt,
     )
-    return transpose_refined_themes(refined_themes)
+    return refined_themes, _
 
 
 async def theme_target_alignment(
@@ -330,7 +335,7 @@ async def theme_target_alignment(
     batch_size: int = 10000,
     prompt_template: str | Path | PromptTemplate = "theme_target_alignment",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Align themes to target number using an LLM.
 
     This function processes refined themes to consolidate them into a target number of
@@ -354,10 +359,10 @@ async def theme_target_alignment(
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: A single-row DataFrame where:
-            - Each column represents a unique theme (identified by topic_id)
-            - The values contain the aligned theme descriptions
-            - The format is optimized for subsequent theme mapping operations
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
 
     Note:
         The function adds sequential response_ids to the input DataFrame and
@@ -365,19 +370,10 @@ async def theme_target_alignment(
         processing.
     """
     logger.info(
-        f"Running theme target alignment on {len(refined_themes_df.columns)} themes compressing to {target_n_themes} themes"
+        f"Running theme target alignment on {len(refined_themes_df)} themes compressing to {target_n_themes} themes"
     )
-    refined_themes_df = refined_themes_df.T.rename(columns={0: "topic"})
-    refined_themes_df["response_id"] = range(len(refined_themes_df))
-
-    def transpose_aligned_themes(aligned_themes: pd.DataFrame):
-        """Transpose topics for increased legibility."""
-        transposed_df = pd.DataFrame(
-            [aligned_themes["topic"].to_numpy()], columns=aligned_themes["topic_id"]
-        )
-        return transposed_df
-
-    aligned_themes = await batch_and_run(
+    refined_themes_df["response_id"] = refined_themes_df.index + 1
+    aligned_themes, _ = await batch_and_run(
         refined_themes_df,
         prompt_template,
         llm,
@@ -386,7 +382,7 @@ async def theme_target_alignment(
         system_prompt=system_prompt,
         target_n_themes=target_n_themes,
     )
-    return transpose_aligned_themes(aligned_themes)
+    return aligned_themes, _
 
 
 async def theme_mapping(
@@ -397,7 +393,7 @@ async def theme_mapping(
     batch_size: int = 20,
     prompt_template: str | Path | PromptTemplate = "theme_mapping",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Map survey responses to refined themes using an LLM.
 
     This function analyzes each survey response and determines which of the refined
@@ -419,19 +415,34 @@ async def theme_mapping(
             Defaults to CONSULTATION_SYSTEM_PROMPT.
 
     Returns:
-        pd.DataFrame: DataFrame containing the original responses enriched with
-            theme mapping results, ensuring all responses are mapped through ID integrity checks.
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processed by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
+
     """
     logger.info(
-        f"Running theme mapping on {len(responses_df)} responses using {len(refined_themes_df.columns)} themes"
+        f"Running theme mapping on {len(responses_df)} responses using {len(refined_themes_df)} themes"
     )
-    return await batch_and_run(
+
+    def transpose_refined_themes(refined_themes: pd.DataFrame):
+        """Transpose topics for increased legibility."""
+        transposed_df = pd.DataFrame(
+            [refined_themes["topic"].to_numpy()], columns=refined_themes["topic_id"]
+        )
+        return transposed_df
+
+    mapping, _ = await batch_and_run(
         responses_df,
         prompt_template,
         llm,
         batch_size=batch_size,
         question=question,
-        refined_themes=refined_themes_df.to_dict(orient="records"),
-        response_id_integrity_check=True,
+        refined_themes=transpose_refined_themes(refined_themes_df).to_dict(
+            orient="records"
+        ),
+        validation_check=True,
+        task_validation_model=ThemeMappingOutput,
         system_prompt=system_prompt,
     )
+    return mapping, _

themefinder/llm_batch_processor.py

@@ -1,14 +1,24 @@
 import asyncio
 import json
 import logging
+import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional, Type
 
+import openai
 import pandas as pd
+import tiktoken
 from langchain_core.prompts import PromptTemplate
 from langchain_core.runnables import Runnable
-from tenacity import before, retry, stop_after_attempt, wait_random_exponential
+from pydantic import BaseModel, ValidationError
+from tenacity import (
+    before,
+    before_sleep_log,
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
 
 from .themefinder_logging import logger
 
@@ -16,63 +26,82 @@ from .themefinder_logging import logger
 @dataclass
 class BatchPrompt:
     prompt_string: str
-    response_ids: list[str]
+    response_ids: list[int]
 
 
 async def batch_and_run(
-    responses_df: pd.DataFrame,
+    input_df: pd.DataFrame,
     prompt_template: str | Path | PromptTemplate,
     llm: Runnable,
     batch_size: int = 10,
     partition_key: str | None = None,
-    response_id_integrity_check: bool = False,
+    validation_check: bool = False,
+    task_validation_model: Type[BaseModel] = None,
     **kwargs: Any,
-) -> pd.DataFrame:
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Process a DataFrame of responses in batches using an LLM.
 
     Args:
-        responses_df (pd.DataFrame): DataFrame containing responses to be processed.
+        input_df (pd.DataFrame): DataFrame containing input to be processed.
             Must include a 'response_id' column.
         prompt_template (Union[str, Path, PromptTemplate]): Template for LLM prompts.
             Can be a string (file path), Path object, or PromptTemplate.
         llm (Runnable): LangChain Runnable instance that will process the prompts.
-        batch_size (int, optional): Number of responses to process in each batch.
+        batch_size (int, optional): Number of input rows to process in each batch.
             Defaults to 10.
-        partition_key (str | None, optional): Optional column name to group responses
+        partition_key (str | None, optional): Optional column name to group input rows
             before batching. Defaults to None.
-        response_id_integrity_check (bool, optional): If True, verifies that all input
-            response IDs are present in LLM output and retries failed responses individually.
+        validation_check (bool, optional): If True, verifies that all input
+            response IDs are present in LLM output and validates the rows against the validation model,
+            failed rows are retried individually.
             If False, no integrity checking or retrying occurs. Defaults to False.
+        task_validation_model (Type[BaseModel]): the pydanctic model to validate each row against
         **kwargs (Any): Additional keyword arguments to pass to the prompt template.
 
     Returns:
         pd.DataFrame: DataFrame containing the original responses merged with the
             LLM-processed results.
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing two DataFrames:
+                - The first DataFrame contains the rows that were successfully processes by the LLM
+                - The second DataFrame contains the rows that could not be processed by the LLM
     """
+
     logger.info(f"Running batch and run with batch size {batch_size}")
     prompt_template = convert_to_prompt_template(prompt_template)
-    batched_response_dfs = batch_responses(
-        responses_df, batch_size=batch_size, partition_key=partition_key
+    batch_prompts = generate_prompts(
+        prompt_template,
+        input_df,
+        batch_size=batch_size,
+        partition_key=partition_key,
+        **kwargs,
     )
-    batch_prompts = generate_prompts(batched_response_dfs, prompt_template, **kwargs)
-    llm_responses, failed_ids = await call_llm(
+    processed_rows, failed_ids = await call_llm(
         batch_prompts=batch_prompts,
         llm=llm,
-        response_id_integrity_check=response_id_integrity_check,
+        validation_check=validation_check,
+        task_validation_model=task_validation_model,
     )
-    processed_responses = process_llm_responses(llm_responses, responses_df)
+    processed_results = process_llm_responses(processed_rows, input_df)
+
     if failed_ids:
-        new_df = responses_df[responses_df["response_id"].astype(str).isin(failed_ids)]
-        processed_failed_responses = await batch_and_run(
-            responses_df=new_df,
-            prompt_template=prompt_template,
+        retry_df = input_df[input_df["response_id"].isin(failed_ids)]
+        retry_prompts = generate_prompts(
+            prompt_template, retry_df, batch_size=1, **kwargs
+        )
+        retry_results, unprocessable_ids = await call_llm(
+            batch_prompts=retry_prompts,
             llm=llm,
-            batch_size=1,
-            partition_key=partition_key,
-            **kwargs,
+            validation_check=validation_check,
+            task_validation_model=task_validation_model,
         )
-        return pd.concat(objs=[processed_failed_responses, processed_responses])
-    return processed_responses
+        retry_processed_results = process_llm_responses(retry_results, retry_df)
+        unprocessable_df = retry_df.loc[retry_df["response_id"].isin(unprocessable_ids)]
+        processed_results = pd.concat([processed_results, retry_processed_results])
+    else:
+        unprocessable_df = pd.DataFrame()
+    return processed_results, unprocessable_df
 
 
 def load_prompt_from_file(file_path: str | Path) -> str:
@@ -117,81 +146,150 @@ def convert_to_prompt_template(prompt_template: str | Path | PromptTemplate):
     return template
 
 
-def batch_responses(
-    responses_df: pd.DataFrame, batch_size: int = 10, partition_key: str | None = None
+def partition_dataframe(
+    df: pd.DataFrame, partition_key: Optional[str]
+) -> list[pd.DataFrame]:
+    """Splits the DataFrame into partitions based on the partition_key if provided."""
+    if partition_key:
+        return [group.reset_index(drop=True) for _, group in df.groupby(partition_key)]
+    return [df]
+
+
+def split_overflowing_batch(
+    batch: pd.DataFrame, allowed_tokens: int
 ) -> list[pd.DataFrame]:
-    """Split a DataFrame into batches, optionally partitioned by a key column.
+    """
+    Splits a DataFrame batch into smaller sub-batches such that each sub-batch's total token count
+    does not exceed the allowed token limit.
 
     Args:
-        responses_df (pd.DataFrame): Input DataFrame to be split into batches.
-        batch_size (int, optional): Maximum number of rows in each batch. Defaults to 10.
-        partition_key (str | None, optional): Column name to group by before batching.
-            If provided, ensures rows with the same partition key value stay together
-            and each group is batched separately. Defaults to None.
+        batch (pd.DataFrame): The input DataFrame to split.
+        allowed_tokens (int): The maximum allowed number of tokens per sub-batch.
 
     Returns:
-        list[pd.DataFrame]: List of DataFrame batches, where each batch contains
-            at most batch_size rows. If partition_key is used, rows within each
-            partition are kept together and batched separately.
+        list[pd.DataFrame]: A list of sub-batches, each within the token limit.
     """
-    if partition_key:
-        grouped = responses_df.groupby(partition_key)
-        batches = []
-        for _, group in grouped:
-            group_batches = [
-                group.iloc[i : i + batch_size].reset_index(drop=True)
-                for i in range(0, len(group), batch_size)
-            ]
-            batches.extend(group_batches)
-        return batches
-
-    return [
-        responses_df.iloc[i : i + batch_size].reset_index(drop=True)
-        for i in range(0, len(responses_df), batch_size)
-    ]
+    sub_batches = []
+    current_indices = []
+    current_token_sum = 0
+    token_counts = batch.apply(
+        lambda row: calculate_string_token_length(row.to_json()), axis=1
+    ).tolist()
+
+    for i, token_count in enumerate(token_counts):
+        if token_count > allowed_tokens:
+            logging.warning(
+                f"Row at index {batch.index[i]} exceeds allowed token limit ({token_count} > {allowed_tokens}). Skipping row."
+            )
+            continue
+
+        if current_token_sum + token_count > allowed_tokens:
+            if current_indices:
+                sub_batch = batch.iloc[current_indices].reset_index(drop=True)
+                if not sub_batch.empty:
+                    sub_batches.append(sub_batch)
+            current_indices = [i]
+            current_token_sum = token_count
+        else:
+            current_indices.append(i)
+            current_token_sum += token_count
+
+    if current_indices:
+        sub_batch = batch.iloc[current_indices].reset_index(drop=True)
+        if not sub_batch.empty:
+            sub_batches.append(sub_batch)
+    return sub_batches
+
+
+def batch_task_input_df(
+    df: pd.DataFrame,
+    allowed_tokens: int,
+    batch_size: int,
+    partition_key: Optional[str] = None,
+) -> list[pd.DataFrame]:
+    """
+    Partitions and batches a DataFrame according to a token limit and batch size, optionally using a partition key. Batches that exceed the token limit are further split.
+
+    Args:
+        df (pd.DataFrame): The input DataFrame to batch.
+        allowed_tokens (int): Maximum allowed tokens per batch.
+        batch_size (int): Maximum number of rows per batch before token filtering.
+        partition_key (Optional[str], optional): Column name to partition the DataFrame by.
+            Defaults to None.
+
+    Returns:
+        list[pd.DataFrame]: A list of batches, each within the specified token and size limits.
+    """
+    batches = []
+    partitions = partition_dataframe(df, partition_key)
+
+    for partition in partitions:
+        partition_batches = [
+            partition.iloc[i : i + batch_size].reset_index(drop=True)
+            for i in range(0, len(partition), batch_size)
+        ]
+        for batch in partition_batches:
+            batch_length = calculate_string_token_length(batch.to_json())
+            if batch_length <= allowed_tokens:
+                batches.append(batch)
+            else:
+                sub_batches = split_overflowing_batch(batch, allowed_tokens)
+                batches.extend(sub_batches)
+    return batches
 
 
 def generate_prompts(
-    response_dfs: list[pd.DataFrame], prompt_template: PromptTemplate, **kwargs: Any
+    prompt_template: PromptTemplate,
+    input_data: pd.DataFrame,
+    batch_size: int = 50,
+    max_prompt_length: int = 50_000,
+    partition_key: str | None = None,
+    **kwargs,
 ) -> list[BatchPrompt]:
-    """Generate a list of BatchPrompts from DataFrames using a prompt template.
+    """
+    Generate a list of BatchPrompt objects by splitting the input DataFrame into batches
+    and formatting each batch using a prompt template.
+
+    The function first calculates the token length of the prompt template to determine
+    the allowed tokens available for the input data. It then splits the input data into batches,
+    optionally partitioning by a specified key. Each batch is then formatted into a prompt string
+    using the provided prompt template, and a BatchPrompt is created containing the prompt string
+    and a list of response IDs from the batch.
 
     Args:
-        response_dfs (list[pd.DataFrame]): List of DataFrames, each containing a batch
-            of responses to be processed. Each DataFrame must include a 'response_id' column.
-        prompt_template (PromptTemplate): LangChain PromptTemplate object used to format
-            the prompts for each batch.
-        **kwargs (Any): Additional keyword arguments to pass to the prompt template's
-            format method.
+        prompt_template (PromptTemplate): An object with a 'template' attribute and a 'format' method
+            used to create a prompt string from a list of response dictionaries.
+        input_data (pd.DataFrame): A DataFrame containing the input responses, with at least a
+            'response_id' column.
+        batch_size (int, optional): Maximum number of rows to include in each batch. Defaults to 50.
+        max_prompt_length (int, optional): The maximum total token length allowed for the prompt,
+            including both the prompt template and the input data. Defaults to 50,000.
+        partition_key (str | None, optional): Column name used to partition the DataFrame before batching.
+            If provided, the DataFrame will be grouped by this key so that rows with the same value
+            remain in the same batch. Defaults to None.
+        **kwargs: Additional keyword arguments to pass to the prompt template's format method.
 
     Returns:
-        list[BatchPrompt]: List of BatchPrompt objects, each containing:
-            - prompt_string: Formatted prompt text for the batch
-            - response_ids: List of response IDs included in the batch
-
-    Note:
-        The function converts each DataFrame to a list of dictionaries and passes it
-        to the prompt template as the 'responses' variable.
+        list[BatchPrompt]: A list of BatchPrompt objects where each object contains:
+            - prompt_string: The formatted prompt string for a batch.
+            - response_ids: A list of response IDs corresponding to the rows in that batch.
     """
-    batched_prompts = []
-    for df in response_dfs:
-        prompt = prompt_template.format(
-            responses=df.to_dict(orient="records"), **kwargs
-        )
-        response_ids = df["response_id"].astype(str).to_list()
-        batched_prompts.append(
-            BatchPrompt(prompt_string=prompt, response_ids=response_ids)
-        )
-
-    return batched_prompts
+    prompt_token_length = calculate_string_token_length(prompt_template.template)
+    allowed_tokens_for_data = max_prompt_length - prompt_token_length
+    batches = batch_task_input_df(
+        input_data, allowed_tokens_for_data, batch_size, partition_key
+    )
+    prompts = [build_prompt(prompt_template, batch, **kwargs) for batch in batches]
+    return prompts
 
 
 async def call_llm(
     batch_prompts: list[BatchPrompt],
     llm: Runnable,
     concurrency: int = 10,
-    response_id_integrity_check: bool = False,
-):
+    validation_check: bool = False,
+    task_validation_model: Optional[Type[BaseModel]] = None,
+) -> tuple[list[dict], list[int]]:
     """Process multiple batches of prompts concurrently through an LLM with retry logic.
 
     Args:
@@ -200,9 +298,10 @@ async def call_llm(
         llm (Runnable): LangChain Runnable instance that will process the prompts.
         concurrency (int, optional): Maximum number of simultaneous LLM calls allowed.
             Defaults to 10.
-        response_id_integrity_check (bool, optional): If True, verifies that all input
+        validation_check (bool, optional): If True, verifies that all input
             response IDs are present in the LLM output. Failed batches are discarded and
             their IDs are returned for retry. Defaults to False.
+        task_validation_model (Type[BaseModel]): The Pydantic model to check the LLM outputs against
 
     Returns:
         tuple[list[dict[str, Any]], set[str]]: A tuple containing:
@@ -215,69 +314,76 @@ async def call_llm(
         - Concurrency is managed via asyncio.Semaphore to prevent overwhelming the LLM
     """
     semaphore = asyncio.Semaphore(concurrency)
-    failed_ids: set = set()
 
     @retry(
         wait=wait_random_exponential(min=1, max=20),
         stop=stop_after_attempt(6),
         before=before.before_log(logger=logger, log_level=logging.DEBUG),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
         reraise=True,
     )
-    async def async_llm_call(batch_prompt):
+    async def async_llm_call(batch_prompt) -> tuple[list[dict], list[int]]:
         async with semaphore:
-            response = await llm.ainvoke(batch_prompt.prompt_string)
-            parsed_response = json.loads(response.content)
-
-            if response_id_integrity_check and not check_response_integrity(
-                batch_prompt.response_ids, parsed_response
-            ):
-                # discard this response but keep track of failed response ids
-                failed_ids.update(batch_prompt.response_ids)
-                return None
-
-            return parsed_response
+            try:
+                llm_response = await llm.ainvoke(batch_prompt.prompt_string)
+                all_results = json.loads(llm_response.content)
+            except (openai.BadRequestError, json.JSONDecodeError) as e:
+                failed_ids = batch_prompt.response_ids
+                logger.warning(e)
+                return [], failed_ids
+
+            if validation_check:
+                failed_ids = get_missing_response_ids(
+                    batch_prompt.response_ids, all_results
+                )
+                validated_results, invalid_rows = validate_task_data(
+                    all_results["responses"], task_validation_model
+                )
+                failed_ids.extend([r["response_id"] for r in invalid_rows])
+                return validated_results, failed_ids
+            else:
+                # Flatten the list to align with valid output format
+                return [r for r in all_results["responses"]], []
 
     results = await asyncio.gather(
         *[async_llm_call(batch_prompt) for batch_prompt in batch_prompts]
     )
-    successful_responses = [
-        r for r in results if r is not None
-    ]  # ignore discarded responses
-    return (successful_responses, failed_ids)
+    valid_inputs = [row for result, _ in results for row in result]
+    failed_response_ids = [
+        failed_response_id
+        for _, batch_failures in results
+        for failed_response_id in batch_failures
+    ]
 
+    return valid_inputs, failed_response_ids
 
-def check_response_integrity(
-    input_response_ids: set[str], parsed_response: dict
-) -> bool:
-    """Verify that all input response IDs are present in the LLM's parsed response.
+
+def get_missing_response_ids(
+    input_response_ids: list[int], parsed_response: dict
+) -> list[int]:
+    """Identify which response IDs are missing from the LLM's parsed response.
 
     Args:
         input_response_ids (set[str]): Set of response IDs that were included in the
-            original prompt sent to the LLM.
+            original prompt.
         parsed_response (dict): Parsed response from the LLM containing a 'responses' key
             with a list of dictionaries, each containing a 'response_id' field.
 
     Returns:
-        bool: True if all input response IDs are present in the parsed response and
-            no additional IDs are present, False otherwise.
+        set[str]: Set of response IDs that are missing from the parsed response.
     """
-    response_ids_set = set(input_response_ids)
 
+    response_ids_set = {int(response_id) for response_id in input_response_ids}
     returned_ids_set = {
-        str(
-            element["response_id"]
-        )  # treat ids as strings to match response_ids_in_each_prompt
+        int(element["response_id"])
         for element in parsed_response["responses"]
         if element.get("response_id", False)
     }
-    # assumes: all input ids ought to be present in output
-    if returned_ids_set != response_ids_set:
-        logger.info("Failed integrity check")
-        logger.info(
-            f"Present in original but not returned from LLM: {response_ids_set - returned_ids_set}. Returned in LLM but not present in original: {returned_ids_set - response_ids_set}"
-        )
-        return False
-    return True
+
+    missing_ids = list(response_ids_set - returned_ids_set)
+    if missing_ids:
+        logger.info(f"Missing response IDs from LLM output: {missing_ids}")
+    return missing_ids
 
 
 def process_llm_responses(
@@ -298,13 +404,87 @@ def process_llm_responses(
             - If no response_id in LLM output: DataFrame containing only the LLM results
     """
     responses.loc[:, "response_id"] = responses["response_id"].astype(int)
-    unpacked_responses = [
-        response
-        for batch_response in llm_responses
-        for response in batch_response.get("responses", [])
-    ]
-    task_responses = pd.DataFrame(unpacked_responses)
+    task_responses = pd.DataFrame(llm_responses)
     if "response_id" in task_responses.columns:
         task_responses["response_id"] = task_responses["response_id"].astype(int)
         return responses.merge(task_responses, how="inner", on="response_id")
     return task_responses
+
+
+def calculate_string_token_length(input_text: str, model: str = None) -> int:
+    """
+    Calculates the number of tokens in a given string using the specified model's tokenizer.
+
+    Args:
+        input_text (str): The input string to tokenize.
+        model (str, optional): The model name used for tokenization. If not provided,
+            uses the MODEL_NAME environment variable or defaults to "gpt-4o".
+
+    Returns:
+        int: The number of tokens in the input string.
+    """
+    # Use the MODEL_NAME env var if no model is provided; otherwise default to "gpt-4o"
+    model = model or os.environ.get("MODEL_NAME", "gpt-4o")
+    tokenizer_encoding = tiktoken.encoding_for_model(model)
+    number_of_tokens = len(tokenizer_encoding.encode(input_text))
+    return number_of_tokens
+
+
+def build_prompt(
+    prompt_template: PromptTemplate, input_batch: pd.DataFrame, **kwargs
+) -> BatchPrompt:
+    """
+    Constructs a BatchPrompt by formatting a prompt template with a batch of responses.
+
+    The function converts the input DataFrame batch into a list of dictionaries (one per row) and passes
+    this list to the prompt template's format method under the key 'responses', along with any additional
+    keyword arguments. It also extracts the 'response_id' column from the batch,
+    and uses these to create the BatchPrompt.
+
+    Args:
+        prompt_template (PromptTemplate): An object with a 'template' attribute and a 'format' method that is used
+            to generate the prompt string.
+        input_batch (pd.DataFrame): A DataFrame containing the batch of responses, which must include a 'response_id'
+            column.
+        **kwargs: Additional keyword arguments to pass to the prompt template's format method.
+
+    Returns:
+        BatchPrompt: An object containing:
+            - prompt_string: The formatted prompt string for the batch.
+            - response_ids: A list of response IDs (as strings) corresponding to the responses in the batch.
+    """
+    prompt = prompt_template.format(
+        responses=input_batch.to_dict(orient="records"), **kwargs
+    )
+    response_ids = input_batch["response_id"].astype(int).to_list()
+    return BatchPrompt(prompt_string=prompt, response_ids=response_ids)
+
+
+def validate_task_data(
+    task_data: pd.DataFrame | list[dict], task_validation_model: Type[BaseModel] = None
+) -> tuple[list[dict], list[dict]]:
+    """
+    Validate each row in task_output against the provided Pydantic model.
+
+    Returns:
+        valid: a list of validated records (dicts).
+        invalid: a list of records (dicts) that failed validation.
+    """
+
+    records = (
+        task_data.to_dict(orient="records")
+        if isinstance(task_data, pd.DataFrame)
+        else task_data
+    )
+
+    if task_validation_model:
+        valid_records, invalid_records = [], []
+        for record in records:
+            try:
+                task_validation_model(**record)
+                valid_records.append(record)
+            except ValidationError as e:
+                invalid_records.append(record)
+                logger.info(f"Failed Validation: {e}")
+        return valid_records, invalid_records
+    return records, []

themefinder/models.py

@@ -0,0 +1,138 @@
+from pydantic import BaseModel, Field, model_validator
+
+
+def validate_non_empty_fields(model: BaseModel) -> BaseModel:
+    """
+    Validate that all string fields in the model are non-empty (after stripping)
+    and that list fields are not empty.
+
+    Args:
+        model (BaseModel): A Pydantic model instance.
+
+    Returns:
+        BaseModel: The same model if validation passes.
+
+    Raises:
+        ValueError: If any string field is empty or any list field is empty.
+    """
+    for field_name, value in model.__dict__.items():
+        if isinstance(value, str) and not value.strip():
+            raise ValueError(f"{field_name} cannot be empty or only whitespace")
+        if isinstance(value, list) and not value:
+            raise ValueError(f"{field_name} cannot be an empty list")
+    return model
+
+
+def validate_position(model: BaseModel) -> BaseModel:
+    """
+    Validate that the model's 'position' field is one of the allowed values.
+
+    Args:
+        model (BaseModel): A Pydantic model instance with a 'position' attribute.
+
+    Returns:
+        BaseModel: The same model if validation passes.
+
+    Raises:
+        ValueError: If the 'position' field is not one of the allowed values.
+    """
+    allowed_positions = {"AGREEMENT", "DISAGREEMENT", "UNCLEAR"}
+    if model.position not in allowed_positions:
+        raise ValueError(f"position must be one of {allowed_positions}")
+    return model
+
+
+def validate_stances(model: BaseModel) -> BaseModel:
+    """
+    Validate that every stance in the model's 'stances' field is allowed.
+
+    Args:
+        model (BaseModel): A Pydantic model instance with a 'stances' attribute.
+
+    Returns:
+        BaseModel: The same model if validation passes.
+
+    Raises:
+        ValueError: If any stance is not among the allowed stances.
+    """
+    allowed_stances = {"POSITIVE", "NEGATIVE"}
+    for stance in model.stances:
+        if stance not in allowed_stances:
+            raise ValueError(f"stances must be one of {allowed_stances}")
+    return model
+
+
+def validate_mapping_stance_lengths(model: BaseModel) -> BaseModel:
+    """
+    Validate that the lengths of the model's 'stances' and 'labels' fields match.
+
+    Args:
+        model (BaseModel): A Pydantic model instance with 'stances' and 'labels' attributes.
+
+    Returns:
+        BaseModel: The same model if validation passes.
+
+    Raises:
+        ValueError: If the lengths of 'stances' and 'labels' do not match.
+    """
+    if len(model.stances) != len(model.labels):
+        raise ValueError("'stances' must have the same length as 'labels'")
+    return model
+
+
+def validate_mapping_unique_labels(model: BaseModel) -> BaseModel:
+    """
+    Validate that the model's 'labels' field contains unique values.
+
+    Args:
+        model (BaseModel): A Pydantic model instance with a 'labels' attribute.
+
+    Returns:
+        BaseModel: The same model if validation passes.
+
+    Raises:
+        ValueError: If 'labels' contains duplicate values.
+    """
+    if len(model.labels) != len(set(model.labels)):
+        raise ValueError("'labels' must be unique")
+    return model
+
+
+class SentimentAnalysisOutput(BaseModel):
+    response_id: int = Field(gt=0)
+    position: str
+
+    @model_validator(mode="after")
+    def run_validations(self) -> "SentimentAnalysisOutput":
+        """
+        Run all validations for SentimentAnalysisOutput.
+
+        Validates that:
+         - 'position' is one of the allowed values.
+         - No fields are empty or only whitespace (for strings) and no lists are empty.
+        """
+        validate_position(self)
+        validate_non_empty_fields(self)
+        return self
+
+
+class ThemeMappingOutput(BaseModel):
+    response_id: int = Field(gt=0)
+    labels: list[str]
+    reasons: list[str]
+    stances: list[str]
+
+    @model_validator(mode="after")
+    def run_validations(self) -> "ThemeMappingOutput":
+        """
+        Run all validations for ThemeMappingOutput.
+
+        Validates that:
+         - 'stances' are only 'POSITIVE' or 'NEGATIVE'.
+         - The 'stances' and 'labels' have matching lengths.
+         - 'labels' are unique.
+        """
+        validate_stances(self)
+        validate_mapping_stance_lengths(self)
+        validate_mapping_unique_labels(self)
+        return self

themefinder/prompts/sentiment_analysis.txt

@@ -3,8 +3,8 @@
 You will receive a list of RESPONSES, each containing a response_id and a response.
 Your job is to analyze each response to the QUESTION below and decide:
 
-POSITION - is the response agreeing or disagreeing or is it unclear about the change being proposed in the question.
-Choose one from [agreement, disagreement, unclear]
+POSITION - is the response AGREEING or DISAGREEING or is it UNCLEAR about the change being proposed in the question.
+Choose one from [AGREEMENT, DISAGREEMENT, UNCLEAR]
 
 The final output should be in the following JSON format:
 
@@ -24,20 +24,23 @@ You MUST include every response ID in the output.
 If the response can not be labelled return empty sections where appropriate but you MUST return an entry
 with the correct response ID for each input object
 
+You MUST pick one of the given POSITION values.
+You MUST not return an empty value for the POSITION of a response.
+
 ## EXAMPLE
 Example 1:
 Question: \n What are your thoughts on the proposed government changes to the policy about reducing school holidays?
 Response: \n as a parent I have no idea why you would make this change. I guess you were thinking about increasing productivity but any productivity gains would be totally offset by the decrease in family time. \n
 
 Output:
-POSITION: disagreement
+POSITION: DISAGREEMENT
 
 Example 2:
 Question: \n What are your thoughts on the proposed government changes to the policy about reducing school holidays?
 Response: \n I think this is a great idea, our children will learn more if they are in school more \n
 
 Output:
-POSITION: agreement
+POSITION: AGREEMENT
 
 Example 3:
 Question: \n What are your thoughts on the proposed government changes to the policy about reducing school holidays?
@@ -45,7 +48,7 @@ Response: \n it will be good for our children to be around their friends more bu
 less time with their children \n
 
 Output:
-POSITION: unclear
+POSITION: UNCLEAR
 
 
 QUESTION: \n {question}

themefinder/prompts/theme_mapping.txt

@@ -17,7 +17,7 @@ Your task is to analyze each response and decide which topics are present. Guide
     - There is no limit on how many topics can be assigned to a response.
     - For each assignment provide a single rationale for why you have chosen the label.
     - For each topic identified in a response, indicate whether the response expresses a positive or negative stance toward that topic (options: 'POSITIVE' or 'NEGATIVE')
-    - If a response contains both positive and negative statements about a topic within the same response, choose the stance that receives more emphasis or appears more central to the argument
+    - You MUST use either 'POSTIVE' or 'NEGATIVE'
     - The order of reasons and stances must align with the order of labels (e.g., stance_a applies to topic_a)
 
 You MUST include every response ID in the output.
@@ -30,13 +30,13 @@ The final output should be in the following JSON format:
 {{
   "responses": [
     {{
-      "response_id": "response_id_1",
+      "response_id": response_id_1,
       "reasons": ["reason_a", "reason_b"],
       "labels": ["topic_a", "topic_b"],
       "stances": ["stance_a", "stance_b"],
     }},
     {{
-      "response_id": "response_id_2",
+      "response_id": response_id_2,
       "reasons": ["reason_c"],
       "labels": ["topic_c"],
       "stances": ["stance_c"],

themefinder/prompts/theme_refinement.txt

@@ -1,13 +1,12 @@
 {system_prompt}
 
-You are tasked with refining and neutralizing a list of topics generated from responses to a question. 
-Your goal is to transform opinionated topics into neutral, well-structured, and distinct topics while preserving the essential information.
+You are tasked with refining a list of topics generated from responses to a question. 
 
 ## Input
-You will receive a list of OPINIONATED TOPICS. These topics explicitly tie opinions to whether a person agrees or disagrees with the question.
+You will receive a list of TOPICS. These topics explicitly tie opinions to whether a person agrees or disagrees with the question.
 
 ## Output
-You will produce a list of NEUTRAL TOPICS based on the input. Each neutral topic should have two parts:
+You will produce a list of CLEAR STANCE TOPICS based on the input. Each topic should have two parts:
 1. A brief, clear topic label (3-7 words)
 2. A more detailed topic description (1-2 sentences)
 
@@ -17,10 +16,11 @@ You will produce a list of NEUTRAL TOPICS based on the input. Each neutral topic
    - Preserve all key information, details and concepts from the original topics.
    - Ensure no significant details are lost in the refinement process.
 
-2. Neutrality:
-   - Remove all language indicating agreement or disagreement.
-   - Present topics objectively without favoring any particular stance.
-   - Avoid phrases like "supporters believe" or "critics argue".
+2. Clear Stance Formulation:
+   - Reformulate topics to express a clear stance that can be agreed or disagreed with.
+   - Use direct language like "Increased risk of X" rather than "X"
+   - Avoid double negatives and ambiguous phrasing.
+   - Phrase topics as definitive statements.
 
 3. Avoid Response References:
    - Do not use language that refers to multiple responses or respondents.
@@ -39,16 +39,15 @@ You will produce a list of NEUTRAL TOPICS based on the input. Each neutral topic
 
 ## Process
 
-1. Analyze the OPINIONATED TOPICS to identify key themes and information.
+1. Analyze the TOPICS to identify key themes and information.
 2. Group closely related topics together.
 3. For each group or individual topic:
    a. Distill the core concept, removing any bias or opinion.
    b. Create a neutral, concise topic label.
    c. Write a more detailed description that provides context without taking sides.
 4. Review the entire list to ensure distinctiveness and adjust as needed.
-5. Double-check that all topics are truly neutral and free of response references.
-6. Assign each output topic a topic_id a single uppercase letters (starting from 'A')
-7. Combine the topic label and description with a colon separator
+5. Assign each output topic a topic_id a single uppercase letters (starting from 'A', for the 27th element use AA)
+6. Combine the topic label and description with a colon separator
 
 Return your output in the following JSON format:
 {{
@@ -61,21 +60,6 @@ Return your output in the following JSON format:
 }}
 
 
-## EXAMPLE
 
-OPINIONATED TOPIC:
-"Economic impact: Many respondents who support the policy believe it will create jobs and boost the economy, it could raise GDP by 2%. [source_topic_count: 15]"
-
-NEUTRAL TOPIC:
-{{
-  "topic_id": "A",
-  "topic": "Economic Impact on Employment: The policy's potential effects on job creation and overall economic growth, including potential for a 2% increase in GDP.",
-  "source_topic_count": 15
-}}
-
-Remember, your goal is to create a list of neutral, informative, and distinct topics that accurately represent the content of the original opinionated topics without any bias or references to responses.
-
-
-
-OPINIONATED TOPIC:
+TOPICS:
 {responses}

{themefinder-0.5.4.dist-info → themefinder-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: themefinder
-Version: 0.5.4
+Version: 0.6.2
 Summary: A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses.
 License: MIT
 Author: i.AI
@@ -100,7 +100,7 @@ system_prompt = "You are an AI evaluation tool analyzing survey responses about
 # Run the function to find themes
 # We use asyncio to query LLM endpoints asynchronously, so we need to await our function
 async def main():
-    result = await find_themes(responses_df, llm, question, system_prompt)
+    result = await find_themes(responses_df, llm, question, system_prompt=system_prompt)
     print(result)
 
 if __name__ == "__main__":
@@ -155,3 +155,4 @@ The documentation is [© Crown copyright](https://www.nationalarchives.gov.uk/in
 ## Feedback
 
 If you have feedback on this package, please fill in our [feedback form](https://forms.gle/85xUSMvxGzSSKQ499) or contact us with questions or feedback at packages@cabinetoffice.gov.uk.
+

themefinder-0.6.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+themefinder/__init__.py,sha256=wSpW2fEnC4gTzbeNC78nSD3DpJq43-h_H-LK_cqt1cw,327
+themefinder/core.py,sha256=u1DY9gbzn-tFhQS3hrXQ8_1mIbR-iBWYVAdKeAX1BdE,18304
+themefinder/llm_batch_processor.py,sha256=OrFEl1nSi5ninbSZSiE1HFMcYZiQ-NzuYPj_iDcPPoE,19988
+themefinder/models.py,sha256=Y5-okndYwtBO09n_qUlYNVmHRVNEnJviArQZukm8Ox8,4251
+themefinder/prompts/consultation_system_prompt.txt,sha256=_A07oY_an4hnRx-9pQ0y-TLXJz0dd8vDI-MZne7Mdb4,89
+themefinder/prompts/sentiment_analysis.txt,sha256=9-LkdR95JTHXRKUXknAgNf86uVdv6jSaXMf-OtFL9_0,1948
+themefinder/prompts/theme_condensation.txt,sha256=DB4pqUmMpo0OG4AZWGTj0FfLFfjbX6wOMUr44HBxZ1o,2433
+themefinder/prompts/theme_generation.txt,sha256=JMXuNojxdSAcxPRU1Jg12Xunv_dX4hNvXYU2pXMWTAw,2500
+themefinder/prompts/theme_mapping.txt,sha256=YcRGMkuTyTPzPQPtsDY31DUwX60c8AdmdHKw0XeUejQ,2258
+themefinder/prompts/theme_refinement.txt,sha256=hBXwZnNZmhmoEFXpY5OJinp-7xxdoDRf_5LmgrilYgc,2713
+themefinder/prompts/theme_target_alignment.txt,sha256=-_ghr4--KAN6Tz8ExO9s2IXvI6pjWaEA_nG5L83GV5I,1035
+themefinder/themefinder_logging.py,sha256=n5SUQovEZLC4skEbxicjz_fOGF9mOk3S-Wpj5uXsaL8,314
+themefinder-0.6.2.dist-info/LICENCE,sha256=C9ULIN0ctF60ZxUWH_hw1H434bDLg49Z-Qzn6BUHgqs,1060
+themefinder-0.6.2.dist-info/METADATA,sha256=gI9Hp754EjopJQWw0QZIPb9dex8TalPMGnorUEOJlp0,6498
+themefinder-0.6.2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+themefinder-0.6.2.dist-info/RECORD,,

{themefinder-0.5.4.dist-info → themefinder-0.6.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.2
 Root-Is-Purelib: true
 Tag: py3-none-any

themefinder-0.5.4.dist-info/RECORD

@@ -1,15 +0,0 @@
-themefinder/__init__.py,sha256=p6QoCgA-BYWljk8yPOeTgkNcN5m_gA_o3Q86Eh0QjSM,327
-themefinder/core.py,sha256=yH68-DtpIv0jX__LnjuBaKJn01hj-VurW3WnFxk0wMQ,17537
-themefinder/llm_batch_processor.py,sha256=SDDeMJeX1J3u7FGFddRhVSxty6U8lFVXwG4eNI_0C5o,12573
-themefinder/prompts/consultation_system_prompt.txt,sha256=_A07oY_an4hnRx-9pQ0y-TLXJz0dd8vDI-MZne7Mdb4,89
-themefinder/prompts/sentiment_analysis.txt,sha256=e3DcUKga6pSFcfeo2TAq8x9LXk0YDV-D7P2gtymcyuc,1832
-themefinder/prompts/theme_condensation.txt,sha256=DB4pqUmMpo0OG4AZWGTj0FfLFfjbX6wOMUr44HBxZ1o,2433
-themefinder/prompts/theme_generation.txt,sha256=JMXuNojxdSAcxPRU1Jg12Xunv_dX4hNvXYU2pXMWTAw,2500
-themefinder/prompts/theme_mapping.txt,sha256=nb_D7gwKGd8BzrAlzSZC3mQIPYaCRXdE6XmoJaJEKZQ,2405
-themefinder/prompts/theme_refinement.txt,sha256=_NVHdXBfqCFX2u0R5oZEqWQo70MAjJ5nXQfZ7p_HRAM,3528
-themefinder/prompts/theme_target_alignment.txt,sha256=-_ghr4--KAN6Tz8ExO9s2IXvI6pjWaEA_nG5L83GV5I,1035
-themefinder/themefinder_logging.py,sha256=n5SUQovEZLC4skEbxicjz_fOGF9mOk3S-Wpj5uXsaL8,314
-themefinder-0.5.4.dist-info/LICENCE,sha256=C9ULIN0ctF60ZxUWH_hw1H434bDLg49Z-Qzn6BUHgqs,1060
-themefinder-0.5.4.dist-info/METADATA,sha256=JKSxdzARGcJ-OJwrd5ScuPzm4Uln2cBQ_SnrxFAhQLQ,6483
-themefinder-0.5.4.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-themefinder-0.5.4.dist-info/RECORD,,