themefinder 0.3.1__tar.gz → 0.5.2__tar.gz

{themefinder-0.3.1 → themefinder-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: themefinder
-Version: 0.3.1
+Version: 0.5.2
 Summary: A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses.
 License: MIT
 Author: i.AI
@@ -128,6 +128,9 @@ ThemeFinder's pipeline consists of five distinct stages, each utilizing a specia
 - Leverages standardisation prompts to normalise theme descriptions
 - Creates clear, consistent theme definitions through structured refinement
 
+### Theme target alignment
+- Optional step to consolidate themes down to a target number
+
 ### Theme mapping
 - Utilizes classification prompts to map individual responses to refined themes
 - Supports multiple theme assignments per response through detailed analysis

{themefinder-0.3.1 → themefinder-0.5.2}/README.md

@@ -98,6 +98,9 @@ ThemeFinder's pipeline consists of five distinct stages, each utilizing a specia
 - Leverages standardisation prompts to normalise theme descriptions
 - Creates clear, consistent theme definitions through structured refinement
 
+### Theme target alignment
+- Optional step to consolidate themes down to a target number
+
 ### Theme mapping
 - Utilizes classification prompts to map individual responses to refined themes
 - Supports multiple theme assignments per response through detailed analysis

{themefinder-0.3.1 → themefinder-0.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "themefinder"
-version = "0.3.1"
+version = "0.5.2"
 description = "A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses."
 authors = ["i.AI <packages@cabinetoffice.gov.uk>"]
 packages = [{include = "themefinder", from = "src"}]

{themefinder-0.3.1 → themefinder-0.5.2}/src/themefinder/core.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 
 import pandas as pd
@@ -7,7 +8,6 @@ from langchain_core.runnables import Runnable
 from .llm_batch_processor import batch_and_run, load_prompt_from_file
 from .themefinder_logging import logger
 
-
 CONSULTATION_SYSTEM_PROMPT = load_prompt_from_file("consultation_system_prompt")
 
 
@@ -15,7 +15,9 @@ async def find_themes(
     responses_df: pd.DataFrame,
     llm: Runnable,
     question: str,
+    target_n_themes: int | None = None,
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
+    verbose: bool = True,
 ) -> dict[str, pd.DataFrame]:
     """Process survey responses through a multi-stage theme analysis pipeline.
 
@@ -24,14 +26,19 @@ async def find_themes(
     2. Initial theme generation
     3. Theme condensation (combining similar themes)
     4. Theme refinement
-    5. Mapping responses to refined themes
+    5. Theme target alignment (optional, if target_n_themes is specified)
+    6. Mapping responses to refined themes
 
     Args:
         responses_df (pd.DataFrame): DataFrame containing survey responses
         llm (Runnable): Language model instance for text analysis
         question (str): The survey question
+        target_n_themes (int | None, optional): Target number of themes to consolidate to.
+            If None, skip theme target alignment step. Defaults to None.
         system_prompt (str): System prompt to guide the LLM's behavior.
             Defaults to CONSULTATION_SYSTEM_PROMPT.
+        verbose (bool): Whether to show information messages during processing.
+            Defaults to True.
 
     Returns:
         dict[str, pd.DataFrame]: Dictionary containing results from each pipeline stage:
@@ -42,6 +49,8 @@ async def find_themes(
             - refined_topics: DataFrame with refined theme definitions
             - mapping: DataFrame mapping responses to final themes
     """
+    logger.setLevel(logging.INFO if verbose else logging.CRITICAL)
+
     sentiment_df = await sentiment_analysis(
         responses_df,
         llm,
@@ -63,6 +72,14 @@ async def find_themes(
         question=question,
         system_prompt=system_prompt,
     )
+    if target_n_themes is not None:
+        refined_theme_df = await theme_target_alignment(
+            refined_theme_df,
+            llm,
+            question=question,
+            target_n_themes=target_n_themes,
+            system_prompt=system_prompt,
+        )
     mapping_df = await theme_mapping(
         sentiment_df,
         llm,
@@ -79,8 +96,8 @@ async def find_themes(
         "question": question,
         "sentiment": sentiment_df,
         "topics": theme_df,
-        "condensed_topics": condensed_theme_df,
-        "refined_topics": refined_theme_df,
+        "condensed_themes": condensed_theme_df,
+        "refined_themes": refined_theme_df,
         "mapping": mapping_df,
     }
 
@@ -89,7 +106,7 @@ async def sentiment_analysis(
     responses_df: pd.DataFrame,
     llm: Runnable,
     question: str,
-    batch_size: int = 10,
+    batch_size: int = 20,
     prompt_template: str | Path | PromptTemplate = "sentiment_analysis",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
 ) -> pd.DataFrame:
@@ -104,7 +121,7 @@ async def sentiment_analysis(
         llm (Runnable): Language model instance to use for sentiment analysis.
         question (str): The survey question.
         batch_size (int, optional): Number of responses to process in each batch.
-            Defaults to 10.
+            Defaults to 20.
         prompt_template (str | Path | PromptTemplate, optional): Template for structuring
             the prompt to the LLM. Can be a string identifier, path to template file,
             or PromptTemplate instance. Defaults to "sentiment_analysis".
@@ -180,9 +197,10 @@ async def theme_condensation(
     themes_df: pd.DataFrame,
     llm: Runnable,
     question: str,
-    batch_size: int = 10000,
+    batch_size: int = 100,
     prompt_template: str | Path | PromptTemplate = "theme_condensation",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
+    **kwargs,
 ) -> pd.DataFrame:
     """Condense and combine similar themes identified from survey responses.
 
@@ -195,7 +213,7 @@ async def theme_condensation(
         llm (Runnable): Language model instance to use for theme condensation.
         question (str): The survey question.
         batch_size (int, optional): Number of themes to process in each batch.
-            Defaults to 10000.
+            Defaults to 100.
         prompt_template (str | Path | PromptTemplate, optional): Template for structuring
             the prompt to the LLM. Can be a string identifier, path to template file,
             or PromptTemplate instance. Defaults to "theme_condensation".
@@ -206,17 +224,43 @@ async def theme_condensation(
         pd.DataFrame: DataFrame containing the condensed themes, where similar topics
             have been combined into broader categories.
     """
-    logger.info(f"Running theme condensation on {len(themes_df)} topics")
+    logger.info(f"Running theme condensation on {len(themes_df)} responses")
     themes_df["response_id"] = range(len(themes_df))
-    return await batch_and_run(
+
+    n_themes = themes_df.shape[0]
+    while n_themes > batch_size:
+        logger.info(
+            f"{n_themes} larger than batch size, using recursive theme condensation"
+        )
+        themes_df = await batch_and_run(
+            themes_df,
+            prompt_template,
+            llm,
+            batch_size=batch_size,
+            question=question,
+            system_prompt=system_prompt,
+            **kwargs,
+        )
+        themes_df = themes_df.sample(frac=1).reset_index(drop=True)
+        themes_df["response_id"] = range(len(themes_df))
+        if len(themes_df) == n_themes:
+            logger.info("Themes no longer being condensed")
+            break
+        n_themes = themes_df.shape[0]
+
+    themes_df = await batch_and_run(
         themes_df,
         prompt_template,
         llm,
         batch_size=batch_size,
         question=question,
         system_prompt=system_prompt,
+        **kwargs,
     )
 
+    logger.info(f"Final number of condensed themes: {themes_df.shape[0]}")
+    return themes_df
+
 
 async def theme_refinement(
     condensed_themes_df: pd.DataFrame,
@@ -257,10 +301,10 @@ async def theme_refinement(
         transposes the output for improved readability and easier downstream
         processing.
     """
-    logger.info(f"Running topic refinement on {len(condensed_themes_df)} responses")
+    logger.info(f"Running theme refinement on {len(condensed_themes_df)} responses")
     condensed_themes_df["response_id"] = range(len(condensed_themes_df))
 
-    def transpose_refined_topics(refined_themes: pd.DataFrame):
+    def transpose_refined_themes(refined_themes: pd.DataFrame):
         """Transpose topics for increased legibility."""
         transposed_df = pd.DataFrame(
             [refined_themes["topic"].to_numpy()], columns=refined_themes["topic_id"]
@@ -275,7 +319,74 @@ async def theme_refinement(
         question=question,
         system_prompt=system_prompt,
     )
-    return transpose_refined_topics(refined_themes)
+    return transpose_refined_themes(refined_themes)
+
+
+async def theme_target_alignment(
+    refined_themes_df: pd.DataFrame,
+    llm: Runnable,
+    question: str,
+    target_n_themes: int = 10,
+    batch_size: int = 10000,
+    prompt_template: str | Path | PromptTemplate = "theme_target_alignment",
+    system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
+) -> pd.DataFrame:
+    """Align themes to target number using an LLM.
+
+    This function processes refined themes to consolidate them into a target number of
+    distinct categories while preserving all significant details and perspectives.
+    It transforms the output format for improved readability by transposing the
+    results into a single-row DataFrame where columns represent individual themes.
+
+    Args:
+        refined_themes_df (pd.DataFrame): DataFrame containing the refined themes
+            from the previous pipeline stage.
+        llm (Runnable): Language model instance to use for theme alignment.
+        question (str): The survey question.
+        target_n_themes (int, optional): Target number of themes to consolidate to.
+            Defaults to 10.
+        batch_size (int, optional): Number of themes to process in each batch.
+            Defaults to 10000.
+        prompt_template (str | Path | PromptTemplate, optional): Template for structuring
+            the prompt to the LLM. Can be a string identifier, path to template file,
+            or PromptTemplate instance. Defaults to "theme_target_alignment".
+        system_prompt (str): System prompt to guide the LLM's behavior.
+            Defaults to CONSULTATION_SYSTEM_PROMPT.
+
+    Returns:
+        pd.DataFrame: A single-row DataFrame where:
+            - Each column represents a unique theme (identified by topic_id)
+            - The values contain the aligned theme descriptions
+            - The format is optimized for subsequent theme mapping operations
+
+    Note:
+        The function adds sequential response_ids to the input DataFrame and
+        transposes the output for improved readability and easier downstream
+        processing.
+    """
+    logger.info(
+        f"Running theme target alignment on {len(refined_themes_df.columns)} themes compressing to {target_n_themes} themes"
+    )
+    refined_themes_df = refined_themes_df.T.rename(columns={0: "topic"})
+    refined_themes_df["response_id"] = range(len(refined_themes_df))
+
+    def transpose_aligned_themes(aligned_themes: pd.DataFrame):
+        """Transpose topics for increased legibility."""
+        transposed_df = pd.DataFrame(
+            [aligned_themes["topic"].to_numpy()], columns=aligned_themes["topic_id"]
+        )
+        return transposed_df
+
+    aligned_themes = await batch_and_run(
+        refined_themes_df,
+        prompt_template,
+        llm,
+        batch_size=batch_size,
+        question=question,
+        system_prompt=system_prompt,
+        target_n_themes=target_n_themes,
+    )
+    return transpose_aligned_themes(aligned_themes)
 
 
 async def theme_mapping(

{themefinder-0.3.1 → themefinder-0.5.2}/src/themefinder/llm_batch_processor.py

@@ -174,7 +174,6 @@ def generate_prompts(
         to the prompt template as the 'responses' variable.
     """
     batched_prompts = []
-
     for df in response_dfs:
         prompt = prompt_template.format(
             responses=df.to_dict(orient="records"), **kwargs
@@ -219,7 +218,7 @@ async def call_llm(
     failed_ids: set = set()
 
     @retry(
-        wait=wait_random_exponential(min=1, max=60),
+        wait=wait_random_exponential(min=1, max=20),
         stop=stop_after_attempt(6),
         before=before.before_log(logger=logger, log_level=logging.DEBUG),
         reraise=True,
@@ -275,7 +274,7 @@ def check_response_integrity(
     if returned_ids_set != response_ids_set:
         logger.info("Failed integrity check")
         logger.info(
-            f"Present in original but not returned from LLM: {response_ids_set - returned_ids_set}. Returned in LLM but not present in original: {returned_ids_set -response_ids_set}"
+            f"Present in original but not returned from LLM: {response_ids_set - returned_ids_set}. Returned in LLM but not present in original: {returned_ids_set - response_ids_set}"
         )
         return False
     return True

{themefinder-0.3.1 → themefinder-0.5.2}/src/themefinder/prompts/sentiment_analysis.txt

@@ -6,7 +6,7 @@ Your job is to analyze each response to the QUESTION below and decide:
 POSITION - is the response agreeing or disagreeing or is it unclear about the change being proposed in the question.
 Choose one from [agreement, disagreement, unclear]
 
-You should only return a response in strict json and nothing else. The final output should be in the following JSON format:
+The final output should be in the following JSON format:
 
 {{"responses": [
     {{
@@ -20,6 +20,11 @@ You should only return a response in strict json and nothing else. The final out
     ...
 ]}}
 
+You MUST include every response ID in the output.
+If the response can not be labelled return empty sections where appropriate but you MUST return an entry
+with the correct response ID for each input object
+
+## EXAMPLE
 Example 1:
 Question: \n What are your thoughts on the proposed government changes to the policy about reducing school holidays?
 Response: \n as a parent I have no idea why you would make this change. I guess you were thinking about increasing productivity but any productivity gains would be totally offset by the decrease in family time. \n

themefinder-0.5.2/src/themefinder/prompts/theme_condensation.txt

@@ -0,0 +1,37 @@
+{system_prompt}
+
+Below is a question and a list of topics extracted from answers to that question. Each topic has a topic_label and a topic_description.
+
+Your task is to analyze these topics and produce a refined list that:
+1. Identifies and preserves core themes that appear frequently
+2. Combines redundant topics while maintaining nuanced differences
+3. Ensures the final list represents the full spectrum of viewpoints present in the original data
+
+Guidelines for Topic Analysis:
+- Begin by identifying distinct concept clusters in the topics
+- Consider the context of the question when determining topic relevance
+- Look for complementary perspectives that could enrich understanding of the same core concept
+- Consider the key ideas behind themes when merging, don't simply focus on the words used in the label and description
+
+For each topic in your output:
+1. Choose a clear, representative label that captures the essence of the combined or preserved topic
+2. Write a concise description that incorporates key insights from all constituent topics, this should only be a single sentence
+
+Return at most 30 topics
+
+The final output should be in the following JSON format:
+
+{{"responses": [
+    {{"topic_label": "{{label for condensed topic 1}}", "topic_description": "{{description for condensed topic 1}}"}},
+    {{"topic_label": "{{label for condensed topic 2}}", "topic_description": "{{description for condensed topic 2}}"}},
+    {{"topic_label": "{{label for condensed topic 3}}", "topic_description": "{{description for condensed topic 3}}"}},
+    // Additional topics as necessary
+]}}
+
+[Question]
+
+{question}
+
+[Themes]
+
+{responses}

themefinder-0.5.2/src/themefinder/prompts/theme_generation.txt

@@ -0,0 +1,70 @@
+{system_prompt}
+
+Below is a question and a list of responses to that question.
+
+Your task is to analyze the RESPONSES below and extract TOPICS such that: 
+1. Each topic summarizes a point of view expressed in the responses 
+2. Every distinct and relevant point of view in the responses should be captured by a topic 
+3. Each topic has a topic_label which summarizes the topic in a few words
+4. Each topic has a topic_description which gives more detail about the topic in one or two sentences
+5. The position field should just be the sentiment stated, and is either "agreement" or "disagreement" or "unclear"
+6. There should be no duplicate topics 
+
+The topics identified will be used by policy makers to understand what the public like and don't like about the proposals. 
+
+Here is an example of how to extract topics from some responses
+
+The final output should be in the following JSON format:
+
+{{"responses": [
+    {{
+        "topic_label": "{{label_1}}",
+        "topic_description": "{{description_1}}",
+        "position": "{{position_1}}"
+    }},
+    {{
+        "topic_label": "{{label_2}}",
+        "topic_description": "{{description_2}}",
+        "position": "{{position_2}}"
+    }},
+    // Additional topics as necessary
+]}}
+
+## EXAMPLE
+
+QUESTION 
+What are your views on the proposed change by the government to introduce a 2% tax on fast food meat products. 
+
+RESPONSES 
+[
+    {{"response": "I wish the government would stop interfering in the lves of its citizens. It only ever makes things worse. This change will just cost us all more money, and especially poorer people", "position": "disagreement"}},
+    {{"response": "Even though it will make people eat more healthier, I beleibe the government should interfer less and not more!", "position": "disagreement"}},
+    {{"response": "I hate grapes", "position": "disagreement"}},
+]
+
+OUTPUTS
+
+{{"responses": [
+    {{
+        "topic_label": "Government overreach",
+        "topic_description": "The proposals would result in government interfering too much with citizen's lives",
+        "position": "disagreement"
+    }},
+    {{
+        "topic_label": "Regressive change",
+        "topic_description": "The change would have a larger negative impact on poorer people",
+        "position": "disagreement"
+    }},
+    {{
+        "topic_label": "Health",
+        "topic_description": "The change would result in people eating healthier diets",
+        "position": "disagreement"
+    }},
+]}}
+
+
+QUESTION:
+{question}
+
+RESPONSES:
+{responses}

{themefinder-0.3.1 → themefinder-0.5.2}/src/themefinder/prompts/theme_mapping.txt

@@ -20,6 +20,9 @@ Your task is to analyze each response and decide which topics are present. Guide
     - If a response contains both positive and negative statements about a topic within the same response, choose the stance that receives more emphasis or appears more central to the argument
     - The order of reasons and stances must align with the order of labels (e.g., stance_a applies to topic_a)
 
+You MUST include every response ID in the output.
+If the response can not be labelled return empty sections where appropriate but you MUST return an entry
+with the correct response ID for each input object
 
 The final output should be in the following JSON format:
 

{themefinder-0.3.1 → themefinder-0.5.2}/src/themefinder/prompts/theme_refinement.txt

@@ -1,6 +1,7 @@
 {system_prompt}
 
-You are tasked with refining and neutralizing a list of topics generated from responses to a question. Your goal is to transform opinionated topics into neutral, well-structured, and distinct topics while preserving the essential information.
+You are tasked with refining and neutralizing a list of topics generated from responses to a question. 
+Your goal is to transform opinionated topics into neutral, well-structured, and distinct topics while preserving the essential information.
 
 ## Input
 You will receive a list of OPINIONATED TOPICS. These topics explicitly tie opinions to whether a person agrees or disagrees with the question.
@@ -60,7 +61,7 @@ Return your output in the following JSON format:
 }}
 
 
-## Example
+## EXAMPLE
 
 OPINIONATED TOPIC:
 "Economic impact: Many respondents who support the policy believe it will create jobs and boost the economy, it could raise GDP by 2%."

themefinder-0.5.2/src/themefinder/prompts/theme_target_alignment.txt

@@ -0,0 +1,26 @@
+{system_prompt}
+Input: You will receive a JSON array of themes, where each theme contains a description of a topic or concept.
+
+Goal: Consolidate these themes into approximately {target_n_themes} distinct categories by:
+1. Identifying and combining similar or overlapping themes
+2. Preserving all significant details and perspectives
+3. Creating clear, comprehensive descriptions for each merged theme
+
+Requirements:
+- Each consolidated theme should capture all relevant information from its source themes
+- Final descriptions should be concise but thorough
+- The merged themes should be distinct from each other with minimal overlap
+
+Return your output in the following JSON format:
+
+{{
+   "responses": [
+       {{"topic_id": "A", "topic": "{{topic label 1}}: {{topic description 1}}"}},
+       {{"topic_id": "B", "topic": "{{topic label 2}}: {{topic description 2}}"}},
+       {{"topic_id": "C", "topic": "{{topic label 3}}: {{topic description 3}}"}},
+      // Additional topics as necessary
+   ]
+}}
+
+Themes to analyze: 
+{responses}

themefinder-0.3.1/src/themefinder/prompts/theme_condensation.txt

@@ -1,42 +0,0 @@
-{system_prompt}
-
-Below is a question and a list of topics extracted from answers to that question. Each topic has a topic_label and a topic_description.
-
-Your task is to analyze these topics and produce a refined list that:
-1. Identifies and preserves core themes that appear frequently
-2. Captures unique perspectives that may only appear once but offer valuable insights
-3. Combines truly redundant topics while maintaining nuanced differences
-4. Ensures the final list represents the full spectrum of viewpoints present in the original data
-
-Guidelines for Topic Analysis:
-- Begin by identifying distinct concept clusters in the topics
-- When a topic appears only once, evaluate its unique contribution before deciding to merge or preserve it
-- Consider the context of the question when determining topic relevance
-- Look for complementary perspectives that could enrich understanding of the same core concept
-- Preserve specific examples or concrete applications that illustrate abstract concepts
-- Maintain granularity where different aspects of the same broader theme offer distinct insights
-
-The topics you are analyzing are all extracted from answers with the same position, where "position" means that the answer agrees ("Y") or disagrees ("N") with the question.
-
-For each topic in your output:
-1. Choose a clear, representative label that captures the essence of the combined or preserved topic
-2. Write a comprehensive description that incorporates key insights from all constituent topics
-3. Ensure the description maintains specific examples or unique angles from the original topics
-4. Include the shared position value
-
-The final output should be in the following JSON format:
-
-{{"responses": [
-    {{"topic_label": "{{label for condensed topic 1}}", "topic_description": "{{description for condensed topic 1}}", "position": {{the position given below}}"}},
-    {{"topic_label": "{{label for condensed topic 2}}", "topic_description": "{{description for condensed topic 2}}", "position": {{the position given below}}"}},
-    {{"topic_label": "{{label for condensed topic 3}}", "topic_description": "{{description for condensed topic 3}}", "position": {{the position given below}}"}},
-    // Additional topics as necessary
-]}}
-
-[Question]
-
-{question}
-
-[Themes]
-
-{responses}

themefinder-0.3.1/src/themefinder/prompts/theme_generation.txt

@@ -1,70 +0,0 @@
-{system_prompt}
-
-Your task is to analyse RESPONSES below and extract TOPICS such that: 
-1. Each topic summarises points of view expressed in the responses 
-2. Every distinct and relevant point of view in the responses should be captured by a topic 
-3. Each topic has a topic_label which summarises the topic in a few words
-4. Each topic has a topic_description which gives more detail about the topic in one or two sentences
-5. The position field should just be the sentiment stated, and is either "agreement" or "disagreement"
-6. There should be no duplicate topics 
-
-The topics identified will be used by policy makers to understand what the public like and don't like about the proposals. 
-
-Here is an example of how to extract topics from some responses
-
-EXAMPLE:
-
-    POSITION 
-    disagreement
-
-    QUESTION 
-    What are your views on the proposed change by the government to introduce a 2% tax on fast food meat products. 
-
-    RESPONSES 
-    [
-        {{"response": "I wish the government would stop interfering in the lves of its citizens. It only ever makes things worse. This change will just cost us all more money, and especially poorer people", "position": "disagreement"}},
-        {{"response": "Even though it will make people eat more healthier, I beleibe the government should interfer less and not more!", "position": "disagreement"}},
-        {{"response": "I hate grapes", "position": "disagreement"}},
-    ]
-
-    OUTPUTS
-
-    {{"responses": [
-        {{
-            "topic_label": "Government overreach",
-            "topic_description": "Some people thought the proposals would result in government interfering too much with citizen's lives",
-            "position": "disagreement"
-        }},
-        {{
-            "topic_label": "Regressive change",
-            "topic_description": "Some people thought the change would have a larger negative impact on poorer people",
-            "position": "disagreement"
-        }},
-        {{
-            "topic_label": "Health",
-            "topic_description": "Some people thought the change would result in people eating healthier diets",
-            "position": "disagreement"
-        }},
-    ]}}
-
-You should only return a response in strict json and nothing else. The final output should be in the following JSON format:
-
-{{"responses": [
-    {{
-        "topic_label": "{{label_1}}",
-        "topic_description": "{{description_1}}",
-        "position": "{{position_1}}"
-    }},
-    {{
-        "topic_label": "{{label_2}}",
-        "topic_description": "{{description_2}}",
-        "position": "{{position_2}}"
-    }},
-    // Additional topics as necessary
-]}}
-
-QUESTION:
-{question}
-
-RESPONSES:
-{responses}