themefinder 0.5.2__tar.gz → 0.5.4__tar.gz

{themefinder-0.5.2 → themefinder-0.5.4}/PKG-INFO

@@ -1,17 +1,18 @@
 Metadata-Version: 2.3
 Name: themefinder
-Version: 0.5.2
+Version: 0.5.4
 Summary: A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses.
 License: MIT
 Author: i.AI
 Author-email: packages@cabinetoffice.gov.uk
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.10,<3.13
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
 Requires-Dist: boto3 (>=1.29,<2.0)

{themefinder-0.5.2 → themefinder-0.5.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "themefinder"
-version = "0.5.2"
+version = "0.5.4"
 description = "A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses."
 authors = ["i.AI <packages@cabinetoffice.gov.uk>"]
 packages = [{include = "themefinder", from = "src"}]
@@ -17,7 +17,7 @@ classifiers = [
 
 
 [tool.poetry.dependencies]
-python = ">=3.12,<4.0"
+python = ">=3.10,<3.13"
 langchain = "*"
 langchain-openai = "0.1.17"
 pandas = "^2.2.2"

{themefinder-0.5.2 → themefinder-0.5.4}/src/themefinder/core.py

@@ -95,7 +95,7 @@ async def find_themes(
     return {
         "question": question,
         "sentiment": sentiment_df,
-        "topics": theme_df,
+        "themes": theme_df,
         "condensed_themes": condensed_theme_df,
         "refined_themes": refined_theme_df,
         "mapping": mapping_df,
@@ -197,7 +197,7 @@ async def theme_condensation(
     themes_df: pd.DataFrame,
     llm: Runnable,
     question: str,
-    batch_size: int = 100,
+    batch_size: int = 75,
     prompt_template: str | Path | PromptTemplate = "theme_condensation",
     system_prompt: str = CONSULTATION_SYSTEM_PROMPT,
     **kwargs,
@@ -224,7 +224,7 @@ async def theme_condensation(
         pd.DataFrame: DataFrame containing the condensed themes, where similar topics
             have been combined into broader categories.
     """
-    logger.info(f"Running theme condensation on {len(themes_df)} responses")
+    logger.info(f"Running theme condensation on {len(themes_df)} themes")
     themes_df["response_id"] = range(len(themes_df))
 
     n_themes = themes_df.shape[0]

themefinder-0.5.4/src/themefinder/prompts/theme_condensation.txt

@@ -0,0 +1,50 @@
+{system_prompt}
+
+Below is a question and a list of topics extracted from answers to that question. Each topic has a topic_label, topic_description, and may have a source_topic_count field indicating how many original topics it represents.
+
+Your task is to analyze these topics and produce a refined list that:
+1. Identifies and preserves core themes that appear frequently
+2. Combines redundant topics while maintaining nuanced differences
+3. Ensures the final list represents the full spectrum of viewpoints present in the original data
+4. Tracks the total number of original topics combined into each new topic
+
+Guidelines for Topic Analysis:
+- Begin by identifying distinct concept clusters in the topics
+- Consider the context of the question when determining topic relevance
+- Look for complementary perspectives that could enrich understanding of the same core concept
+- Consider the key ideas behind themes when merging, don't simply focus on the words used in the label and description
+- When combining topics:
+  * For topics without a source_topic_count field, assume count = 1
+  * For topics with source_topic_count, use their existing count
+  * The new topic's count should be the sum of all combined topics' counts
+
+For each topic in your output:
+1. Choose a clear, representative label that captures the essence of the combined or preserved topic
+2. Write a concise description that incorporates key insights from all constituent topics, this should only be a single sentence
+3. Include the total count of original topics combined by summing the source_topic_counts of merged topics (or 1 for topics without a count)
+
+The final output should be in the following JSON format:
+
+{{"responses": [
+    {{"topic_label": "{{label for condensed topic 1}}", 
+      "topic_description": "{{description for condensed topic 1}}",
+      "source_topic_count": {{sum of source_topic_counts from combined topics}}
+    }},
+    {{"topic_label": "{{label for condensed topic 2}}", 
+      "topic_description": "{{description for condensed topic 2}}",
+      "source_topic_count": {{sum of source_topic_counts from combined topics}}
+    }},
+    {{"topic_label": "{{label for condensed topic 3}}", 
+      "topic_description": "{{description for condensed topic 3}}",
+      "source_topic_count": {{sum of source_topic_counts from combined topics}}
+    }},
+    // Additional topics as necessary
+]}}
+
+[Question]
+
+{question}
+
+[Themes]
+
+{responses}

{themefinder-0.5.2 → themefinder-0.5.4}/src/themefinder/prompts/theme_mapping.txt

@@ -12,7 +12,7 @@ You will be given:
 Your task is to analyze each response and decide which topics are present. Guidelines:
     - You can only assign to a response to a topic in the provided TOPIC LIST
     - A response doesn't need to exactly match the language used in the TOPIC LIST, it should be considered a match if it expresses a similar sentiment.
-    - You must use the alphabetic 'topic_id' to indicate which topic you have assigned.
+    - You must use the alphabetic 'topic_id' to indicate which topic you have assigned. Do not use the full topic description
     - Each response can be assigned to multiple topics if it matches more than one topic from the TOPIC LIST.
     - There is no limit on how many topics can be assigned to a response.
     - For each assignment provide a single rationale for why you have chosen the label.
@@ -22,7 +22,8 @@ Your task is to analyze each response and decide which topics are present. Guide
 
 You MUST include every response ID in the output.
 If the response can not be labelled return empty sections where appropriate but you MUST return an entry
-with the correct response ID for each input object
+with the correct response ID for each input object.
+You must only return the alphabetic topic_ids in the labels section.
 
 The final output should be in the following JSON format:
 

{themefinder-0.5.2 → themefinder-0.5.4}/src/themefinder/prompts/theme_refinement.txt

@@ -53,9 +53,9 @@ You will produce a list of NEUTRAL TOPICS based on the input. Each neutral topic
 Return your output in the following JSON format:
 {{
    "responses": [
-       {{"topic_id": "A", "topic": "{{topic label 1}}: {{topic description 1}}"}},
-       {{"topic_id": "B", "topic": "{{topic label 2}}: {{topic description 2}}"}},
-       {{"topic_id": "C", "topic": "{{topic label 3}}: {{topic description 3}}"}},
+       {{"topic_id": "A", "topic": "{{topic label 1}}: {{topic description 1}}", "source_topic_count": {{count1}}}},
+       {{"topic_id": "B", "topic": "{{topic label 2}}: {{topic description 2}}", "source_topic_count": {{count2}}}},
+       {{"topic_id": "C", "topic": "{{topic label 3}}: {{topic description 3}}", "source_topic_count": {{count3}}}},
       // Additional topics as necessary
    ]
 }}
@@ -64,11 +64,14 @@ Return your output in the following JSON format:
 ## EXAMPLE
 
 OPINIONATED TOPIC:
-"Economic impact: Many respondents who support the policy believe it will create jobs and boost the economy, it could raise GDP by 2%."
+"Economic impact: Many respondents who support the policy believe it will create jobs and boost the economy, it could raise GDP by 2%. [source_topic_count: 15]"
 
 NEUTRAL TOPIC:
-Topic Label: Economic Impact on Employment
-Description: The policy's potential effects on job creation and overall economic growth, including potential for a 2% increase in GDP.
+{{
+  "topic_id": "A",
+  "topic": "Economic Impact on Employment: The policy's potential effects on job creation and overall economic growth, including potential for a 2% increase in GDP.",
+  "source_topic_count": 15
+}}
 
 Remember, your goal is to create a list of neutral, informative, and distinct topics that accurately represent the content of the original opinionated topics without any bias or references to responses.
 

themefinder-0.5.2/src/themefinder/prompts/theme_condensation.txt

@@ -1,37 +0,0 @@
-{system_prompt}
-
-Below is a question and a list of topics extracted from answers to that question. Each topic has a topic_label and a topic_description.
-
-Your task is to analyze these topics and produce a refined list that:
-1. Identifies and preserves core themes that appear frequently
-2. Combines redundant topics while maintaining nuanced differences
-3. Ensures the final list represents the full spectrum of viewpoints present in the original data
-
-Guidelines for Topic Analysis:
-- Begin by identifying distinct concept clusters in the topics
-- Consider the context of the question when determining topic relevance
-- Look for complementary perspectives that could enrich understanding of the same core concept
-- Consider the key ideas behind themes when merging, don't simply focus on the words used in the label and description
-
-For each topic in your output:
-1. Choose a clear, representative label that captures the essence of the combined or preserved topic
-2. Write a concise description that incorporates key insights from all constituent topics, this should only be a single sentence
-
-Return at most 30 topics
-
-The final output should be in the following JSON format:
-
-{{"responses": [
-    {{"topic_label": "{{label for condensed topic 1}}", "topic_description": "{{description for condensed topic 1}}"}},
-    {{"topic_label": "{{label for condensed topic 2}}", "topic_description": "{{description for condensed topic 2}}"}},
-    {{"topic_label": "{{label for condensed topic 3}}", "topic_description": "{{description for condensed topic 3}}"}},
-    // Additional topics as necessary
-]}}
-
-[Question]
-
-{question}
-
-[Themes]
-
-{responses}