themefinder 0.6.3__tar.gz → 0.7.1__tar.gz

{themefinder-0.6.3 → themefinder-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: themefinder
-Version: 0.6.3
+Version: 0.7.1
 Summary: A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses.
 License: MIT
 Author: i.AI
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
 Requires-Dist: boto3 (>=1.29,<2.0)
 Requires-Dist: langchain
-Requires-Dist: langchain-openai (==0.1.17)
+Requires-Dist: langchain-openai
 Requires-Dist: langfuse (==2.29.1)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
@@ -169,5 +169,5 @@ The documentation is [© Crown copyright](https://www.nationalarchives.gov.uk/in
 
 ## Feedback
 
-If you have feedback on this package, please fill in our [feedback form](https://forms.gle/85xUSMvxGzSSKQ499) or contact us with questions or feedback at packages@cabinetoffice.gov.uk.
+Contact us with questions or feedback at packages@cabinetoffice.gov.uk.
 

{themefinder-0.6.3 → themefinder-0.7.1}/README.md

@@ -138,4 +138,4 @@ The documentation is [© Crown copyright](https://www.nationalarchives.gov.uk/in
 
 ## Feedback
 
-If you have feedback on this package, please fill in our [feedback form](https://forms.gle/85xUSMvxGzSSKQ499) or contact us with questions or feedback at packages@cabinetoffice.gov.uk.
+Contact us with questions or feedback at packages@cabinetoffice.gov.uk.

{themefinder-0.6.3 → themefinder-0.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "themefinder"
-version = "0.6.3"
+version = "0.7.1"
 description = "A topic modelling Python package designed for analysing one-to-many question-answer data eg free-text survey responses."
 authors = ["i.AI <packages@cabinetoffice.gov.uk>"]
 packages = [{include = "themefinder", from = "src"}]
@@ -19,7 +19,7 @@ classifiers = [
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
 langchain = "*"
-langchain-openai = "0.1.17"
+langchain-openai = "*"
 pandas = "^2.2.2"
 python-dotenv = "^1.0.1"
 langfuse = "2.29.1"

{themefinder-0.6.3 → themefinder-0.7.1}/src/themefinder/__init__.py

@@ -1,6 +1,7 @@
 from .core import (
     find_themes,
     sentiment_analysis,
+    theme_clustering,
     theme_condensation,
     theme_generation,
     theme_mapping,
@@ -12,11 +13,12 @@ from .core import (
 __all__ = [
     "find_themes",
     "sentiment_analysis",
-    "theme_generation",
+    "theme_clustering",
     "theme_condensation",
+    "theme_generation",
+    "theme_mapping",
     "theme_refinement",
     "theme_target_alignment",
-    "theme_mapping",
     "detail_detection",
 ]
 __version__ = "0.1.0"

{themefinder-0.6.3 → themefinder-0.7.1}/src/themefinder/core.py

@@ -5,16 +5,19 @@ import pandas as pd
 from langchain_core.prompts import PromptTemplate
 from langchain.schema.runnable import RunnableWithFallbacks
 
-from .llm_batch_processor import batch_and_run, load_prompt_from_file
-from .models import (
+from themefinder.llm_batch_processor import batch_and_run, load_prompt_from_file
+from themefinder.models import (
     SentimentAnalysisResponses,
     ThemeGenerationResponses,
     ThemeCondensationResponses,
     ThemeRefinementResponses,
     ThemeMappingResponses,
     DetailDetectionResponses,
+    HierarchicalClusteringResponse,
+    ThemeNode,
 )
-from .themefinder_logging import logger
+from themefinder.theme_clustering_agent import ThemeClusteringAgent
+from themefinder.themefinder_logging import logger
 
 CONSULTATION_SYSTEM_PROMPT = load_prompt_from_file("consultation_system_prompt")
 
@@ -114,9 +117,7 @@ async def find_themes(
     )
 
     logger.info("Finished finding themes")
-    logger.info(
-        "Provide feedback or report bugs: https://forms.gle/85xUSMvxGzSSKQ499 or packages@cabinetoffice.gov.uk"
-    )
+    logger.info("Provide feedback or report bugs: packages@cabinetoffice.gov.uk")
     return {
         "question": question,
         "sentiment": sentiment_df,
@@ -309,6 +310,87 @@ async def theme_condensation(
     return themes_df, _
 
 
+def theme_clustering(
+    themes_df: pd.DataFrame,
+    llm: RunnableWithFallbacks,
+    max_iterations: int = 5,
+    target_themes: int = 10,
+    significance_percentage: float = 10.0,
+    return_all_themes: bool = False,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Perform hierarchical clustering of themes using an agentic approach.
+
+    This function takes a DataFrame of themes and uses the ThemeClusteringAgent
+    to iteratively merge similar themes into a hierarchical structure, then
+    selects the most significant themes based on a threshold.
+
+    Args:
+        themes_df (pd.DataFrame): DataFrame containing themes with columns:
+            - topic_id: Unique identifier for each theme
+            - topic_label: Short descriptive label for the theme
+            - topic_description: Detailed description of the theme
+            - source_topic_count: Number of source responses for this theme
+        llm (RunnableWithFallbacks): Language model instance configured with
+            structured output for HierarchicalClusteringResponse
+        max_iterations (int, optional): Maximum number of clustering iterations.
+            Defaults to 5.
+        target_themes (int, optional): Target number of themes to cluster down to.
+            Defaults to 10.
+        significance_percentage (float, optional): Percentage threshold for
+            selecting significant themes. Defaults to 10.0.
+        return_all_themes (bool, optional): If True, returns all clustered themes.
+            If False, returns only significant themes. Defaults to False.
+
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame]:
+            A tuple containing:
+                - DataFrame of clustered themes (all or significant based on return_all_themes)
+                - Empty DataFrame (for consistency with other functions)
+    """
+    logger.info(f"Starting hierarchical clustering of {len(themes_df)} themes")
+
+    # Convert DataFrame to ThemeNode objects
+    initial_themes = [
+        ThemeNode(
+            topic_id=row["topic_id"],
+            topic_label=row["topic_label"],
+            topic_description=row["topic_description"],
+            source_topic_count=row["source_topic_count"],
+        )
+        for _, row in themes_df.iterrows()
+    ]
+
+    # Initialize clustering agent with structured output LLM
+    agent = ThemeClusteringAgent(
+        llm.with_structured_output(HierarchicalClusteringResponse), initial_themes
+    )
+
+    # Perform clustering
+    logger.info(
+        f"Clustering themes with max_iterations={max_iterations}, target_themes={target_themes}"
+    )
+    all_themes_df = agent.cluster_themes(
+        max_iterations=max_iterations, target_themes=target_themes
+    )
+
+    # Return appropriate themes based on parameter
+    if return_all_themes:
+        logger.info(
+            f"Clustering complete: returning all {len(all_themes_df)} clustered themes"
+        )
+        return all_themes_df, pd.DataFrame()
+    else:
+        # Select significant themes
+        logger.info(
+            f"Selecting themes with significance_percentage={significance_percentage}%"
+        )
+        selected_themes_df = agent.select_themes(significance_percentage)
+        logger.info(
+            f"Clustering complete: returning {len(selected_themes_df)} significant themes"
+        )
+        return selected_themes_df, pd.DataFrame()
+
+
 async def theme_refinement(
     condensed_themes_df: pd.DataFrame,
     llm: RunnableWithFallbacks,

{themefinder-0.6.3 → themefinder-0.7.1}/src/themefinder/llm_batch_processor.py

@@ -19,7 +19,7 @@ from tenacity import (
     wait_random_exponential,
 )
 
-from .themefinder_logging import logger
+from themefinder.themefinder_logging import logger
 
 
 @dataclass

{themefinder-0.6.3 → themefinder-0.7.1}/src/themefinder/models.py

@@ -349,3 +349,67 @@ class DetailDetectionResponses(ValidatedModel):
         if len(response_ids) != len(set(response_ids)):
             raise ValueError("Response IDs must be unique")
         return self
+
+
+class ThemeNode(ValidatedModel):
+    """Model for topic nodes created during hierarchical clustering"""
+
+    topic_id: str = Field(
+        ...,
+        description="Short alphabetic ID (e.g. 'A', 'B', 'C') - iteration prefix will be added automatically",
+    )
+    topic_label: str = Field(
+        ..., description="4-5 word label encompassing merged child topics"
+    )
+    topic_description: str = Field(
+        ..., description="1-2 sentences combining key aspects of child topics"
+    )
+    source_topic_count: int = Field(gt=0, description="Sum of all child topic counts")
+    parent_id: Optional[str] = Field(
+        default=None,
+        description="Internal field: ID of parent topic node, managed by clustering agent, not set by LLM",
+    )
+    children: List[str] = Field(
+        default_factory=list, description="List of topic_ids of merged child topics"
+    )
+
+    @model_validator(mode="after")
+    def run_validations(self) -> "ThemeNode":
+        """Validate topic node constraints"""
+        if self.children:
+            # Each parent must have at least 2 children
+            if len(self.children) < 2:
+                raise ValueError("Each topic node must have at least 2 children")
+            # Validate children are unique
+            if len(self.children) != len(set(self.children)):
+                raise ValueError("Child topic IDs must be unique")
+
+        return self
+
+
+class HierarchicalClusteringResponse(ValidatedModel):
+    """Model for hierarchical clustering agent response"""
+
+    parent_themes: List[ThemeNode] = Field(
+        default=[],
+        description="List of parent themes created by merging similar themes",
+    )
+    should_terminate: bool = Field(
+        ...,
+        description="True if no more meaningful clustering is possible, false otherwise",
+    )
+
+    @model_validator(mode="after")
+    def run_validations(self) -> "HierarchicalClusteringResponse":
+        """Validate clustering response constraints"""
+        self.validate_non_empty_fields()
+
+        # Validate that no child appears in multiple parents
+        all_children = []
+        for parent in self.parent_themes:
+            all_children.extend(parent.children)
+
+        if len(all_children) != len(set(all_children)):
+            raise ValueError("Each child theme can have at most one parent")
+
+        return self

themefinder-0.7.1/src/themefinder/prompts/agentic_theme_clustering.txt

@@ -0,0 +1,31 @@
+Analyze these topics and identify which ones should be merged based on semantic similarity.
+Your goal is to significantly reduce the number of topics by creating meaningful parent topics.
+Be aggressive in finding opportunities to merge topics that share any semantic relationship.
+
+TOPICS:
+{themes_json}
+
+For each group of similar topics that should be merged, create a new parent topic.
+
+Guidelines:
+- Each parent topic must have at least 2 children, it can have more than 2 if appropriate
+- Each child topic can have at most 1 parent
+- topic_id should be a simple alphabetic ID (e.g. 'A', 'B', 'C') - the iteration prefix will be added automatically
+- Be creative and look for higher-level abstractions that can combine seemingly different topics
+- When creating parent topics, follow these naming rules:
+    * The label should read naturally as a single coherent topic
+    * Choose labels that can encompass broader categories of topics
+    * If merging different topics, the topic with the higher source_topic_count should dominate the label
+    * Never combine different topics with "and" or "/" in the label
+- topic_description must be 1 or 2 sentences that:
+    * preserves key information from the child topics
+- source_topic_count must be the sum of all child topic counts
+- children must be a list of valid topic_ids from the input
+- should_terminate should only be true if ALL of these conditions are met:
+    * There are fewer than 10 active topics remaining
+    * The remaining topics are fundamentally incompatible semantically
+    * Any further merging would create meaninglessly broad categories
+
+If no topics should be merged in this iteration but future iterations might still yield meaningful merges, set should_terminate to false with an empty parent_themes list.
+
+If no topics should be merged and the termination conditions are met, set should_terminate to true with an empty parent_themes list.

themefinder-0.7.1/src/themefinder/prompts/detail_detection.txt

@@ -0,0 +1,31 @@
+{system_prompt}
+
+You will receive a list of RESPONSES, each containing a response_id and a response.
+Your job is to analyze each response to the QUESTION below and decide if a response contains rich evidence.
+You MUST include every response ID in the output.
+
+A response is evidence-rich only if it satisfies both of the following:
+
+Relevance and depth:
+    - It clearly answers the question
+    - AND provides insights that go beyond generic opinion, such as nuanced reasoning, contextual explanation, or argumentation that could inform decision-making
+
+Substantive evidence, including at least one of:
+    - Specific, verifiable facts or data (e.g., statistics, dates, named reports or studies)
+    - Concrete, illustrative examples that clearly support a broader claim
+    - Detailed personal or professional experiences that include contextual information (e.g., roles, locations, timelines)
+
+Do NOT classify a response as evidence-rich if it:
+    - Uses vague or general language with no supporting detail
+    - Restates commonly known points without adding new information
+    - Shares personal anecdotes without sufficient context or a clear takeaway
+
+Before answering, ask: Would this response provide useful input to someone drafting policy, beyond what is already commonly known or expected?
+
+For each response, determine:
+EVIDENCE_RICH - does the response contain significant evidence as defined above?
+Choose one from ['YES', 'NO']
+
+
+QUESTION: \n {question}
+RESPONSES: \n {responses}

{themefinder-0.6.3 → themefinder-0.7.1}/src/themefinder/prompts/theme_refinement.txt

@@ -6,10 +6,11 @@ You are tasked with refining a list of topics generated from responses to a ques
 You will receive a list of TOPICS. These topics explicitly tie opinions to whether a person agrees or disagrees with the question.
 
 ## Output
-You will produce a list of CLEAR STANCE TOPICS based on the input. Each topic should have two parts:
-1. A brief, clear topic label (3-7 words)
-2. A more detailed topic description (1-2 sentences)
-3. The source_topic_count field should be included for each topic and should reflect the number of original source topics that were merged to create this refined topic. If multiple source topics were combined, sum their individual counts. If only one source topic was used, simply retain its original count value.
+You will produce a list of CLEAR STANCE TOPICS based on the input. Each topic should have four parts:
+1. A topic_id that is an uppercase letter (starting from 'A', for the 27th element use AA)
+2. A brief, clear topic label (3-7 words)
+3. A more detailed topic description (1-2 sentences)
+4. The source_topic_count field should be included for each topic and should reflect the number of original source topics that were merged to create this refined topic. If multiple source topics were combined, sum their individual counts. If only one source topic was used, simply retain its original count value.
 
 
 ## Guidelines
@@ -48,7 +49,7 @@ You will produce a list of CLEAR STANCE TOPICS based on the input. Each topic sh
    b. Create a neutral, concise topic label.
    c. Write a more detailed description that provides context without taking sides.
 4. Review the entire list to ensure distinctiveness and adjust as needed.
-5. Assign each output topic a topic_id a single uppercase letters (starting from 'A', for the 27th element use AA)
+5. Assign each output topic a topic_id that is an uppercase letter (starting from 'A', for the 27th element use AA)
 6. Combine the topic label and description with a colon separator
 
 TOPICS:

themefinder-0.7.1/src/themefinder/theme_clustering_agent.py

@@ -0,0 +1,332 @@
+"""Theme clustering agent for hierarchical topic organization.
+
+This module provides the ThemeClusteringAgent class for performing iterative
+hierarchical clustering of topics using a language model.
+"""
+
+import json
+import logging
+from typing import Dict, List, Any
+
+import pandas as pd
+from langchain.schema.runnable import Runnable
+from tenacity import (
+    before,
+    before_sleep_log,
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from .models import ThemeNode
+from .llm_batch_processor import load_prompt_from_file
+from .themefinder_logging import logger
+
+
+class ThemeClusteringAgent:
+    """Agent for performing hierarchical clustering of topics using language models.
+
+    This class manages the iterative process of merging similar topics into a
+    hierarchical structure using an LLM to identify semantic relationships and
+    create meaningful parent-child topic relationships.
+
+    Attributes:
+        llm: Language model configured with structured output for clustering
+        themes: Dictionary mapping topic IDs to ThemeNode objects
+        active_themes: Set of topic IDs that are currently active for clustering
+        current_iteration: Current iteration number in the clustering process
+    """
+
+    def __init__(self, llm: Runnable, themes: List[ThemeNode]) -> None:
+        """Initialize the clustering agent with an LLM and initial themes.
+
+        Args:
+            llm: Language model instance configured with structured output
+                for HierarchicalClusteringResponse
+            themes: List of ThemeNode objects to be clustered
+        """
+        self.llm = llm
+        self.themes: Dict[str, ThemeNode] = {}
+        for theme in themes:
+            self.themes[theme.topic_id] = theme
+        self.active_themes = set(self.themes.keys())
+        self.current_iteration = 0
+
+    def _format_prompt(self) -> str:
+        """Format the clustering prompt with current active themes.
+
+        Creates a JSON representation of all currently active themes and
+        formats them into the clustering prompt template.
+
+        Returns:
+            str: Formatted prompt string ready for LLM processing
+        """
+        themes_for_prompt = []
+        for active_id in self.active_themes:
+            theme_dict = {
+                "topic_id": self.themes[active_id].topic_id,
+                "topic_label": self.themes[active_id].topic_label,
+                "topic_description": self.themes[active_id].topic_description,
+            }
+            themes_for_prompt.append(theme_dict)
+        themes_json = json.dumps(themes_for_prompt, indent=2)
+
+        # Load the clustering prompt template
+        prompt_template = load_prompt_from_file("agentic_theme_clustering")
+        return prompt_template.format(
+            themes_json=themes_json, iteration=self.current_iteration
+        )
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=2),
+        stop=stop_after_attempt(3),
+        before=before.before_log(logger=logger, log_level=logging.DEBUG),
+        before_sleep=before_sleep_log(logger, logging.ERROR),
+        reraise=True,
+    )
+    def cluster_iteration(self) -> None:
+        """Perform one iteration of hierarchical theme clustering.
+
+        Uses the configured LLM to identify semantically similar themes
+        and merge them into parent themes. Updates the theme hierarchy
+        and active theme set based on the clustering results.
+
+        The method includes retry logic to handle transient API failures
+        and will automatically retry up to 3 times with exponential backoff.
+
+        Side Effects:
+            - Creates new parent ThemeNode objects in self.themes
+            - Updates parent_id relationships for child themes
+            - Modifies self.active_themes set
+            - Increments self.current_iteration
+        """
+        prompt = self._format_prompt()
+        response = self.llm.invoke(prompt)
+        # The response is already a parsed dictionary when using with_structured_output
+        result = response
+        for i, parent in enumerate(result["parent_themes"]):
+            new_theme_id = f"{chr(65 + i)}_{self.current_iteration}"
+            children = [c for c in parent["children"] if c in self.active_themes]
+            for child in children:
+                self.themes[child].parent_id = new_theme_id
+            total_source_count = sum(
+                self.themes[child_id].source_topic_count for child_id in children
+            )
+            new_theme = ThemeNode(
+                topic_id=new_theme_id,
+                topic_label=parent["topic_label"],
+                topic_description=parent["topic_description"],
+                source_topic_count=total_source_count,
+                children=children,
+            )
+            self.themes[new_theme_id] = new_theme
+            self.active_themes.add(new_theme_id)
+            for child in children:
+                self.active_themes.remove(child)
+        self.current_iteration += 1
+
+    def cluster_themes(
+        self, max_iterations: int = 5, target_themes: int = 5
+    ) -> pd.DataFrame:
+        """Perform hierarchical clustering to reduce themes to target number.
+
+        Iteratively merges similar themes using the clustering agent until
+        either the maximum iterations is reached or the target number of
+        themes is achieved. Creates a root node to represent the complete
+        hierarchy.
+
+        Args:
+            max_iterations: Maximum number of clustering iterations to perform
+            target_themes: Target number of themes to cluster down to
+
+        Returns:
+            pd.DataFrame: DataFrame containing all theme nodes (excluding root)
+                with their hierarchical relationships and metadata
+        """
+        logger.info(f"Starting clustering with {len(self.active_themes)} active themes")
+        while (
+            self.current_iteration <= max_iterations
+            and len(self.active_themes) > target_themes
+        ):
+            self.cluster_iteration()
+            logger.info(
+                f"After {self.current_iteration} iterations {len(self.active_themes)} active themes remaining"
+            )
+        root_node = ThemeNode(
+            topic_id="0",
+            topic_label="All Topics",
+            topic_description="",
+            source_topic_count=sum(
+                self.themes[theme_id].source_topic_count
+                for theme_id in self.active_themes
+            ),
+            children=list(self.active_themes),
+        )
+        self.themes["0"] = root_node
+        for theme in self.active_themes:
+            self.themes[theme].parent_id = "0"
+
+        # Convert all themes (except root) to DataFrame
+        theme_nodes_dicts = [
+            node.model_dump() for node in self.themes.values() if node.topic_id != "0"
+        ]
+        return pd.DataFrame(theme_nodes_dicts)
+
+    def convert_themes_to_tree_json(self) -> str:
+        """Convert themes into a hierarchical JSON structure for visualization.
+
+        Creates a nested JSON structure starting from the root node (ID '0')
+        that represents the complete theme hierarchy. Each node includes
+        metadata and references to its children.
+
+        Returns:
+            str: JSON string representing the hierarchical tree structure
+                suitable for JavaScript tree visualization libraries
+        """
+
+        def build_tree(node: ThemeNode) -> Dict[str, Any]:
+            return {
+                "id": node.topic_id,
+                "name": node.topic_label,
+                "description": node.topic_description,
+                "value": node.source_topic_count,
+                "children": [
+                    build_tree(self.themes[child_id])
+                    for child_id in node.children
+                    if child_id in self.themes
+                ],
+            }
+
+        tree_data = build_tree(self.themes["0"])
+        return json.dumps(tree_data, indent=2)
+
+    def select_significant_themes(
+        self, significance_threshold: int, total_responses: int
+    ) -> Dict[str, Any]:
+        """Select significant themes using depth-first traversal.
+
+        Performs a depth-first search on the theme hierarchy to identify
+        themes that meet the significance threshold. Prioritizes leaf nodes
+        when possible, but selects parent nodes when children don't meet
+        the threshold.
+
+        Args:
+            significance_threshold: Minimum source_topic_count for significance
+            total_responses: Total number of responses across all themes
+
+        Returns:
+            Dict containing selected theme nodes and metadata
+        """
+        # Track selected nodes
+        selected_nodes: List[Dict[str, Any]] = []
+
+        # Perform the DFS selection
+        self._traverse_tree(self.themes["0"], selected_nodes, significance_threshold)
+
+        # Format the final result
+        result = {"selected_nodes": selected_nodes, "total_responses": total_responses}
+
+        return result
+
+    def _traverse_tree(
+        self,
+        node: ThemeNode,
+        selected_nodes: List[Dict[str, Any]],
+        significance_threshold: int,
+    ) -> bool:
+        """Recursively traverse theme tree to select significant nodes.
+
+        Implements depth-first traversal logic for theme selection:
+        1. For leaf nodes: always select
+        2. For parent nodes: select if no significant children exist
+        3. For significant children: recursively process them
+
+        Args:
+            node: Current ThemeNode being processed
+            selected_nodes: List to accumulate selected theme dictionaries
+            significance_threshold: Minimum source_topic_count for significance
+
+        Returns:
+            bool: True if this node or descendants were selected, False otherwise
+        """
+        # Base case: if node has no children (leaf node)
+        if not node.children:
+            selected_nodes.append(
+                {
+                    "id": node.topic_id,
+                    "name": node.topic_label,
+                    "value": node.source_topic_count,
+                }
+            )
+            return True
+
+        # Check if any children are significant
+        has_significant_children = any(
+            self.themes[child_id].source_topic_count >= significance_threshold
+            for child_id in node.children
+            if child_id in self.themes
+        )
+
+        # If no significant children, select this node
+        if not has_significant_children:
+            selected_nodes.append(
+                {
+                    "id": node.topic_id,
+                    "name": node.topic_label,
+                    "value": node.source_topic_count,
+                }
+            )
+            return True
+
+        # If significant children exist, recursively process them
+        any_selected = False
+        for child_id in node.children:
+            if child_id in self.themes:
+                if self._traverse_tree(
+                    self.themes[child_id], selected_nodes, significance_threshold
+                ):
+                    any_selected = True
+
+        # If none of the children were selected, select this node
+        if not any_selected:
+            selected_nodes.append(
+                {
+                    "id": node.topic_id,
+                    "name": node.topic_label,
+                    "value": node.source_topic_count,
+                }
+            )
+            return True
+
+        return any_selected
+
+    def select_themes(self, significance_percentage: float) -> pd.DataFrame:
+        """Select themes that meet the significance threshold.
+
+        Calculates the significance threshold based on the percentage of total
+        responses and returns only themes that meet or exceed this threshold.
+        Excludes the root node from results.
+
+        Args:
+            significance_percentage: Percentage (0-100) of total responses
+                required for a theme to be considered significant
+
+        Returns:
+            pd.DataFrame: DataFrame containing significant theme data,
+                excluding the root node (topic_id='0')
+        """
+        total_responses = self.themes["0"].source_topic_count
+        # Convert percentage to absolute threshold
+        significance_threshold = int(total_responses * (significance_percentage / 100))
+
+        # Filter themes that meet the significance threshold
+        significant_themes = [
+            theme_node
+            for theme_node in self.themes.values()
+            if theme_node.source_topic_count >= significance_threshold
+        ]
+        # Convert significant themes to DataFrame, excluding root node
+        theme_nodes_dicts = [
+            node.model_dump() for node in significant_themes if node.topic_id != "0"
+        ]
+        return pd.DataFrame(theme_nodes_dicts)

themefinder-0.6.3/src/themefinder/prompts/detail_detection.txt

@@ -1,19 +0,0 @@
-{system_prompt}
-
-You will receive a list of RESPONSES, each containing a response_id and a response.
-Your job is to analyze each response to the QUESTION below and decide if a response contains rich evidence.
-You MUST include every response ID in the output.
-
-Evidence-rich responses contain one or more of the following:
-- Specific facts or figures that shed new light on the issue (e.g., statistics, percentages, measurements, dates)
-- Concrete examples and specific insights that could inform decision-making
-- Detailed personal or professional experiences with clear contextual information or specific incidents
-In addition to the above an evidence rich response should answer the question and provide deeper insights than an average response.
-
-For each response, determine:
-EVIDENCE_RICH - does the response contain significant evidence as defined above?
-Choose one from ['YES', 'NO']
-
-
-QUESTION: \n {question}
-RESPONSES: \n {responses}