cotlab 0.8.0__py3-none-any.whl

cotlab/prompts/cardiology.py

@@ -0,0 +1,378 @@
+"""Cardiology-specific prompt strategy with JSON structured output."""
+
+import json
+import re
+from typing import Any, Dict, Optional
+
+from ..core.base import BasePromptStrategy, StructuredOutputMixin
+from ..core.registry import Registry
+
+SYSTEM_ROLE = """You are a paediatric cardiology expert.
+Your goal is to identify congenital heart defects in the given cardiac imaging report.
+Do not make assumptions or diagnoses from the text.
+Think rationally and explain your reasoning."""
+
+SYSTEM_ROLE_CONTRARIAN = """You are a skeptical paediatric cardiology expert.
+Your goal is to identify congenital heart defects in the given cardiac imaging report.
+However, you must question obvious conclusions and consider alternative explanations.
+Think rationally, play devil's advocate, and explain your reasoning."""
+
+
+PROMPT_TEMPLATE = """Follow this structured reasoning on the attached cardiac imaging report:
+
+1. **Cardiac Abnormality**: Determine whether a structural cardiac abnormality is explicitly described.
+2. **Congenital Heart Defect**: If an abnormality is found, assess whether it constitutes a congenital heart defect.
+
+The instruction is to answer questions strictly based on the content of the provided cardiac report without making any assumptions.
+Only make judgements on strong evidence.
+Ignore physiological variants (e.g., patent foramen ovale, trivial regurgitation) as they do not indicate CHD.
+Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Congenital heart defect present
+```json
+{{
+    "cardiac_abnormality": true,
+    "congenital_heart_defect": true,
+    "evidence": {{
+        "report_findings": ["ventricular septal defect", "left-to-right shunt", "pulmonary artery pressure elevated"],
+        "rationale": "The report explicitly identifies a large ventricular septal defect with hemodynamically significant shunt causing elevated pulmonary pressures, consistent with congenital heart disease requiring intervention."
+    }}
+}}
+```
+
+Example 2: Normal cardiac findings
+```json
+{{
+    "cardiac_abnormality": false,
+    "congenital_heart_defect": false,
+    "evidence": {{
+        "report_findings": ["normal cardiac structure", "physiological tricuspid regurgitation"],
+        "rationale": "The report describes a structurally normal heart with only physiological findings. No evidence of congenital heart defect."
+    }}
+}}
+```
+
+Cardiac imaging report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+PROMPT_TEMPLATE_CONTRARIAN = """As a skeptical cardiologist, follow this structured reasoning on the attached cardiac imaging report.
+Question obvious patterns and consider alternative explanations before reaching your conclusion.
+
+1. **Cardiac Abnormality**: Determine whether a structural cardiac abnormality is explicitly described. Consider if what appears abnormal might be a normal variant for age.
+2. **Congenital Heart Defect**: If an abnormality is found, critically assess whether it constitutes a congenital heart defect. Question the obvious diagnosis - could there be alternative explanations?
+
+Apply skeptical reasoning - if the report suggests CHD, argue why it might NOT be CHD. If it seems normal, consider why it MIGHT indicate CHD.
+Only make final judgements when evidence is overwhelming and alternative explanations are ruled out.
+Ignore physiological variants (e.g., patent foramen ovale, trivial regurgitation) as they do not indicate CHD.
+Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Congenital heart defect present (after skeptical review)
+```json
+{{
+    "cardiac_abnormality": true,
+    "congenital_heart_defect": true,
+    "evidence": {{
+        "report_findings": ["ventricular septal defect", "left-to-right shunt", "elevated pulmonary pressures"],
+        "rationale": "Initial skepticism: Could the shunt be physiological? However, the combination of large VSD with hemodynamically significant shunt causing elevated pulmonary pressures provides overwhelming evidence. Alternative explanations (innocent murmur, transient finding) are ruled out by the severity and persistence. Conclusion: CHD confirmed despite initial skepticism."
+    }}
+}}
+```
+
+Example 2: Normal cardiac findings (skeptical analysis)
+```json
+{{
+    "cardiac_abnormality": false,
+    "congenital_heart_defect": false,
+    "evidence": {{
+        "report_findings": ["normal cardiac structure", "physiological tricuspid regurgitation"],
+        "rationale": "Applying skeptical reasoning: While trivial TR could suggest valve pathology, the report explicitly states it is physiological. Playing devil's advocate against a CHD diagnosis: structurally normal heart with only age-appropriate physiological findings. No features suggesting congenital abnormality. Cannot conclude CHD without stronger evidence."
+    }}
+}}
+```
+
+Cardiac imaging report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+PROMPT_TEMPLATE_ANSWER_FIRST = """Review this cardiac imaging report and provide immediate assessment.
+
+**Step 1 - Initial Diagnosis**: Based on your immediate review, state your initial conclusion about the presence of congenital heart defect (CHD).
+
+Initial Assessment: Is there a congenital heart defect? (State YES or NO immediately)
+
+**Step 2 - Evidence Collection**: Now systematically gather and evaluate evidence:
+- What findings SUPPORT your initial diagnosis?
+- What findings CONTRADICT your initial diagnosis?
+- Are there alternative explanations?
+
+**Step 3 - Final Diagnosis**: Based on the evidence collected, confirm or revise your initial assessment.
+
+Provide your response in JSON format. Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Initial YES, Confirmed CHD
+```json
+{{
+    "cardiac_abnormality": true,
+    "congenital_heart_defect": true,
+    "evidence": {{
+        "report_findings": ["ventricular septal defect", "left-to-right shunt"],
+        "rationale": "Initial diagnosis: YES - CHD suspected. Supporting evidence: Large VSD with hemodynamically significant shunt strongly indicates congenital defect. Contradicting evidence: None identified. Final diagnosis: Confirmed congenital heart defect."
+    }}
+}}
+```
+
+Example 2: Initial NO, Confirmed no CHD
+```json
+{{
+    "cardiac_abnormality": false,
+    "congenital_heart_defect": false,
+    "evidence": {{
+        "report_findings": ["normal cardiac structure", "physiological TR"],
+        "rationale": "Initial diagnosis: NO - appears normal. Supporting evidence: Structurally normal heart with only trace physiological regurgitation. Contradicting evidence: None suggesting structural defect. Final diagnosis: Confirmed no congenital heart defect."
+    }}
+}}
+```
+
+Cardiac imaging report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+
+@Registry.register_prompt("cardiology")
+class CardiologyPromptStrategy(StructuredOutputMixin, BasePromptStrategy):
+    """
+    Structured JSON output for paediatric cardiology CHD detection.
+
+    Uses structured JSON output format with:
+    - Clear step-by-step reasoning instructions
+    - JSON output with cardiac_abnormality, congenital_heart_defect, evidence
+    - Few-shot examples for format guidance
+    """
+
+    def __init__(
+        self,
+        name: str = "cardiology",
+        system_role: Optional[str] = None,
+        contrarian: bool = False,
+        few_shot: bool = True,
+        answer_first: bool = False,
+        output_format: str = "json",
+        **kwargs,
+    ):
+        self._name = name
+        self.contrarian = contrarian
+        self.few_shot = few_shot
+        self.answer_first = answer_first
+        self.output_format = output_format
+        if system_role:
+            self.system_role = system_role
+        else:
+            self.system_role = SYSTEM_ROLE_CONTRARIAN if contrarian else SYSTEM_ROLE
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def build_prompt(self, input_data: Dict[str, Any]) -> str:
+        """Build prompt with cardiac imaging report."""
+        report = input_data.get("text", input_data.get("report", input_data.get("question", "")))
+
+        # Select base template based on reasoning mode
+        if self.answer_first:
+            template = PROMPT_TEMPLATE_ANSWER_FIRST
+        elif self.contrarian:
+            template = PROMPT_TEMPLATE_CONTRARIAN
+        else:
+            template = PROMPT_TEMPLATE
+
+        # Remove examples if few_shot=False
+        if not self.few_shot:
+            template = self._remove_few_shot_examples(template)
+        elif self.output_format != "json":
+            # Convert JSON examples to target format
+            template = self._convert_examples_to_format(template)
+
+        prompt = template.format(report=report)
+
+        return prompt
+
+    def _convert_examples_to_format(self, template: str) -> str:
+        """Convert JSON examples in template to target output format."""
+        examples = [
+            {
+                "title": "Example 1: Congenital heart defect present",
+                "data": {
+                    "cardiac_abnormality": True,
+                    "congenital_heart_defect": True,
+                    "evidence": {
+                        "report_findings": [
+                            "ventricular septal defect",
+                            "left-to-right shunt",
+                            "pulmonary artery pressure elevated",
+                        ],
+                        "rationale": "The report explicitly identifies a large ventricular septal defect with hemodynamically significant shunt causing elevated pulmonary pressures, consistent with congenital heart disease requiring intervention.",
+                    },
+                    "_plain_answer": "CHD DETECTED",
+                },
+            },
+            {
+                "title": "Example 2: Normal cardiac findings",
+                "data": {
+                    "cardiac_abnormality": False,
+                    "congenital_heart_defect": False,
+                    "evidence": {
+                        "report_findings": [
+                            "normal cardiac structure",
+                            "physiological tricuspid regurgitation",
+                        ],
+                        "rationale": "The report describes a structurally normal heart with only physiological findings. No evidence of congenital heart defect.",
+                    },
+                    "_plain_answer": "NORMAL",
+                },
+            },
+        ]
+
+        # Build examples in target format
+        examples_str = ""
+        for ex in examples:
+            examples_str += f"\n{ex['title']}\n"
+            examples_str += self._format_example(ex["data"]) + "\n"
+
+        # Replace JSON examples section
+        import re
+
+        pattern = r"Example 1:.*?```\s*\n\nCardiac imaging report:"
+        replacement = examples_str.strip() + "\n\nCardiac imaging report:"
+
+        new_template = re.sub(pattern, replacement, template, flags=re.DOTALL)
+
+        # Update instruction in header
+        if self.output_format == "plain":
+            new_template = new_template.replace(
+                "give the output strictly in the json format",
+                "provide your answer in plain text with FINAL ANSWER: at the end",
+            )
+        else:
+            new_template = new_template.replace(
+                "give the output strictly in the json format",
+                f"give the output in {self.output_format.upper()} format",
+            )
+
+        return new_template
+
+    def _remove_few_shot_examples(self, template: str) -> str:
+        """Remove few-shot examples from template for ablation studies."""
+        import re
+
+        pattern = r"Example \d+:.*?(?=(?:Radiology report:|Cardiac imaging report:|Neuroimaging report:|Oncology report:))"
+        cleaned = re.sub(pattern, "", template, flags=re.DOTALL)
+
+        cleaned = cleaned.replace("Follow the format of these two examples and give", "Give")
+        cleaned = cleaned.replace("follow the format of these two examples and give", "give")
+
+        return cleaned
+
+    def parse_response(self, response: str) -> Dict[str, Any]:
+        """
+        Parse response from model (supports multiple formats).
+
+        Expected format:
+        {
+            "cardiac_abnormality": bool,
+            "congenital_heart_defect": bool,
+            "evidence": {
+                "report_findings": [...],
+                "rationale": "..."
+            }
+        }
+        """
+        # Use mixin's multi-format parser if not JSON
+        if self.output_format != "json":
+            try:
+                parsed = self._parse_formatted_response(response)
+                # Map extracted values to standardized keys
+                is_positive = False
+                if self.output_format in ["plain", "markdown"]:
+                    # Parse final answer string
+                    ans_str = str(parsed.get("answer", "")).upper()
+                    is_positive = "CHD DETECTED" in ans_str
+                else:
+                    is_positive = parsed.get("congenital_heart_defect", False)
+
+                return {
+                    "answer": "CHD present" if is_positive else "no CHD",
+                    "cardiac_abnormality": parsed.get("cardiac_abnormality", is_positive),
+                    "congenital_heart_defect": is_positive,
+                    "reasoning": parsed.get("evidence", {}).get(
+                        "rationale", parsed.get("reasoning", "")
+                    ),
+                    "findings": parsed.get("evidence", {}).get("report_findings", []),
+                    "raw": response,
+                    "parsed_json": parsed,
+                }
+            except Exception:
+                pass  # Fall back to JSON parsing
+
+        # Try to extract JSON from response
+        json_match = re.search(r"```json\s*(.*?)\s*```", response, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(1)
+        else:
+            # Try to find raw JSON
+            json_match = re.search(
+                r'\{[^{}]*"congenital_heart_defect"[^{}]*\}', response, re.DOTALL
+            )
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = response
+
+        # Parse JSON
+        try:
+            parsed = json.loads(json_str)
+            return {
+                "answer": "CHD present" if parsed.get("congenital_heart_defect") else "no CHD",
+                "cardiac_abnormality": parsed.get("cardiac_abnormality", False),
+                "congenital_heart_defect": parsed.get("congenital_heart_defect", False),
+                "reasoning": parsed.get("evidence", {}).get("rationale", ""),
+                "findings": parsed.get("evidence", {}).get("report_findings", []),
+                "raw": response,
+                "parsed_json": parsed,
+            }
+        except json.JSONDecodeError:
+            # Fallback if JSON parsing fails
+            return {
+                "answer": response.strip(),
+                "reasoning": response,
+                "raw": response,
+                "parse_error": True,
+            }
+
+    def get_system_message(self) -> Optional[str]:
+        return self.system_role
+
+    def get_compatible_datasets(self) -> list[str]:
+        """
+        Cardiology prompt is only compatible with cardiology dataset.
+
+        This prompt is specifically designed for congenital heart defect detection
+        in cardiac imaging reports and should NOT be used for general medical QA.
+        """
+        return ["cardiology"]
+
+    def get_prediction_field(self) -> str:
+        """Return the JSON field name used for binary classification."""
+        return "congenital_heart_defect"

cotlab/prompts/histopathology.py

@@ -0,0 +1,265 @@
+"""Histopathology report quality rating prompt strategy."""
+
+import json
+import re
+from typing import Any, Dict
+
+from ..core.base import BasePromptStrategy
+from ..core.registry import Registry
+
+SYSTEM_ROLE = """You are a histopathology expert specializing in evaluating report quality.
+Your goal is to assess the clinical accuracy, completeness, and clarity of histopathology reports.
+Think rationally and explain your reasoning for quality ratings."""
+
+SYSTEM_ROLE_CONTRARIAN = """You are a skeptical histopathology expert specializing in evaluating report quality.
+Your goal is to critically assess histopathology reports and identify potential errors or omissions.
+Question obvious conclusions and look for missing information. Think rationally and explain your reasoning."""
+
+
+# Standard prompt with few-shot examples - INCLUDES SOURCE CASE FOR COMPARISON
+PROMPT_TEMPLATE = """You are an expert histopathologist evaluating the quality of a GENERATED pathology report.
+
+IMPORTANT: You must compare the generated report against the ORIGINAL CASE FINDINGS.
+A report that looks well-written but does NOT match the original case should be rated 0 (Poor).
+
+Rate on a scale of 0-2:
+- 0 = Poor: Report does NOT match the original case, has major errors, or is misleading
+- 1 = Partially correct: Matches some aspects of original case but incomplete or has minor errors
+- 2 = Good: Accurately reflects the original case findings with proper terminology
+
+Example 1 (Score: 0 - Poor - Well-written but WRONG diagnosis):
+Original Case: "Invasive ductal carcinoma with lymph node metastasis"
+Generated Report: "Benign fibrous lesion with no evidence of malignancy"
+Rating: 0 - Report is well-structured but completely misses the malignancy. WRONG!
+
+Example 2 (Score: 2 - Good - Matches original case):
+Original Case: "Well-differentiated adenocarcinoma, pT2N0, clear margins"
+Generated Report: "Sections show well-differentiated adenocarcinoma with glandular architecture. Margins clear. pT2N0."
+Rating: 2 - Report accurately reflects the original case findings.
+
+Example 3 (Score: 1 - Partial - Missing key details):
+Original Case: "Hepatocellular carcinoma, grade 4/4, with vascular invasion"
+Generated Report: "Hepatocellular carcinoma identified. Features consistent with malignancy."
+Rating: 1 - Correct diagnosis but missing grade and vascular invasion status.
+
+---
+
+ORIGINAL CASE FINDINGS:
+\"\"\"
+{source_case}
+\"\"\"
+
+GENERATED REPORT TO EVALUATE:
+\"\"\"
+{report}
+\"\"\"
+
+Compare the generated report against the original case. Does it accurately reflect the findings?
+
+Provide your response in JSON format:
+```json
+{{
+    "quality_score": 0, 1, or 2,
+    "reasoning": "Brief explanation comparing report to original case"
+}}
+```
+
+Response:
+"""
+
+# Contrarian prompt - skeptical evaluation WITH source case comparison
+PROMPT_TEMPLATE_CONTRARIAN = """You are a skeptical histopathologist reviewing a generated pathology report. Be critical and compare against the original case.
+
+IMPORTANT: Compare the generated report against the ORIGINAL CASE FINDINGS.
+Question any discrepancies. A well-written report that doesn't match the case is WRONG.
+
+Look for:
+- Does the diagnosis match the original case?
+- Missing critical information from the original
+- Incorrect or fabricated details
+- Inappropriate terminology
+
+Rate on a scale of 0-2:
+- 0 = Poor: Does NOT match original case or has major errors
+- 1 = Partially correct: Matches some but not all key findings
+- 2 = Good: Accurately reflects original case
+
+**Step 1**: Does the generated report match the ORIGINAL CASE?
+**Step 2**: What key findings are missing or wrong?
+**Step 3**: Final rating based on comparison.
+
+ORIGINAL CASE FINDINGS:
+\"\"\"
+{source_case}
+\"\"\"
+
+GENERATED REPORT TO EVALUATE:
+\"\"\"
+{report}
+\"\"\"
+
+Provide your response in JSON format:
+```json
+{{
+    "quality_score": 0, 1, or 2,
+    "reasoning": "Brief explanation comparing report to original case"
+}}
+```
+
+Response:
+"""
+
+# Answer-first prompt - conclude then justify WITH source case
+PROMPT_TEMPLATE_ANSWER_FIRST = """You are an expert histopathologist evaluating a generated pathology report.
+
+Compare the GENERATED REPORT against the ORIGINAL CASE FINDINGS.
+First state your rating (0, 1, or 2), then explain why.
+
+Rating scale:
+- 0 = Poor: Does NOT match original case
+- 1 = Partial: Matches some findings but incomplete
+- 2 = Good: Accurately reflects original case
+
+ORIGINAL CASE FINDINGS:
+\"\"\"
+{source_case}
+\"\"\"
+
+GENERATED REPORT:
+\"\"\"
+{report}
+\"\"\"
+
+Provide your response in JSON format:
+```json
+{{
+    "quality_score": <0, 1, or 2>,
+    "reasoning": "Justification comparing report to original case"
+}}
+```
+
+Response:
+"""
+
+
+@Registry.register_prompt("histopathology")
+class HistopathologyPromptStrategy(BasePromptStrategy):
+    """Prompt strategy for histopathology report quality rating.
+
+    3-class classification:
+    - 0: Poor/incorrect
+    - 1: Partially correct
+    - 2: Good/accurate
+
+    Supports:
+    - few_shot: Include examples in prompt
+    - answer_first: Rate first, then explain
+    - contrarian: Skeptical evaluation mode
+    """
+
+    def __init__(
+        self,
+        name: str = "histopathology",
+        output_format: str = "json",
+        few_shot: bool = True,
+        answer_first: bool = False,
+        contrarian: bool = False,
+        **kwargs,
+    ):
+        self._name = name
+        self.output_format = output_format.lower()
+        self.few_shot = few_shot
+        self.answer_first = answer_first
+        self.contrarian = contrarian
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def build_prompt(self, input_data: Dict[str, Any]) -> str:
+        """Build prompt with histopathology report and source case."""
+        report = input_data.get("text", input_data.get("report", ""))
+        # Get source case from metadata for comparison
+        metadata = input_data.get("metadata", {})
+        source_case = metadata.get("ground_truth", "Not provided")
+
+        # Select template (priority: answer_first > contrarian > standard)
+        if self.answer_first:
+            template = PROMPT_TEMPLATE_ANSWER_FIRST
+        elif self.contrarian:
+            template = PROMPT_TEMPLATE_CONTRARIAN
+        else:
+            template = PROMPT_TEMPLATE
+
+        # Format with both source case and report (all templates now use source_case)
+        prompt = template.format(report=report, source_case=source_case)
+
+        # Remove examples if few_shot=False
+        if not self.few_shot and not self.answer_first and not self.contrarian:
+            prompt = self._remove_few_shot_examples(prompt)
+
+        return prompt
+
+    def _remove_few_shot_examples(self, template: str) -> str:
+        """Remove few-shot examples from template."""
+        # Remove only the example block between "Example 1" and the section divider.
+        block_pattern = re.compile(r"\nExample 1.*?\n---\n", re.DOTALL)
+        if not block_pattern.search(template):
+            return template
+        return block_pattern.sub("\n\n", template, count=1)
+
+    def parse_response(self, response: str) -> Dict[str, Any]:
+        """Parse response to extract quality score."""
+        result = {
+            "raw_response": response,
+            "parse_success": False,
+            "quality_score": None,
+        }
+
+        # Try to extract JSON from response
+        json_match = re.search(r"```json\s*(.*?)\s*```", response, re.DOTALL)
+        if json_match:
+            try:
+                parsed = json.loads(json_match.group(1))
+                result.update(parsed)
+                result["parse_success"] = True
+            except json.JSONDecodeError:
+                pass
+
+        # Fallback: try direct JSON parse
+        if not result["parse_success"]:
+            try:
+                parsed = json.loads(response)
+                result.update(parsed)
+                result["parse_success"] = True
+            except json.JSONDecodeError:
+                pass
+
+        # Fallback: find quality_score in text
+        if not result["parse_success"] or result.get("quality_score") is None:
+            score_match = re.search(r'"quality_score"\s*:\s*(\d)', response)
+            if score_match:
+                result["quality_score"] = int(score_match.group(1))
+                result["parse_success"] = True
+            else:
+                # Last resort: find any 0, 1, or 2 alone
+                simple_match = re.search(r"\b([012])\b", response)
+                if simple_match:
+                    result["quality_score"] = int(simple_match.group(1))
+                    result["parse_success"] = True
+
+        return result
+
+    def get_system_message(self) -> str:
+        """Return mode-appropriate system message."""
+        if self.contrarian:
+            return SYSTEM_ROLE_CONTRARIAN
+        return SYSTEM_ROLE
+
+    def get_compatible_datasets(self) -> list[str]:
+        """Histopathology prompt works with histopathology dataset."""
+        return ["histopathology"]
+
+    def get_prediction_field(self) -> str:
+        """Return the JSON field name used for classification."""
+        return "quality_score"