cotlab 0.8.0__py3-none-any.whl

cotlab/prompts/pubmedqa.py

@@ -0,0 +1,173 @@
+"""PubMedQA prompt strategy.
+
+This strategy is designed for the PubMedQA dataset which requires answering
+Yes/No/Maybe to research questions based on abstract contexts.
+"""
+
+import json
+import re
+from typing import Any, Dict, Optional
+
+from ..core.base import BasePromptStrategy, StructuredOutputMixin
+from ..core.registry import Registry
+
+
+@Registry.register_prompt("pubmedqa")
+class PubMedQAPromptStrategy(BasePromptStrategy, StructuredOutputMixin):
+    """Prompt strategy for PubMedQA (Yes/No/Maybe)."""
+
+    SYSTEM_ROLE = """You are a medical researcher evaluating scientific studies.
+You must answer the research question based ONLY on the provided context.
+The valid answers are 'yes', 'no', or 'maybe'."""
+
+    SYSTEM_ROLE_CONTRARIAN = """You are a skeptical peer reviewer evaluating a study's claims.
+Be highly critical. Do not accept the abstract's conclusions unless the evidence in the context is irrefutable.
+Look for limitations, confounding factors, or ambiguity that would make the answer 'maybe' instead of a definite 'yes' or 'no'."""
+
+    PROMPT_TEMPLATE = """## Research Context
+{text}
+
+## Instructions
+
+1. Read the research question and abstract context carefully.
+2. Determine if the context provides enough evidence to answer the question.
+3. If the answer is clearly supported by the findings, answer 'yes' or 'no'.
+4. If the answer is ambiguous, not fully supported, or conditional, answer 'maybe'.
+
+{format_instructions}"""
+
+    FEW_SHOT_EXAMPLES = [
+        {
+            "text": """Question: Is laparoscopic appendectomy safe in pregnant women?
+
+Context: A retrospective review of 300 pregnant women who underwent appendectomy (150 laparoscopic, 150 open). There was no significant difference in fetal loss or preterm delivery between groups. Laparoscopic group had shorter hospital stay and lower wound infection rates.""",
+            "reasoning": "The context provides a direct comparison between laparoscopic and open appendectomy in a substantial cohort. The key safety outcomes (fetal loss, preterm delivery) showed no significant difference, while secondary outcomes favored laparoscopy. This supports an affirmative answer regarding safety.",
+            "answer": "yes",
+        },
+        {
+            "text": """Question: Does Vitamin D supplementation prevent cancer?
+
+Context: A systematic review of 10 randomized trials found inconsistent results. Three trials showed benefit, four showed no effect, and three were inconclusive. Heterogeneity in dosage and population was high.""",
+            "reasoning": "The context describes 'inconsistent results' from a systematic review, with some trials showing benefit and others not. This ambiguity and lack of definitive consensus means a simple 'yes' or 'no' is not fully supported by the provided evidence.",
+            "answer": "maybe",
+        },
+    ]
+
+    def __init__(
+        self,
+        name: str = "pubmedqa",
+        output_format: str = "json",
+        few_shot: bool = True,
+        contrarian: bool = False,
+        answer_first: bool = False,
+        **kwargs,
+    ):
+        self._name = name
+        self.output_format = output_format
+        self.few_shot = few_shot
+        self.contrarian = contrarian
+        self.answer_first = answer_first
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def get_system_prompt(self) -> str:
+        return self.SYSTEM_ROLE_CONTRARIAN if self.contrarian else self.SYSTEM_ROLE
+
+    def build_prompt(
+        self,
+        inputs: Dict[str, Any],
+        few_shot: Optional[bool] = None,
+        **kwargs,
+    ) -> str:
+        # Loaders provide text as "Question: ...\n\nContext: ..."
+        text = inputs.get("text", "")
+
+        use_few_shot = few_shot if few_shot is not None else self.few_shot
+        format_instructions = self._get_format_instructions()
+
+        # Build few-shot examples
+        examples_str = ""
+        if use_few_shot:
+            examples_str = self._build_few_shot_examples()
+
+        prompt = self.PROMPT_TEMPLATE.format(
+            text=text,
+            format_instructions=format_instructions,
+        )
+
+        if examples_str:
+            prompt = f"## Examples\n\n{examples_str}\n\n{prompt}"
+
+        return prompt
+
+    def _build_few_shot_examples(self) -> str:
+        examples = []
+        for i, ex in enumerate(self.FEW_SHOT_EXAMPLES, 1):
+            if self.answer_first:
+                example = f"### Example {i}\n\n{ex['text']}\n\n**Answer:** {ex['answer']}\n\n**Reasoning:** {ex['reasoning']}"
+            else:
+                example = f"### Example {i}\n\n{ex['text']}\n\n**Reasoning:** {ex['reasoning']}\n\n**Answer:** {ex['answer']}"
+            examples.append(example)
+        return "\n\n".join(examples)
+
+    def _get_format_instructions(self) -> str:
+        if self.output_format == "json":
+            if self.answer_first:
+                return """Respond with a JSON object in this exact format:
+```json
+{"answer": "yes/no/maybe", "reasoning": "Explain your finding based on the context"}
+```
+The "answer" field MUST be one of: "yes", "no", "maybe"."""
+            else:
+                return """Respond with a JSON object in this exact format:
+```json
+{"reasoning": "Explain your finding based on the context", "answer": "yes/no/maybe"}
+```
+The "answer" field MUST be one of: "yes", "no", "maybe"."""
+        else:
+            if self.answer_first:
+                return "State your final answer as 'yes', 'no', or 'maybe', followed by your reasoning."
+            else:
+                return "Provide your reasoning, then state your final answer as 'yes', 'no', or 'maybe'."
+
+    def parse_response(self, response: str) -> Dict[str, Any]:
+        """Parse model response to extract answer(yes/no/maybe)."""
+        result = {"answer": None, "reasoning": None, "raw_response": response}
+
+        # Clean response
+        clean_response = response.strip()
+
+        # Try JSON parsing
+        try:
+            json_match = re.search(r"\{[^{}]*\}", clean_response, re.DOTALL)
+            if json_match:
+                parsed = json.loads(json_match.group())
+                ans = parsed.get("answer", "").lower().strip()
+                if ans in ["yes", "no", "maybe"]:
+                    result["answer"] = ans
+                result["reasoning"] = parsed.get("reasoning", "")
+                if result["answer"]:
+                    return result
+        except json.JSONDecodeError:
+            pass
+
+        # Fallback regex
+        patterns = [
+            r"answer[:\s]*[\"']?(yes|no|maybe)[\"']?",
+            r"\b(yes|no|maybe)\b",
+        ]
+
+        for p in patterns:
+            matches = re.findall(p, clean_response, re.IGNORECASE)
+            if matches:
+                # Take the last match as it's likely the conclusion
+                result["answer"] = matches[-1].lower()
+                break
+
+        result["reasoning"] = response
+        return result
+
+    def get_prediction_field(self) -> str:
+        return "answer"

cotlab/prompts/radiology.py

@@ -0,0 +1,414 @@
+"""Radiology-specific prompt strategy with JSON structured output."""
+
+import json
+import re
+from typing import Any, Dict, Optional
+
+from ..core.base import BasePromptStrategy, StructuredOutputMixin
+from ..core.registry import Registry
+
+SYSTEM_ROLE = """You are a radiology expert specialised in paedeatric radiology.
+Your goal is to identify incidence of pathological fractures in the given radiology report.
+Do not make assumptions or diagnoses from the text.
+Think rationally and explain your reasoning."""
+
+SYSTEM_ROLE_CONTRARIAN = """You are a skeptical radiology expert specialised in paedeatric radiology.
+Your goal is to identify incidence of pathological fractures in the given radiology report.
+However, you must question obvious conclusions and consider alternative explanations.
+Think rationally, play devil's advocate, and explain your reasoning."""
+
+
+PROMPT_TEMPLATE = """Follow this structured reasoning on the attached radiology report:
+
+1. **Fracture Mention**: Determine whether a bone fracture is explicitly described in the report.
+2. **Pathological Nature**: If a bone fracture is mentioned, assess the nature of the fracture, check if there is a strong, direct and explicit indication that this fracture is of pathological nature.
+
+The instruction is to answer questions strictly based on the content of the provided radiology report without making any assumptions or future projections/possibilities.
+Only make judgements on strong reason.
+Ignore non-bone fractures (e.g., device or lead fractures) as they do not indicate bone pathology.
+Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Fracture present and signs of pathological fracture
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": true,
+    "evidence": {{
+        "report_findings": ["bilateral clavicle fractures", "right clavicle fracture shows some periosteal reaction and callus formation"],
+        "rationale": "The report explicitly mentions bilateral clavicle fractures. The description of the right clavicle fracture includes periosteal reaction and callus formation, which are indicative of a pathological fracture."
+    }}
+}}
+```
+
+Example 2: Fracture present and signs of non-pathological fracture
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": false,
+    "evidence": {{
+        "report_findings": ["displaced fracture of the right sixth posterolateral rib"],
+        "rationale": "The report mentions a displaced rib fracture, but does not provide any information that suggests it is a pathological fracture. Therefore, I cannot conclude that it is a pathological fracture."
+    }}
+}}
+```
+
+Radiology report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+PROMPT_TEMPLATE_CONTRARIAN = """As a skeptical diagnostician, follow this structured reasoning on the attached radiology report.
+Question obvious patterns and consider alternative explanations before reaching your conclusion.
+
+1. **Fracture Mention**: Determine whether a bone fracture is explicitly described in the report. Consider if what appears to be a fracture might be a normal variant or imaging artifact.
+2. **Pathological Nature**: If a bone fracture is mentioned, critically assess whether it is truly pathological. Question the obvious diagnosis - could there be alternative explanations for the findings?
+
+The instruction is to answer questions strictly based on the content of the provided radiology report.
+Apply skeptical reasoning - if the report suggests a pathological fracture, argue why it might NOT be pathological. If it seems non-pathological, consider why it MIGHT be pathological.
+Only make final judgements when the evidence is overwhelming and alternative explanations are ruled out.
+Ignore non-bone fractures (e.g., device or lead fractures) as they do not indicate bone pathology.
+Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Fracture present and signs of pathological fracture (after skeptical review)
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": true,
+    "evidence": {{
+        "report_findings": ["bilateral clavicle fractures", "right clavicle fracture shows some periosteal reaction and callus formation"],
+        "rationale": "Initial skepticism: Could these findings represent normal bone remodeling? However, the bilateral nature and explicit mention of periosteal reaction and callus formation in the context of reported pathology provide overwhelming evidence. Alternative explanations (trauma, normal variant) are ruled out by the report's clinical context. Conclusion: pathological fracture confirmed despite initial skepticism."
+    }}
+}}
+```
+
+Example 2: Fracture present but questioning pathological nature (skeptical analysis)
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": false,
+    "evidence": {{
+        "report_findings": ["displaced fracture of the right sixth posterolateral rib"],
+        "rationale": "The report mentions a displaced rib fracture. Applying skeptical reasoning: While displacement might suggest pathology, it could equally result from mechanical trauma. The report provides no explicit indicators of metabolic bone disease, no mention of osteopenia, no description of abnormal bone architecture. Playing devil's advocate against a pathological diagnosis: the lack of corroborating features means this is more likely traumatic. Cannot conclude pathological fracture without stronger evidence."
+    }}
+}}
+```
+
+Radiology report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+PROMPT_TEMPLATE_ANSWER_FIRST = """Review this radiology report and provide immediate assessment.
+
+**Step 1 - Initial Diagnosis**: Based on your immediate review, state your initial conclusion about the presence of a pathological fracture.
+
+Initial Assessment: Is this a pathological fracture? (State YES or NO immediately)
+
+**Step 2 - Evidence Collection**: Now systematically gather and evaluate evidence:
+- What findings SUPPORT your initial diagnosis?
+- What findings CONTRADICT your initial diagnosis?
+- Are there alternative explanations?
+
+**Step 3 - Final Diagnosis**: Based on the evidence collected, confirm or revise your initial assessment.
+
+Provide your response in JSON format. Follow the format of these two examples and give the output strictly in the json format.
+
+Example 1: Initial YES, Confirmed pathological
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": true,
+    "evidence": {{
+        "report_findings": ["bilateral clavicle fractures", "periosteal reaction", "callus formation"],
+        "rationale": "Initial diagnosis: YES - pathological fracture suspected. Supporting evidence: bilateral nature of fractures and periosteal reaction strongly indicate pathological process. Contradicting evidence: None identified. Final diagnosis: Confirmed pathological fracture."
+    }}
+}}
+```
+
+Example 2: Initial NO, Confirmed non-pathological
+```json
+{{
+    "fracture_mentioned": true,
+    "pathological_fracture": false,
+    "evidence": {{
+        "report_findings": ["displaced rib fracture"],
+        "rationale": "Initial diagnosis: NO - likely traumatic. Supporting evidence: single displaced rib fracture with no additional pathological features. Contradicting evidence: None suggesting metabolic disease. Final diagnosis: Confirmed non-pathological fracture."
+    }}
+}}
+```
+
+Radiology report:
+\"\"\"
+{report}
+\"\"\"
+
+Response:
+"""
+
+
+@Registry.register_prompt("radiology")
+class RadiologyPromptStrategy(StructuredOutputMixin, BasePromptStrategy):
+    """
+    Structured JSON output for radiology pathological fracture detection.
+
+    Uses structured JSON output format with:
+    - Clear step-by-step reasoning instructions
+    - JSON output with fracture_mentioned, pathological_fracture, evidence
+    - Few-shot examples for format guidance
+    """
+
+    def __init__(
+        self,
+        name: str = "radiology",
+        system_role: Optional[str] = None,
+        contrarian: bool = False,
+        few_shot: bool = True,
+        answer_first: bool = False,
+        output_format: str = "json",
+        **kwargs,
+    ):
+        self._name = name
+        self.contrarian = contrarian
+        self.few_shot = few_shot
+        self.answer_first = answer_first
+        self.output_format = output_format
+        # Choose system role based on contrarian mode
+        if system_role:
+            self.system_role = system_role
+        else:
+            self.system_role = SYSTEM_ROLE_CONTRARIAN if contrarian else SYSTEM_ROLE
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def build_prompt(self, input_data: Dict[str, Any]) -> str:
+        """Build prompt with radiology report."""
+        report = input_data.get("text", input_data.get("report", input_data.get("question", "")))
+
+        # Select base template based on reasoning mode
+        if self.answer_first:
+            template = PROMPT_TEMPLATE_ANSWER_FIRST
+        elif self.contrarian:
+            template = PROMPT_TEMPLATE_CONTRARIAN
+        else:
+            template = PROMPT_TEMPLATE
+
+        # Remove examples if few_shot=False
+        if not self.few_shot:
+            template = self._remove_few_shot_examples(template)
+        elif self.output_format != "json":
+            # Convert JSON examples to target format
+            template = self._convert_examples_to_format(template)
+
+        prompt = template.format(report=report)
+
+        return prompt
+
+    def _convert_examples_to_format(self, template: str) -> str:
+        """Convert JSON examples in template to target output format."""
+        # Example data for radiology
+        examples = [
+            {
+                "title": "Example 1: Fracture present and signs of pathological fracture",
+                "data": {
+                    "fracture_mentioned": True,
+                    "pathological_fracture": True,
+                    "evidence": {
+                        "report_findings": [
+                            "bilateral clavicle fractures",
+                            "periosteal reaction and callus formation",
+                        ],
+                        "rationale": "The report explicitly mentions bilateral clavicle fractures with periosteal reaction and callus formation, indicative of a pathological fracture.",
+                    },
+                    "_plain_answer": "PATHOLOGICAL",
+                },
+            },
+            {
+                "title": "Example 2: Fracture present but non-pathological",
+                "data": {
+                    "fracture_mentioned": True,
+                    "pathological_fracture": False,
+                    "evidence": {
+                        "report_findings": [
+                            "displaced fracture of the right sixth posterolateral rib"
+                        ],
+                        "rationale": "The report mentions a displaced rib fracture, but provides no information suggesting it is pathological.",
+                    },
+                    "_plain_answer": "NON-PATHOLOGICAL",
+                },
+            },
+        ]
+
+        # Build examples in target format
+        examples_str = ""
+        for ex in examples:
+            examples_str += f"\n{ex['title']}\n"
+            examples_str += self._format_example(ex["data"]) + "\n"
+
+        # Replace JSON examples section
+        import re
+
+        pattern = r"Example 1:.*?```\s*\n\nRadiology report:"
+        replacement = examples_str.strip() + "\n\nRadiology report:"
+
+        new_template = re.sub(pattern, replacement, template, flags=re.DOTALL)
+
+        # Also update the format instruction in the header
+        if self.output_format == "plain":
+            new_template = new_template.replace(
+                "give the output strictly in the json format",
+                "provide your answer in plain text with FINAL ANSWER: at the end",
+            )
+        else:
+            new_template = new_template.replace(
+                "give the output strictly in the json format",
+                f"give the output in {self.output_format.upper()} format",
+            )
+
+        return new_template
+
+    def _remove_few_shot_examples(self, template: str) -> str:
+        """Remove few-shot examples from template for ablation studies."""
+        import re
+
+        # Remove everything from "Example 1:" to just before "Radiology report:" or similar
+        # Match: Example 1: ... Example 2: ... up until the report section
+        pattern = r"Example \d+:.*?(?=(?:Radiology report:|Cardiac imaging report:|Neuroimaging report:|Oncology report:))"
+        cleaned = re.sub(pattern, "", template, flags=re.DOTALL)
+
+        # Clean up instruction text that references examples
+        cleaned = cleaned.replace("Follow the format of these two examples and give", "Give")
+        cleaned = cleaned.replace("follow the format of these two examples and give", "give")
+
+        return cleaned
+
+    def parse_response(self, response: str) -> Dict[str, Any]:
+        """
+        Parse response from model (supports multiple formats).
+
+        Expected format:
+        {
+            "fracture_mentioned": bool,
+            "pathological_fracture": bool,
+            "evidence": {
+                "report_findings": [...],
+                "rationale": "..."
+            }
+        }
+        """
+        # Plain text parsing - extract FINAL ANSWER
+        if self.output_format == "plain":
+            # Look for FINAL ANSWER pattern
+            final_match = re.search(
+                r"FINAL ANSWER:\s*(PATHOLOGICAL|NON-PATHOLOGICAL|NO FRACTURE)",
+                response,
+                re.IGNORECASE,
+            )
+            if final_match:
+                answer_text = final_match.group(1).upper()
+                is_pathological = answer_text == "PATHOLOGICAL"
+                has_fracture = answer_text != "NO FRACTURE"
+                return {
+                    "answer": "pathological" if is_pathological else "non-pathological",
+                    "fracture_mentioned": has_fracture,
+                    "pathological_fracture": is_pathological,
+                    "reasoning": response,
+                    "findings": [],
+                    "raw": response,
+                }
+            # Fallback: look for pathological/non-pathological keywords
+            if (
+                "pathological fracture" in response.lower()
+                and "non-pathological" not in response.lower()
+            ):
+                return {
+                    "answer": "pathological",
+                    "fracture_mentioned": True,
+                    "pathological_fracture": True,
+                    "reasoning": response,
+                    "raw": response,
+                }
+
+        # Use mixin's multi-format parser if not JSON/plain
+        if self.output_format not in ("json", "plain"):
+            try:
+                parsed = self._parse_formatted_response(response)
+
+                is_pathological = parsed.get("pathological_fracture")
+                if is_pathological is None and parsed.get("answer"):
+                    ans = str(parsed.get("answer", "")).upper()
+                    is_pathological = "PATHOLOGICAL" in ans and "NON-PATHOLOGICAL" not in ans
+
+                # Derive fracture_mentioned from answer (if diagnosis given, fracture was mentioned)
+                has_fracture = parsed.get("fracture_mentioned", is_pathological is not None)
+
+                return {
+                    "answer": "pathological" if is_pathological else "non-pathological",
+                    "fracture_mentioned": has_fracture,
+                    "pathological_fracture": is_pathological
+                    if is_pathological is not None
+                    else False,
+                    "reasoning": parsed.get(
+                        "reasoning", parsed.get("evidence", {}).get("rationale", "")
+                    ),
+                    "findings": parsed.get("evidence", {}).get("report_findings", []),
+                    "raw": response,
+                    "parsed_json": parsed,
+                }
+            except Exception:
+                pass  # Fall back to JSON parsing
+
+        # Original JSON parsing logic
+        json_match = re.search(r"```json\s*(.*?)\s*```", response, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(1)
+        else:
+            json_match = re.search(r'\{[^{}]*"fracture_mentioned"[^{}]*\}', response, re.DOTALL)
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = response
+
+        try:
+            parsed = json.loads(json_str)
+            return {
+                "answer": "pathological"
+                if parsed.get("pathological_fracture")
+                else "non-pathological",
+                "fracture_mentioned": parsed.get("fracture_mentioned", False),
+                "pathological_fracture": parsed.get("pathological_fracture", False),
+                "reasoning": parsed.get("evidence", {}).get("rationale", ""),
+                "findings": parsed.get("evidence", {}).get("report_findings", []),
+                "raw": response,
+                "parsed_json": parsed,
+            }
+        except json.JSONDecodeError:
+            return {
+                "answer": response.strip(),
+                "reasoning": response,
+                "raw": response,
+                "parse_error": True,
+            }
+
+    def get_system_message(self) -> Optional[str]:
+        return self.system_role
+
+    def get_compatible_datasets(self) -> list[str]:
+        """
+        Radiology prompt is only compatible with radiology dataset.
+
+        This prompt is specifically designed for pathological fracture detection
+        in radiology reports and should NOT be used for general medical QA.
+        """
+        return ["radiology"]
+
+    def get_prediction_field(self) -> str:
+        """Return the JSON field name used for binary classification."""
+        return "pathological_fracture"