open-pharma-plugins 2.2.0__py3-none-any.whl

open_pharma_plugins_hcp_intelligence/batch_cli.py

@@ -0,0 +1,221 @@
+"""Batch enrichment for hcp-intelligence accounts.
+
+Calls tool handle() functions directly (no MCP server) to gather data from
+PubMed, ClinicalTrials.gov, ORCID, NIH RePORTER, and web search, then
+optionally synthesizes structured profiles through OpenRouter's OpenAI-compatible API.
+
+Usage:
+  # Dry run — list what would be processed
+  python3 scripts/batch_enrich.py --dry-run
+
+  # Enrich one account (raw data only)
+  python3 scripts/batch_enrich.py --ids HCP-AU-001
+
+  # Enrich all Singapore HCPs with schema-validated DeepSeek synthesis via OpenRouter
+  python3 scripts/batch_enrich.py --country Singapore --account-type HCP \\
+      --synthesize
+
+  # Process a user-curated CSV into a private output directory
+  python3 scripts/batch_enrich.py --input-file ./accounts.csv \\
+      --output-dir ./data/hcp-intelligence --synthesize
+
+  # Resume after interruption
+  python3 scripts/batch_enrich.py --resume --synthesize
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Literal, Sequence, cast
+
+from open_pharma_plugins_hcp_intelligence.batch import (
+    DEFAULT_OPENROUTER_BASE_URL,
+    DEFAULT_REASONING_EFFORT,
+    DEFAULT_SYNTHESIS_MODEL,
+    DEFAULT_SYNTHESIS_TIMEOUT_SECONDS,
+    HCO_TOOLS,
+    HCP_TOOLS,
+    BatchOptions,
+    BatchOutcome,
+    BatchPlan,
+    BatchUsageError,
+    plan_batch,
+    run_batch,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Batch enrichment for hcp-intelligence accounts.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+
+    filtering = parser.add_argument_group("filtering")
+    filtering.add_argument(
+        "--input-file",
+        help="Account CSV using the bundled fixture columns (default: bundled sample_accounts.csv)",
+    )
+    filtering.add_argument("--country", help="Filter by country")
+    filtering.add_argument(
+        "--account-type",
+        type=str.upper,
+        choices=("HCP", "HCO"),
+        help="Filter by HCP or HCO",
+    )
+    filtering.add_argument("--ids", nargs="+", help="Process specific account IDs only")
+
+    execution = parser.add_argument_group("execution")
+    execution.add_argument("--concurrency", type=int, default=5, help="Parallel accounts (default: 5)")
+    execution.add_argument("--resume", action="store_true", help="Skip accounts whose output JSON already exists")
+    execution.add_argument("--dry-run", action="store_true", help="List accounts without processing")
+    execution.add_argument(
+        "--write-back",
+        action="store_true",
+        help="Persist results to the demo enrichment store (off by default for --input-file)",
+    )
+
+    synthesis = parser.add_argument_group("synthesis")
+    synthesis.add_argument("--synthesize", action="store_true", help="Enable LLM profile synthesis")
+    synthesis.add_argument(
+        "--base-url",
+        default=None,
+        help="OpenAI-compatible API base URL (default: OPENROUTER_BASE_URL or OpenRouter)",
+    )
+    synthesis.add_argument(
+        "--api-key-env",
+        default="OPENROUTER_API_KEY",
+        help="Config/environment key holding the API key (default: OPENROUTER_API_KEY)",
+    )
+    synthesis.add_argument(
+        "--model",
+        default=DEFAULT_SYNTHESIS_MODEL,
+        help=f"Model name (default: {DEFAULT_SYNTHESIS_MODEL})",
+    )
+    synthesis.add_argument(
+        "--reasoning-effort",
+        choices=("high", "xhigh"),
+        default=DEFAULT_REASONING_EFFORT,
+        help=f"Reasoning effort for extraction/synthesis (default: {DEFAULT_REASONING_EFFORT})",
+    )
+    synthesis.add_argument(
+        "--synthesis-timeout-seconds",
+        type=float,
+        default=DEFAULT_SYNTHESIS_TIMEOUT_SECONDS,
+        help=f"Per-request synthesis timeout (default: {DEFAULT_SYNTHESIS_TIMEOUT_SECONDS:g})",
+    )
+
+    output = parser.add_argument_group("output")
+    output.add_argument(
+        "--output-dir",
+        default="./batch_output",
+        help="Directory for raw results (default: ./batch_output)",
+    )
+    return parser
+
+
+def _options_from_args(args: argparse.Namespace) -> BatchOptions:
+    from shared.env import get_env
+
+    base_url = args.base_url or get_env("OPENROUTER_BASE_URL", DEFAULT_OPENROUTER_BASE_URL)
+    return BatchOptions(
+        input_file=args.input_file,
+        output_dir=args.output_dir,
+        country=args.country,
+        account_type=args.account_type,
+        ids=tuple(args.ids or ()),
+        concurrency=args.concurrency,
+        resume=args.resume,
+        write_back=args.write_back,
+        synthesize=args.synthesize,
+        base_url=base_url or DEFAULT_OPENROUTER_BASE_URL,
+        api_key_env=args.api_key_env,
+        model=args.model,
+        reasoning_effort=cast(Literal["high", "xhigh"], args.reasoning_effort),
+        synthesis_timeout_seconds=args.synthesis_timeout_seconds,
+    )
+
+
+def render_dry_run(plan: BatchPlan) -> None:
+    hcp_count = sum(account["account_type"] == "HCP" for account in plan.accounts)
+    hco_count = sum(account["account_type"] == "HCO" for account in plan.accounts)
+    print(f"Input: {plan.input_path}")
+    print(f"Output: {plan.output_dir}")
+    print(f"Selected: {len(plan.accounts)} total (HCP: {hcp_count}, HCO: {hco_count})")
+    print(f"Would process {len(plan.accounts)} account(s):\n")
+    for account in plan.accounts:
+        tools = HCP_TOOLS if account.get("account_type", "").upper() == "HCP" else HCO_TOOLS
+        print(
+            f"  {account['id']:16s} {account['name']:40s} "
+            f"{account['account_type']:4s} {account['country']:12s} ({len(tools)} tools)"
+        )
+    if plan.options.synthesize:
+        print(f"\nSynthesis: {plan.options.model} via {plan.options.base_url}")
+        print(f"Provider: {plan.options.base_url}")
+        print(f"Model: {plan.options.model}")
+        print(
+            f"Reasoning effort: {plan.options.reasoning_effort}; "
+            f"timeout: {plan.options.synthesis_timeout_seconds:g}s; SDK retries: 0"
+        )
+        print(f"Timeout: {plan.options.synthesis_timeout_seconds:g}s")
+    else:
+        print("\nSynthesis: disabled")
+        print(f"Provider: {plan.options.base_url}")
+        print(f"Model: {plan.options.model}")
+        print(f"Reasoning effort: {plan.options.reasoning_effort}")
+        print(f"Timeout: {plan.options.synthesis_timeout_seconds:g}s")
+        print("SDK retries: 0")
+
+    print("\nPlanned artifacts:")
+    for account in plan.accounts:
+        print(f"  {account['id']}.json")
+    print("  batch_summary.csv")
+    print("  batch_manifest.json")
+    print("\nNo external calls were made.")
+    print("Data sharing: execution would send search query terms to configured public data providers.")
+    if plan.options.synthesize:
+        print(
+            "Data sharing (synthesis): selected account fields and gathered evidence "
+            f"would be sent to {plan.options.base_url}."
+        )
+    if not plan.accounts:
+        print("No accounts match the filters.")
+
+
+def render_outcome(outcome: BatchOutcome) -> None:
+    if not outcome.manifest:
+        print("No accounts match the filters.")
+        return
+
+    summary = outcome.manifest["summary"]
+    print(
+        "\nDone. "
+        f"completed={summary['completed']}  partial={summary['partial']}  "
+        f"failed={summary['failed']}  skipped={summary['skipped']}"
+    )
+    summary_csv = outcome.manifest["outputs"]["summary_csv"]
+    print(f"Summary CSV: {summary_csv['path']}")
+    print(f"Manifest: {(outcome.output_dir / 'batch_manifest.json').resolve()}")
+    print(f"Output directory: {outcome.output_dir.resolve()}")
+    if summary_csv["status"] == "failed":
+        print(summary_csv["error"], file=sys.stderr)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        plan = plan_batch(_options_from_args(args))
+        if args.dry_run:
+            render_dry_run(plan)
+            return 0
+        outcome = run_batch(plan)
+    except BatchUsageError as exc:
+        parser.error(str(exc))
+    render_outcome(outcome)
+    return outcome.exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

open_pharma_plugins_hcp_intelligence/batch_csv.py

@@ -0,0 +1,206 @@
+"""Stable CSV projection for HCP Intelligence batch artifacts."""
+
+from __future__ import annotations
+
+import codecs
+import csv
+import hashlib
+import io
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from pydantic import ValidationError
+
+from open_pharma_plugins_hcp_intelligence.models import HcoProfile, HcpProfile
+from shared.filesystem import atomic_write_bytes
+
+CSV_SCHEMA_VERSION = 1
+SUMMARY_FILENAME = "batch_summary.csv"
+SUMMARY_COLUMNS = (
+    "account_id",
+    "account_type",
+    "input_name",
+    "input_specialty",
+    "input_country",
+    "input_institution",
+    "status",
+    "profile_validated",
+    "profile_completeness",
+    "profile_name",
+    "profile_specialty",
+    "profile_country",
+    "current_title",
+    "organization_type",
+    "affiliations",
+    "qualifications",
+    "research_or_clinical_focus",
+    "professional_roles",
+    "key_publication_count",
+    "clinical_trial_count",
+    "active_grant_count",
+    "congress_activity_count",
+    "source_count",
+    "source_urls",
+    "tools_failed",
+    "error",
+    "json_file",
+)
+
+_NUMERIC_COLUMNS = {
+    "profile_completeness",
+    "key_publication_count",
+    "clinical_trial_count",
+    "active_grant_count",
+    "congress_activity_count",
+    "source_count",
+}
+
+
+def _join_unique(values: Iterable[str]) -> str:
+    return " | ".join(dict.fromkeys(value for value in values if value))
+
+
+def _safe_text(value: object) -> str:
+    text = "" if value is None else str(value)
+    return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
+
+
+def _result_values(value: object) -> Iterable[str]:
+    if isinstance(value, str):
+        return (value,)
+    if isinstance(value, Sequence):
+        return (str(item) for item in value if item is not None)
+    return ()
+
+
+def _blank_row(account: Mapping[str, str], result: Mapping[str, Any]) -> dict[str, str | int | float]:
+    account_id = account["id"]
+    return {
+        "account_id": account_id,
+        "account_type": account["account_type"],
+        "input_name": account["name"],
+        "input_specialty": account.get("specialty", ""),
+        "input_country": account["country"],
+        "input_institution": account.get("institution", ""),
+        "status": result.get("status", ""),
+        "profile_validated": "false",
+        "profile_completeness": "",
+        "profile_name": "",
+        "profile_specialty": "",
+        "profile_country": "",
+        "current_title": "",
+        "organization_type": "",
+        "affiliations": "",
+        "qualifications": "",
+        "research_or_clinical_focus": "",
+        "professional_roles": "",
+        "key_publication_count": "",
+        "clinical_trial_count": "",
+        "active_grant_count": "",
+        "congress_activity_count": "",
+        "source_count": "",
+        "source_urls": "",
+        "tools_failed": _join_unique(_result_values(result.get("tools_failed", []))),
+        "error": result.get("error", ""),
+        "json_file": f"{account_id}.json",
+    }
+
+
+def _populate_hcp(row: dict[str, str | int | float], profile: HcpProfile) -> None:
+    row.update(
+        {
+            "profile_validated": "true",
+            "profile_completeness": profile.profile_completeness,
+            "profile_name": profile.full_name,
+            "profile_specialty": profile.specialty,
+            "profile_country": profile.country,
+            "current_title": profile.current_title.value if profile.current_title else "",
+            "affiliations": _join_unique(claim.value for claim in profile.affiliations),
+            "qualifications": _join_unique(claim.value for claim in profile.qualifications),
+            "research_or_clinical_focus": _join_unique(claim.value for claim in profile.research_interests),
+            "professional_roles": _join_unique(claim.value for claim in profile.professional_roles),
+            "key_publication_count": len(profile.key_publications),
+            "clinical_trial_count": len(profile.clinical_trial_involvement),
+            "active_grant_count": len(profile.active_grants),
+            "congress_activity_count": len(profile.congress_activity),
+        }
+    )
+
+
+def _populate_hco(row: dict[str, str | int | float], profile: HcoProfile) -> None:
+    row.update(
+        {
+            "profile_validated": "true",
+            "profile_completeness": profile.profile_completeness,
+            "profile_name": profile.name,
+            "profile_country": profile.country,
+            "organization_type": profile.organization_type.value if profile.organization_type else "",
+            "affiliations": _join_unique(claim.value for claim in profile.notable_affiliations),
+            "qualifications": _join_unique(claim.value for claim in profile.accreditations),
+            "research_or_clinical_focus": _join_unique(
+                claim.value for claim in (*profile.clinical_focus_areas, *profile.research_focus)
+            ),
+            "key_publication_count": 0,
+            "clinical_trial_count": len(profile.active_clinical_trials),
+            "active_grant_count": len(profile.institutional_grants),
+            "congress_activity_count": 0,
+        }
+    )
+
+
+def build_summary_rows(
+    accounts: Sequence[Mapping[str, str]],
+    results: Sequence[Mapping[str, Any]],
+    artifacts: Mapping[str, Mapping[str, Any]],
+) -> list[dict[str, str | int | float]]:
+    """Build rows from artifacts already ownership-validated by the batch engine."""
+    results_by_id = {result["account_id"]: result for result in results}
+    rows: list[dict[str, str | int | float]] = []
+    for account in accounts:
+        account_id = account["id"]
+        result = results_by_id.get(account_id, {})
+        row = _blank_row(account, result)
+        artifact = artifacts.get(account_id, {})
+        raw_profile = artifact.get("synthesized_profile")
+        if raw_profile is not None:
+            try:
+                if account["account_type"] == "HCP":
+                    profile = HcpProfile.model_validate(raw_profile)
+                    _populate_hcp(row, profile)
+                    sources = profile.sources_consulted
+                else:
+                    hco_profile = HcoProfile.model_validate(raw_profile)
+                    _populate_hco(row, hco_profile)
+                    sources = hco_profile.sources_consulted
+            except ValidationError:
+                pass
+            else:
+                source_urls = list(dict.fromkeys(source.url for source in sources if source.url))
+                row["source_count"] = len(source_urls)
+                row["source_urls"] = _join_unique(source_urls)
+
+        rows.append(
+            {
+                column: row[column] if column in _NUMERIC_COLUMNS else _safe_text(row[column])
+                for column in SUMMARY_COLUMNS
+            }
+        )
+    return rows
+
+
+def write_summary_csv(path: Path, rows: Sequence[Mapping[str, object]]) -> dict[str, object]:
+    """Atomically write the stable UTF-8 BOM CSV and return exact-byte metadata."""
+    buffer = io.StringIO(newline="")
+    writer = csv.DictWriter(buffer, fieldnames=SUMMARY_COLUMNS, lineterminator="\r\n")
+    writer.writeheader()
+    writer.writerows(rows)
+    payload = codecs.BOM_UTF8 + buffer.getvalue().encode("utf-8")
+    written = atomic_write_bytes(path, payload)
+    return {
+        "status": "completed",
+        "path": str(written.resolve()),
+        "schema_version": CSV_SCHEMA_VERSION,
+        "row_count": len(rows),
+        "sha256": hashlib.sha256(payload).hexdigest(),
+    }

open_pharma_plugins_hcp_intelligence/fixtures/sample_accounts.csv

@@ -0,0 +1,27 @@
+id,name,specialty,country,account_type,institution
+HCP-AU-001,Sarah Chen,Medical Oncology,Australia,HCP,Peter MacCallum Cancer Centre
+HCP-AU-002,James Mitchell,Radiation Oncology,Australia,HCP,Chris O'Brien Lifehouse
+HCP-AU-003,Priya Sharma,Haematology-Oncology,Australia,HCP,Royal Melbourne Hospital
+HCP-AU-004,David Lee,Surgical Oncology,Australia,HCP,Westmead Hospital
+HCP-AU-005,Emily Watson,Paediatric Oncology,Australia,HCP,The Children's Hospital at Westmead
+HCP-AU-006,Michael Zhou,Medical Oncology,Australia,HCP,Royal Prince Alfred Hospital
+HCP-AU-007,Rachel O'Brien,Breast Oncology,Australia,HCP,Mater Hospital Brisbane
+HCP-AU-008,Andrew Tan,Immuno-Oncology,Australia,HCP,Olivia Newton-John Cancer Centre
+HCP-AU-009,Lisa Nguyen,Gynaecological Oncology,Australia,HCP,Royal Women's Hospital Melbourne
+HCP-AU-010,Thomas Baker,Thoracic Oncology,Australia,HCP,The Prince Charles Hospital
+HCP-SG-001,Wei Lin Tan,Medical Oncology,Singapore,HCP,National Cancer Centre Singapore
+HCP-SG-002,David Lee,Radiation Oncology,Singapore,HCP,National University Cancer Institute
+HCP-SG-003,Mei Ling Wong,Haematology-Oncology,Singapore,HCP,Singapore General Hospital
+HCP-SG-004,Rajesh Kumar,Surgical Oncology,Singapore,HCP,National University Hospital
+HCP-SG-005,Nur Aisyah Rahman,Paediatric Oncology,Singapore,HCP,KK Women's and Children's Hospital
+HCP-SG-006,Cheng Wei Lim,Medical Oncology,Singapore,HCP,Parkway Cancer Centre
+HCP-SG-007,Hui Fen Ong,Breast Oncology,Singapore,HCP,National Cancer Centre Singapore
+HCP-SG-008,Arun Patel,Thoracic Oncology,Singapore,HCP,National University Cancer Institute
+HCO-AU-001,Peter MacCallum Cancer Centre,Oncology,Australia,HCO,
+HCO-AU-002,Chris O'Brien Lifehouse,Oncology,Australia,HCO,
+HCO-AU-003,Olivia Newton-John Cancer Wellness and Research Centre,Oncology,Australia,HCO,
+HCO-AU-004,Royal Melbourne Hospital,General - Oncology Services,Australia,HCO,
+HCO-SG-001,National Cancer Centre Singapore,Oncology,Singapore,HCO,
+HCO-SG-002,National University Cancer Institute Singapore,Oncology,Singapore,HCO,
+HCO-SG-003,Parkway Cancer Centre,Oncology,Singapore,HCO,
+HCO-SG-004,Mount Elizabeth Hospital,General - Oncology Department,Singapore,HCO,