synthneura 2.0.0__py3-none-any.whl

synthneura/core/config.py

@@ -0,0 +1,44 @@
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class Settings:
+    db_path: str
+    sink: str
+    log_level: str
+    output_path: Optional[str]
+    output_format: str
+    summary_path: Optional[str]
+    changes_path: Optional[str]
+    max_results: int
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    db_path = os.getenv("SYNTHNEURA_DB_PATH", "trials.db")
+    sink = os.getenv("SYNTHNEURA_SINK", "sqlite")
+    log_level = os.getenv("SYNTHNEURA_LOG_LEVEL", "INFO")
+    output_path = os.getenv("SYNTHNEURA_OUTPUT_PATH")
+    output_format = os.getenv("SYNTHNEURA_OUTPUT_FORMAT", "json")
+    summary_path = os.getenv("SYNTHNEURA_SUMMARY_PATH")
+    changes_path = os.getenv("SYNTHNEURA_CHANGES_PATH")
+    max_results_raw = os.getenv("SYNTHNEURA_MAX_RESULTS", "10")
+
+    try:
+        max_results = int(max_results_raw)
+    except ValueError:
+        max_results = 10
+
+    return Settings(
+        db_path=db_path,
+        sink=sink,
+        log_level=log_level,
+        output_path=output_path,
+        output_format=output_format,
+        summary_path=summary_path,
+        changes_path=changes_path,
+        max_results=max_results,
+    )

synthneura/core/logger.py

@@ -0,0 +1,46 @@
+import logging
+import sys
+from typing import Any
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Create or retrieve a configured logger.
+    """
+    logger = logging.getLogger(name)
+
+    if logger.handlers:
+        # Logger already configured
+        return logger
+
+    # Inherit root level by default so CLI/user config controls verbosity.
+    logger.setLevel(logging.NOTSET)
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(logging.NOTSET)
+    formatter = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
+    )
+    handler.setFormatter(formatter)
+
+    logger.addHandler(handler)
+    logger.propagate = False
+
+    return logger
+
+
+def set_log_level(level: Any) -> None:
+    """
+    Set the log level for root + existing module loggers.
+    """
+    if isinstance(level, str):
+        resolved = getattr(logging, level.upper(), logging.INFO)
+    else:
+        resolved = int(level)
+
+    logging.getLogger().setLevel(resolved)
+    for logger in logging.Logger.manager.loggerDict.values():
+        if isinstance(logger, logging.Logger):
+            logger.setLevel(resolved)
+            for handler in logger.handlers:
+                handler.setLevel(resolved)

synthneura/core/schemas.py

@@ -0,0 +1,47 @@
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ClinicalTrial(BaseModel):
+    """
+    Canonical internal representation of a clinical trial
+    normalized from ClinicalTrials.gov (API v2 or legacy).
+    """
+
+    # Core identifiers
+    nct_id: str
+
+    # Descriptive fields (often missing in real-world data)
+    title: Optional[str] = None
+    status: Optional[str] = None
+    phase: Optional[str] = None
+    sponsor: Optional[str] = None
+
+    # Lists are defaulted to empty to avoid runtime errors
+    conditions: List[str] = []
+    interventions: List[str] = []
+    outcomes: List[str] = []
+
+
+class TopCount(BaseModel):
+    value: str
+    count: int
+
+
+class Summary(BaseModel):
+    """
+    Aggregated summary statistics for a set of trials.
+    """
+
+    total_trials: int
+    status_counts: Dict[str, int] = Field(default_factory=dict)
+    phase_counts: Dict[str, int] = Field(default_factory=dict)
+    top_conditions: List[TopCount] = Field(default_factory=list)
+    top_sponsors: List[TopCount] = Field(default_factory=list)
+    top_interventions: List[TopCount] = Field(default_factory=list)
+    change_counts: Dict[str, int] = Field(default_factory=dict)
+    run_id: Optional[str] = None
+    ingested_at: Optional[str] = None
+    source: Optional[str] = None
+    query: Optional[str] = None

synthneura/ingestion/clinical_trials.py

@@ -0,0 +1,73 @@
+from typing import Any, Dict, List
+
+import requests
+
+from synthneura.core.logger import get_logger
+
+CTG_V2_STUDIES_URL = "https://clinicaltrials.gov/api/v2/studies"
+
+logger = get_logger(__name__)
+
+
+def fetch_trials(query: str, max_results: int = 10) -> List[Dict[str, Any]]:
+    """
+    Fetch clinical trials from ClinicalTrials.gov API v2.
+    Returns a list of study objects (dicts).
+    """
+    logger.info("Fetching ClinicalTrials.gov trials")
+    logger.debug("Query=%s max_results=%s", query, max_results)
+
+    params: dict[str, str] = {
+        "query.term": query,
+        "pageSize": str(max_results),
+        "format": "json",
+        "countTotal": "true",
+    }
+
+    try:
+        response = requests.get(
+            CTG_V2_STUDIES_URL,
+            params=params,
+            timeout=30,
+        )
+    except requests.RequestException as exc:
+        logger.exception("HTTP request to ClinicalTrials.gov failed")
+        raise RuntimeError("Failed to contact ClinicalTrials.gov") from exc
+
+    logger.info(
+        "ClinicalTrials.gov response status=%s url=%s",
+        response.status_code,
+        response.url,
+    )
+
+    content_type = response.headers.get("Content-Type", "")
+    if "text/html" in content_type.lower():
+        logger.error(
+            "Received HTML instead of JSON (status=%s). Response preview=%s",
+            response.status_code,
+            response.text[:200],
+        )
+        raise ValueError(
+            f"ClinicalTrials.gov returned HTML (not JSON). "
+            f"Status={response.status_code}. Check endpoint/params."
+        )
+
+    try:
+        response.raise_for_status()
+    except requests.HTTPError:
+        logger.error(
+            "HTTP error from ClinicalTrials.gov status=%s body=%s",
+            response.status_code,
+            response.text[:200],
+        )
+        raise
+
+    data = response.json()
+    studies = data.get("studies", [])
+
+    if not studies:
+        logger.warning("No trials found for query=%s", query)
+        raise ValueError(f"No trials found for query: {query}")
+
+    logger.info("Fetched %d trials from ClinicalTrials.gov", len(studies))
+    return studies

synthneura/services/pipeline.py

@@ -0,0 +1,141 @@
+from typing import Any, Dict, List
+
+from synthneura.core.logger import get_logger
+from synthneura.core.schemas import ClinicalTrial
+
+logger = get_logger(__name__)
+
+
+def _as_list(x: Any) -> List[str]:
+    """Coerce strings / lists / missing values into a list[str]."""
+    if x is None:
+        return []
+    if isinstance(x, str):
+        return [x]
+    if isinstance(x, list):
+        out: List[str] = []
+        for item in x:
+            if item is None:
+                continue
+            if isinstance(item, str):
+                out.append(item)
+            elif isinstance(item, dict):
+                # Try common name fields
+                name = item.get("name") or item.get("measure") or item.get("title")
+                if name:
+                    out.append(str(name))
+            else:
+                out.append(str(item))
+        return out
+    if isinstance(x, dict):
+        # legacy containers like {"Condition": [...]}
+        for key in ("Condition", "Intervention", "PrimaryOutcome"):
+            if key in x:
+                return _as_list(x.get(key))
+        # fallback: stringify dict
+        return [str(x)]
+    return [str(x)]
+
+
+def normalize_trial(raw_trial: Dict[str, Any]) -> ClinicalTrial:
+    """
+    Normalize raw trial data into the ClinicalTrial schema.
+    Supports:
+      - ClinicalTrials.gov API v2 (preferred)
+      - Legacy v1 'FullStudies' shape (fallback)
+    """
+    # --- Detect v1 wrapper ---
+    if "Study" in raw_trial:
+        ps = raw_trial["Study"]["ProtocolSection"]
+
+        nct_id = ps["IdentificationModule"]["NCTId"]
+        title = ps["IdentificationModule"].get("BriefTitle") or ps[
+            "IdentificationModule"
+        ].get("OfficialTitle")
+        status = ps.get("StatusModule", {}).get("OverallStatus")
+        phase = ps.get("DesignModule", {}).get("Phase")
+        sponsor = ps.get("SponsorModule", {}).get("LeadSponsor", {}).get("Name")
+
+        conditions = _as_list(
+            ps.get("ConditionsModule", {}).get("ConditionList", {}).get("Condition")
+        )
+        interventions = _as_list(
+            ps.get("ArmsInterventionsModule", {})
+            .get("InterventionList", {})
+            .get("Intervention")
+        )
+        outcomes = _as_list(
+            ps.get("OutcomesModule", {})
+            .get("PrimaryOutcomeList", {})
+            .get("PrimaryOutcome")
+        )
+
+        logger.debug(
+            "Normalized v1 trial nct_id=%s title=%s", nct_id, (title or "")[:60]
+        )
+
+        return ClinicalTrial(
+            nct_id=nct_id,
+            title=title,
+            status=status,
+            phase=phase,
+            sponsor=sponsor,
+            conditions=conditions,
+            interventions=interventions,
+            outcomes=outcomes,
+        )
+
+    # --- v2 path ---
+    ps = raw_trial.get("protocolSection", {}) or {}
+
+    ident = ps.get("identificationModule", {}) or {}
+    status_mod = ps.get("statusModule", {}) or {}
+    design_mod = ps.get("designModule", {}) or {}
+    sponsor_mod = ps.get("sponsorsModule", {}) or {}
+    cond_mod = ps.get("conditionsModule", {}) or {}
+    arms_mod = ps.get("armsInterventionsModule", {}) or {}
+    outcomes_mod = ps.get("outcomesModule", {}) or {}
+
+    nct_id = ident.get("nctId")
+    title = ident.get("briefTitle") or ident.get("officialTitle")
+    status = status_mod.get("overallStatus")
+
+    phases = design_mod.get("phases")
+    phase = ", ".join(_as_list(phases)) if phases else None
+
+    sponsor = (sponsor_mod.get("leadSponsor", {}) or {}).get("name")
+
+    conditions = _as_list(cond_mod.get("conditions"))
+    interventions = _as_list(arms_mod.get("interventions"))
+
+    primary_outcomes = outcomes_mod.get("primaryOutcomes")
+    outcomes = _as_list(primary_outcomes)
+
+    if not nct_id:
+        logger.error(
+            "Missing nctId in trial record. Keys=%s",
+            list(raw_trial.keys()),
+        )
+        raise ValueError(
+            "Missing nctId in trial record "
+            "(unexpected ClinicalTrials.gov response shape)."
+        )
+
+    # Helpful warnings (non-fatal)
+    if not title:
+        logger.warning("Missing title for nct_id=%s", nct_id)
+    if not status:
+        logger.warning("Missing status for nct_id=%s", nct_id)
+
+    logger.debug("Normalized v2 trial nct_id=%s title=%s", nct_id, (title or "")[:60])
+
+    return ClinicalTrial(
+        nct_id=nct_id,
+        title=title,
+        status=status,
+        phase=phase,
+        sponsor=sponsor,
+        conditions=conditions,
+        interventions=interventions,
+        outcomes=outcomes,
+    )

synthneura/services/summary.py

@@ -0,0 +1,60 @@
+from collections import Counter
+from typing import Any, Dict, Iterable, List, Optional
+
+from synthneura.core.schemas import Summary, TopCount
+
+
+def _as_str_list(values: Iterable[Any]) -> List[str]:
+    out: List[str] = []
+    for value in values:
+        if value is None:
+            continue
+        if isinstance(value, str):
+            if value.strip():
+                out.append(value.strip())
+            continue
+        if isinstance(value, list):
+            out.extend(_as_str_list(value))
+            continue
+        out.append(str(value))
+    return out
+
+
+def _top_counts(values: Iterable[str], limit: int = 5) -> List[TopCount]:
+    counts = Counter(values)
+    return [
+        TopCount(value=value, count=count) for value, count in counts.most_common(limit)
+    ]
+
+
+def summarize_trials(
+    trials: List[Dict[str, Any]],
+    *,
+    run_meta: Optional[Dict[str, str]] = None,
+    change_counts: Optional[Dict[str, int]] = None,
+) -> Summary:
+    statuses = _as_str_list(trial.get("status") for trial in trials)
+    phases = _as_str_list(trial.get("phase") for trial in trials)
+    sponsors = _as_str_list(trial.get("sponsor") for trial in trials)
+
+    conditions: List[str] = []
+    interventions: List[str] = []
+    for trial in trials:
+        conditions.extend(_as_str_list(trial.get("conditions", [])))
+        interventions.extend(_as_str_list(trial.get("interventions", [])))
+
+    run_meta = run_meta or {}
+
+    return Summary(
+        total_trials=len(trials),
+        status_counts=dict(Counter(statuses)),
+        phase_counts=dict(Counter(phases)),
+        top_conditions=_top_counts(conditions),
+        top_sponsors=_top_counts(sponsors),
+        top_interventions=_top_counts(interventions),
+        change_counts=change_counts or {},
+        run_id=run_meta.get("run_id"),
+        ingested_at=run_meta.get("ingested_at"),
+        source=run_meta.get("source"),
+        query=run_meta.get("query"),
+    )

synthneura/storage/__init__.py

@@ -0,0 +1,10 @@
+from synthneura.storage.base import ChangeRecord, RunMeta, StoreResult, TrialSink
+from synthneura.storage.sqlite_sink import SQLiteTrialSink
+
+__all__ = [
+    "ChangeRecord",
+    "RunMeta",
+    "StoreResult",
+    "TrialSink",
+    "SQLiteTrialSink",
+]

synthneura/storage/base.py

@@ -0,0 +1,36 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+from synthneura.core.schemas import ClinicalTrial
+
+RunMeta = Dict[str, str]
+
+
+@dataclass(frozen=True)
+class ChangeRecord:
+    nct_id: str
+    change_status: str
+    changed_fields: List[str]
+    before: Dict[str, Any]
+    after: Dict[str, Any]
+
+
+@dataclass(frozen=True)
+class StoreResult:
+    counts: Dict[str, int]
+    changes: List[ChangeRecord]
+
+
+class TrialSink(ABC):
+    @abstractmethod
+    def store_trials(
+        self,
+        trials: List[ClinicalTrial],
+        raw_trials: List[Dict[str, Any]],
+        run_meta: RunMeta,
+    ) -> StoreResult:
+        """
+        Persist trials and return counts for new/updated/unchanged/failed.
+        """
+        raise NotImplementedError

synthneura/storage/sqlite_sink.py

@@ -0,0 +1,552 @@
+import json
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Tuple
+
+from synthneura.core.logger import get_logger
+from synthneura.core.schemas import ClinicalTrial
+from synthneura.storage.base import ChangeRecord, RunMeta, StoreResult, TrialSink
+
+logger = get_logger(__name__)
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _ensure_schema(conn: sqlite3.Connection) -> None:
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS clinical_trials (
+            nct_id TEXT PRIMARY KEY,
+            title TEXT,
+            status TEXT,
+            phase TEXT,
+            sponsor TEXT,
+            conditions TEXT,
+            interventions TEXT,
+            outcomes TEXT,
+            raw_json TEXT,
+            source TEXT,
+            query TEXT,
+            ingested_at TEXT,
+            run_id TEXT
+        )
+        """
+    )
+    conn.commit()
+
+    cursor.execute("PRAGMA table_info(clinical_trials)")
+    existing = {row[1] for row in cursor.fetchall()}
+    expected = {
+        "raw_json",
+        "source",
+        "query",
+        "ingested_at",
+        "run_id",
+    }
+    missing = expected - existing
+    for column in sorted(missing):
+        cursor.execute(f"ALTER TABLE clinical_trials ADD COLUMN {column} TEXT")
+    if missing:
+        conn.commit()
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS clinical_trials_history (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            nct_id TEXT,
+            title TEXT,
+            status TEXT,
+            phase TEXT,
+            sponsor TEXT,
+            conditions TEXT,
+            interventions TEXT,
+            outcomes TEXT,
+            raw_json TEXT,
+            source TEXT,
+            query TEXT,
+            ingested_at TEXT,
+            run_id TEXT,
+            valid_from TEXT,
+            valid_to TEXT
+        )
+        """
+    )
+    conn.commit()
+
+    cursor.execute("PRAGMA table_info(clinical_trials_history)")
+    existing_history = {row[1] for row in cursor.fetchall()}
+    expected_history = {
+        "nct_id",
+        "title",
+        "status",
+        "phase",
+        "sponsor",
+        "conditions",
+        "interventions",
+        "outcomes",
+        "raw_json",
+        "source",
+        "query",
+        "ingested_at",
+        "run_id",
+        "valid_from",
+        "valid_to",
+    }
+    missing_history = expected_history - existing_history
+    for column in sorted(missing_history):
+        cursor.execute(f"ALTER TABLE clinical_trials_history ADD COLUMN {column} TEXT")
+    if missing_history:
+        conn.commit()
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS runs (
+            run_id TEXT PRIMARY KEY,
+            ingested_at TEXT,
+            source TEXT,
+            query TEXT,
+            total INTEGER,
+            new_count INTEGER,
+            updated_count INTEGER,
+            unchanged_count INTEGER,
+            failed_count INTEGER
+        )
+        """
+    )
+    conn.commit()
+
+    cursor.execute("PRAGMA table_info(runs)")
+    existing_runs = {row[1] for row in cursor.fetchall()}
+    expected_runs = {
+        "ingested_at",
+        "source",
+        "query",
+        "total",
+        "new_count",
+        "updated_count",
+        "unchanged_count",
+        "failed_count",
+    }
+    missing_runs = expected_runs - existing_runs
+    for column in sorted(missing_runs):
+        column_type = "INTEGER"
+        if column in {"ingested_at", "source", "query"}:
+            column_type = "TEXT"
+        cursor.execute(f"ALTER TABLE runs ADD COLUMN {column} {column_type}")
+    if missing_runs:
+        conn.commit()
+
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS trial_diffs (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            nct_id TEXT,
+            run_id TEXT,
+            ingested_at TEXT,
+            changed_fields TEXT,
+            before_json TEXT,
+            after_json TEXT
+        )
+        """
+    )
+    conn.commit()
+
+    cursor.execute("PRAGMA table_info(trial_diffs)")
+    existing_diffs = {row[1] for row in cursor.fetchall()}
+    expected_diffs = {
+        "nct_id",
+        "run_id",
+        "ingested_at",
+        "changed_fields",
+        "before_json",
+        "after_json",
+    }
+    missing_diffs = expected_diffs - existing_diffs
+    for column in sorted(missing_diffs):
+        cursor.execute(f"ALTER TABLE trial_diffs ADD COLUMN {column} TEXT")
+    if missing_diffs:
+        conn.commit()
+
+    cursor.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_clinical_trials_nct_id
+        ON clinical_trials(nct_id)
+        """
+    )
+    cursor.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_clinical_trials_history_nct_id_valid_from
+        ON clinical_trials_history(nct_id, valid_from)
+        """
+    )
+    cursor.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_trial_diffs_nct_id_ingested_at
+        ON trial_diffs(nct_id, ingested_at)
+        """
+    )
+    conn.commit()
+
+
+def _trial_core_values(trial: ClinicalTrial) -> Tuple[Optional[str], ...]:
+    return (
+        trial.title,
+        trial.status,
+        trial.phase,
+        trial.sponsor,
+        ",".join(trial.conditions or []),
+        ",".join(trial.interventions or []),
+        ",".join(trial.outcomes or []),
+    )
+
+
+def _core_to_dict(core: Tuple[Optional[str], ...]) -> Dict[str, Any]:
+    return {
+        "title": core[0],
+        "status": core[1],
+        "phase": core[2],
+        "sponsor": core[3],
+        "conditions": core[4],
+        "interventions": core[5],
+        "outcomes": core[6],
+    }
+
+
+def _changed_fields(
+    existing: Tuple[Optional[str], ...], updated: Tuple[Optional[str], ...]
+) -> List[str]:
+    fields = [
+        "title",
+        "status",
+        "phase",
+        "sponsor",
+        "conditions",
+        "interventions",
+        "outcomes",
+    ]
+    changed: List[str] = []
+    for idx, field in enumerate(fields):
+        existing_value = existing[idx] or ""
+        updated_value = updated[idx] or ""
+        if existing_value != updated_value:
+            changed.append(field)
+    return changed
+
+
+def _raw_nct_id(raw_trial: Dict[str, Any]) -> Optional[str]:
+    if "Study" in raw_trial:
+        return (
+            raw_trial.get("Study", {})
+            .get("ProtocolSection", {})
+            .get("IdentificationModule", {})
+            .get("NCTId")
+        )
+    return (
+        raw_trial.get("protocolSection", {})
+        .get("identificationModule", {})
+        .get("nctId")
+    )
+
+
+class SQLiteTrialSink(TrialSink):
+    def __init__(self, db_path: str = "trials.db") -> None:
+        self.db_path = db_path
+
+    def fetch_previous_run_time(self, current_run_id: str) -> Optional[str]:
+        conn = sqlite3.connect(self.db_path)
+        try:
+            _ensure_schema(conn)
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT ingested_at
+                FROM runs
+                WHERE run_id != ?
+                ORDER BY ingested_at DESC
+                LIMIT 1
+                """,
+                (current_run_id,),
+            )
+            row = cursor.fetchone()
+        finally:
+            conn.close()
+        if row:
+            return row[0]
+        return None
+
+    def fetch_diffs_between(
+        self, start_time: Optional[str], end_time: Optional[str]
+    ) -> List[ChangeRecord]:
+        if not end_time:
+            return []
+        conn = sqlite3.connect(self.db_path)
+        try:
+            _ensure_schema(conn)
+            cursor = conn.cursor()
+            if start_time:
+                cursor.execute(
+                    """
+                    SELECT nct_id, changed_fields, before_json, after_json
+                    FROM trial_diffs
+                    WHERE ingested_at > ? AND ingested_at <= ?
+                    ORDER BY ingested_at DESC
+                    """,
+                    (start_time, end_time),
+                )
+            else:
+                cursor.execute(
+                    """
+                    SELECT nct_id, changed_fields, before_json, after_json
+                    FROM trial_diffs
+                    WHERE ingested_at <= ?
+                    ORDER BY ingested_at DESC
+                    """,
+                    (end_time,),
+                )
+            rows = cursor.fetchall()
+        finally:
+            conn.close()
+
+        changes: List[ChangeRecord] = []
+        for row in rows:
+            nct_id, changed_fields_json, before_json, after_json = row
+            changes.append(
+                ChangeRecord(
+                    nct_id=nct_id,
+                    change_status="updated",
+                    changed_fields=json.loads(changed_fields_json or "[]"),
+                    before=json.loads(before_json or "{}"),
+                    after=json.loads(after_json or "{}"),
+                )
+            )
+        return changes
+
+    def store_trials(
+        self,
+        trials: List[ClinicalTrial],
+        raw_trials: List[Dict[str, Any]],
+        run_meta: RunMeta,
+    ) -> StoreResult:
+        logger.debug("Storing %d trials in SQLite db=%s", len(trials), self.db_path)
+
+        raw_by_id = {}
+        for raw_trial in raw_trials:
+            nct_id = _raw_nct_id(raw_trial)
+            if nct_id:
+                raw_by_id[nct_id] = raw_trial
+
+        counts = {"new": 0, "updated": 0, "unchanged": 0, "failed": 0}
+        changes: List[ChangeRecord] = []
+        conn = sqlite3.connect(self.db_path)
+        try:
+            _ensure_schema(conn)
+            cursor = conn.cursor()
+            for trial in trials:
+                cursor.execute(
+                    """
+                    SELECT
+                        title,
+                        status,
+                        phase,
+                        sponsor,
+                        conditions,
+                        interventions,
+                        outcomes,
+                        raw_json,
+                        source,
+                        query,
+                        ingested_at,
+                        run_id
+                    FROM clinical_trials
+                    WHERE nct_id = ?
+                    """,
+                    (trial.nct_id,),
+                )
+                existing = cursor.fetchone()
+                existing_core = existing[:7] if existing else None
+                existing_raw_json = existing[7] if existing else None
+                existing_source = existing[8] if existing else None
+                existing_query = existing[9] if existing else None
+                existing_ingested_at = existing[10] if existing else None
+                existing_run_id = existing[11] if existing else None
+                new_core = _trial_core_values(trial)
+
+                if existing is None:
+                    change_status = "new"
+                elif existing_core == new_core:
+                    change_status = "unchanged"
+                else:
+                    change_status = "updated"
+
+                ingested_at_value = run_meta.get("ingested_at") or _utc_now()
+                if change_status == "unchanged" and existing_ingested_at:
+                    ingested_at_value = existing_ingested_at
+                raw_json = json.dumps(
+                    raw_by_id.get(trial.nct_id, {}), ensure_ascii=True
+                )
+                if trial.nct_id not in raw_by_id and existing_raw_json is not None:
+                    raw_json = existing_raw_json
+                if change_status == "updated" and existing is not None:
+                    changed_fields = _changed_fields(
+                        existing_core or new_core, new_core
+                    )
+                    before = _core_to_dict(existing_core or new_core)
+                    after = _core_to_dict(new_core)
+                    changes.append(
+                        ChangeRecord(
+                            nct_id=trial.nct_id,
+                            change_status=change_status,
+                            changed_fields=changed_fields,
+                            before=before,
+                            after=after,
+                        )
+                    )
+                    cursor.execute(
+                        """
+                        INSERT INTO trial_diffs
+                        (
+                            nct_id,
+                            run_id,
+                            ingested_at,
+                            changed_fields,
+                            before_json,
+                            after_json
+                        )
+                        VALUES (?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            trial.nct_id,
+                            run_meta.get("run_id"),
+                            ingested_at_value,
+                            json.dumps(changed_fields, ensure_ascii=True),
+                            json.dumps(before, ensure_ascii=True),
+                            json.dumps(after, ensure_ascii=True),
+                        ),
+                    )
+                    (
+                        existing_title,
+                        existing_status,
+                        existing_phase,
+                        existing_sponsor,
+                        existing_conditions,
+                        existing_interventions,
+                        existing_outcomes,
+                    ) = (
+                        existing_core or new_core
+                    )
+                    cursor.execute(
+                        """
+                        INSERT INTO clinical_trials_history
+                        (
+                            nct_id,
+                            title,
+                            status,
+                            phase,
+                            sponsor,
+                            conditions,
+                            interventions,
+                            outcomes,
+                            raw_json,
+                            source,
+                            query,
+                            ingested_at,
+                            run_id,
+                            valid_from,
+                            valid_to
+                        )
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            trial.nct_id,
+                            existing_title,
+                            existing_status,
+                            existing_phase,
+                            existing_sponsor,
+                            existing_conditions,
+                            existing_interventions,
+                            existing_outcomes,
+                            existing_raw_json,
+                            existing_source,
+                            existing_query,
+                            existing_ingested_at,
+                            existing_run_id,
+                            existing_ingested_at,
+                            ingested_at_value,
+                        ),
+                    )
+                if change_status != "unchanged":
+                    cursor.execute(
+                        """
+                        INSERT OR REPLACE INTO clinical_trials
+                        (
+                            nct_id,
+                            title,
+                            status,
+                            phase,
+                            sponsor,
+                            conditions,
+                            interventions,
+                            outcomes,
+                            raw_json,
+                            source,
+                            query,
+                            ingested_at,
+                            run_id
+                        )
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            trial.nct_id,
+                            trial.title,
+                            trial.status,
+                            trial.phase,
+                            trial.sponsor,
+                            ",".join(trial.conditions or []),
+                            ",".join(trial.interventions or []),
+                            ",".join(trial.outcomes or []),
+                            raw_json,
+                            run_meta.get("source"),
+                            run_meta.get("query"),
+                            ingested_at_value,
+                            run_meta.get("run_id"),
+                        ),
+                    )
+                counts[change_status] += 1
+            conn.commit()
+            run_id = run_meta.get("run_id")
+            if run_id:
+                cursor.execute(
+                    """
+                    INSERT OR REPLACE INTO runs
+                    (
+                        run_id,
+                        ingested_at,
+                        source,
+                        query,
+                        total,
+                        new_count,
+                        updated_count,
+                        unchanged_count,
+                        failed_count
+                    )
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        run_id,
+                        run_meta.get("ingested_at"),
+                        run_meta.get("source"),
+                        run_meta.get("query"),
+                        len(trials),
+                        counts["new"],
+                        counts["updated"],
+                        counts["unchanged"],
+                        counts["failed"],
+                    ),
+                )
+                conn.commit()
+        finally:
+            conn.close()
+        return StoreResult(counts=counts, changes=changes)

synthneura/ui/cli.py

@@ -0,0 +1,282 @@
+import csv
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from uuid import uuid4
+
+import click
+
+from synthneura.core.config import get_settings
+from synthneura.core.logger import get_logger, set_log_level
+from synthneura.ingestion.clinical_trials import fetch_trials
+from synthneura.services.pipeline import normalize_trial
+from synthneura.services.summary import summarize_trials
+from synthneura.storage.sqlite_sink import SQLiteTrialSink
+
+logger = get_logger(__name__)
+
+
+def _trial_to_dict(trial: Any) -> Dict[str, Any]:
+    if hasattr(trial, "model_dump"):
+        return trial.model_dump()
+    if hasattr(trial, "dict"):
+        return trial.dict()
+    return dict(trial)
+
+
+def _write_json(path: Path, trials: List[Dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(trials, f, indent=2, ensure_ascii=True)
+
+
+def _write_json_payload(path: Path, payload: Dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2, ensure_ascii=True)
+
+
+def _write_csv(path: Path, trials: List[Dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not trials:
+        path.write_text("", encoding="utf-8")
+        return
+    fieldnames = sorted({key for trial in trials for key in trial.keys()})
+    with path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(trials)
+
+
+def _change_to_dict(change: Any) -> Dict[str, Any]:
+    return {
+        "nct_id": change.nct_id,
+        "change_status": change.change_status,
+        "changed_fields": change.changed_fields,
+        "before": change.before,
+        "after": change.after,
+    }
+
+
+@click.command()
+@click.option("--query", required=True, help="Search query for clinical trials.")
+@click.option(
+    "--db-path",
+    default=get_settings().db_path,
+    show_default=True,
+    help="Path to the SQLite database.",
+)
+@click.option(
+    "--sink",
+    default=get_settings().sink,
+    show_default=True,
+    type=click.Choice(["sqlite"], case_sensitive=False),
+    help="Storage backend for persisted trials.",
+)
+@click.option(
+    "--log-level",
+    default=get_settings().log_level,
+    show_default=True,
+    type=click.Choice(
+        ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False
+    ),
+    help="Logging verbosity.",
+)
+@click.option(
+    "--max-results",
+    default=get_settings().max_results,
+    show_default=True,
+    type=int,
+    help="Maximum number of trials to fetch.",
+)
+@click.option(
+    "--output-path",
+    default=get_settings().output_path,
+    help="Optional path to write results (JSON or CSV).",
+)
+@click.option(
+    "--output-format",
+    default=get_settings().output_format,
+    show_default=True,
+    type=click.Choice(["json", "csv"], case_sensitive=False),
+    help="Output format when --output-path is provided.",
+)
+@click.option(
+    "--summary-path",
+    default=get_settings().summary_path,
+    help="Optional path to write summary (JSON).",
+)
+@click.option(
+    "--changes-path",
+    default=get_settings().changes_path,
+    help="Optional path to write change summary (JSON).",
+)
+@click.option(
+    "--since-last-run",
+    is_flag=True,
+    help="When set, change summary uses diffs since the previous run.",
+)
+def run(
+    query: str,
+    db_path: str,
+    sink: str,
+    log_level: str,
+    max_results: int,
+    output_path: Optional[str],
+    output_format: str,
+    summary_path: Optional[str],
+    changes_path: Optional[str],
+    since_last_run: bool,
+) -> None:
+    """
+    Run the SynthNeura pipeline to fetch, normalize, and store clinical trials.
+    """
+    # Set runtime log level for this run
+    set_log_level(log_level)
+    logger.info("Starting SynthNeura CLI run")
+    logger.info(
+        "query=%s sink=%s log_level=%s",
+        query,
+        sink.lower(),
+        log_level.upper(),
+    )
+
+    click.echo(f"Fetching trials for query: {query}")
+
+    try:
+        raw_trials = fetch_trials(query, max_results=max_results)
+
+        if not raw_trials:
+            logger.warning("No trials returned from fetch_trials for query=%s", query)
+            click.echo(
+                f"No trials found for the query: {query}."
+                f"Try a broader or different query."
+            )
+            return
+
+        click.echo(f"Fetched {len(raw_trials)} trials.")
+        logger.info("Fetched %d raw trials", len(raw_trials))
+
+        stored = 0
+        run_id = uuid4().hex
+        ingested_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
+        run_meta = {
+            "run_id": run_id,
+            "ingested_at": ingested_at,
+            "source": "clinicaltrials.gov",
+            "query": query,
+        }
+        normalized_trials: List[Dict[str, Any]] = []
+        trial_models = []
+        for raw_trial in raw_trials:
+            trial = normalize_trial(raw_trial)
+            trial_models.append(trial)
+            normalized_trials.append(_trial_to_dict(trial))
+
+            # User-friendly output
+            click.echo(
+                f"- {trial.nct_id} | "
+                f"{trial.status or 'UNKNOWN'} | "
+                f"{trial.phase or 'NA'} | "
+                f"{(trial.title or '')[:80]}"
+            )
+
+        if sink.lower() == "sqlite":
+            sink_impl = SQLiteTrialSink(db_path)
+        else:
+            raise ValueError(f"Unsupported sink: {sink}")
+
+        store_result = sink_impl.store_trials(trial_models, raw_trials, run_meta)
+        change_counts = store_result.counts
+        stored = len(trial_models)
+
+        logger.info(
+            "Completed run. stored=%d new=%d updated=%d unchanged=%d",
+            stored,
+            change_counts["new"],
+            change_counts["updated"],
+            change_counts["unchanged"],
+        )
+        click.echo("Pipeline completed successfully.")
+
+        summary = summarize_trials(
+            normalized_trials, run_meta=run_meta, change_counts=change_counts
+        )
+        summary_dict = _trial_to_dict(summary)
+
+        click.echo(
+            f"Summary: total={summary.total_trials} "
+            f"status_count={len(summary.status_counts)} "
+            f"phase_count={len(summary.phase_counts)} "
+            f"new={change_counts['new']} "
+            f"updated={change_counts['updated']} "
+            f"unchanged={change_counts['unchanged']}"
+        )
+
+        if output_path:
+            path = Path(output_path)
+            if output_format.lower() == "json":
+                _write_json_payload(
+                    path, {"summary": summary_dict, "trials": normalized_trials}
+                )
+            else:
+                _write_csv(path, normalized_trials)
+            logger.info("Wrote output %s to %s", output_format.lower(), path)
+            click.echo(f"Wrote {output_format.lower()} output to {path}")
+
+        if summary_path:
+            summary_file = Path(summary_path)
+        elif output_path and output_format.lower() == "csv":
+            summary_file = Path(f"{output_path}.summary.json")
+        else:
+            summary_file = None
+
+        if summary_file:
+            _write_json_payload(summary_file, summary_dict)
+            logger.info("Wrote summary to %s", summary_file)
+            click.echo(f"Wrote summary to {summary_file}")
+
+        if since_last_run and sink.lower() == "sqlite":
+            previous_time = sink_impl.fetch_previous_run_time(run_id)
+            change_records = [
+                _change_to_dict(change)
+                for change in sink_impl.fetch_diffs_between(
+                    previous_time, run_meta["ingested_at"]
+                )
+            ]
+        else:
+            change_records = [
+                _change_to_dict(change) for change in store_result.changes
+            ]
+        change_records.sort(
+            key=lambda record: len(record.get("changed_fields", [])), reverse=True
+        )
+        changes_payload = {
+            "run": run_meta,
+            "changes": change_records,
+            "top_changed": change_records[:5],
+        }
+
+        if changes_path:
+            changes_file = Path(changes_path)
+        elif output_path:
+            changes_file = Path(f"{output_path}.changes.json")
+        else:
+            changes_file = None
+
+        if changes_file:
+            _write_json_payload(changes_file, changes_payload)
+            logger.info("Wrote change summary to %s", changes_file)
+            click.echo(f"Wrote change summary to {changes_file}")
+
+    except ValueError as e:
+        logger.warning("ValueError during run: %s", e)
+        click.echo(f"Error: {e}")
+    except Exception as e:
+        logger.exception("Unexpected error during run")
+        click.echo(f"An unexpected error occurred: {e}")
+
+
+if __name__ == "__main__":
+    run()

synthneura-2.0.0.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: synthneura
+Version: 2.0.0
+Requires-Dist: click>=8.1
+Requires-Dist: pydantic<3,>=1.10
+Requires-Dist: requests>=2.31
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"

synthneura-2.0.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+synthneura/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synthneura/core/config.py,sha256=0dlVnYPlnFJEZRmrHaYLV1_zr2571zBeDJgW3Oyer20,1229
+synthneura/core/logger.py,sha256=l5233qwvz8MjV_ezdstUJPjFR2PiuxidilMQ83YaNmk,1223
+synthneura/core/schemas.py,sha256=giYuEq8Zf5zsov0IzQ3ktRTlz9_8cJHjtvVRdPmgpdo,1336
+synthneura/ingestion/clinical_trials.py,sha256=of408fbDDYjV2mdGhm0afFm2R69yZ6m3sHFIKMbv1VY,2146
+synthneura/services/pipeline.py,sha256=Yes3Ld4imzLhGa2MXXhn1jUATlZ3UhxMRsn8cT6Zd58,4551
+synthneura/services/summary.py,sha256=LFyiipU7wa-0EEatjU_ajBZizaP51gfwyguAeqe1Db4,1936
+synthneura/storage/__init__.py,sha256=q_9hlFMVlOjgUcyEKtSiCIFP96Wi1-IlTm_32hl8VIs,250
+synthneura/storage/base.py,sha256=rc7J3VIOFiNhan0fu2gpdpgaXF8OkceVcqxOed4yzYg,795
+synthneura/storage/sqlite_sink.py,sha256=h0D-6mPpVfjCoiMRan1jtURRK6KxxiaX-cC_lUucXP0,17982
+synthneura/ui/cli.py,sha256=6rVv-kteR1cI9CqXinjmIuZ0E6oMCeWmIB_SiZZCjVk,9000
+synthneura-2.0.0.dist-info/METADATA,sha256=Swx9TFRFvc5VeFJS5prqStIG-1rCr_PcRfwZTWP1nQk,204
+synthneura-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synthneura-2.0.0.dist-info/top_level.txt,sha256=6XwiS6dxlBMStS2v71SEMv8_Kba31uXwIbkIlgr4-Ic,11
+synthneura-2.0.0.dist-info/RECORD,,

synthneura-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

synthneura-2.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+synthneura