circex 0.1.0__py3-none-any.whl

circex/__init__.py

@@ -0,0 +1,3 @@
+"""Circex: LLM-based extractor for GCN optical circulars."""
+
+__version__ = "0.1.0"

circex/__main__.py

@@ -0,0 +1,4 @@
+from circex.cli import app
+
+if __name__ == "__main__":
+    app()

circex/bot/__init__.py

@@ -0,0 +1,23 @@
+"""SkyPortal poster bot — maps CircularExtraction to SkyPortal API writes.
+
+See docs/design_skyportal_bot.md. The mapping (skyportal_map) is pure and
+offline-testable; the poster (poster) defaults to dry-run and only hits a live
+SkyPortal with an explicit token + flag.
+"""
+
+from circex.bot.aggregate import aggregate_event, gather_by_xref
+from circex.bot.skyportal_map import (
+    PhotometryPoint,
+    SkyPortalActions,
+    SourceUpsert,
+    to_actions,
+)
+
+__all__ = [
+    "PhotometryPoint",
+    "SkyPortalActions",
+    "SourceUpsert",
+    "aggregate_event",
+    "gather_by_xref",
+    "to_actions",
+]

circex/bot/aggregate.py

@@ -0,0 +1,185 @@
+"""Aggregate an event's circulars into ONE SkyPortal source.
+
+The single-circular path (`circex post`) posts one bulletin at a time. A real
+transient is described across many circulars — a discovery circular carries the
+position, follow-ups carry the light curve. `aggregate_event` fuses them:
+
+  - position preferring the refined optical-counterpart circular over a coarse
+    gamma-ray trigger box (which can be degrees off),
+  - photometry unioned across every circular (each point keeps its own source
+    circular id in altdata),
+  - deduped so a re-run or an overlapping report doesn't double-post,
+  - a single redshift if any circular states a valid one.
+
+`gather_by_xref` discovers an event's circulars from a seed by walking the GCN
+cross-reference graph the extractor already recovers — no event-search API needed.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable
+from datetime import datetime
+from typing import Any
+
+from circex.bot.skyportal_map import SkyPortalActions, SourceUpsert, to_actions
+from circex.extract.protocol import Circular, Extractor
+from circex.extract.regex.regex_events import extract_gcn_xrefs_with_positions
+from circex.schema import Event
+
+# Prefer a refined optical-counterpart position over a coarse gamma-ray trigger box.
+# Both kinds of circular carry coordinates, but the trigger box can be degrees off
+# (e.g. a Fermi-GBM localization vs the arcsec MASTER OT position).
+_REFINED_POSITION_CUES = re.compile(
+    r"optical\s+(?:counterpart|afterglow|transient)|\bOT\b|counterpart|afterglow"
+    r"|arcsec|UVOT|XRT\s+position|enhanced\s+XRT|refined\s+(?:position|localization)"
+    r"|MASTER\s+OT|discover",
+    re.IGNORECASE,
+)
+_COARSE_POSITION_CUES = re.compile(
+    r"\bGBM\b|\bGRM\b|ECLAIRs|real-?time\s+localization|initial\s+localization"
+    r"|error\s+(?:radius|circle)\s+of\s+\d|gamma-?ray\s+(?:burst\s+)?(?:localization|detection)",
+    re.IGNORECASE,
+)
+
+
+def gather_by_xref(
+    seed_id: int,
+    fetch: Callable[[int], dict[str, Any] | None],
+    max_hops: int = 1,
+) -> list[dict[str, Any]]:
+    """Collect an event's circular records by BFS over the GCN cross-reference graph.
+
+    `fetch(circular_id)` returns a record ({circularId, subject, body, eventId})
+    or None (404). `max_hops` bounds the walk: 1 = the seed plus everything it
+    cites (usually the whole flurry). Records are returned in discovery order.
+    """
+    seen: dict[int, dict[str, Any]] = {}
+    frontier = [seed_id]
+    hops = 0
+    while frontier and hops <= max_hops:
+        next_frontier: list[int] = []
+        for cid in frontier:
+            if cid in seen:
+                continue
+            record = fetch(cid)
+            if record is None:
+                continue
+            seen[cid] = record
+            for xid, _, _ in extract_gcn_xrefs_with_positions(str(record.get("body") or "")):
+                if xid not in seen:
+                    next_frontier.append(xid)
+        frontier = next_frontier
+        hops += 1
+    return list(seen.values())
+
+
+def _position_score(record: dict[str, Any]) -> int:
+    """+1 if the circular's text reads like a refined/optical position, -1 if coarse."""
+    text = f"{record.get('subject', '') or ''}\n{record.get('body', '') or ''}"
+    score = 0
+    if _REFINED_POSITION_CUES.search(text):
+        score += 1
+    if _COARSE_POSITION_CUES.search(text):
+        score -= 1
+    return score
+
+
+def _record_to_circular(record: dict[str, Any], trigger_time: datetime | None) -> Circular:
+    return Circular(
+        circular_id=int(record.get("circularId") or 0),
+        subject=str(record.get("subject") or ""),
+        body=str(record.get("body") or ""),
+        event_id=record.get("eventId") or None,
+        trigger_time=trigger_time,
+    )
+
+
+def _dedup_key(point: Any) -> tuple[Any, ...]:
+    return (point.obj_id, point.filter, point.magsys, round(point.mjd, 5), point.mag)
+
+
+def aggregate_event(
+    records: Iterable[dict[str, Any]],
+    extractor: Extractor,
+    *,
+    trigger_time: datetime | None = None,
+    instrument_map: dict[str, int] | None = None,
+    default_instrument_id: int | None = None,
+    group_ids: list[int] | None = None,
+    event_name: str | list[str] | None = None,
+) -> SkyPortalActions:
+    """Fuse an event's circulars into one source + a deduped, attributed light curve."""
+    group_ids = group_ids or []
+    records = list(records)
+    extractions = [extractor.extract(_record_to_circular(r, trigger_time)) for r in records]
+
+    # Position: prefer the refined optical-counterpart position over a coarse
+    # trigger box. Rank localization-bearing circulars by text cues; ties keep
+    # discovery order. Falls back to the first localization when none score.
+    candidates = [
+        (_position_score(rec), -i, ext.localization)
+        for i, (rec, ext) in enumerate(zip(records, extractions, strict=True))
+        if ext.localization is not None
+    ]
+    localization = max(candidates, key=lambda c: (c[0], c[1]))[2] if candidates else None
+    # Event name: caller override, else the first one any circular names.
+    name = event_name
+    if name is None:
+        name = next(
+            (e.event.event_name for e in extractions if e.event and e.event.event_name), None
+        )
+
+    photometry: list[Any] = []
+    comments: list[str] = []
+    skipped = 0
+    redshift: tuple[float, float | None] | None = None
+    event = Event(event_name=name) if name is not None else None
+
+    for extraction in extractions:
+        if event is not None:
+            extraction.event = event
+        extraction.localization = localization
+        actions = to_actions(
+            extraction,
+            instrument_map=instrument_map,
+            default_instrument_id=default_instrument_id,
+            group_ids=group_ids,
+        )
+        photometry.extend(actions.photometry)
+        comments.extend(actions.comments)
+        skipped += actions.skipped_rows
+        if redshift is None and actions.redshift is not None:
+            redshift = actions.redshift
+
+    # Dedup photometry (idempotent re-runs; overlapping reports).
+    seen: set[tuple[Any, ...]] = set()
+    deduped = []
+    for point in photometry:
+        key = _dedup_key(point)
+        if key not in seen:
+            seen.add(key)
+            deduped.append(point)
+
+    obj_id = deduped[0].obj_id if deduped else None
+    source: SourceUpsert | None = None
+    if obj_id is not None and localization is not None:
+        source = SourceUpsert(
+            id=obj_id, ra=localization.ra, dec=localization.dec, group_ids=group_ids
+        )
+
+    # Unique comments, order preserved.
+    seen_c: set[str] = set()
+    uniq_comments: list[str] = []
+    for comment in comments:
+        if comment not in seen_c:
+            seen_c.add(comment)
+            uniq_comments.append(comment)
+
+    return SkyPortalActions(
+        source=source,
+        photometry=deduped,
+        redshift=redshift,
+        comments=uniq_comments,
+        skipped_rows=skipped,
+    )

circex/bot/poster.py

@@ -0,0 +1,119 @@
+"""Turn SkyPortalActions into HTTP writes — or, by default, print them (dry-run).
+
+Live posting requires BOTH an explicit `live=True` and a `token`; otherwise the
+poster only renders the payloads it would send. This guards an outward-facing,
+hard-to-reverse action (writing to a shared SkyPortal instance).
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+from circex.bot.skyportal_map import SkyPortalActions
+
+log = structlog.get_logger(__name__)
+
+
+@dataclass
+class SkyPortalPoster:
+    """Posts (or dry-runs) SkyPortalActions against a SkyPortal instance."""
+
+    base_url: str = "https://skyportal.example/api"
+    token: str | None = None
+    live: bool = False
+    timeout: float = 30.0
+    continue_on_error: bool = False  # log + continue instead of raising (unattended use)
+
+    def post(self, actions: SkyPortalActions) -> list[dict[str, Any]]:
+        """Apply the actions. Returns the list of {method, path, payload} planned.
+
+        In dry-run (default) nothing is sent. With `live=True` and a token, each
+        planned request is executed in order: source upsert, photometry,
+        redshift patch, comments.
+        """
+        plan = self._plan(actions)
+        if not (self.live and self.token):
+            for req in plan:
+                log.info(
+                    "skyportal_dryrun",
+                    method=req["method"],
+                    path=req["path"],
+                    payload=req["payload"],
+                )
+            return plan
+        self._execute(plan)
+        return plan
+
+    def _plan(self, actions: SkyPortalActions) -> list[dict[str, Any]]:
+        plan: list[dict[str, Any]] = []
+        if actions.source is not None:
+            plan.append(
+                {"method": "POST", "path": "/sources", "payload": actions.source.to_payload()}
+            )
+            obj_id = actions.source.id
+            for point in actions.photometry:
+                plan.append(
+                    {"method": "POST", "path": "/photometry", "payload": point.to_payload()}
+                )
+            if actions.redshift is not None:
+                z, z_err = actions.redshift
+                plan.append(
+                    {
+                        "method": "PATCH",
+                        "path": f"/sources/{obj_id}",
+                        "payload": {"redshift": z, "redshift_error": z_err},
+                    }
+                )
+            for text in actions.comments:
+                plan.append(
+                    {
+                        "method": "POST",
+                        "path": f"/sources/{obj_id}/comments",
+                        "payload": {"text": text},
+                    }
+                )
+        return plan
+
+    def _execute(self, plan: list[dict[str, Any]]) -> None:
+        import requests  # local import: only needed for live posting
+
+        headers = {"Authorization": f"token {self.token}", "Content-Type": "application/json"}
+        for req in plan:
+            url = self.base_url.rstrip("/") + req["path"]
+            try:
+                resp = requests.request(
+                    req["method"], url, headers=headers,
+                    data=json.dumps(req["payload"]), timeout=self.timeout,
+                )
+                resp.raise_for_status()
+                # SkyPortal also signals failure as HTTP 200 with status="error".
+                body = resp.json()
+                if isinstance(body, dict) and body.get("status") == "error":
+                    raise RuntimeError(str(body.get("message", "skyportal error"))[:200])
+            except Exception as exc:
+                if not self.continue_on_error:
+                    raise
+                # Unattended: one bad request (e.g. a filter the instrument lacks)
+                # must not kill the stream.
+                log.warning(
+                    "skyportal_post_failed",
+                    method=req["method"], path=req["path"], error=str(exc)[:200],
+                )
+                continue
+            log.info(
+                "skyportal_posted",
+                method=req["method"], path=req["path"], status=resp.status_code,
+            )
+
+
+def render_plan(plan: list[dict[str, Any]]) -> str:
+    """Human-readable dump of a planned request list (for CLI dry-run output)."""
+    lines: list[str] = []
+    for req in plan:
+        lines.append(f"{req['method']} {req['path']}")
+        lines.append("  " + json.dumps(req["payload"], ensure_ascii=False))
+    return "\n".join(lines)

circex/bot/skyportal_map.py

@@ -0,0 +1,257 @@
+"""Map a CircularExtraction to SkyPortal API writes.
+
+Pure functions, no network. `to_actions()` returns a SkyPortalActions bundle —
+a source upsert, photometry points, an optional redshift patch, and comments —
+shaped as the dicts SkyPortal's REST API expects. The poster turns these into
+HTTP calls (or prints them in dry-run). See docs/design_skyportal_bot.md.
+
+Everything the ICARE work added feeds in here: obs_mjd -> mjd, bandpass ->
+filter, telescope_canonical -> instrument_id, is_detection, redshift bounds ->
+comment, provenance -> altdata.note.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from circex.extract.regex.mag_table import (
+    infer_bandpass,
+    infer_mag_system,
+    normalize_filter,
+)
+from circex.schema import CircularExtraction, PhotometryExt
+
+# mag_system (our enum) -> SkyPortal magsys (lowercase). STMag has no direct
+# SkyPortal equivalent; map to the closest and flag it in a comment upstream.
+_MAGSYS = {"AB": "ab", "Vega": "vega", "STMag": "ab"}
+
+
+def _obj_id(extraction: CircularExtraction) -> str | None:
+    """SkyPortal source id from the event name (spaces removed). None if unnamed.
+
+    Prefers an AT/optical designation over a bare GRB/GW trigger when the
+    extraction carries a list (the optical name is what SkyPortal keys on).
+    """
+    if extraction.event is None or extraction.event.event_name is None:
+        return None
+    name = extraction.event.event_name
+    names = name if isinstance(name, list) else [name]
+    optical = [n for n in names if re.match(r"(?i)^(AT|SN)\s?\d", n)]
+    chosen = optical[0] if optical else (names[0] if names else None)
+    return re.sub(r"\s+", "", chosen) if chosen else None
+
+
+@dataclass(frozen=True)
+class SourceUpsert:
+    """`POST /api/sources` payload."""
+
+    id: str
+    ra: float | None
+    dec: float | None
+    group_ids: list[int] = field(default_factory=list)
+
+    def to_payload(self) -> dict[str, Any]:
+        out: dict[str, Any] = {"id": self.id}
+        if self.ra is not None:
+            out["ra"] = self.ra
+        if self.dec is not None:
+            out["dec"] = self.dec
+        if self.group_ids:
+            out["group_ids"] = self.group_ids
+        return out
+
+
+@dataclass(frozen=True)
+class PhotometryPoint:
+    """`POST /api/photometry` payload for one row."""
+
+    obj_id: str
+    mjd: float
+    filter: str
+    magsys: str
+    instrument_id: int | None
+    mag: float | None
+    magerr: float | None
+    limiting_mag: float | None
+    altdata: dict[str, Any] = field(default_factory=dict)
+
+    def to_payload(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "obj_id": self.obj_id,
+            "mjd": self.mjd,
+            "filter": self.filter,
+            "magsys": self.magsys,
+            "mag": self.mag,
+            "magerr": self.magerr,
+            "limiting_mag": self.limiting_mag,
+        }
+        if self.instrument_id is not None:
+            out["instrument_id"] = self.instrument_id
+        if self.altdata:
+            out["altdata"] = self.altdata
+        return out
+
+
+@dataclass(frozen=True)
+class SkyPortalActions:
+    """Everything to post for one circular."""
+
+    source: SourceUpsert | None
+    photometry: list[PhotometryPoint]
+    redshift: tuple[float, float | None] | None  # (z, z_err)
+    comments: list[str]
+    skipped_rows: int  # photometry rows we could not post (no mjd/filter)
+
+
+def _provenance_note(extraction: CircularExtraction, path: str) -> str | None:
+    span = extraction.provenance.get(path)
+    return f'{path}: "{span.snippet}"' if span is not None else None
+
+
+def to_actions(
+    extraction: CircularExtraction,
+    *,
+    instrument_map: dict[str, int] | None = None,
+    default_instrument_id: int | None = None,
+    group_ids: list[int] | None = None,
+) -> SkyPortalActions:
+    """Build the SkyPortal write bundle for one extraction.
+
+    `instrument_map` maps `telescope_canonical` -> SkyPortal instrument_id
+    (ICARE's table). `default_instrument_id` is the generic GCN instrument used
+    when a telescope isn't in the map (matching ICARE's fall-back). SkyPortal's
+    photometry endpoint REQUIRES an instrument_id, so a row that resolves to
+    neither a mapped nor a default id cannot be posted — it becomes a comment.
+    """
+    instrument_map = instrument_map or {}
+    group_ids = group_ids or []
+
+    obj_id = _obj_id(extraction)
+    loc = extraction.localization
+    ra = loc.ra if loc is not None else None
+    dec = loc.dec if loc is not None else None
+
+    photometry: list[PhotometryPoint] = []
+    comments: list[str] = []
+    skipped = 0
+
+    # A NEW SkyPortal source requires ra/dec. A follow-up circular usually has no
+    # position (it lives in the discovery circular), so guard against emitting a
+    # positionless source-create that SkyPortal would reject with a 400.
+    source: SourceUpsert | None = None
+    if obj_id is not None and ra is not None and dec is not None:
+        source = SourceUpsert(id=obj_id, ra=ra, dec=dec, group_ids=group_ids)
+    elif obj_id is not None:
+        comments.append(
+            f"Source {obj_id} not created: no RA/Dec in this circular "
+            f"(position comes from the discovery circular)."
+        )
+
+    for idx, row in enumerate(extraction.photometry):
+        point = _row_to_point(extraction, obj_id, idx, row, instrument_map, default_instrument_id)
+        if point is None:
+            skipped += 1
+        else:
+            photometry.append(point)
+
+    # Redshift: scalar only. Bounds (in notes) become a comment, not a value.
+    redshift: tuple[float, float | None] | None = None
+    if extraction.redshift is not None and extraction.redshift.redshift is not None:
+        err = extraction.redshift.redshift_error
+        z_err = err if isinstance(err, float) else None
+        redshift = (extraction.redshift.redshift, z_err)
+        if (note := _provenance_note(extraction, "redshift")) is not None:
+            comments.append(f"Redshift z={extraction.redshift.redshift} from {note}")
+
+    # Bound redshifts and other extractor notes -> comment.
+    for note in extraction.extraction_meta.notes:
+        comments.append(f"Note (not posted as a value): {note}")
+
+    if skipped:
+        comments.append(
+            f"{skipped} photometry row(s) could not be posted "
+            f"(missing observation time, filter, or instrument_id); "
+            f"kept in the extraction only."
+        )
+
+    return SkyPortalActions(
+        source=source,
+        photometry=photometry,
+        redshift=redshift,
+        comments=comments,
+        skipped_rows=skipped,
+    )
+
+
+def _effective_band(row: PhotometryExt) -> tuple[str | None, str]:
+    """Canonical (bandpass, magsys) for a row.
+
+    The deterministic filter crosswalk is authoritative for recognized filters
+    and OVERRIDES the extractor's values — an LLM may mislabel a band (e.g.
+    Mistral tagging Cousins "Rc" as sdssr/ab when it is bessellr/vega). Falls
+    back to the row's own bandpass/mag_system when the filter isn't recognized.
+    """
+    base = normalize_filter(row.filter) if row.filter else None
+    band = infer_bandpass(base) if base else None
+    if band is not None:
+        system = infer_mag_system(base) if base else None
+        return band, _MAGSYS.get(system or "", "ab")
+    return row.bandpass, _MAGSYS.get(row.mag_system or "", "ab")
+
+
+def _row_to_point(
+    extraction: CircularExtraction,
+    obj_id: str | None,
+    idx: int,
+    row: PhotometryExt,
+    instrument_map: dict[str, int],
+    default_instrument_id: int | None,
+) -> PhotometryPoint | None:
+    """One photometry row -> a SkyPortal point, or None if unpostable.
+
+    A row needs an obj_id, an obs_mjd, a resolvable bandpass, AND an
+    instrument_id (mapped, or the generic default) — SkyPortal requires all of
+    these. Otherwise it cannot be posted.
+    """
+    band, magsys = _effective_band(row)
+    mapped = instrument_map.get(row.telescope_canonical) if row.telescope_canonical else None
+    instrument_id = mapped if mapped is not None else default_instrument_id
+    if obj_id is None or row.obs_mjd is None or band is None or instrument_id is None:
+        return None
+
+    # SkyPortal's photometry endpoint REQUIRES a non-null limiting_mag for
+    # mag-space points. Circulars often report a detection with no explicit
+    # per-point limit, so fall back to the detection mag itself — a conservative,
+    # truthful depth floor (the field was seen at least this faint) — and flag it.
+    limiting_mag = row.limiting_mag
+    limit_assumed = False
+    if limiting_mag is None:
+        if row.mag is None:
+            return None  # neither a detection nor a stated limit — nothing to post
+        limiting_mag = row.mag
+        limit_assumed = True
+
+    altdata: dict[str, Any] = {}
+    if (note := _provenance_note(extraction, f"photometry[{idx}]")) is not None:
+        altdata["note"] = note
+    altdata["circex_circular_id"] = extraction.circular_id
+    if mapped is None:
+        # Fell back to the generic instrument; record what we actually saw.
+        altdata["instrument_fallback"] = True
+        if row.telescope:
+            altdata["telescope_as_written"] = row.telescope
+    if limit_assumed:
+        altdata["limiting_mag_assumed"] = True
+    return PhotometryPoint(
+        obj_id=obj_id,
+        mjd=row.obs_mjd,
+        filter=band,
+        magsys=magsys,
+        instrument_id=instrument_id,
+        mag=row.mag,
+        magerr=row.mag_error,
+        limiting_mag=limiting_mag,
+        altdata=altdata,
+    )

circex/cache/__init__.py

@@ -0,0 +1,5 @@
+"""LLM response cache."""
+
+from circex.cache.llm import LLMCache, cache_key
+
+__all__ = ["LLMCache", "cache_key"]