codex-cissor 0.2.0__py3-none-any.whl

codex_cissor/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

codex_cissor/activity.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .config import state_dir
+
+
+def activity_path() -> Path:
+    return state_dir() / "activity.jsonl"
+
+
+def log_event(kind: str, session=None, **fields) -> None:
+    p = activity_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    rec = {"ts": datetime.now(timezone.utc).isoformat(timespec="seconds"), "kind": kind}
+    if session is not None:
+        rec["session"] = str(session)
+    for k, v in fields.items():
+        if v is not None:
+            rec[k] = v
+    with open(p, "a", encoding="utf-8") as f:
+        f.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
+
+
+def read_events(limit: int = 50, session: str | None = None, channel: str | None = None, project: str | None = None) -> list[dict]:
+    p = activity_path()
+    if not p.exists():
+        return []
+    out: list[dict] = []
+    with open(p, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if session and session not in (rec.get("session") or ""):
+                continue
+            if channel and channel != rec.get("channel") and channel not in str(rec.get("chain") or ""):
+                continue
+            if project and project not in (rec.get("project") or ""):
+                continue
+            out.append(rec)
+    return out[-limit:]
+
+
+def stats(events: list[dict]) -> dict:
+    from collections import Counter
+
+    by_kind = Counter(e.get("kind", "?") for e in events)
+    saved = sum(
+        (e.get("tokens_before") or 0) - (e.get("tokens_after") or 0)
+        for e in events
+        if e.get("kind") == "edit"
+    )
+    return {
+        "events": len(events),
+        "by_kind": dict(by_kind),
+        "tokens_saved_total": saved,
+        "sessions_touched": len({e.get("session") for e in events if e.get("session")}),
+        "channels": sorted({e["channel"] for e in events if e.get("channel")}),
+        "first_ts": events[0].get("ts") if events else None,
+        "last_ts": events[-1].get("ts") if events else None,
+    }

codex_cissor/asks.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .rollout import Rollout, load_rollout
+from .summarize import _transcript, chat
+from .tokens import count_text
+
+ASK_SYSTEM = (
+    "You answer questions strictly from the material provided. Cite item numbers, turn numbers, or line "
+    "anchors from the material when they support the answer. If the material does not contain the answer, "
+    "say so plainly instead of guessing. Be dense and concrete."
+)
+
+
+def asks_dir() -> Path:
+    return state_dir_path() / "asks"
+
+
+def state_dir_path() -> Path:
+    from .config import state_dir
+
+    return state_dir()
+
+
+def _new_id() -> str:
+    return "a" + uuid.uuid4().hex[:10]
+
+
+def save_store(store: dict) -> Path:
+    d = asks_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    p = d / f"{store['id']}.json"
+    store["updated"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    p.write_text(json.dumps(store, indent=2, ensure_ascii=False), encoding="utf-8")
+    return p
+
+
+def load_ask(ref: str) -> dict | None:
+    d = asks_dir()
+    if not d.exists():
+        return None
+    for p in sorted(d.glob("*.json")):
+        if p.stem == ref or p.stem.startswith(ref):
+            try:
+                return json.loads(p.read_text(encoding="utf-8"))
+            except Exception:
+                continue
+    return None
+
+
+def list_asks() -> list[dict]:
+    d = asks_dir()
+    if not d.exists():
+        return []
+    out = []
+    for p in sorted(d.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True):
+        try:
+            s = json.loads(p.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+        out.append(
+            {
+                "id": s.get("id"),
+                "model": s.get("model"),
+                "turns": len(s.get("turns", [])),
+                "target": _target_label(s.get("target", {})),
+                "updated": s.get("updated"),
+                "last_question": (s.get("turns") or [{}])[-1].get("q", "")[:80],
+            }
+        )
+    return out
+
+
+def _target_label(t: dict) -> str:
+    kind = t.get("kind", "?")
+    if kind == "file":
+        return f"file:{t.get('path', '')}"
+    if kind == "session":
+        ref = t.get("ref", "")
+        extra = ""
+        if t.get("selection"):
+            extra = " " + t["selection"]
+        return f"session:{ref}{extra} scope={t.get('scope', 'auto')}"
+    return kind
+
+
+def resolve_file_target(path: Path) -> dict:
+    raw = path.read_bytes()
+    if path.suffix.lower() == ".pdf" or raw[:4] == b"%PDF":
+        raise SystemExit(
+            f"{path.name} is a PDF — cissor ask reads text only. Convert it to text/markdown first "
+            f"(e.g. with a parse tool or pdftotext), then ask the converted file."
+        )
+    if b"\x00" in raw[:8192]:
+        raise SystemExit(f"{path.name} looks binary; ask supports text files only (convert first if needed)")
+    return {"kind": "file", "path": str(path)}
+
+
+def resolve_session_target(doc: Rollout, selection_spec: dict) -> dict:
+    return {
+        "kind": "session",
+        "ref": str(doc.path),
+        "session_id": doc.session_id,
+        "scope": selection_spec.get("scope", "auto"),
+        "selection": selection_spec.get("selection", ""),
+    }
+
+
+def collect_content(target: dict, cfg: dict) -> str:
+    if target["kind"] == "file":
+        return Path(target["path"]).read_text(encoding="utf-8", errors="replace")
+    from .selection import resolve_selection
+
+    doc = load_rollout(target["ref"])
+    items = doc.build_items(cfg.get("encoding", "o200k_base"), scope=target.get("scope", "auto"))
+    spec = target.get("selection") or ""
+    if spec.strip():
+        kwargs = json.loads(spec)
+        sel = resolve_selection(doc, items, **kwargs)
+        items = sel.ordered_items(items)
+    return _transcript(items)
+
+
+def content_hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
+
+
+def _chunk_text(content: str, max_chunk_tokens: int, encoding_name: str) -> list[str]:
+    paragraphs = content.split("\n\n")
+    chunks: list[str] = []
+    cur: list[str] = []
+    cur_tokens = 0
+    for para in paragraphs:
+        p_tokens = count_text(para, encoding_name)
+        if cur and cur_tokens + p_tokens > max_chunk_tokens:
+            chunks.append("\n\n".join(cur))
+            cur = []
+            cur_tokens = 0
+        if p_tokens > max_chunk_tokens:
+            if cur:
+                chunks.append("\n\n".join(cur))
+                cur = []
+                cur_tokens = 0
+            line = ""
+            for line_part in para.split("\n"):
+                if line and count_text(line + "\n" + line_part, encoding_name) > max_chunk_tokens:
+                    chunks.append(line)
+                    line = line_part
+                else:
+                    line = (line + "\n" + line_part) if line else line_part
+            if line:
+                cur = [line]
+                cur_tokens = count_text(line, encoding_name)
+            continue
+        cur.append(para)
+        cur_tokens += p_tokens
+    if cur:
+        chunks.append("\n\n".join(cur))
+    return chunks
+
+
+def ask_question(
+    store: dict,
+    content: str,
+    question: str,
+    cfg: dict,
+    model: str,
+) -> str:
+    store["content_hash"] = content_hash(content)
+
+    encoding_name = cfg.get("encoding", "o200k_base")
+    max_input = int(cfg.get("ask_max_input_tokens", 120000))
+    chunk_tokens = int(cfg.get("chunk_tokens", 48000))
+    n_tokens = count_text(content, encoding_name)
+    prior = []
+    for t in store.get("turns", []):
+        prior.append({"role": "user", "content": t["q"]})
+        prior.append({"role": "assistant", "content": t["a"]})
+
+    def _finish(answer: str) -> str:
+        _record_turn(store, question, answer)
+        return answer
+
+    if n_tokens <= max_input:
+        messages = [{"role": "system", "content": ASK_SYSTEM}]
+        messages += prior
+        messages.append(
+            {
+                "role": "user",
+                "content": f"MATERIAL ({n_tokens:,} tokens):\n\n{content}\n\nQUESTION: {question}",
+            }
+        )
+        return _finish(chat(cfg, model, messages))
+
+    chunks = _chunk_text(content, chunk_tokens, encoding_name)
+    map_notes: list[str] = []
+    for i, chunk in enumerate(chunks, 1):
+        messages = [
+            {"role": "system", "content": "You extract facts relevant to a question from a fragment of a larger document. Quote exact values, names, and anchors. If nothing relevant exists, say 'nothing relevant'."},
+            {"role": "user", "content": f"QUESTION TO ANSWER: {question}\n\nFRAGMENT {i}/{len(chunks)} of the document:\n\n{chunk}"},
+        ]
+        map_notes.append(f"## Fragment {i}/{len(chunks)}\n\n{chat(cfg, model, messages)}")
+
+    messages = [{"role": "system", "content": ASK_SYSTEM}]
+    messages += prior
+    messages.append(
+        {
+            "role": "user",
+            "content": (
+                f"The document is too large ({n_tokens:,} tokens) so it was scanned in {len(chunks)} fragments; "
+                f"below are the fragments' relevant extracts.\n\n"
+                f"--- EXTRACTS ---\n\n" + "\n\n".join(map_notes) +
+                f"\n\n--- ORIGINAL QUESTION ---\n{question}\n\n"
+                "Answer the question from these extracts. Say which fragments support each claim. "
+                "If the extracts don't contain the answer, say so."
+            ),
+        }
+    )
+    return _finish(chat(cfg, model, messages))
+
+
+def _record_turn(store: dict, q: str, a: str) -> None:
+    store.setdefault("turns", []).append({"q": q, "a": a, "at": datetime.now(timezone.utc).isoformat(timespec="seconds")})