hypgraph 0.1.0__py3-none-any.whl

hypgraph/__init__.py

@@ -0,0 +1,28 @@
+"""
+HypGraph — a research record that can't flatter itself.
+
+    Don't believe the hype. Test the hyp.
+
+    >>> from hypgraph import Graph, Policy
+    >>> g = Graph(policy=Policy(reviewers={"ada"}))          # in memory; Graph("study.hypgraph") for a file
+    >>> me = g.author("sam")
+    >>> inv = me.investigation("Plant growth", "Find out whether light colour changes how fast cress grows.")["id"]
+    >>> h = me.add("hypothesis", inv, statement="Blue light makes cress grow faster than red light.",
+    ...            predictions=["Blue-lit trays are taller after 7 days."])
+    >>> h["state"]["sentence"]
+    'active hypothesis · unchecked · no live support · unverified report · no external replication'
+
+Standing is derived on every read, never stored; every write is an event in a SHA-256 hash chain; claims carry
+their numbers; reviews pin versions; independence is counted, not assumed. See https://hypgraph.com.
+"""
+
+__version__ = "0.1.0"
+
+from .bundle import verify_bundle  # noqa: E402
+from .graph import Author, Graph, GraphError, file_sha256  # noqa: E402
+from .policy import Policy  # noqa: E402
+from .schema import FORMAT, describe  # noqa: E402
+from .store import verify_file  # noqa: E402
+
+__all__ = ["Graph", "GraphError", "Policy", "Author", "verify_bundle", "verify_file", "file_sha256", "describe",
+           "FORMAT", "__version__"]

hypgraph/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

hypgraph/bundle.py

@@ -0,0 +1,216 @@
+"""
+Bundles: a record you can hand to someone else, who can check it without trusting you.
+
+``export()`` writes the whole graph; ``export(inv)`` one investigation. Every event is included as the **exact string
+its hash was taken over**, and every version body and edge's data as the exact string stored, so verifying is just
+SHA-256 over strings plus a few lookups — any language can do it (the site does it in the browser with WebCrypto).
+
+    {"format": "hypgraph/1-bundle", "complete": true, "scope": {...}, "policy": {...},
+     "events":   [{"seq", "hash", "body"}],                 # body = canonical JSON of {at, actor, kind, ref, payload, prev}
+     "nodes":    [{"id", "type", "code", "inv", "created_at", "created_by"}],
+     "versions": [{"node", "v", "at", "by", "reason", "body"}],   # body = the stored canonical string
+     "edges":    [{"id", "src", "src_v", "rel", "dst", "dst_v", "data", "at", "by", "why"}],   # data likewise
+     "withdrawals": [...], "retractions": [...], "leases": [...],
+     "derived":  {...}}                                     # standing at export time, for viewers; never trusted
+
+A scoped bundle (one investigation out of a shared graph) redacts the events outside it to {seq, hash, prev}: their
+content can't be checked, so a change hidden among them (a withdrawal, say) would not show. ``verify_bundle`` says so
+(``complete: false``). To publish a study, keep it in a graph of its own and export all of it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING, Any
+
+from . import __version__
+from .store import GENESIS, event_body, now
+
+if TYPE_CHECKING:
+    from .graph import Graph
+
+BUNDLE_FORMAT = "hypgraph/1-bundle"
+
+
+def _sha(s: str) -> str:
+    return hashlib.sha256(s.encode()).hexdigest()
+
+
+def export(g: "Graph", inv: str = "", *, derived: bool = True) -> dict:
+    with g._snap_lock:
+        con = g._connect()
+        rows = lambda q, *a: [dict(zip([c[0] for c in cur.description], r))  # noqa: E731
+                              for cur in [con.execute(q, a)] for r in cur.fetchall()]
+        nodes = rows("SELECT id, type, code, inv, created_at, created_by FROM nodes ORDER BY rowid")
+        if inv:
+            if not any(n["id"] == inv and n["type"] == "investigation" for n in nodes):
+                raise KeyError(inv)
+            nodes = [n for n in nodes if n["id"] == inv or n["inv"] == inv]
+        ids = {n["id"] for n in nodes}
+        versions = [v for v in rows("SELECT node, v, at, by, reason, body FROM versions ORDER BY rowid")
+                    if v["node"] in ids]
+        edges = [e for e in rows("SELECT id, src, src_v, rel, dst, dst_v, data, at, by, why FROM edges ORDER BY id")
+                 if e["src"] in ids and e["dst"] in ids]
+        eids = {e["id"] for e in edges}
+        withdrawals = [w for w in rows("SELECT node, at, by, why FROM withdrawals ORDER BY rowid") if w["node"] in ids]
+        retractions = [r for r in rows("SELECT edge, at, by, why FROM retractions ORDER BY rowid") if r["edge"] in eids]
+        leases = [x for x in rows("SELECT id, node, agent, task, kind, at, expires, note FROM leases ORDER BY id")
+                  if x["node"] in ids]
+        events, redacted = [], 0
+        for r in rows("SELECT seq, at, actor, kind, ref, payload, prev, hash FROM events ORDER BY seq"):
+            p = json.loads(r["payload"])
+            mine = not inv or _event_in(r["kind"], r["ref"], p, ids, eids)
+            if mine:
+                events.append({"seq": r["seq"], "hash": r["hash"],
+                               "body": event_body(r["at"], r["actor"], r["kind"], r["ref"], p, r["prev"])})
+            else:
+                redacted += 1
+                events.append({"seq": r["seq"], "hash": r["hash"], "prev": r["prev"], "redacted": True})
+    title = next((json.loads(v["body"]).get("title") for v in versions if v["node"] == inv), "") if inv else ""
+    out: dict[str, Any] = {
+        "format": BUNDLE_FORMAT, "generator": f"hypgraph {__version__}",
+        "exported_at": now(),
+        "complete": redacted == 0, "redacted_events": redacted,
+        "scope": {"inv": inv, "title": title} if inv else {"inv": "", "title": "whole graph"},
+        "policy": g.policy.to_dict(), "head": events[-1]["hash"] if events else GENESIS,
+        "events": events, "nodes": nodes, "versions": versions, "edges": edges,
+        "withdrawals": withdrawals, "retractions": retractions, "leases": leases,
+    }
+    if derived:
+        out["derived"] = derive_views(g, [inv] if inv else [n["id"] for n in nodes if n["type"] == "investigation"])
+    return out
+
+
+def _event_in(kind: str, ref: str, p: dict, ids: set, eids: set) -> bool:
+    if kind in ("link", "retract"):
+        return int(p.get("edge", -1)) in eids or (kind == "link" and ref in ids and p.get("dst") in ids)
+    return ref in ids
+
+
+def derive_views(g: "Graph", invs: list[str]) -> dict:
+    """What a viewer needs without running the engine: standing of every node and the per-investigation views."""
+    views = {"investigations": [i for i in g.investigations() if i["id"] in invs], "stats": g.stats(), "by_inv": {}}
+    for i in invs:
+        views["by_inv"][i] = {"subgraph": g.subgraph(i), "work": g.work_map(i), "next": g.next_experiments(i, 20),
+                              "report": g.report(i)}
+    views["verify"] = g.verify()
+    return views
+
+
+def verify_bundle(b: dict) -> dict:
+    """Check a bundle string-for-string: the chain, then every row against the event that wrote it."""
+    if not isinstance(b, dict) or b.get("format") != BUNDLE_FORMAT:
+        return {"ok": False, "problem": f"not a {BUNDLE_FORMAT} bundle"}
+    prev, checked, red = GENESIS, 0, 0
+    creates, revs, reviews, links, withdraws, retracts = {}, {}, {}, {}, set(), set()
+    for ev in sorted(b.get("events") or [], key=lambda e: e["seq"]):
+        if ev.get("redacted"):
+            if ev.get("prev") != prev:
+                return {"ok": False, "broken_at": ev["seq"], "problem": "chain broken (redacted event)"}
+            prev, red = ev["hash"], red + 1
+            continue
+        if _sha(ev["body"]) != ev["hash"]:
+            return {"ok": False, "broken_at": ev["seq"], "problem": "event body does not match its hash"}
+        body = json.loads(ev["body"])
+        if body.get("prev") != prev:
+            return {"ok": False, "broken_at": ev["seq"], "problem": "chain broken (prev does not match)"}
+        prev, checked = ev["hash"], checked + 1
+        k, ref, p = body["kind"], body["ref"], body["payload"]
+        if k == "create":
+            creates[ref] = p
+            revs[(ref, int(p.get("v", 1)))] = p.get("body_sha")
+        elif k == "revise":
+            revs[(ref, int(p.get("v")))] = p.get("body_sha")
+        elif k == "review":
+            reviews[p["review"]] = p
+            revs[(p["review"], 1)] = p.get("body_sha")
+        elif k == "link":
+            links[int(p["edge"])] = p
+        elif k == "withdraw":
+            withdraws.add(ref)
+        elif k == "retract":
+            retracts.add(int(p["edge"]))
+    if b.get("head") and b["head"] != prev:
+        return {"ok": False, "problem": "the chain does not end at the bundle's head"}
+    for n in b.get("nodes") or []:
+        c = creates.get(n["id"])
+        if n["type"] == "review":
+            if n["id"] not in reviews:
+                return {"ok": False, "problem": f"review {n['id']} has no event"}
+        elif not c or (c.get("type"), c.get("code"), c.get("inv")) != (n["type"], n["code"], n["inv"]):
+            return {"ok": False, "problem": f"node {n['id']} differs from its create event"}
+    for v in b.get("versions") or []:
+        want = revs.get((v["node"], v["v"]))
+        if want is None:
+            return {"ok": False, "problem": f"version {v['node']} v{v['v']} has no event"}
+        if _sha(v["body"]) != want:
+            return {"ok": False, "problem": f"version {v['node']} v{v['v']} differs from its event"}
+    for e in b.get("edges") or []:
+        want = links.get(e["id"])
+        if want is None:
+            return {"ok": False, "problem": f"edge {e['id']} has no event"}
+        if (want.get("rel"), want.get("dst"), want.get("dst_v")) != (e["rel"], e["dst"], e["dst_v"]) or \
+                _sha(e["data"]) != want.get("data_sha"):
+            return {"ok": False, "problem": f"edge {e['id']} differs from its event"}
+    for w in b.get("withdrawals") or []:
+        if w["node"] not in withdraws:
+            return {"ok": False, "problem": f"withdrawal of {w['node']} has no event"}
+    for r in b.get("retractions") or []:
+        if int(r["edge"]) not in retracts:
+            return {"ok": False, "problem": f"retraction of edge {r['edge']} has no event"}
+    out = {"ok": True, "checked": checked, "head": prev, "complete": red == 0, "redacted": red,
+           "nodes": len(b.get("nodes") or []), "edges": len(b.get("edges") or [])}
+    if red:
+        out["warning"] = (f"{red} event(s) outside this investigation are redacted: their content can't be checked, so "
+                          "a change hidden among them (a withdrawal, say) would not show. The full record rules that "
+                          "out.")
+    return out
+
+
+def load(g: "Graph", b: dict) -> None:
+    """Replay a complete bundle into an empty graph, row for row — the result verifies exactly as the original."""
+    v = verify_bundle(b)
+    if not v["ok"]:
+        raise ValueError(f"bundle does not verify: {v['problem']}")
+    if not v["complete"]:
+        raise ValueError("a scoped bundle has redacted events and can't be replayed; view it, or ask for the full one")
+    with g._lock:
+        con = g._connect()
+        if con.execute("SELECT COUNT(*) FROM events").fetchone()[0]:
+            raise ValueError("load a bundle into an empty graph")
+        con.execute("BEGIN IMMEDIATE")
+        try:
+            for n in b["nodes"]:
+                con.execute("INSERT INTO nodes (id, type, code, inv, created_at, created_by) VALUES (?,?,?,?,?,?)",
+                            (n["id"], n["type"], n["code"], n["inv"], n["created_at"], n["created_by"]))
+            for x in b["versions"]:
+                con.execute("INSERT INTO versions (node, v, at, by, reason, body) VALUES (?,?,?,?,?,?)",
+                            (x["node"], x["v"], x["at"], x["by"], x["reason"], x["body"]))
+            for e in b["edges"]:
+                con.execute("INSERT INTO edges (id, src, src_v, rel, dst, dst_v, data, at, by, why) "
+                            "VALUES (?,?,?,?,?,?,?,?,?,?)", (e["id"], e["src"], e["src_v"], e["rel"], e["dst"],
+                                                             e["dst_v"], e["data"], e["at"], e["by"], e["why"]))
+            for w in b["withdrawals"]:
+                con.execute("INSERT INTO withdrawals (node, at, by, why) VALUES (?,?,?,?)",
+                            (w["node"], w["at"], w["by"], w["why"]))
+            for r in b["retractions"]:
+                con.execute("INSERT INTO retractions (edge, at, by, why) VALUES (?,?,?,?)",
+                            (r["edge"], r["at"], r["by"], r["why"]))
+            for x in b["leases"]:
+                con.execute("INSERT INTO leases (id, node, agent, task, kind, at, expires, note) "
+                            "VALUES (?,?,?,?,?,?,?,?)", (x["id"], x["node"], x["agent"], x["task"], x["kind"],
+                                                         x["at"], x["expires"], x["note"]))
+            for ev in b["events"]:
+                body = json.loads(ev["body"])
+                con.execute("INSERT INTO events (seq, at, actor, kind, ref, payload, prev, hash) "
+                            "VALUES (?,?,?,?,?,?,?,?)", (ev["seq"], body["at"], body["actor"], body["kind"],
+                                                         body["ref"], json.dumps(body["payload"], sort_keys=True,
+                                                                                 ensure_ascii=False,
+                                                                                 separators=(",", ":")),
+                                                         body["prev"], ev["hash"]))
+            con.execute("COMMIT")
+        except BaseException:
+            con.execute("ROLLBACK")
+            raise
+    g._invalidate()

hypgraph/checks.py

@@ -0,0 +1,140 @@
+"""
+Mechanical checks on a claim before it enters the record. Each rule exists because the mistake behind it was made:
+
+- **every number accounted for** — a number in the statement must match a measurement bound to a result field, the
+  scope, or one of the investigation's declared constants ("all 49 in 6 of 6" when the result said 5 of 6);
+- **absolutes need k of n** — "never", "all", "proves" need a measurement whose k is 0 or n;
+- **exclusions need power** — "X is ruled out" needs a search shown to recover a planted answer, else it's
+  inconclusive;
+- **conventions don't hedge** — agreed notation isn't "could be";
+- **scope and evidence** — a finding says under what conditions it holds, and links what supports it.
+
+Spelled-out two … twenty count as numbers ("two practice ciphers" when the record had one). Digits in quotes are
+names ('63 63 63', "04"), not quantities.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from .derive import Snap, producing_run
+
+_NUM = re.compile(r"(?<![\w.])(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d+))?(\s*%)?(?![\w])")
+_ABSOLUTE = re.compile(r"\b(never|always|all|every|none|impossible|cannot|can't|proves?|proven|certainly|"
+                       r"guaranteed?)\b", re.I)
+_HEDGE = re.compile(r"\b(could|might|may|likely|probably|perhaps|possibly|suggests?|consistent with|"
+                    r"aligns? with|plausibl\w*)\b", re.I)
+_WORDS = {w: i for i, w in enumerate("two three four five six seven eight nine ten eleven twelve thirteen fourteen "
+                                     "fifteen sixteen seventeen eighteen nineteen twenty".split(), start=2)}
+_WORDNUM = re.compile(r"\b(" + "|".join(_WORDS) + r")\b", re.I)   # 'one' is left out: too common as a pronoun
+_LABEL = re.compile(r"""(['"`‘“])[0-9][0-9 ]{0,30}(['"`’”])""")
+
+
+def numbers(text: str) -> list[tuple[float, int, bool, str]]:
+    """(value, decimals printed, is_percent, as printed) for each number in a text."""
+    out = []
+    for m in _NUM.finditer(text or ""):
+        whole, frac, pct = m.group(1), m.group(2), m.group(3)
+        v = float(whole.replace(",", "") + ("." + frac if frac else ""))
+        out.append((v, len(frac or ""), bool(pct), m.group(0).strip()))
+    for m in _WORDNUM.finditer(text or ""):
+        out.append((float(_WORDS[m.group(1).lower()]), 0, False, m.group(0)))
+    return out
+
+
+def _matches(printed: tuple, candidates: list[float]) -> bool:
+    v, dec, pct, _ = printed
+    for c in candidates:
+        for x in ((c, c * 100) if pct or c <= 1 else (c,)):
+            if round(x, dec) == v or (dec == 0 and abs(x - v) < 0.5 and round(x) == v):
+                return True
+    return False
+
+
+def _measure_values(m: dict) -> list[float]:
+    vals = []
+    for k in ("value", "k", "n", "lo", "hi"):
+        x = m.get(k)
+        if isinstance(x, (int, float)) and not isinstance(x, bool):
+            vals.append(float(x))
+    for x in m.get("interval") or []:
+        if isinstance(x, (int, float)):
+            vals.append(float(x))
+    return vals
+
+
+def result_value(s: Snap, rid: str, key: str) -> Optional[dict]:
+    r = s.nodes.get(rid)
+    if not r or r["type"] != "result":
+        return None
+    v = (r["body"].get("values") or {}).get(key)
+    if isinstance(v, (int, float)):
+        return {"value": v}
+    return v if isinstance(v, dict) else None
+
+
+def claim_problems(s: Snap, body: dict, support: list[str], inv: str) -> list[str]:
+    """What's wrong with a claim, as instructions to fix it (empty = it may enter the record). ``support``: the ids
+    of nodes that support it or that it is derived from."""
+    problems = []
+    kind = body.get("kind") or "finding"
+    stmt = body.get("statement", "")
+    scope = body.get("scope", "")
+    invb = s.nodes.get(inv, {}).get("body", {}) if inv else {}
+    if invb.get("strict_claims") is False:
+        return []
+    if kind in ("finding", "exclusion") and len(scope.strip()) < 10:
+        problems.append("scope: say under which conditions this holds (the data, method, sample) — at least a sentence")
+    if kind == "convention" and _HEDGE.search(stmt):
+        problems.append("a convention states agreed notation; a hedged interpretation (could / suggests / consistent "
+                        "with) is a finding or a lead")
+    support = [x for x in support if x in s.nodes]
+    if kind in ("finding", "exclusion", "inconclusive") and not support:
+        problems.append("link the evidence: at least one result, claim or source that supports it "
+                        "(links: {\"rel\": \"supports\", \"from\": <result id>, \"why\": …})")
+    meas = body.get("measurements") or []
+    coverage: list[float] = []
+    for i, m in enumerate(meas):
+        if not isinstance(m, dict) or not m.get("result") or not m.get("key"):
+            problems.append(f"measurement {i + 1}: give {{name, value (or k/n), result: <result id>, key: <field>}}")
+            continue
+        rv = result_value(s, str(m["result"]), str(m["key"]))
+        if rv is None:
+            problems.append(f"measurement {m.get('name') or i + 1}: result {m['result']} has no value '{m['key']}'")
+            continue
+        for k in ("value", "k", "n"):
+            if k in m and k in rv and isinstance(m[k], (int, float)) and isinstance(rv[k], (int, float)):
+                dec = len(str(m[k]).split(".")[1]) if "." in str(m[k]) else 0
+                if round(float(rv[k]), dec) != round(float(m[k]), dec) and not (
+                        rv[k] <= 1 < m[k] and round(float(rv[k]) * 100, dec) == round(float(m[k]), dec)):
+                    problems.append(f"measurement {m.get('name') or i + 1}: says {k}={m[k]} but result {m['result']} "
+                                    f"has {rv[k]}")
+        coverage += _measure_values(m) + _measure_values(rv)
+    allowed = coverage + [v for v, *_ in numbers(scope)] + [float(c) for c in (invb.get("constants") or [])
+                                                            if isinstance(c, (int, float))]
+    loose = [p for p in numbers(_LABEL.sub(" ", stmt)) if not _matches(p, allowed)]
+    if loose:
+        problems.append("numbers in the statement that no bound measurement, the scope or the investigation's "
+                        f"constants account for: {', '.join(p[3] for p in loose[:8])} — bind them to result fields "
+                        "(measurements) or move design constants into the scope")
+    for w in {m.group(0).lower() for m in _ABSOLUTE.finditer(stmt)}:
+        exact = any(isinstance(m, dict) and m.get("n") and m.get("k") in (0, m.get("n")) for m in meas)
+        if not exact:
+            problems.append(f"'{w}' is absolute: back it with a k/n measurement where k is 0 or n, and keep the "
+                            "scope to the trials actually run — or say 'rarely' / 'in k of n'")
+    if kind == "exclusion":
+        powered = False
+        for sid in support:
+            src = s.nodes[sid]
+            if src["type"] == "result" and src["body"].get("power") == "demonstrated":
+                powered = True
+            run = producing_run(s, src["id"]) if src["type"] == "result" else None
+            for e in s.out.get(run or "", []):
+                if e["rel"] == "executes" and s.nodes[e["dst"]]["body"].get("power_check"):
+                    powered = True
+        if not powered:
+            problems.append("an exclusion needs a search that recovered a planted answer (a result with "
+                            "power=demonstrated, or a run of an experiment with a power_check) — otherwise file it "
+                            "as inconclusive")
+    return problems