PhantomReason 0.1.2__py3-none-any.whl

phantomreason/__init__.py

@@ -0,0 +1,20 @@
+from .model import PhantomLanguageModel, run_interactive_test
+from .evaluate import run_evaluation
+from .service import PhantomAgentService, build_server
+from .storage import EXTRA_WORDS_PATH, GLOBAL_INDEX, GLOBAL_VOCAB, MODEL_STATE_PATH, vocabadder
+
+__version__ = "0.1.2"
+
+__all__ = [
+    "EXTRA_WORDS_PATH",
+    "GLOBAL_INDEX",
+    "GLOBAL_VOCAB",
+    "MODEL_STATE_PATH",
+    "PhantomAgentService",
+    "PhantomLanguageModel",
+    "__version__",
+    "build_server",
+    "run_evaluation",
+    "run_interactive_test",
+    "vocabadder",
+]

phantomreason/corpus.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import html
+import re
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+
+DEFAULT_USER_AGENT = "phantomreason/1.0 (+symbolic-trace-ingest)"
+DEFAULT_TIMEOUT_SECONDS = 15.0
+
+
+def normalize_text(raw_text: str) -> str:
+    cleaned = html.unescape(raw_text)
+    cleaned = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", cleaned)
+    cleaned = re.sub(r"(?s)<[^>]+>", " ", cleaned)
+    cleaned = cleaned.replace("\r", "\n")
+    cleaned = re.sub(r"[^\S\n]+", " ", cleaned)
+    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+    return cleaned.strip()
+
+
+def tokenize_words(text: str) -> list[str]:
+    return re.findall(r"[a-zA-Z']+", text.lower())
+
+
+def iter_sentences(
+    text: str,
+    min_words: int = 3,
+    max_words: int = 24,
+    max_sentences: int | None = None,
+) -> list[str]:
+    normalized = normalize_text(text)
+    if not normalized:
+        return []
+    sentence_like = re.split(r"(?<=[.!?])\s+|\n+", normalized)
+    collected: list[str] = []
+    seen: set[str] = set()
+    for part in sentence_like:
+        words = tokenize_words(part)
+        if len(words) < min_words:
+            continue
+        while words:
+            chunk = words[:max_words]
+            words = words[max_words:]
+            if len(chunk) < min_words:
+                break
+            sentence = " ".join(chunk)
+            if sentence in seen:
+                continue
+            seen.add(sentence)
+            collected.append(sentence)
+            if max_sentences is not None and len(collected) >= max_sentences:
+                return collected
+    return collected
+
+
+def load_text_file(path: str | Path) -> str:
+    return Path(path).read_text(encoding="utf-8")
+
+
+def fetch_url_text(
+    url: str,
+    timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    user_agent: str = DEFAULT_USER_AGENT,
+) -> str:
+    request = urllib.request.Request(url, headers={"User-Agent": user_agent})
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        charset = response.headers.get_content_charset() or "utf-8"
+        payload = response.read()
+    return payload.decode(charset, errors="replace")
+
+
+def extract_dictionary_facts(text: str, max_entries: int = 256) -> list[str]:
+    facts: list[str] = []
+    seen: set[str] = set()
+    for raw_line in normalize_text(text).splitlines():
+        line = raw_line.strip(" -:\t")
+        if not line:
+            continue
+        match = re.match(r"^([A-Za-z][A-Za-z' -]{1,48})\s*[:\-]\s*(.+)$", line)
+        if not match:
+            continue
+        term = " ".join(tokenize_words(match.group(1)))
+        definition_tokens = tokenize_words(match.group(2))
+        if not term or len(definition_tokens) < 2:
+            continue
+        definition = " ".join(definition_tokens[:10])
+        sentence = f"{term} means {definition}"
+        if sentence in seen:
+            continue
+        seen.add(sentence)
+        facts.append(sentence)
+        if len(facts) >= max_entries:
+            break
+    return facts
+
+
+def load_local_corpus_candidates(root: Path) -> list[Path]:
+    candidates: list[Path] = []
+    for relative in ("words.txt", "words"):
+        path = root / relative
+        if path.exists() and path.is_file():
+            candidates.append(path)
+    return candidates
+
+
+__all__ = [
+    "DEFAULT_TIMEOUT_SECONDS",
+    "DEFAULT_USER_AGENT",
+    "extract_dictionary_facts",
+    "fetch_url_text",
+    "iter_sentences",
+    "load_local_corpus_candidates",
+    "load_text_file",
+    "normalize_text",
+    "tokenize_words",
+]

phantomreason/evaluate.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from .model import PhantomLanguageModel
+
+
+def require(condition: bool, label: str, failures: list[str]) -> None:
+    if not condition:
+        failures.append(label)
+
+
+def run_evaluation() -> dict[str, object]:
+    model = PhantomLanguageModel(dim=512, sparsity=47)
+    training_text = (
+        "vector means an ordered list used for state. "
+        "artist is a person who creates stories. "
+        "sun rises east. "
+        "north star guides sailors nightly. "
+        "gardeners water orchids daily."
+    )
+    model.train_on_text(training_text, epochs=1, persist=False)
+    failures: list[str] = []
+
+    parsed = model.understand_input("north star guides sailors nightly")
+    require(parsed["subject"] == "north star", "parse subject for two-word noun phrase", failures)
+    require(parsed["predicate"] == "guides", "parse predicate for two-word noun phrase", failures)
+
+    parsed = model.understand_input("what is vector?")
+    require(parsed["subject"] == "vector", "parse copular question subject", failures)
+    require(parsed["predicate"] == "is", "parse copular question predicate", failures)
+
+    routed = model.route_prompt("what is vector?")
+    require(routed["fact_answer"] is not None, "fact recall for learned definition", failures)
+
+    model.register_fact("vector", "means", ["symbolic", "state"])
+    require(
+        not model._fact_is_active("fact:vector|means|ordered list"),
+        "older contradictory fact should be inactive",
+        failures,
+    )
+    require(
+        model._fact_is_active("fact:vector|means|symbolic state"),
+        "new contradictory fact should stay active",
+        failures,
+    )
+    latest = model._direct_fact_match(subject="vector", predicate="means")
+    require(
+        latest is not None and latest["object_tokens"] == ["symbolic", "state"],
+        "latest contradictory fact should dominate recall",
+        failures,
+    )
+
+    before_decay = model._fact_strength_value("fact:vector|means|symbolic state")
+    model.apply_fact_decay(keep_recent=0, amount=1)
+    after_decay = model._fact_strength_value("fact:vector|means|symbolic state")
+    require(after_decay <= before_decay, "decay should not increase fact strength", failures)
+
+    if failures:
+        return {
+            "status": "FAIL",
+            "failures": failures,
+            "fact_count": model.fact_count(),
+            "fact_answer": routed["fact_answer"],
+            "parse": model.render_understanding(model.understand_input("north star guides sailors nightly")),
+        }
+
+    return {
+        "status": "PASS",
+        "failures": [],
+        "fact_count": model.fact_count(),
+        "fact_answer": routed["fact_answer"],
+        "parse": model.render_understanding(model.understand_input("north star guides sailors nightly")),
+    }
+
+
+def main() -> int:
+    result = run_evaluation()
+    print(result["status"])
+    for item in result["failures"]:
+        print(f"- {item}")
+    if result["status"] == "PASS":
+        print(f"fact_count={result['fact_count']}")
+        print(f"fact_answer={result['fact_answer']}")
+        print(f"parse={result['parse']}")
+        return 0
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())