aka-kg 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aka_kg-0.1.0/PKG-INFO +97 -0
- aka_kg-0.1.0/README.md +74 -0
- aka_kg-0.1.0/pyproject.toml +24 -0
- aka_kg-0.1.0/setup.cfg +4 -0
- aka_kg-0.1.0/src/aka/__init__.py +76 -0
- aka_kg-0.1.0/src/aka/aka_loop.py +56 -0
- aka_kg-0.1.0/src/aka/concept_layer.py +273 -0
- aka_kg-0.1.0/src/aka/knowledge_graph.py +238 -0
- aka_kg-0.1.0/src/aka/llm_environment.py +126 -0
- aka_kg-0.1.0/src/aka/persistence.py +136 -0
- aka_kg-0.1.0/src/aka/py.typed +0 -0
- aka_kg-0.1.0/src/aka/question_generator.py +241 -0
- aka_kg-0.1.0/src/aka/strategy_selector.py +118 -0
- aka_kg-0.1.0/src/aka_kg.egg-info/PKG-INFO +97 -0
- aka_kg-0.1.0/src/aka_kg.egg-info/SOURCES.txt +18 -0
- aka_kg-0.1.0/src/aka_kg.egg-info/dependency_links.txt +1 -0
- aka_kg-0.1.0/src/aka_kg.egg-info/requires.txt +17 -0
- aka_kg-0.1.0/src/aka_kg.egg-info/top_level.txt +1 -0
- aka_kg-0.1.0/tests/test_learning.py +339 -0
- aka_kg-0.1.0/tests/test_smoke.py +313 -0
aka_kg-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aka-kg
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Autonomous Knowledge Acquisition: a frozen LLM outer-loop library with graph-driven question selection
|
|
5
|
+
Author-email: Vanthony Laster <support@tyrantsole.com>
|
|
6
|
+
License: OpenSource
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Dist: networkx>=3.0
|
|
11
|
+
Provides-Extra: llm
|
|
12
|
+
Requires-Dist: litellm>=1.0; extra == "llm"
|
|
13
|
+
Provides-Extra: full
|
|
14
|
+
Requires-Dist: aka-kg[llm]; extra == "full"
|
|
15
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "full"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
18
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
19
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
20
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
21
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
22
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
23
|
+
|
|
24
|
+
\# aka — Autonomous Knowledge Acquisition
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
A minimal, modular library for \*\*autonomous knowledge acquisition\*\*: a frozen
|
|
29
|
+
|
|
30
|
+
LLM is used as an environment, while a lightweight outer learner decides what
|
|
31
|
+
|
|
32
|
+
to ask next, extracts concepts from the answers, and builds a knowledge graph
|
|
33
|
+
|
|
34
|
+
whose structural gaps drive the next round of questions.
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
The design follows the "outer layer" architecture validated in
|
|
39
|
+
|
|
40
|
+
\[Dream-Layer](https://github.com/aggarwamnew/dream-layer): a frozen LLM plus a
|
|
41
|
+
|
|
42
|
+
separate learning loop that observes, forms concepts, and selects questioning
|
|
43
|
+
|
|
44
|
+
strategies. The key empirical finding it operationalizes is that
|
|
45
|
+
|
|
46
|
+
\*\*feedback-driven questioning produces substantially denser knowledge graphs
|
|
47
|
+
|
|
48
|
+
than structure-only or random baselines\*\*, and that keyword accumulation
|
|
49
|
+
|
|
50
|
+
without semantic representation fails to build real connections.
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
\---
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
\## What it does
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
Given a seed corpus and any chat-completion LLM, `aka` will:
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
1\. Extract semantic features from text and promote frequently-seen ones into
|
|
67
|
+
|
|
68
|
+
  stable \*\*concepts\*\* (Module 1).
|
|
69
|
+
|
|
70
|
+
2\. Build an undirected \*\*knowledge graph\*\* where edges encode shared features,
|
|
71
|
+
|
|
72
|
+
  weighted by Jaccard similarity (Module 2).
|
|
73
|
+
|
|
74
|
+
3\. Read structural gaps from that graph and turn them into LLM-ready
|
|
75
|
+
|
|
76
|
+
  \*\*question prompts\*\* using five fixed strategies (Module 3).
|
|
77
|
+
|
|
78
|
+
4\. Choose which strategy to fire next using \*\*Discounted Thompson Sampling\*\*,
|
|
79
|
+
|
|
80
|
+
  rewarded by the information gain each answer produces in the graph
|
|
81
|
+
|
|
82
|
+
  (Module 4).
|
|
83
|
+
|
|
84
|
+
5\. Run the whole thing against a \*\*cached, retried, provider-neutral LLM\*\*
|
|
85
|
+
|
|
86
|
+
  wrapper (Module 5a), with \*\*atomic checkpointing\*\* so runs resume cleanly
|
|
87
|
+
|
|
88
|
+
  (Module 5b).
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
The loop in one line:
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
`observe(text)` -> `ConceptLayer` (features & concepts) -> `KnowledgeGraph` (`GraphDelta` & Jaccard edges) -> `QuestionGenerator` (5 strategies) -> `StrategySelector` (Discounted Thompson Sampling) -> repeat.
|
aka_kg-0.1.0/README.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
\# aka — Autonomous Knowledge Acquisition
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
A minimal, modular library for \*\*autonomous knowledge acquisition\*\*: a frozen
|
|
6
|
+
|
|
7
|
+
LLM is used as an environment, while a lightweight outer learner decides what
|
|
8
|
+
|
|
9
|
+
to ask next, extracts concepts from the answers, and builds a knowledge graph
|
|
10
|
+
|
|
11
|
+
whose structural gaps drive the next round of questions.
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
The design follows the "outer layer" architecture validated in
|
|
16
|
+
|
|
17
|
+
\[Dream-Layer](https://github.com/aggarwamnew/dream-layer): a frozen LLM plus a
|
|
18
|
+
|
|
19
|
+
separate learning loop that observes, forms concepts, and selects questioning
|
|
20
|
+
|
|
21
|
+
strategies. The key empirical finding it operationalizes is that
|
|
22
|
+
|
|
23
|
+
\*\*feedback-driven questioning produces substantially denser knowledge graphs
|
|
24
|
+
|
|
25
|
+
than structure-only or random baselines\*\*, and that keyword accumulation
|
|
26
|
+
|
|
27
|
+
without semantic representation fails to build real connections.
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
\---
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
\## What it does
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Given a seed corpus and any chat-completion LLM, `aka` will:
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
1\. Extract semantic features from text and promote frequently-seen ones into
|
|
44
|
+
|
|
45
|
+
  stable \*\*concepts\*\* (Module 1).
|
|
46
|
+
|
|
47
|
+
2\. Build an undirected \*\*knowledge graph\*\* where edges encode shared features,
|
|
48
|
+
|
|
49
|
+
  weighted by Jaccard similarity (Module 2).
|
|
50
|
+
|
|
51
|
+
3\. Read structural gaps from that graph and turn them into LLM-ready
|
|
52
|
+
|
|
53
|
+
  \*\*question prompts\*\* using five fixed strategies (Module 3).
|
|
54
|
+
|
|
55
|
+
4\. Choose which strategy to fire next using \*\*Discounted Thompson Sampling\*\*,
|
|
56
|
+
|
|
57
|
+
  rewarded by the information gain each answer produces in the graph
|
|
58
|
+
|
|
59
|
+
  (Module 4).
|
|
60
|
+
|
|
61
|
+
5\. Run the whole thing against a \*\*cached, retried, provider-neutral LLM\*\*
|
|
62
|
+
|
|
63
|
+
  wrapper (Module 5a), with \*\*atomic checkpointing\*\* so runs resume cleanly
|
|
64
|
+
|
|
65
|
+
  (Module 5b).
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
The loop in one line:
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
`observe(text)` -> `ConceptLayer` (features & concepts) -> `KnowledgeGraph` (`GraphDelta` & Jaccard edges) -> `QuestionGenerator` (5 strategies) -> `StrategySelector` (Discounted Thompson Sampling) -> repeat.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "aka-kg"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Autonomous Knowledge Acquisition: a frozen LLM outer-loop library with graph-driven question selection"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "OpenSource"}
|
|
12
|
+
authors = [{name = "Vanthony Laster", email = "support@tyrantsole.com"}]
|
|
13
|
+
dependencies = ["numpy>=1.24", "networkx>=3.0"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
llm = ["litellm>=1.0"]
|
|
17
|
+
full = ["aka-kg[llm]", "sentence-transformers>=2.0"]
|
|
18
|
+
dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.4", "mypy>=1.0", "build>=1.0", "twine>=5.0"]
|
|
19
|
+
|
|
20
|
+
[tool.setuptools.packages.find]
|
|
21
|
+
where = ["src"]
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.package-data]
|
|
24
|
+
aka = ["py.typed"]
|
aka_kg-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""aka — Autonomous Knowledge Acquisition.
|
|
2
|
+
|
|
3
|
+
A frozen LLM outer-loop library: extract concepts from text, build a knowledge
|
|
4
|
+
graph from shared features, generate graph-gap-driven questions, and select
|
|
5
|
+
questioning strategies with Discounted Thompson Sampling.
|
|
6
|
+
|
|
7
|
+
Quickstart
|
|
8
|
+
----------
|
|
9
|
+
>>> from aka import ConceptLayer, KnowledgeGraph, QuestionGenerator, Strategy
|
|
10
|
+
>>> from aka import StrategySelector, LLMEnvironment
|
|
11
|
+
>>> env = LLMEnvironment(model="gpt-4o-mini")
|
|
12
|
+
>>> layer, kg = ConceptLayer(), KnowledgeGraph()
|
|
13
|
+
>>> layer.observe("Fish have gills and live in water.")
|
|
14
|
+
>>> kg.update_from_snapshot(layer.snapshot())
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
# -- Module 1: concept layer --------------------------------------------------
|
|
20
|
+
from aka.concept_layer import (
|
|
21
|
+
Concept,
|
|
22
|
+
ConceptLayer,
|
|
23
|
+
Embedder,
|
|
24
|
+
Feature,
|
|
25
|
+
default_embedder,
|
|
26
|
+
extract_features,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# -- Module 2: knowledge graph -------------------------------------------------
|
|
30
|
+
from aka.knowledge_graph import GraphDelta, KnowledgeGraph
|
|
31
|
+
|
|
32
|
+
# -- Module 3: question generation ---------------------------------------------
|
|
33
|
+
from aka.question_generator import Question, QuestionGenerator, Strategy
|
|
34
|
+
|
|
35
|
+
# -- Module 4: strategy selection ----------------------------------------------
|
|
36
|
+
from aka.strategy_selector import ArmState, StrategySelector
|
|
37
|
+
|
|
38
|
+
# -- Module 5a: LLM environment ------------------------------------------------
|
|
39
|
+
from aka.llm_environment import LLMEnvironment
|
|
40
|
+
|
|
41
|
+
# -- Module 5b: persistence ----------------------------------------------------
|
|
42
|
+
from aka.persistence import (
|
|
43
|
+
AKAState,
|
|
44
|
+
restore_graph,
|
|
45
|
+
restore_layer,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
__version__ = "0.1.0"
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
# concept layer
|
|
52
|
+
"ConceptLayer",
|
|
53
|
+
"Concept",
|
|
54
|
+
"Feature",
|
|
55
|
+
"Embedder",
|
|
56
|
+
"default_embedder",
|
|
57
|
+
"extract_features",
|
|
58
|
+
# knowledge graph
|
|
59
|
+
"KnowledgeGraph",
|
|
60
|
+
"GraphDelta",
|
|
61
|
+
# question generation
|
|
62
|
+
"QuestionGenerator",
|
|
63
|
+
"Question",
|
|
64
|
+
"Strategy",
|
|
65
|
+
# strategy selection
|
|
66
|
+
"StrategySelector",
|
|
67
|
+
"ArmState",
|
|
68
|
+
# LLM environment
|
|
69
|
+
"LLMEnvironment",
|
|
70
|
+
# persistence
|
|
71
|
+
"AKAState",
|
|
72
|
+
"restore_layer",
|
|
73
|
+
"restore_graph",
|
|
74
|
+
# metadata
|
|
75
|
+
"__version__",
|
|
76
|
+
]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# aka_loop.py (updated)
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from .concept_layer import ConceptLayer
|
|
4
|
+
from .knowledge_graph import KnowledgeGraph
|
|
5
|
+
from .question_generator import QuestionGenerator, Strategy
|
|
6
|
+
from .strategy_selector import StrategySelector
|
|
7
|
+
from .llm_environment import LLMEnvironment
|
|
8
|
+
from .persistence import AKAState, restore_layer, restore_graph
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def run_aka(
|
|
12
|
+
seed_corpus: list[str],
|
|
13
|
+
env: LLMEnvironment,
|
|
14
|
+
steps: int = 100,
|
|
15
|
+
checkpoint_path: Path | str = ".aka_state.json",
|
|
16
|
+
resume: bool = False,
|
|
17
|
+
):
|
|
18
|
+
state = AKAState(checkpoint_path)
|
|
19
|
+
|
|
20
|
+
if resume and (saved := state.load()):
|
|
21
|
+
layer = restore_layer(saved["concept_layer"])
|
|
22
|
+
kg = restore_graph(saved["knowledge_graph"])
|
|
23
|
+
selector = AKAState.restore_selector(saved["selector"], list(Strategy))
|
|
24
|
+
start_step = saved["step"]
|
|
25
|
+
print(f"Resumed from step {start_step}")
|
|
26
|
+
else:
|
|
27
|
+
layer = ConceptLayer(promotion_threshold=3)
|
|
28
|
+
kg = KnowledgeGraph()
|
|
29
|
+
selector = StrategySelector(strategies=list(Strategy))
|
|
30
|
+
start_step = 0
|
|
31
|
+
for text in seed_corpus:
|
|
32
|
+
layer.observe(text)
|
|
33
|
+
kg.update_from_snapshot(layer.snapshot())
|
|
34
|
+
|
|
35
|
+
qg = QuestionGenerator(kg)
|
|
36
|
+
|
|
37
|
+
for step in range(start_step, steps):
|
|
38
|
+
candidates = {s: qg.generate(s) for s in Strategy}
|
|
39
|
+
strategy = selector.select(candidates)
|
|
40
|
+
if strategy is None:
|
|
41
|
+
print(f"Step {step}: no candidates, stopping")
|
|
42
|
+
break
|
|
43
|
+
|
|
44
|
+
question = candidates[strategy]
|
|
45
|
+
answer = env.query(question.prompt)
|
|
46
|
+
|
|
47
|
+
layer.observe(answer)
|
|
48
|
+
delta = kg.update_from_snapshot(layer.snapshot())
|
|
49
|
+
selector.update(strategy, delta.information_gain())
|
|
50
|
+
|
|
51
|
+
# Checkpoint every 10 steps
|
|
52
|
+
if (step + 1) % 10 == 0:
|
|
53
|
+
state.save(layer, kg, selector, step + 1)
|
|
54
|
+
|
|
55
|
+
state.save(layer, kg, selector, steps)
|
|
56
|
+
return layer, kg, selector
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# concept_layer.py
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Callable
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# ---------------------------------------------------------------------------
|
|
13
|
+
# Data structures
|
|
14
|
+
# ---------------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Feature:
|
|
18
|
+
"""A normalized semantic unit extracted from text."""
|
|
19
|
+
key: str # canonical short label, e.g. "has_gills"
|
|
20
|
+
embedding: np.ndarray # L2-normalized vector
|
|
21
|
+
count: int = 1 # how many times observed
|
|
22
|
+
examples: list[str] = field(default_factory=list) # source snippets
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Concept:
|
|
27
|
+
"""A promoted, stable knowledge node."""
|
|
28
|
+
id: str
|
|
29
|
+
label: str
|
|
30
|
+
embedding: np.ndarray
|
|
31
|
+
feature_keys: set[str] = field(default_factory=set)
|
|
32
|
+
created_at: int = 0 # step index when born
|
|
33
|
+
observations: int = 1
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
# Embedding backend (pluggable)
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
class Embedder:
|
|
41
|
+
"""Wraps any sentence-embedding function. Keeps the layer model-agnostic."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, fn: Callable[[str], np.ndarray], dim: int):
|
|
44
|
+
self._fn = fn
|
|
45
|
+
self.dim = dim
|
|
46
|
+
|
|
47
|
+
def __call__(self, text: str) -> np.ndarray:
|
|
48
|
+
v = np.asarray(self._fn(text), dtype=np.float32)
|
|
49
|
+
if v.shape != (self.dim,):
|
|
50
|
+
raise ValueError(f"Embedder returned {v.shape}, expected ({self.dim},)")
|
|
51
|
+
# L2 normalize - critical to prevent magnitude drift (mode collapse guard #1)
|
|
52
|
+
n = np.linalg.norm(v)
|
|
53
|
+
return v / n if n > 1e-8 else v
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def default_embedder() -> Embedder:
|
|
57
|
+
"""Tiny hash-based fallback so the module runs without external deps.
|
|
58
|
+
Swap for sentence-transformers / OpenAI embeddings in production."""
|
|
59
|
+
dim = 256
|
|
60
|
+
|
|
61
|
+
def _hash_embed(text: str) -> np.ndarray:
|
|
62
|
+
vec = np.zeros(dim, dtype=np.float32)
|
|
63
|
+
for tok in re.findall(r"[a-z0-9]+", text.lower()):
|
|
64
|
+
h = int(hashlib.md5(tok.encode()).hexdigest(), 16)
|
|
65
|
+
vec[h % dim] += 1.0
|
|
66
|
+
return vec
|
|
67
|
+
|
|
68
|
+
return Embedder(_hash_embed, dim)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Feature extraction
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
_STOP = {
|
|
76
|
+
"a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
|
|
77
|
+
"of", "to", "in", "on", "at", "for", "with", "by", "from", "as",
|
|
78
|
+
"and", "or", "but", "if", "then", "than", "that", "this", "these",
|
|
79
|
+
"it", "its", "they", "them", "their", "we", "you", "i", "he", "she",
|
|
80
|
+
"do", "does", "did", "have", "has", "had", "can", "could", "will",
|
|
81
|
+
"would", "should", "may", "might", "must", "not", "no", "yes",
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
_CUE_PATTERNS = [
|
|
85
|
+
re.compile(r"\b(?P<subj>[\w\- ]+?)\s+(?:has|have|possesses|exhibits)\s+(?P<obj>[\w\- ]+)", re.I),
|
|
86
|
+
re.compile(r"\b(?P<subj>[\w\- ]+?)\s+(?:can|could|is able to)\s+(?P<obj>[\w\- ]+)", re.I),
|
|
87
|
+
re.compile(r"\b(?P<subj>[\w\- ]+?)\s+(?:is|are)\s+(?:a|an|the)?\s*(?P<obj>[\w\- ]+)", re.I),
|
|
88
|
+
re.compile(r"\b(?P<subj>[\w\- ]+?)\s+(?:uses|produces|generates|creates)\s+(?P<obj>[\w\- ]+)", re.I),
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _tokenize(text: str) -> list[str]:
|
|
93
|
+
return [t for t in re.findall(r"[a-z0-9]+", text.lower()) if t not in _STOP and len(t) > 2]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def extract_features(text: str, max_features: int = 12) -> list[str]:
|
|
97
|
+
"""Return canonical short feature keys, e.g. ['has_gills', 'aquatic', 'respires']."""
|
|
98
|
+
features: list[str] = []
|
|
99
|
+
|
|
100
|
+
# 1. Cue-based relational features
|
|
101
|
+
for pat in _CUE_PATTERNS:
|
|
102
|
+
for m in pat.finditer(text):
|
|
103
|
+
obj_tokens = _tokenize(m.group("obj"))
|
|
104
|
+
if obj_tokens:
|
|
105
|
+
features.append("has_" + obj_tokens[0])
|
|
106
|
+
features.extend(obj_tokens)
|
|
107
|
+
|
|
108
|
+
# 2. Content tokens as standalone features
|
|
109
|
+
features.extend(_tokenize(text))
|
|
110
|
+
|
|
111
|
+
# 3. Deduplicate, preserve order, cap length
|
|
112
|
+
seen, out = set(), []
|
|
113
|
+
for f in features:
|
|
114
|
+
if f not in seen:
|
|
115
|
+
seen.add(f)
|
|
116
|
+
out.append(f)
|
|
117
|
+
if len(out) >= max_features:
|
|
118
|
+
break
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------------------------------------------------------------------
|
|
123
|
+
# Concept formation layer
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
class ConceptLayer:
|
|
127
|
+
"""Stateful layer that turns text into Features and promotes them to Concepts.
|
|
128
|
+
|
|
129
|
+
Parameters
|
|
130
|
+
----------
|
|
131
|
+
embedder : Embedder
|
|
132
|
+
novelty_threshold : float
|
|
133
|
+
Cosine *distance* above which an observation spawns a NEW concept
|
|
134
|
+
instead of merging into an existing one. 0.35 ~= "fairly different".
|
|
135
|
+
promotion_threshold : int
|
|
136
|
+
How many times a feature must be seen before it becomes a Concept.
|
|
137
|
+
max_concepts : int
|
|
138
|
+
Hard cap; when hit, we force-merge instead of spawning.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
def __init__(
|
|
142
|
+
self,
|
|
143
|
+
embedder: Embedder | None = None,
|
|
144
|
+
novelty_threshold: float = 0.35,
|
|
145
|
+
promotion_threshold: int = 3,
|
|
146
|
+
max_concepts: int = 512,
|
|
147
|
+
):
|
|
148
|
+
self.embedder = embedder or default_embedder()
|
|
149
|
+
self.novelty_threshold = novelty_threshold
|
|
150
|
+
self.promotion_threshold = promotion_threshold
|
|
151
|
+
self.max_concepts = max_concepts
|
|
152
|
+
|
|
153
|
+
self.features: dict[str, Feature] = {}
|
|
154
|
+
self.concepts: dict[str, Concept] = {}
|
|
155
|
+
self._cooccurring: dict[str, set[str]] = {}
|
|
156
|
+
self._step = 0
|
|
157
|
+
|
|
158
|
+
def observe(self, text: str) -> dict:
|
|
159
|
+
"""Run one observation through the full pipeline."""
|
|
160
|
+
self._step += 1
|
|
161
|
+
report = {
|
|
162
|
+
"step": self._step,
|
|
163
|
+
"new_features": [],
|
|
164
|
+
"promoted_concepts": [],
|
|
165
|
+
"matched_concepts": [],
|
|
166
|
+
"noisy_features": [],
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
# Stage 1: extract
|
|
170
|
+
feature_keys = extract_features(text)
|
|
171
|
+
valid_keys: list[str] = []
|
|
172
|
+
|
|
173
|
+
# Stage 2: embed + novelty check per feature
|
|
174
|
+
for key in feature_keys:
|
|
175
|
+
emb = self.embedder(key.replace("_", " "))
|
|
176
|
+
|
|
177
|
+
if np.linalg.norm(emb) < 0.5:
|
|
178
|
+
report["noisy_features"].append(key)
|
|
179
|
+
continue
|
|
180
|
+
|
|
181
|
+
valid_keys.append(key)
|
|
182
|
+
|
|
183
|
+
if key in self.features:
|
|
184
|
+
f = self.features[key]
|
|
185
|
+
f.count += 1
|
|
186
|
+
f.embedding = self._blend(f.embedding, emb, weight=1.0 / f.count)
|
|
187
|
+
if len(f.examples) < 3:
|
|
188
|
+
f.examples.append(text[:120])
|
|
189
|
+
else:
|
|
190
|
+
self.features[key] = Feature(key=key, embedding=emb, examples=[text[:120]])
|
|
191
|
+
report["new_features"].append(key)
|
|
192
|
+
|
|
193
|
+
valid_set = set(valid_keys)
|
|
194
|
+
for key in valid_keys:
|
|
195
|
+
cooc = self._cooccurring.setdefault(key, set())
|
|
196
|
+
cooc.update(valid_set - {key})
|
|
197
|
+
|
|
198
|
+
# Stage 3: promotion - features seen enough times become concepts
|
|
199
|
+
for key, feat in self.features.items():
|
|
200
|
+
if feat.count < self.promotion_threshold:
|
|
201
|
+
continue
|
|
202
|
+
existing = self._find_concept(feat.embedding)
|
|
203
|
+
if existing is None:
|
|
204
|
+
if len(self.concepts) < self.max_concepts:
|
|
205
|
+
c = self._spawn_concept(feat)
|
|
206
|
+
report["promoted_concepts"].append(c.id)
|
|
207
|
+
else:
|
|
208
|
+
self._merge_into(existing, feat)
|
|
209
|
+
if existing.id not in report["matched_concepts"]:
|
|
210
|
+
report["matched_concepts"].append(existing.id)
|
|
211
|
+
|
|
212
|
+
return report
|
|
213
|
+
|
|
214
|
+
def snapshot(self) -> dict:
|
|
215
|
+
"""Serializable state for the knowledge graph module."""
|
|
216
|
+
return {
|
|
217
|
+
"step": self._step,
|
|
218
|
+
"features": {
|
|
219
|
+
k: {"key": v.key, "count": v.count, "embedding": v.embedding.tolist()}
|
|
220
|
+
for k, v in self.features.items()
|
|
221
|
+
},
|
|
222
|
+
"concepts": {
|
|
223
|
+
k: {
|
|
224
|
+
"id": v.id,
|
|
225
|
+
"label": v.label,
|
|
226
|
+
"embedding": v.embedding.tolist(),
|
|
227
|
+
"feature_keys": sorted(v.feature_keys),
|
|
228
|
+
"created_at": v.created_at,
|
|
229
|
+
"observations": v.observations,
|
|
230
|
+
}
|
|
231
|
+
for k, v in self.concepts.items()
|
|
232
|
+
},
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
@staticmethod
|
|
236
|
+
def _blend(old: np.ndarray, new: np.ndarray, weight: float) -> np.ndarray:
|
|
237
|
+
"""Running average, re-normalized - keeps embeddings on the unit sphere."""
|
|
238
|
+
merged = (1 - weight) * old + weight * new
|
|
239
|
+
n = np.linalg.norm(merged)
|
|
240
|
+
return merged / n if n > 1e-8 else old
|
|
241
|
+
|
|
242
|
+
def _find_concept(self, emb: np.ndarray) -> Concept | None:
|
|
243
|
+
if not self.concepts:
|
|
244
|
+
return None
|
|
245
|
+
best, best_sim = None, -1.0
|
|
246
|
+
for c in self.concepts.values():
|
|
247
|
+
sim = float(np.dot(c.embedding, emb))
|
|
248
|
+
if sim > best_sim:
|
|
249
|
+
best, best_sim = c, sim
|
|
250
|
+
if best is not None and (1.0 - best_sim) < self.novelty_threshold:
|
|
251
|
+
return best
|
|
252
|
+
return None
|
|
253
|
+
|
|
254
|
+
def _spawn_concept(self, feat: Feature) -> Concept:
|
|
255
|
+
cid = f"c{len(self.concepts) + 1:04d}"
|
|
256
|
+
cooc = self._cooccurring.get(feat.key, set())
|
|
257
|
+
c = Concept(
|
|
258
|
+
id=cid,
|
|
259
|
+
label=feat.key,
|
|
260
|
+
embedding=feat.embedding.copy(),
|
|
261
|
+
feature_keys={feat.key} | cooc,
|
|
262
|
+
created_at=self._step,
|
|
263
|
+
)
|
|
264
|
+
self.concepts[cid] = c
|
|
265
|
+
return c
|
|
266
|
+
|
|
267
|
+
def _merge_into(self, concept: Concept, feat: Feature) -> None:
|
|
268
|
+
cooc = self._cooccurring.get(feat.key, set())
|
|
269
|
+
concept.feature_keys.update({feat.key} | cooc)
|
|
270
|
+
concept.observations += 1
|
|
271
|
+
concept.embedding = self._blend(
|
|
272
|
+
concept.embedding, feat.embedding, weight=1.0 / concept.observations
|
|
273
|
+
)
|