verimem 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- engram/__init__.py +44 -0
- engram/_call_telemetry.py +27 -0
- engram/_compat.py +99 -0
- engram/_hang_watchdog.py +87 -0
- engram/_proc_quiet.py +37 -0
- engram/_singleton_guard.py +109 -0
- engram/_sqlite_pragma.py +25 -0
- engram/_telemetry_prefixes.py +97 -0
- engram/active_learning.py +114 -0
- engram/adaptive_threshold.py +57 -0
- engram/admission_cleanup.py +248 -0
- engram/admission_gate.py +188 -0
- engram/agent.py +103 -0
- engram/agent_scope.py +108 -0
- engram/agent_specialization.py +80 -0
- engram/agent_workload.py +60 -0
- engram/airgap.py +115 -0
- engram/analogy.py +134 -0
- engram/ann_cache.py +118 -0
- engram/ann_index.py +82 -0
- engram/anomaly_detection.py +114 -0
- engram/anti_confab_gate.py +783 -0
- engram/anti_confabulation.py +411 -0
- engram/apply_recommendations.py +105 -0
- engram/audit_summary.py +71 -0
- engram/audit_tail.py +78 -0
- engram/auto_dream_trigger.py +284 -0
- engram/auto_dream_worker.py +437 -0
- engram/backup.py +511 -0
- engram/bench_corpus_scale.py +103 -0
- engram/bench_harness.py +732 -0
- engram/betweenness_cache.py +162 -0
- engram/bm25_rank.py +95 -0
- engram/briefing.py +364 -0
- engram/briefing_by_project.py +131 -0
- engram/briefing_stats.py +155 -0
- engram/capability_token.py +263 -0
- engram/causal_extract.py +121 -0
- engram/causal_skill_mine.py +63 -0
- engram/chain_complexity.py +36 -0
- engram/chain_render.py +79 -0
- engram/chain_validate.py +87 -0
- engram/chain_visualize.py +38 -0
- engram/cli.py +2373 -0
- engram/client.py +279 -0
- engram/code.py +736 -0
- engram/codebase_ingest.py +283 -0
- engram/coding_reflection.py +188 -0
- engram/coherence_check.py +183 -0
- engram/community_detector.py +276 -0
- engram/compilation.py +311 -0
- engram/compose_macro.py +68 -0
- engram/config.py +553 -0
- engram/consolidation.py +535 -0
- engram/context_engine.py +102 -0
- engram/contradiction.py +641 -0
- engram/conversation_ingest.py +251 -0
- engram/corpus_diff.py +118 -0
- engram/corpus_health_metrics.py +149 -0
- engram/corpus_health_score.py +194 -0
- engram/corpus_size.py +68 -0
- engram/correction_velocity.py +138 -0
- engram/corroboration.py +135 -0
- engram/counterfactual_rollout.py +60 -0
- engram/cross_agent_consensus.py +81 -0
- engram/cross_encoder_rerank.py +133 -0
- engram/curate_pipeline.py +121 -0
- engram/daemon_runner.py +224 -0
- engram/daemon_spawn.py +132 -0
- engram/dashboard.py +186 -0
- engram/dashboard_overview.py +65 -0
- engram/dashboard_overview_v2.py +51 -0
- engram/dashboard_routes/__init__.py +54 -0
- engram/dashboard_routes/active_memory.py +227 -0
- engram/dashboard_routes/auth.py +143 -0
- engram/dashboard_routes/chat.py +157 -0
- engram/dashboard_routes/episodes.py +42 -0
- engram/dashboard_routes/events.py +142 -0
- engram/dashboard_routes/health.py +12 -0
- engram/dashboard_routes/layout.py +115 -0
- engram/dashboard_routes/lineage.py +50 -0
- engram/dashboard_routes/memory_map.py +491 -0
- engram/dashboard_routes/settings.py +529 -0
- engram/dashboard_routes/skills.py +197 -0
- engram/dashboard_routes/welcome.py +88 -0
- engram/dashboard_widget.py +176 -0
- engram/decay.py +79 -0
- engram/decay_job.py +217 -0
- engram/decay_simulate.py +43 -0
- engram/dentate_gyrus.py +104 -0
- engram/derivation_detect.py +79 -0
- engram/documents.py +275 -0
- engram/dream.py +908 -0
- engram/dream_community_hook.py +102 -0
- engram/dream_emergence_hook.py +134 -0
- engram/dream_stuck_hook.py +98 -0
- engram/dream_thompson_hook.py +88 -0
- engram/editfmt.py +286 -0
- engram/embedding.py +395 -0
- engram/embedding_quantize.py +92 -0
- engram/emerging_briefing.py +98 -0
- engram/emerging_patterns.py +78 -0
- engram/emerging_skill_register.py +263 -0
- engram/encode_service.py +378 -0
- engram/engram_syscall_mcp.py +234 -0
- engram/entity_extract_lite.py +121 -0
- engram/entity_kg.py +1133 -0
- engram/entity_populate.py +163 -0
- engram/episode.py +130 -0
- engram/episode_batch_get.py +40 -0
- engram/episode_classify.py +97 -0
- engram/episode_clusters.py +82 -0
- engram/episode_dedup.py +160 -0
- engram/episode_diff.py +56 -0
- engram/episode_recent_failures.py +37 -0
- engram/episode_replay.py +65 -0
- engram/episode_rollup.py +104 -0
- engram/episode_summary.py +55 -0
- engram/episodes_with_skill.py +59 -0
- engram/epistemic_health.py +126 -0
- engram/event_jsonl_log.py +135 -0
- engram/evidence_requirement.py +96 -0
- engram/fact_chain.py +85 -0
- engram/fact_priority.py +81 -0
- engram/facts_aggregate_overall.py +59 -0
- engram/facts_by_confidence.py +44 -0
- engram/facts_cluster_by_topic.py +61 -0
- engram/facts_conflict.py +443 -0
- engram/facts_disagreement.py +90 -0
- engram/facts_export.py +46 -0
- engram/facts_merge.py +78 -0
- engram/facts_recent.py +33 -0
- engram/facts_topic_merge.py +64 -0
- engram/facts_topics.py +57 -0
- engram/failure_clusters.py +79 -0
- engram/failure_diagnosis.py +106 -0
- engram/find_duplicate_facts.py +70 -0
- engram/find_duplicates.py +73 -0
- engram/freshness.py +29 -0
- engram/freshness_check.py +181 -0
- engram/fuse_recall.py +128 -0
- engram/graphviz_export.py +49 -0
- engram/grounding_gate.py +404 -0
- engram/hallucination_rate.py +123 -0
- engram/highway_nodes.py +93 -0
- engram/hippo_pagerank.py +165 -0
- engram/holographic_memory.py +358 -0
- engram/hooks/__init__.py +12 -0
- engram/hooks/pre_tool_use.py +278 -0
- engram/hopfield.py +150 -0
- engram/hot_reload.py +216 -0
- engram/ide.py +1271 -0
- engram/interactive_judge.py +294 -0
- engram/jsonutil.py +60 -0
- engram/justified_memory.py +264 -0
- engram/knowledge_graph_export.py +113 -0
- engram/l1_approval_detector.py +101 -0
- engram/l1_automated_detector.py +133 -0
- engram/l1_completion_detector.py +141 -0
- engram/l1_documentation_detector.py +85 -0
- engram/l1_evidence.py +40 -0
- engram/l1_extended_detector.py +132 -0
- engram/l1_monitored_detector.py +90 -0
- engram/l1_orphan_detector.py +121 -0
- engram/l1_performance_detector.py +268 -0
- engram/l1_production_ready_detector.py +139 -0
- engram/l1_quantitative_detector.py +134 -0
- engram/l1_security_detector.py +125 -0
- engram/l1_tested_detector.py +111 -0
- engram/l1_works_detector.py +142 -0
- engram/lab_live.py +168 -0
- engram/lab_longmemeval_adapter.py +127 -0
- engram/legacy_audit.py +231 -0
- engram/legacy_cleanup.py +133 -0
- engram/lineage_trace.py +267 -0
- engram/live_introspection.py +103 -0
- engram/llm.py +1656 -0
- engram/llm_keywords_augment.py +161 -0
- engram/llm_keywords_batch.py +142 -0
- engram/local_grounding.py +200 -0
- engram/local_relation.py +219 -0
- engram/mcp_server.py +12803 -0
- engram/memory.py +2598 -0
- engram/memory_compaction.py +89 -0
- engram/memory_health_report.py +109 -0
- engram/mesh_memory.py +440 -0
- engram/metacognition.py +97 -0
- engram/metrics_export.py +68 -0
- engram/metrics_one_liner.py +67 -0
- engram/migrations/__init__.py +159 -0
- engram/mode.py +74 -0
- engram/momentum_macro.py +109 -0
- engram/multi_signal_fusion.py +76 -0
- engram/narration.py +214 -0
- engram/narration_llm.py +91 -0
- engram/observability.py +267 -0
- engram/op_supervisor.py +281 -0
- engram/openie.py +327 -0
- engram/oracle.py +127 -0
- engram/outcome_by_skill.py +70 -0
- engram/outcome_pattern.py +75 -0
- engram/outcome_predict.py +95 -0
- engram/outcome_timeseries.py +89 -0
- engram/outlier_summary.py +35 -0
- engram/parallel_drafter.py +89 -0
- engram/ppr_seed.py +131 -0
- engram/predicate_derivation.py +209 -0
- engram/predicate_graph_check.py +119 -0
- engram/preload.py +144 -0
- engram/proactive_step_injector.py +139 -0
- engram/promote_chain.py +82 -0
- engram/prompt_injection.py +288 -0
- engram/prompt_skeleton.py +121 -0
- engram/prompts.py +222 -0
- engram/provenance_validator.py +366 -0
- engram/provider_registry.py +211 -0
- engram/providers.yaml +568 -0
- engram/quantity_match.py +242 -0
- engram/query.py +99 -0
- engram/rank_list_builders.py +159 -0
- engram/reasoning.py +228 -0
- engram/recall_chain.py +87 -0
- engram/recall_usage.py +190 -0
- engram/recommend_actions.py +99 -0
- engram/redaction.py +116 -0
- engram/repomap.py +324 -0
- engram/rerank.py +100 -0
- engram/resonator_cli.py +179 -0
- engram/resonator_memory.py +495 -0
- engram/resonator_text_bridge.py +149 -0
- engram/resource_monitor.py +220 -0
- engram/risk_guard.py +86 -0
- engram/sandbox.py +897 -0
- engram/schema_abstraction.py +132 -0
- engram/scope.py +162 -0
- engram/second_pass_louvain.py +351 -0
- engram/selection.py +277 -0
- engram/self_curation.py +177 -0
- engram/self_heal.py +121 -0
- engram/self_model.py +381 -0
- engram/self_model_refresh.py +153 -0
- engram/semantic.py +4489 -0
- engram/semantic_conflict.py +198 -0
- engram/session_recap.py +76 -0
- engram/settings.py +202 -0
- engram/settings_v2.py +115 -0
- engram/skill.py +826 -0
- engram/skill_archive.py +62 -0
- engram/skill_bottleneck.py +67 -0
- engram/skill_clone.py +61 -0
- engram/skill_co_occurrence.py +67 -0
- engram/skill_combo_mining.py +60 -0
- engram/skill_compile_macro.py +92 -0
- engram/skill_composer.py +147 -0
- engram/skill_cooccurrence_graph.py +62 -0
- engram/skill_crossover.py +80 -0
- engram/skill_diff_render.py +79 -0
- engram/skill_dot.py +72 -0
- engram/skill_draft_persist.py +134 -0
- engram/skill_drafter.py +245 -0
- engram/skill_drafts_list.py +109 -0
- engram/skill_drift.py +87 -0
- engram/skill_emergence_detector.py +364 -0
- engram/skill_export.py +57 -0
- engram/skill_exposure_audit.py +367 -0
- engram/skill_failure_audit.py +57 -0
- engram/skill_health.py +143 -0
- engram/skill_inspect.py +116 -0
- engram/skill_lineage_full.py +88 -0
- engram/skill_lineage_metrics.py +72 -0
- engram/skill_merge_pair.py +83 -0
- engram/skill_name_dedup.py +152 -0
- engram/skill_path.py +64 -0
- engram/skill_promote_from_emerging.py +152 -0
- engram/skill_promote_review.py +94 -0
- engram/skill_promote_threshold.py +60 -0
- engram/skill_provenance.py +52 -0
- engram/skill_recent.py +36 -0
- engram/skill_recommend_failure.py +60 -0
- engram/skill_recover.py +62 -0
- engram/skill_roi.py +57 -0
- engram/skill_semantic_dedup.py +270 -0
- engram/skill_signature.py +54 -0
- engram/skill_stuck_diagnostic.py +114 -0
- engram/skill_usage_decay.py +54 -0
- engram/skill_warmup.py +69 -0
- engram/skills_aggregate_stats.py +45 -0
- engram/skills_orphan.py +44 -0
- engram/skills_search_by_predicate.py +54 -0
- engram/skills_top_failing.py +46 -0
- engram/skills_top_used.py +33 -0
- engram/skills_topology.py +62 -0
- engram/skills_untested.py +35 -0
- engram/sleep.py +1308 -0
- engram/smart_pruning.py +87 -0
- engram/snapshot_at_time.py +88 -0
- engram/sos_compensator.py +240 -0
- engram/stable_partition.py +211 -0
- engram/static/chat.js +408 -0
- engram/static/dashboard.css +559 -0
- engram/static/events.js +76 -0
- engram/static/memory_map.js +402 -0
- engram/static/settings.js +355 -0
- engram/static/skills.js +78 -0
- engram/stats_velocity.py +38 -0
- engram/strips.py +119 -0
- engram/success_factor.py +57 -0
- engram/successor_repr.py +426 -0
- engram/swarm/__init__.py +25 -0
- engram/swarm/bridge.py +196 -0
- engram/swarm/cli.py +207 -0
- engram/swarm/lifecycle.py +136 -0
- engram/swarm/orchestrator.py +241 -0
- engram/swarm/schemas.py +91 -0
- engram/swarm/spawn.py +153 -0
- engram/swarm/state.py +97 -0
- engram/symbolic_inference.py +137 -0
- engram/syntax_pollution.py +197 -0
- engram/syscall_bridge.py +383 -0
- engram/teams/__init__.py +40 -0
- engram/teams/bridge.py +70 -0
- engram/teams/cli.py +456 -0
- engram/teams/harness.py +188 -0
- engram/teams/inbox.py +152 -0
- engram/teams/protocol.py +121 -0
- engram/telemetry_analyzer.py +104 -0
- engram/templates/_layout.html +54 -0
- engram/templates/active_memory.html +196 -0
- engram/templates/chat.html +72 -0
- engram/templates/episode_detail.html +67 -0
- engram/templates/episodes.html +121 -0
- engram/templates/events.html +42 -0
- engram/templates/lineage.html +83 -0
- engram/templates/metrics.html +22 -0
- engram/templates/overview.html +134 -0
- engram/templates/settings.html +199 -0
- engram/templates/skill_detail.html +149 -0
- engram/templates/skills.html +159 -0
- engram/templates/welcome.html +52 -0
- engram/temporal_context.py +166 -0
- engram/temporal_narrative.py +168 -0
- engram/thompson_sampling.py +100 -0
- engram/tier2_judge.py +335 -0
- engram/time_decay.py +109 -0
- engram/time_decay_score.py +89 -0
- engram/tool_registry.py +292 -0
- engram/tools.py +346 -0
- engram/tools_extra.py +1218 -0
- engram/topic_cleanup_suggestions.py +125 -0
- engram/topic_normalization.py +142 -0
- engram/topic_priors.py +109 -0
- engram/trace_alignment.py +370 -0
- engram/trajectory.py +105 -0
- engram/trajectory_diff.py +83 -0
- engram/trajectory_fork.py +62 -0
- engram/trajectory_render.py +96 -0
- engram/transcript_index.py +299 -0
- engram/transcript_ingest.py +221 -0
- engram/transcript_promote.py +81 -0
- engram/trunc.py +103 -0
- engram/trust_calibration.py +82 -0
- engram/trust_calibration_eval.py +163 -0
- engram/trust_report.py +131 -0
- engram/trust_score.py +147 -0
- engram/trust_signal.py +164 -0
- engram/trusted_writer.py +35 -0
- engram/truth_reconciliation.py +351 -0
- engram/tui.py +446 -0
- engram/undo_log.py +300 -0
- engram/validate_claim.py +350 -0
- engram/wake.py +1717 -0
- engram/wake_strategy.py +438 -0
- engram/working_memory.py +189 -0
- engram/world_model.py +194 -0
- hippoagent/__init__.py +100 -0
- verimem/__init__.py +114 -0
- verimem-0.3.0.dist-info/METADATA +1327 -0
- verimem-0.3.0.dist-info/RECORD +382 -0
- verimem-0.3.0.dist-info/WHEEL +5 -0
- verimem-0.3.0.dist-info/entry_points.txt +4 -0
- verimem-0.3.0.dist-info/licenses/LICENSE +21 -0
- verimem-0.3.0.dist-info/top_level.txt +3 -0
engram/__init__.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Engram — persistent memory layer for LLM agents.
|
|
2
|
+
|
|
3
|
+
Architecture: hippocampal-cortical inspired learning loop.
|
|
4
|
+
- Wake: ReAct execution, episodic recording.
|
|
5
|
+
- Sleep: replay, skill synthesis, fitness selection.
|
|
6
|
+
- Skills are persistent artifacts (not weights), inspectable and versioned.
|
|
7
|
+
|
|
8
|
+
Formerly named ``hippoagent``. The legacy ``hippoagent`` package is a
|
|
9
|
+
backward-compat shim that re-exports this module (see top-level
|
|
10
|
+
``hippoagent/__init__.py``); it will be removed ~2026-08-13.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
# Keep in lockstep with pyproject [project].version and .claude-plugin/
|
|
15
|
+
# plugin.json — enforced by tests/test_version_single_source.py (audit#2 C-4).
|
|
16
|
+
__version__ = "0.3.0"
|
|
17
|
+
|
|
18
|
+
# Initialize backward-compat env mirror (HIPPO_* ↔ ENGRAM_*) at import time
|
|
19
|
+
# so the rest of the package — and anyone importing ``engram`` — sees a
|
|
20
|
+
# consistent environment regardless of which prefix the user has set.
|
|
21
|
+
from . import _compat as _compat
|
|
22
|
+
|
|
23
|
+
_compat.init_env_aliases()
|
|
24
|
+
|
|
25
|
+
# ENGRAM_MODE single-knob (subscription | byok | local): derive the lower-level
|
|
26
|
+
# flags (HIPPO_HOSTED / HIPPO_LLM_PROVIDER / HF_HUB_OFFLINE / ...) BEFORE any
|
|
27
|
+
# submodule (config, llm, settings) reads the env. Runs on ANY `import
|
|
28
|
+
# engram[.x]`. setdefault-safe (never clobbers an explicit flag); no-op when
|
|
29
|
+
# ENGRAM_MODE is unset.
|
|
30
|
+
from .mode import apply_engram_mode as _apply_engram_mode # noqa: E402
|
|
31
|
+
|
|
32
|
+
_apply_engram_mode()
|
|
33
|
+
|
|
34
|
+
# Public turnkey SDK — exposed lazily so ``import engram`` stays light (the
|
|
35
|
+
# Memory client pulls in the embedding/semantic stack only when first used).
|
|
36
|
+
__all__ = ["Memory", "Client"]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def __getattr__(name: str): # PEP 562 lazy attribute access
|
|
40
|
+
if name in ("Memory", "Client"):
|
|
41
|
+
from .client import Client, Memory
|
|
42
|
+
|
|
43
|
+
return {"Memory": Memory, "Client": Client}[name]
|
|
44
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Detect cross-LLM call-telemetry episodes.
|
|
2
|
+
|
|
3
|
+
The bridge auto-saves every ask_agy / ask_gemini / ask_claude / … as an episode
|
|
4
|
+
whose ``task_text`` opens with ``[<llm>-call …]``. These are machine telemetry,
|
|
5
|
+
not user tasks: 2026-06-13 they were 123/554 (22%) of the live episode store,
|
|
6
|
+
polluting recall and diluting the emerging/correction/risk quartet.
|
|
7
|
+
|
|
8
|
+
Shared single source of truth for the WRITE-time episode gate (engram.memory
|
|
9
|
+
routes them to a separate ``episode_telemetry`` table) and the READ-time briefing
|
|
10
|
+
filter. Leaf module: imports nothing from ``engram`` → no import cycle.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
_CALL_TELEMETRY_RE = re.compile(
|
|
17
|
+
r"^\s*\[(?:agy|gemini|claude|kimi|deepseek|grok|gpt|qwen|llama|mistral)-call\b",
|
|
18
|
+
re.IGNORECASE,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def is_call_telemetry(task_text: str | None) -> bool:
|
|
23
|
+
"""True when the episode's ``task_text`` is a cross-LLM call record, not a task."""
|
|
24
|
+
return bool(_CALL_TELEMETRY_RE.match(task_text or ""))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
__all__ = ["is_call_telemetry"]
|
engram/_compat.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Backward-compat bridge for the cycle #41 rename (hippoagent → engram).
|
|
2
|
+
|
|
3
|
+
Two surfaces are bridged here so existing user configurations keep working:
|
|
4
|
+
|
|
5
|
+
1. **Environment variables**: every ``HIPPO_*`` env var visible at process
|
|
6
|
+
start is mirrored to ``ENGRAM_*`` (and vice versa) via
|
|
7
|
+
:func:`init_env_aliases`. Existing ``HIPPO_HOSTED=1``, ``HIPPO_DATA_DIR``,
|
|
8
|
+
``HIPPO_AUTH_TOKEN``, etc. continue to work without code changes; new
|
|
9
|
+
``ENGRAM_*`` names are picked up by older code paths through the same
|
|
10
|
+
mirror. The mirror uses :py:meth:`os.environ.setdefault` so an explicit
|
|
11
|
+
value on one side never overrides an explicit value on the other.
|
|
12
|
+
|
|
13
|
+
2. **User data directory**: :func:`data_dir` returns ``~/.engram`` if it
|
|
14
|
+
exists, falls back to ``~/.hippoagent`` if only the old dir exists,
|
|
15
|
+
otherwise creates ``~/.engram``. This keeps existing installations
|
|
16
|
+
reading their data while new installations get the new path.
|
|
17
|
+
|
|
18
|
+
The module is intentionally tiny and stdlib-only — it's imported by
|
|
19
|
+
``engram/__init__.py`` and must not have any heavy dependencies.
|
|
20
|
+
|
|
21
|
+
Scheduled removal: ~2026-08-13 (3 months from rename). After that, all
|
|
22
|
+
code is expected to use ``ENGRAM_*`` env names and the ``~/.engram``
|
|
23
|
+
path; this module can be deleted and ``HIPPO_*`` configs will start
|
|
24
|
+
failing — by then any user still on the old names has had 3 months
|
|
25
|
+
of clear deprecation warnings to migrate.
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import os
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
_PREFIX_OLD = "HIPPO_"
|
|
33
|
+
_PREFIX_NEW = "ENGRAM_"
|
|
34
|
+
|
|
35
|
+
# Old data dir (cycle #1 — #40).
|
|
36
|
+
_OLD_DIR_NAME = ".hippoagent"
|
|
37
|
+
# New data dir (cycle #41+).
|
|
38
|
+
_NEW_DIR_NAME = ".engram"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def init_env_aliases() -> int:
|
|
42
|
+
"""Mirror HIPPO_* ↔ ENGRAM_* env vars (idempotent).
|
|
43
|
+
|
|
44
|
+
For each environment variable starting with ``HIPPO_``, set the
|
|
45
|
+
corresponding ``ENGRAM_`` variant if not already defined. Same in
|
|
46
|
+
reverse for ``ENGRAM_*`` → ``HIPPO_*``. Uses :py:meth:`os.environ.setdefault`
|
|
47
|
+
so explicit user values are never clobbered.
|
|
48
|
+
|
|
49
|
+
Returns the number of mirror entries added (for tests / introspection).
|
|
50
|
+
"""
|
|
51
|
+
added = 0
|
|
52
|
+
# Snapshot keys to avoid mutation-during-iteration warnings.
|
|
53
|
+
snapshot = list(os.environ.items())
|
|
54
|
+
for k, v in snapshot:
|
|
55
|
+
if k.startswith(_PREFIX_OLD):
|
|
56
|
+
new_key = _PREFIX_NEW + k[len(_PREFIX_OLD):]
|
|
57
|
+
if new_key not in os.environ:
|
|
58
|
+
os.environ[new_key] = v
|
|
59
|
+
added += 1
|
|
60
|
+
elif k.startswith(_PREFIX_NEW):
|
|
61
|
+
old_key = _PREFIX_OLD + k[len(_PREFIX_NEW):]
|
|
62
|
+
if old_key not in os.environ:
|
|
63
|
+
os.environ[old_key] = v
|
|
64
|
+
added += 1
|
|
65
|
+
return added
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def data_dir() -> Path:
|
|
69
|
+
"""Return the canonical Engram data directory.
|
|
70
|
+
|
|
71
|
+
Order of preference:
|
|
72
|
+
|
|
73
|
+
1. If ``~/.engram`` exists, use it.
|
|
74
|
+
2. Else if ``~/.hippoagent`` exists, use it (legacy install).
|
|
75
|
+
3. Else create ``~/.engram`` and use it.
|
|
76
|
+
|
|
77
|
+
Never throws — best-effort. Callers should still handle :py:class:`OSError`
|
|
78
|
+
on subsequent disk operations.
|
|
79
|
+
|
|
80
|
+
The env var ``HIPPO_DATA_DIR`` / ``ENGRAM_DATA_DIR`` (whichever is set;
|
|
81
|
+
if both, the new name wins) overrides this default entirely.
|
|
82
|
+
"""
|
|
83
|
+
# Explicit override via env (HIPPO_DATA_DIR or ENGRAM_DATA_DIR).
|
|
84
|
+
override = os.environ.get("ENGRAM_DATA_DIR") or os.environ.get("HIPPO_DATA_DIR")
|
|
85
|
+
if override:
|
|
86
|
+
p = Path(override).expanduser()
|
|
87
|
+
p.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
return p
|
|
89
|
+
|
|
90
|
+
home = Path.home()
|
|
91
|
+
new_dir = home / _NEW_DIR_NAME
|
|
92
|
+
old_dir = home / _OLD_DIR_NAME
|
|
93
|
+
|
|
94
|
+
if new_dir.exists():
|
|
95
|
+
return new_dir
|
|
96
|
+
if old_dir.exists():
|
|
97
|
+
return old_dir
|
|
98
|
+
new_dir.mkdir(parents=True, exist_ok=True)
|
|
99
|
+
return new_dir
|
engram/_hang_watchdog.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Hang watchdog — make intermittent multi-minute MCP hangs DIAGNOSABLE.
|
|
2
|
+
|
|
3
|
+
When a tool call exceeds a wall-clock budget, dump ALL thread stacks to
|
|
4
|
+
``~/.engram/hang-traces/`` so the exact blocking frame is captured in the act
|
|
5
|
+
(``_MODEL_LOCK.acquire`` / socket ``recv`` / sqlite lock / a stale-code path).
|
|
6
|
+
Without this, a hang that only reproduces in a user's specific session is a
|
|
7
|
+
black box.
|
|
8
|
+
|
|
9
|
+
CONTRACT — observability ONLY:
|
|
10
|
+
* never changes dispatch behaviour (the call runs exactly as before),
|
|
11
|
+
* never raises (a broken trace dir must not break the tool),
|
|
12
|
+
* never cancels/returns the call (it only LOGS; fixing is a separate concern),
|
|
13
|
+
* a fast call leaves NO file (the header-only file is cleaned up).
|
|
14
|
+
|
|
15
|
+
faulthandler's timer is process-global, so only ONE call is watched at a time
|
|
16
|
+
(a non-blocking lock); concurrent calls run unwatched rather than clobbering the
|
|
17
|
+
timer. With the synchronous MCP dispatch (one tool body on the loop at a time)
|
|
18
|
+
this watches effectively every call.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import faulthandler
|
|
23
|
+
import os
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
26
|
+
from contextlib import contextmanager
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
_TRACE_DIR = Path(
|
|
30
|
+
os.environ.get("HIPPO_HANG_TRACE_DIR")
|
|
31
|
+
or (Path.home() / ".engram" / "hang-traces")
|
|
32
|
+
)
|
|
33
|
+
# Below this many bytes the file is header-only (nothing was dumped) → delete.
|
|
34
|
+
_HEADER_MAX_BYTES = 300
|
|
35
|
+
_ARMED = threading.Lock()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@contextmanager
|
|
39
|
+
def hang_trace(label: str, budget_s: float):
|
|
40
|
+
"""Wrap a tool call. If it runs longer than ``budget_s`` seconds, append a
|
|
41
|
+
full all-thread stack dump to a per-call file under ``_TRACE_DIR``."""
|
|
42
|
+
if not budget_s or budget_s <= 0:
|
|
43
|
+
yield
|
|
44
|
+
return
|
|
45
|
+
# Process-global faulthandler timer → only one watcher at a time.
|
|
46
|
+
if not _ARMED.acquire(blocking=False):
|
|
47
|
+
yield
|
|
48
|
+
return
|
|
49
|
+
f = None
|
|
50
|
+
path = None
|
|
51
|
+
armed = False
|
|
52
|
+
try:
|
|
53
|
+
_TRACE_DIR.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
safe = "".join(c if c.isalnum() or c in "-_." else "_" for c in str(label))[:40]
|
|
55
|
+
path = _TRACE_DIR / f"hang-{int(time.time())}-{os.getpid()}-{safe}.txt"
|
|
56
|
+
f = open(path, "w", encoding="utf-8")
|
|
57
|
+
f.write(
|
|
58
|
+
f"HANG WATCHDOG tool={label} pid={os.getpid()} budget={budget_s}s\n"
|
|
59
|
+
"the stacks below were dumped because the call exceeded the budget:\n"
|
|
60
|
+
)
|
|
61
|
+
f.flush()
|
|
62
|
+
faulthandler.dump_traceback_later(budget_s, repeat=True, file=f)
|
|
63
|
+
armed = True
|
|
64
|
+
except Exception: # noqa: BLE001 — tracing is best-effort, never break the call
|
|
65
|
+
if f is not None:
|
|
66
|
+
try:
|
|
67
|
+
f.close()
|
|
68
|
+
except Exception: # noqa: BLE001
|
|
69
|
+
pass
|
|
70
|
+
f = None
|
|
71
|
+
try:
|
|
72
|
+
yield
|
|
73
|
+
finally:
|
|
74
|
+
if armed:
|
|
75
|
+
try:
|
|
76
|
+
faulthandler.cancel_dump_traceback_later()
|
|
77
|
+
except Exception: # noqa: BLE001
|
|
78
|
+
pass
|
|
79
|
+
if f is not None:
|
|
80
|
+
try:
|
|
81
|
+
size = f.tell()
|
|
82
|
+
f.close()
|
|
83
|
+
if size <= _HEADER_MAX_BYTES and path is not None:
|
|
84
|
+
path.unlink(missing_ok=True)
|
|
85
|
+
except Exception: # noqa: BLE001
|
|
86
|
+
pass
|
|
87
|
+
_ARMED.release()
|
engram/_proc_quiet.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Cycle #136 (2026-05-17) — Windows console pop-up suppression helper.
|
|
2
|
+
|
|
3
|
+
Aurelio direttiva 2026-05-17: "shell che si aprono e chiudono da sole,
|
|
4
|
+
sono fastidiosissime". Root cause: every ``subprocess.run / Popen`` call
|
|
5
|
+
in ``engram/`` that doesn't pass ``creationflags=CREATE_NO_WINDOW``
|
|
6
|
+
flashes a Windows CMD window on the user's screen. This module exposes
|
|
7
|
+
one tiny helper, ``quiet_popen_kwargs()``, that returns the right
|
|
8
|
+
kwargs for the current platform — empty dict on Linux/macOS so the
|
|
9
|
+
helper is a no-op cross-platform.
|
|
10
|
+
|
|
11
|
+
Usage::
|
|
12
|
+
|
|
13
|
+
from engram._proc_quiet import quiet_popen_kwargs
|
|
14
|
+
subprocess.run(["git", "rev-parse", sha], **quiet_popen_kwargs())
|
|
15
|
+
|
|
16
|
+
The Windows ``CREATE_NO_WINDOW`` constant is only defined when running
|
|
17
|
+
on Windows (it lives in ``subprocess`` only on win32 Python builds).
|
|
18
|
+
We guard the attribute access defensively.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def quiet_popen_kwargs() -> dict[str, Any]:
|
|
28
|
+
"""Return kwargs that suppress the Windows console pop-up.
|
|
29
|
+
|
|
30
|
+
On Windows this is ``{"creationflags": subprocess.CREATE_NO_WINDOW}``;
|
|
31
|
+
on every other platform it is the empty dict (no behaviour change).
|
|
32
|
+
"""
|
|
33
|
+
if sys.platform == "win32":
|
|
34
|
+
flag = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
|
35
|
+
if flag:
|
|
36
|
+
return {"creationflags": flag}
|
|
37
|
+
return {}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Single-instance guard — reap ORPHANED `engram mcp` server processes on startup.
|
|
2
|
+
|
|
3
|
+
Root cause of the recurring multi-minute "Engram froze on a save" incident
|
|
4
|
+
(2026-06-06, empirical via Win32_Process enumeration): Claude Code restarts spawn
|
|
5
|
+
a fresh `engram mcp` server each time, but the OLD ones are never reaped. They
|
|
6
|
+
pile up (6 observed simultaneously); each contends for CPU/disk and drives its own
|
|
7
|
+
embedding work, starving the live server's encode until a save blocks for minutes.
|
|
8
|
+
|
|
9
|
+
This module, called once at server startup, terminates sibling `engram mcp`
|
|
10
|
+
servers that are ORPHANED — their spawning client (Claude Code) is gone.
|
|
11
|
+
|
|
12
|
+
Multi-window SAFE: a sibling whose parent client is still alive is NEVER touched
|
|
13
|
+
(``psutil.Process.parent()`` validates ppid AND create_time, so a recycled PID is
|
|
14
|
+
not mistaken for a live parent). Detached embedding daemons are intentionally
|
|
15
|
+
excluded from the matcher (they have no live parent by design — single-instancing
|
|
16
|
+
them is the daemon's own job, via the encode_service reachability lock).
|
|
17
|
+
|
|
18
|
+
Best-effort: every failure path is swallowed — the guard must NEVER break server
|
|
19
|
+
startup, and psutil is optional.
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
# Matches the engram MCP server in ALL its launch forms:
|
|
27
|
+
# "<path>\engram.exe" mcp | hippo mcp | engram mcp
|
|
28
|
+
# plus the explicit-module substring "engram.mcp_server" (handled separately).
|
|
29
|
+
# Deliberately does NOT match the sibling processes that merely contain "mcp":
|
|
30
|
+
# engram_bridge_mcp.py / clp_mcp_server.py / duo_bridge_mcp.py /
|
|
31
|
+
# critic_orchestrator.mcp_server / engram.encode_service / engram_embedding_daemon.py
|
|
32
|
+
# (none has the `(engram|hippo)[.exe] <whitespace> mcp` token boundary).
|
|
33
|
+
_MCP_CMDLINE_RE = re.compile(
|
|
34
|
+
r'(?:^|[\s"/\\])(?:engram|hippo)(?:\.exe)?["\']?\s+mcp(?:\s|$|["\'])',
|
|
35
|
+
re.IGNORECASE,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _is_engram_mcp_cmdline(cmdline: str) -> bool:
|
|
40
|
+
"""True iff ``cmdline`` is an engram/hippo MCP *server* invocation."""
|
|
41
|
+
if not cmdline:
|
|
42
|
+
return False
|
|
43
|
+
if "engram.mcp_server" in cmdline:
|
|
44
|
+
return True
|
|
45
|
+
return bool(_MCP_CMDLINE_RE.search(cmdline))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _select_orphan_pids(procs, *, self_pid: int) -> list[int]:
|
|
49
|
+
"""Pure selection: given ``procs`` = iterable of
|
|
50
|
+
``(pid: int, cmdline: str, parent_alive: bool)``, return the pids that are
|
|
51
|
+
engram-mcp servers, orphaned (parent NOT alive), and not ``self_pid``."""
|
|
52
|
+
out: list[int] = []
|
|
53
|
+
for pid, cmdline, parent_alive in procs:
|
|
54
|
+
if pid == self_pid:
|
|
55
|
+
continue
|
|
56
|
+
if not _is_engram_mcp_cmdline(cmdline):
|
|
57
|
+
continue
|
|
58
|
+
if not parent_alive:
|
|
59
|
+
out.append(pid)
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _import_psutil():
|
|
64
|
+
"""Return the psutil module, or None if unavailable (indirection for tests)."""
|
|
65
|
+
try:
|
|
66
|
+
import psutil # noqa: PLC0415
|
|
67
|
+
return psutil
|
|
68
|
+
except Exception: # noqa: BLE001
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def reap_orphan_mcp_servers(*, dry_run: bool = False) -> list[int]:
|
|
73
|
+
"""Terminate orphaned sibling `engram mcp` servers. Returns the reaped pids
|
|
74
|
+
(or, under ``dry_run``, the pids that WOULD be reaped). Never raises."""
|
|
75
|
+
psutil = _import_psutil()
|
|
76
|
+
if psutil is None:
|
|
77
|
+
return []
|
|
78
|
+
self_pid = os.getpid()
|
|
79
|
+
candidates = [] # (pid, cmdline, parent_alive, proc)
|
|
80
|
+
try:
|
|
81
|
+
for p in psutil.process_iter(["pid", "cmdline"]):
|
|
82
|
+
try:
|
|
83
|
+
pid = p.info["pid"]
|
|
84
|
+
cmdline = " ".join(p.info.get("cmdline") or [])
|
|
85
|
+
if not _is_engram_mcp_cmdline(cmdline):
|
|
86
|
+
continue
|
|
87
|
+
parent_alive = p.parent() is not None
|
|
88
|
+
candidates.append((pid, cmdline, parent_alive, p))
|
|
89
|
+
except Exception: # noqa: BLE001 — one bad proc never aborts the sweep
|
|
90
|
+
continue
|
|
91
|
+
except Exception: # noqa: BLE001
|
|
92
|
+
return []
|
|
93
|
+
orphan_pids = set(_select_orphan_pids(
|
|
94
|
+
[(pid, cmd, alive) for pid, cmd, alive, _ in candidates],
|
|
95
|
+
self_pid=self_pid,
|
|
96
|
+
))
|
|
97
|
+
reaped: list[int] = []
|
|
98
|
+
for pid, _cmd, _alive, proc in candidates:
|
|
99
|
+
if pid not in orphan_pids:
|
|
100
|
+
continue
|
|
101
|
+
if dry_run:
|
|
102
|
+
reaped.append(pid)
|
|
103
|
+
continue
|
|
104
|
+
try:
|
|
105
|
+
proc.terminate()
|
|
106
|
+
reaped.append(pid)
|
|
107
|
+
except Exception: # noqa: BLE001
|
|
108
|
+
continue
|
|
109
|
+
return reaped
|
engram/_sqlite_pragma.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Single source of truth for the SQLite connection PRAGMA policy.
|
|
2
|
+
|
|
3
|
+
The four DB modules (semantic, memory/episodic, entity_kg, skill) each open
|
|
4
|
+
short-lived per-operation connections with the same PRAGMA set (WAL +
|
|
5
|
+
busy_timeout=60000 + synchronous=NORMAL + foreign_keys). The ``synchronous`` level
|
|
6
|
+
was hard-coded in 5 places; this centralizes the ONE knob a deployment may want to
|
|
7
|
+
tune (production-scaling review 2026-06-20).
|
|
8
|
+
|
|
9
|
+
``synchronous=NORMAL`` (default) is WAL-safe and fast but, between checkpoints, a
|
|
10
|
+
committed-but-uncheckpointed write can be lost on an OS crash / power loss.
|
|
11
|
+
``ENGRAM_SQLITE_SYNCHRONOUS=FULL`` trades write throughput for per-commit fsync
|
|
12
|
+
durability — for deployments that need it. Default keeps current behaviour.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def synchronous_mode() -> str:
|
|
20
|
+
"""Return the SQLite ``synchronous`` level: 'NORMAL' (default) or 'FULL'."""
|
|
21
|
+
v = os.environ.get("ENGRAM_SQLITE_SYNCHRONOUS", "NORMAL").strip().upper()
|
|
22
|
+
return "FULL" if v == "FULL" else "NORMAL"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = ["synchronous_mode"]
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Single source of truth for machine-state / telemetry topic prefixes.
|
|
2
|
+
|
|
3
|
+
A fact whose topic starts with one of these is serialized machine state (event
|
|
4
|
+
bus traffic, metrics, locks, market/cache/citation simulation blobs, agent
|
|
5
|
+
diary exhaust, …), NOT curated natural-language knowledge. Two layers consume
|
|
6
|
+
this list and they must NEVER drift:
|
|
7
|
+
|
|
8
|
+
- WRITE time: ``engram.admission_gate`` routes such a write to the telemetry
|
|
9
|
+
store instead of the curated corpus (ROUTE_TELEMETRY) — it never deletes.
|
|
10
|
+
- READ time: ``engram.semantic`` hides them from a GENERIC (topic=None) recall
|
|
11
|
+
(the cache fast-path SQL, the legacy SQL path, and the cold keyword
|
|
12
|
+
fallback all derive their denylist from this tuple).
|
|
13
|
+
|
|
14
|
+
Before 2026-06-13 these were two hand-maintained lists (a tuple in semantic +
|
|
15
|
+
a regex in admission_gate) that had silently drifted: a live hippo_facts_recall
|
|
16
|
+
surfaced cache/ market/ citations/ blobs that semantic's recall denylist had
|
|
17
|
+
just been extended to hide but the admission gate still admitted. Centralising
|
|
18
|
+
here makes that drift structurally impossible.
|
|
19
|
+
|
|
20
|
+
This module is a LEAF: it imports nothing from ``engram`` so neither
|
|
21
|
+
``semantic`` nor ``admission_gate`` can create an import cycle by reading it.
|
|
22
|
+
|
|
23
|
+
NOT in this list (deliberately curated knowledge, must stay recallable):
|
|
24
|
+
``test/`` (canary fixtures the suite relies on), ``handoff/`` (real operator
|
|
25
|
+
mandates), ``bench/`` (benchmark findings), ``project/``, ``lessons/``,
|
|
26
|
+
``archive/``, ``decisions/``, ``preferences/``, ``emerging_skill/``.
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
#: Topic prefixes treated as machine-state/telemetry. ``startswith`` semantics
|
|
31
|
+
#: (each entry is matched as a literal prefix, not a regex).
|
|
32
|
+
TELEMETRY_TOPIC_PREFIXES: tuple[str, ...] = (
|
|
33
|
+
# original set (SCAN-68, 2026-06-02)
|
|
34
|
+
"bus/", "metric/", "alloc/", "lock/", "tx/", "nego/", "replay/",
|
|
35
|
+
"dialog/voice",
|
|
36
|
+
# 2026-06-13: serialized JSON machine-state / simulation blobs that a LIVE
|
|
37
|
+
# hippo_facts_recall surfaced at score ~0.82, crowding out real knowledge.
|
|
38
|
+
# Per-namespace sampling (B2) confirmed all are JSON state, not NL knowledge.
|
|
39
|
+
"cache/", "market/", "citations/", "obs/", "signal/",
|
|
40
|
+
"dispatch/", "supervisor/", "namespace/", "diary/",
|
|
41
|
+
# 2026-07-02: the real-corpus NLI truth scan surfaced 26 live dream/<id>
|
|
42
|
+
# facts, all serialized scenario state ({"state": "active", ...}) that had
|
|
43
|
+
# slipped past both layers because the prefix was missing.
|
|
44
|
+
"dream/",
|
|
45
|
+
# 2026-07-03: the first KNOWLEDGE-only scan exposed machine state nested
|
|
46
|
+
# under knowledge namespaces (sampled before listing, B2): pin/<fact_id>
|
|
47
|
+
# JSON pin/unpin state; skill/catalog/* auto-generated registry entries
|
|
48
|
+
# ("Pre-installed Engram skill. Path: ...", 49 near-identical rows = 395
|
|
49
|
+
# of 810 scan conflicts); project/recursive-mas/* MAS-worker exhaust
|
|
50
|
+
# (planner/critic run output incl. raw ANSI escapes). Deliberate skills
|
|
51
|
+
# knowledge (emerging_skill/, other skill/ topics) stays recallable.
|
|
52
|
+
"pin/", "skill/catalog/", "project/recursive-mas/",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
#: Fact tiers. `classify_tier` is the single source of truth consumed by the
|
|
56
|
+
#: truth-reconcile guard, the corpus truth scan, and any bench that must not
|
|
57
|
+
#: mix machine state with knowledge (the 2026-07-02 real-corpus scan showed the
|
|
58
|
+
#: residual NLI conflicts were telemetry near-duplicates, not knowledge).
|
|
59
|
+
TIER_TELEMETRY = "telemetry"
|
|
60
|
+
TIER_TEST = "test"
|
|
61
|
+
TIER_DIALOG = "dialog"
|
|
62
|
+
TIER_KNOWLEDGE = "knowledge"
|
|
63
|
+
|
|
64
|
+
#: Suite canaries and lab-experiment material: deliberately recallable (see the
|
|
65
|
+
#: NOT-in-this-list note above) but never a reconcile/scan subject.
|
|
66
|
+
TEST_TOPIC_PREFIXES: tuple[str, ...] = ("test/", "lab/", "project/lab/")
|
|
67
|
+
|
|
68
|
+
#: Verbatim conversation transcripts (e.g. the founding dialog/doc1-* docs):
|
|
69
|
+
#: curated, recallable knowledge of record — but chat turns are not factual
|
|
70
|
+
#: assertions, so a reconcile judge must not supersede/contest them.
|
|
71
|
+
DIALOG_TOPIC_PREFIX = "dialog/"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def classify_tier(topic: str | None) -> str:
|
|
75
|
+
"""Map a fact topic to its tier. Telemetry wins over dialog so that
|
|
76
|
+
``dialog/voice`` (machine exhaust) stays telemetry while ``dialog/doc*``
|
|
77
|
+
transcripts classify as dialog."""
|
|
78
|
+
t = topic or ""
|
|
79
|
+
if any(t.startswith(p) for p in TELEMETRY_TOPIC_PREFIXES):
|
|
80
|
+
return TIER_TELEMETRY
|
|
81
|
+
if any(t.startswith(p) for p in TEST_TOPIC_PREFIXES):
|
|
82
|
+
return TIER_TEST
|
|
83
|
+
if t.startswith(DIALOG_TOPIC_PREFIX):
|
|
84
|
+
return TIER_DIALOG
|
|
85
|
+
return TIER_KNOWLEDGE
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
__all__ = [
|
|
89
|
+
"TELEMETRY_TOPIC_PREFIXES",
|
|
90
|
+
"TEST_TOPIC_PREFIXES",
|
|
91
|
+
"DIALOG_TOPIC_PREFIX",
|
|
92
|
+
"TIER_TELEMETRY",
|
|
93
|
+
"TIER_TEST",
|
|
94
|
+
"TIER_DIALOG",
|
|
95
|
+
"TIER_KNOWLEDGE",
|
|
96
|
+
"classify_tier",
|
|
97
|
+
]
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Cycle 175 (2026-05-22) — Active Learning Design B: stuck-list cron.
|
|
2
|
+
|
|
3
|
+
Implements the empirically-motivated active-learning loop the
|
|
4
|
+
``docs/cycle174_active_learning_design.md`` proposed and Aurelio
|
|
5
|
+
greenlit on 2026-05-22.
|
|
6
|
+
|
|
7
|
+
Empirical motivation (fact ``d778cce2faa8``, verified on the live
|
|
8
|
+
``~/.engram/skills/skills_index.db``):
|
|
9
|
+
- 326 total skills, 71% (233) never trialed
|
|
10
|
+
- 3 candidates stuck at fitness 0.33-0.40
|
|
11
|
+
- candidate→promoted conversion rate: 7/163 = 4.3 %
|
|
12
|
+
|
|
13
|
+
Design B (this module) is the *stuck-list cron*: a deterministic
|
|
14
|
+
``SELECT`` returns the next ids to retry. No bandit, no randomness.
|
|
15
|
+
Design A (warm-up bandit) and Design C (task-driven expansion) are
|
|
16
|
+
deferred to future cycles per the design doc.
|
|
17
|
+
|
|
18
|
+
Falsifiable hypothesis H1 (pre-registered in the design doc):
|
|
19
|
+
Targeted retry over candidates in ``trials ∈ [3, 10]`` and
|
|
20
|
+
``fitness ∈ (0.3, 0.5)`` lifts the candidate→promoted conversion
|
|
21
|
+
from 4.3 % to > 10 % within 20 Auto-Dream cycles.
|
|
22
|
+
|
|
23
|
+
This module is the *selection* primitive only. The decision to actually
|
|
24
|
+
emit a dream task from the selected ids is the caller's job (a thin
|
|
25
|
+
hook in ``engram.auto_dream_trigger.maybe_trigger_dream`` — added in a
|
|
26
|
+
follow-up commit if H1 holds in pilot).
|
|
27
|
+
|
|
28
|
+
Pure-function contract
|
|
29
|
+
----------------------
|
|
30
|
+
``select_stuck_candidates`` opens the skills_index.db read-only, runs
|
|
31
|
+
one SQL query, returns ids ordered oldest-``updated_at`` first (cron
|
|
32
|
+
fairness — every cycle the longest-untouched stuck skill gets a turn).
|
|
33
|
+
No state, no side effects, never raises (missing DB → []).
|
|
34
|
+
|
|
35
|
+
Why SQL not ORM
|
|
36
|
+
---------------
|
|
37
|
+
Bayesian smoothed fitness ``(s+1)/(t+2)`` is cheap in SQL via a
|
|
38
|
+
CAST(...) AS REAL division. Doing the math in Python would require
|
|
39
|
+
loading every candidate skill — wasteful on a corpus of hundreds.
|
|
40
|
+
The SQL prior matches ``Skill.fitness_mean`` *only when* the
|
|
41
|
+
``CONFIG.fitness_prior_alpha = CONFIG.fitness_prior_beta = 1`` (the
|
|
42
|
+
default). If a user customised the prior, this function's filter
|
|
43
|
+
will diverge slightly from ``Skill.fitness_mean``; see the
|
|
44
|
+
``BETA_PRIOR_NOTE`` constant below.
|
|
45
|
+
"""
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import sqlite3
|
|
49
|
+
from pathlib import Path
|
|
50
|
+
|
|
51
|
+
#: Pinned to the default Beta(1, 1) (uniform) prior used by
|
|
52
|
+
#: ``engram.skill.Skill.fitness_mean`` when ``CONFIG.fitness_prior_*``
|
|
53
|
+
#: stays at the package defaults. The SQL filter uses this prior to
|
|
54
|
+
#: keep the band semantics consistent with the dataclass property.
|
|
55
|
+
#: A user who customised the prior should call ``Skill.fitness_mean``
|
|
56
|
+
#: per-row instead of using this fast SQL path.
|
|
57
|
+
BETA_PRIOR_NOTE = "Beta(1,1) — pinned to CONFIG default"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def select_stuck_candidates(
|
|
61
|
+
skill_db: Path | str,
|
|
62
|
+
*,
|
|
63
|
+
min_trials: int = 3,
|
|
64
|
+
max_trials: int = 10,
|
|
65
|
+
fitness_band: tuple[float, float] = (0.3, 0.5),
|
|
66
|
+
max_n: int = 3,
|
|
67
|
+
status: str = "candidate",
|
|
68
|
+
) -> list[str]:
|
|
69
|
+
"""Return the ids of stuck-band candidates eligible for retry.
|
|
70
|
+
|
|
71
|
+
Filters (all conjunctive):
|
|
72
|
+
- ``skills.status = <status>`` (default ``'candidate'``)
|
|
73
|
+
- ``min_trials ≤ skills.trials ≤ max_trials``
|
|
74
|
+
- ``fitness_band[0] < (successes+1)/(trials+2) < fitness_band[1]``
|
|
75
|
+
(strict inequalities — boundary skills are treated as
|
|
76
|
+
outside the "stuck" zone)
|
|
77
|
+
|
|
78
|
+
Ordering: ``updated_at ASC`` (oldest first → cron fairness).
|
|
79
|
+
|
|
80
|
+
Returns up to ``max_n`` ids. Empty list on missing DB or
|
|
81
|
+
SQL error (defensive — this is called from the Auto-Dream
|
|
82
|
+
cooldown path and must never crash a hook).
|
|
83
|
+
"""
|
|
84
|
+
p = Path(skill_db)
|
|
85
|
+
if not p.exists():
|
|
86
|
+
return []
|
|
87
|
+
lo, hi = float(fitness_band[0]), float(fitness_band[1])
|
|
88
|
+
try:
|
|
89
|
+
conn = sqlite3.connect(str(p))
|
|
90
|
+
try:
|
|
91
|
+
conn.row_factory = sqlite3.Row
|
|
92
|
+
rows = conn.execute(
|
|
93
|
+
"""
|
|
94
|
+
SELECT id
|
|
95
|
+
FROM skills
|
|
96
|
+
WHERE status = ?
|
|
97
|
+
AND trials >= ?
|
|
98
|
+
AND trials <= ?
|
|
99
|
+
AND (CAST(successes + 1 AS REAL) / (trials + 2)) > ?
|
|
100
|
+
AND (CAST(successes + 1 AS REAL) / (trials + 2)) < ?
|
|
101
|
+
ORDER BY updated_at ASC
|
|
102
|
+
LIMIT ?
|
|
103
|
+
""",
|
|
104
|
+
(status, int(min_trials), int(max_trials),
|
|
105
|
+
lo, hi, int(max_n)),
|
|
106
|
+
).fetchall()
|
|
107
|
+
finally:
|
|
108
|
+
conn.close()
|
|
109
|
+
except sqlite3.Error:
|
|
110
|
+
return []
|
|
111
|
+
return [str(r["id"]) for r in rows]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
__all__ = ["BETA_PRIOR_NOTE", "select_stuck_candidates"]
|