structverify 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. structverify/__init__.py +83 -0
  2. structverify/adaptation/__init__.py +0 -0
  3. structverify/adaptation/adapter_trainer.py +341 -0
  4. structverify/adaptation/feedback_store.py +31 -0
  5. structverify/adaptation/kosis_crawler.py +317 -0
  6. structverify/adaptation/sample_builder.py +149 -0
  7. structverify/adaptation/synthetic_generator.py +320 -0
  8. structverify/adaptation/update_embeddings.py +178 -0
  9. structverify/agent/__init__.py +21 -0
  10. structverify/agent/builder_agent.py +226 -0
  11. structverify/agent/conformance_agent.py +171 -0
  12. structverify/agent/dependency_planner.py +151 -0
  13. structverify/agent/indexing_agent.py +153 -0
  14. structverify/agent/indexing_planner.py +169 -0
  15. structverify/agent/integration_example.py +182 -0
  16. structverify/agent/loop.py +1165 -0
  17. structverify/agent/memory.py +207 -0
  18. structverify/agent/planner.py +817 -0
  19. structverify/agent/prompts/__init__.py +15 -0
  20. structverify/agent/prompts/planner_prompts.py +219 -0
  21. structverify/agent/prompts/reflect_prompts.py +387 -0
  22. structverify/agent/reflect.py +227 -0
  23. structverify/agent/runtime_agent.py +1272 -0
  24. structverify/agent/schemas.py +262 -0
  25. structverify/agent/source_profiler.py +229 -0
  26. structverify/agent/tools/__init__.py +64 -0
  27. structverify/agent/tools/base.py +222 -0
  28. structverify/agent/tools/calculate.py +244 -0
  29. structverify/agent/tools/catalog_search.py +859 -0
  30. structverify/agent/tools/deep_explore.py +293 -0
  31. structverify/agent/tools/explore_catalog.py +423 -0
  32. structverify/agent/tools/fetch_evidence.py +922 -0
  33. structverify/agent/tools/finish.py +423 -0
  34. structverify/agent/tools/meta_explore.py +267 -0
  35. structverify/agent/tools/query_rewriter.py +134 -0
  36. structverify/agent/tools/read_original.py +144 -0
  37. structverify/agent/tools/replan.py +365 -0
  38. structverify/agent/workspace.py +958 -0
  39. structverify/api.py +804 -0
  40. structverify/config/default.yaml +350 -0
  41. structverify/core/__init__.py +0 -0
  42. structverify/core/config_loader.py +30 -0
  43. structverify/core/pipeline.py +280 -0
  44. structverify/core/schemas.py +362 -0
  45. structverify/detection/__init__.py +26 -0
  46. structverify/detection/_config.py +163 -0
  47. structverify/detection/_llm.py +24 -0
  48. structverify/detection/candidate/__init__.py +1 -0
  49. structverify/detection/candidate/heuristic.py +60 -0
  50. structverify/detection/candidate/llm.py +51 -0
  51. structverify/detection/candidate_scorer.py +81 -0
  52. structverify/detection/claim_detector.py +164 -0
  53. structverify/detection/claims/__init__.py +1 -0
  54. structverify/detection/claims/worthiness.py +142 -0
  55. structverify/detection/domain/__init__.py +1 -0
  56. structverify/detection/domain/classify.py +84 -0
  57. structverify/detection/domain/preview.py +36 -0
  58. structverify/detection/domain/registry.py +99 -0
  59. structverify/detection/domain_classifier.py +75 -0
  60. structverify/detection/prompts/__init__.py +1 -0
  61. structverify/detection/prompts/candidate.py +38 -0
  62. structverify/detection/prompts/claim_worthiness.py +48 -0
  63. structverify/detection/prompts/domain.py +41 -0
  64. structverify/detection/prompts/schema.py +508 -0
  65. structverify/detection/prompts_loader.py +167 -0
  66. structverify/detection/schema/__init__.py +1 -0
  67. structverify/detection/schema/expand.py +83 -0
  68. structverify/detection/schema/induce.py +441 -0
  69. structverify/detection/schema/regenerate.py +162 -0
  70. structverify/detection/schema/temporal_hints.py +130 -0
  71. structverify/detection/schema/validate.py +193 -0
  72. structverify/detection/schema_inductor.py +112 -0
  73. structverify/detection/synthetic_generator.py +270 -0
  74. structverify/explanation/__init__.py +0 -0
  75. structverify/explanation/_config.py +18 -0
  76. structverify/explanation/_llm.py +25 -0
  77. structverify/explanation/explainer.py +183 -0
  78. structverify/explanation/fallback.py +29 -0
  79. structverify/explanation/formatters.py +75 -0
  80. structverify/explanation/prompts/__init__.py +1 -0
  81. structverify/explanation/prompts/match.py +27 -0
  82. structverify/explanation/prompts/mismatch.py +20 -0
  83. structverify/explanation/prompts/multihop.py +16 -0
  84. structverify/explanation/prompts/unverifiable.py +17 -0
  85. structverify/graph/__init__.py +0 -0
  86. structverify/graph/claim_graph.py +226 -0
  87. structverify/graph/document_graph.py +487 -0
  88. structverify/graph/graph_builder.py +238 -0
  89. structverify/graph/graph_multihop.py +335 -0
  90. structverify/graph/graph_store.py +281 -0
  91. structverify/graph/provenance.py +52 -0
  92. structverify/memory/__init__.py +44 -0
  93. structverify/memory/agent_memory.py +142 -0
  94. structverify/memory/embedder.py +69 -0
  95. structverify/memory/exemplar_store.py +241 -0
  96. structverify/memory/normalizer.py +91 -0
  97. structverify/memory/schema.py +119 -0
  98. structverify/memory/storage/__init__.py +29 -0
  99. structverify/memory/storage/jsonl_store.py +117 -0
  100. structverify/memory/working_memory.py +370 -0
  101. structverify/preprocessing/Dockerfile.scraper +27 -0
  102. structverify/preprocessing/__init__.py +0 -0
  103. structverify/preprocessing/extractor.py +574 -0
  104. structverify/preprocessing/pdf/__init__.py +16 -0
  105. structverify/preprocessing/pdf/fields.py +95 -0
  106. structverify/preprocessing/pdf/markdown.py +107 -0
  107. structverify/preprocessing/pdf/models.py +34 -0
  108. structverify/preprocessing/pdf/ocr.py +172 -0
  109. structverify/preprocessing/pdf/pipeline.py +74 -0
  110. structverify/preprocessing/pdf/reader.py +119 -0
  111. structverify/preprocessing/pdf/scoring.py +61 -0
  112. structverify/preprocessing/scraper_sandbox.py +561 -0
  113. structverify/preprocessing/segmenter.py +48 -0
  114. structverify/preprocessing/sir_builder.py +240 -0
  115. structverify/progress.py +591 -0
  116. structverify/retrieval/__init__.py +0 -0
  117. structverify/retrieval/base.py +208 -0
  118. structverify/retrieval/base_connector.py +85 -0
  119. structverify/retrieval/catalog_ranker.py +300 -0
  120. structverify/retrieval/catalog_search.py +583 -0
  121. structverify/retrieval/chunking.py +92 -0
  122. structverify/retrieval/custom_csv_source.py +386 -0
  123. structverify/retrieval/custom_db_source.py +396 -0
  124. structverify/retrieval/custom_docs_source.py +152 -0
  125. structverify/retrieval/dimension_resolver.py +281 -0
  126. structverify/retrieval/evidence_subgraph.py +63 -0
  127. structverify/retrieval/kosis_connector.py +1192 -0
  128. structverify/retrieval/kosis_relevance.py +142 -0
  129. structverify/retrieval/kosis_source.py +1541 -0
  130. structverify/retrieval/query_builder.py +72 -0
  131. structverify/retrieval/registry.py +133 -0
  132. structverify/retrieval/relevance_judge.py +141 -0
  133. structverify/retrieval/row_matcher.py +267 -0
  134. structverify/storage/__init__.py +0 -0
  135. structverify/storage/db_manager.py +157 -0
  136. structverify/storage/dwh_manager.py +92 -0
  137. structverify/storage/init_db.py +99 -0
  138. structverify/storage/raw_storage.py +29 -0
  139. structverify/training/__init__.py +26 -0
  140. structverify/training/curator.py +124 -0
  141. structverify/training/dataset.py +134 -0
  142. structverify/training/doctor.py +99 -0
  143. structverify/training/evalgate.py +96 -0
  144. structverify/training/generate.py +101 -0
  145. structverify/training/loop.py +116 -0
  146. structverify/training/recipe/train_mlx.py +99 -0
  147. structverify/training/recipe/train_qlora.py +104 -0
  148. structverify/training/tasks.py +79 -0
  149. structverify/utils/__init__.py +0 -0
  150. structverify/utils/embedding_client.py +248 -0
  151. structverify/utils/llm_client.py +809 -0
  152. structverify/utils/logger.py +81 -0
  153. structverify/verification/__init__.py +0 -0
  154. structverify/verification/_config.py +45 -0
  155. structverify/verification/adapters.py +405 -0
  156. structverify/verification/conformance.py +117 -0
  157. structverify/verification/decide_verdict.py +216 -0
  158. structverify/verification/decide_verdict_agent.py +454 -0
  159. structverify/verification/growth_diff.py +267 -0
  160. structverify/verification/row_match.py +345 -0
  161. structverify/verification/units.py +64 -0
  162. structverify/verification/verdict_thresholds.py +232 -0
  163. structverify/verification/verifier.py +84 -0
  164. structverify-0.3.0.dist-info/METADATA +903 -0
  165. structverify-0.3.0.dist-info/RECORD +168 -0
  166. structverify-0.3.0.dist-info/WHEEL +5 -0
  167. structverify-0.3.0.dist-info/licenses/LICENSE +21 -0
  168. structverify-0.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,207 @@
1
+ """
2
+ structverify.agent.memory — Agent의 멀티턴 메모리.
3
+
4
+ Memory는 *평문 markdown*으로 저장되며, agent가 매 iteration마다 *전체 읽고
5
+ 새 내용 추가*한다. memory.md가 너무 길어지면 *요약/압축* (Phase D+에서).
6
+
7
+ 저장 자체는 Workspace.append_memory / read_memory를 통하지만, 이 모듈은:
8
+ - 일관된 포맷으로 새 항목 추가 (append_iteration, append_plan_summary 등)
9
+ - LLM에 넘기기 좋은 형태로 read (read_for_llm — 길이 제한 적용)
10
+ - 이미 시도한 action 추적 (중복 방지)
11
+
12
+ Phase A에서는 *저장/읽기 헬퍼*만. 요약(summarize_memory)은 Phase D+.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from structverify.utils.logger import get_logger
18
+ from datetime import datetime, timezone
19
+ from typing import Any
20
+ from uuid import UUID
21
+
22
+ from .workspace import Workspace
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ # ── Append 헬퍼 (일관된 포맷) ─────────────────────────────────────
28
+
29
+ def append_plan_summary(ws: Workspace, claim_id: str | UUID, plan: dict) -> None:
30
+ """Plan Agent 결과를 memory 시작 부분에 기록.
31
+
32
+ Plan Agent (Phase C)가 호출.
33
+ """
34
+ required = plan.get("required_data", [])
35
+ required_lines = []
36
+ for d in required:
37
+ if isinstance(d, dict):
38
+ ind = d.get("indicator", "?")
39
+ t = d.get("time", "?")
40
+ pop = d.get("population", "")
41
+ line = f" - {ind} (time={t}" + (f", pop={pop}" if pop else "") + ")"
42
+ else:
43
+ line = f" - {d}"
44
+ required_lines.append(line)
45
+
46
+ text = (
47
+ "## Initial Plan\n"
48
+ f"Claim type: {plan.get('claim_type', 'unknown')}\n"
49
+ f"Required data:\n" + "\n".join(required_lines) + "\n"
50
+ f"Formula: {plan.get('calculation_formula') or '(none — direct comparison)'}\n"
51
+ f"Fallback: use_original_text={plan.get('fallback', {}).get('use_original_text', False)}\n"
52
+ )
53
+ ws.append_memory(claim_id, text)
54
+
55
+
56
+ def append_iteration(
57
+ ws: Workspace,
58
+ claim_id: str | UUID,
59
+ iteration_num: int,
60
+ action: str,
61
+ action_input: dict,
62
+ observation_summary: str,
63
+ reflection: str | None = None,
64
+ success: bool = True,
65
+ ) -> None:
66
+ """
67
+ 한 iteration 결과를 memory.md에 추가.
68
+
69
+ 포맷:
70
+ ## Iteration {n} — {action}
71
+ Input: {input}
72
+ Result: {summary}
73
+ Reflection: {reflection} (있으면)
74
+ Status: success | failed
75
+ """
76
+ status_marker = "✓" if success else "✗"
77
+ lines = [
78
+ f"## Iteration {iteration_num} — {action} {status_marker}",
79
+ f"Input: {_format_input(action_input)}",
80
+ f"Result: {observation_summary}",
81
+ ]
82
+ if reflection:
83
+ lines.append(f"Reflection: {reflection}")
84
+ lines.append("") # 빈 줄
85
+
86
+ ws.append_memory(claim_id, "\n".join(lines))
87
+
88
+
89
+ def append_final(
90
+ ws: Workspace,
91
+ claim_id: str | UUID,
92
+ verdict: str,
93
+ confidence: float,
94
+ reason: str,
95
+ iterations_used: int,
96
+ ) -> None:
97
+ """최종 판정을 memory 끝에 기록."""
98
+ text = (
99
+ "## Final Verdict\n"
100
+ f"Verdict: {verdict}\n"
101
+ f"Confidence: {confidence:.2f}\n"
102
+ f"Iterations used: {iterations_used}\n"
103
+ f"Reason: {reason}\n"
104
+ )
105
+ ws.append_memory(claim_id, text)
106
+
107
+
108
+ # ── LLM-friendly read ───────────────────────────────────────────
109
+
110
+ def read_for_llm(
111
+ ws: Workspace,
112
+ claim_id: str | UUID,
113
+ max_chars: int = 50_000,
114
+ ) -> str:
115
+ """
116
+ LLM에 넘길 memory 텍스트. 너무 길면 *앞부분(헤더 + Plan)*은 보존하고
117
+ *중간 iteration 일부*를 잘라낸다. 최신 iteration이 가장 중요.
118
+
119
+ Phase D에서 Reflect Agent가 호출.
120
+
121
+ Args:
122
+ max_chars: 글자 수 상한. 보통 LLM token limit의 절반 정도.
123
+ """
124
+ text = ws.read_memory(claim_id)
125
+ if len(text) <= max_chars:
126
+ return text
127
+
128
+ # 헤더(== ~ ## Initial Plan)를 보존, 그 뒤 일부 잘라내고 최신 부분 유지
129
+ plan_header_marker = "## Initial Plan"
130
+ first_iter_marker = "## Iteration 1"
131
+
132
+ header_end = text.find(first_iter_marker)
133
+ if header_end == -1:
134
+ # Plan이 없거나 모름 — 그냥 뒤쪽 max_chars만
135
+ return text[-max_chars:]
136
+
137
+ header = text[:header_end]
138
+ budget_for_recent = max_chars - len(header) - 300 # 안내 텍스트 여유
139
+ if budget_for_recent <= 0:
140
+ # 헤더가 이미 너무 큼
141
+ return text[-max_chars:]
142
+
143
+ recent = text[-budget_for_recent:]
144
+ note = "\n\n[... 중간 iteration 일부 생략됨 — 최신만 표시 ...]\n\n"
145
+ return header + note + recent
146
+
147
+
148
+ # ── 이미 시도한 action 추적 (중복 방지) ─────────────────────────
149
+
150
+ def get_attempted_actions(
151
+ ws: Workspace,
152
+ claim_id: str | UUID,
153
+ ) -> list[dict[str, Any]]:
154
+ """
155
+ 이미 시도한 (action, input) 목록.
156
+
157
+ Reflect Agent (Phase D)가 *같은 검색어 반복 시도 방지*에 사용.
158
+ log.jsonl에서 읽음.
159
+ """
160
+ log = ws.read_log(claim_id)
161
+ attempts: list[dict[str, Any]] = []
162
+ for entry in log:
163
+ action = entry.get("action")
164
+ inp = entry.get("input", {})
165
+ if action:
166
+ attempts.append({"action": action, "input": inp})
167
+ return attempts
168
+
169
+
170
+ def has_attempted(
171
+ ws: Workspace,
172
+ claim_id: str | UUID,
173
+ action: str,
174
+ input_data: dict,
175
+ ) -> bool:
176
+ """동일 action+input이 이미 시도됐는지."""
177
+ attempts = get_attempted_actions(ws, claim_id)
178
+ for a in attempts:
179
+ if a["action"] == action and a["input"] == input_data:
180
+ return True
181
+ return False
182
+
183
+
184
+ # ── 내부 헬퍼 ────────────────────────────────────────────────────
185
+
186
+ def _format_input(inp: dict | None) -> str:
187
+ """간결한 input 표현."""
188
+ if not inp:
189
+ return "(none)"
190
+ if len(inp) == 1:
191
+ k, v = next(iter(inp.items()))
192
+ return f"{k}={v!r}"
193
+ parts = []
194
+ for k, v in inp.items():
195
+ # 값이 너무 길면 잘라냄
196
+ v_str = repr(v)
197
+ if len(v_str) > 80:
198
+ v_str = v_str[:77] + "..."
199
+ parts.append(f"{k}={v_str}")
200
+ return ", ".join(parts)
201
+
202
+
203
+ # ── Phase D+에서 추가될 자리 ──────────────────────────────────────
204
+ #
205
+ # def summarize_memory(ws, claim_id, llm) -> str:
206
+ # """Memory가 너무 길면 LLM으로 요약 (Phase D+)."""
207
+ # ...