dandelion-memory 1.0.0rc15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. benchmarks/__init__.py +3 -0
  2. benchmarks/adapters/__init__.py +29 -0
  3. benchmarks/adapters/ablations.py +406 -0
  4. benchmarks/adapters/base.py +72 -0
  5. benchmarks/adapters/controlled/__init__.py +13 -0
  6. benchmarks/adapters/controlled/core_archival.py +115 -0
  7. benchmarks/adapters/controlled/extract_resolve.py +130 -0
  8. benchmarks/adapters/controlled/hybrid_rag.py +120 -0
  9. benchmarks/adapters/controlled/no_late_binding.py +113 -0
  10. benchmarks/adapters/dandelion.py +94 -0
  11. benchmarks/adapters/native/__init__.py +31 -0
  12. benchmarks/adapters/native/base.py +45 -0
  13. benchmarks/adapters/native/dandelion_native.py +164 -0
  14. benchmarks/adapters/native/dandelion_native_b0.py +165 -0
  15. benchmarks/adapters/native/dandelion_native_b1.py +170 -0
  16. benchmarks/adapters/native/dandelion_native_b2.py +169 -0
  17. benchmarks/adapters/native/dandelion_native_b3.py +171 -0
  18. benchmarks/adapters/native/dandelion_native_c4.py +261 -0
  19. benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
  20. benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
  21. benchmarks/adapters/native/event_evidence_b2.py +616 -0
  22. benchmarks/adapters/native/evidence_b2.py +609 -0
  23. benchmarks/adapters/native/letta_native.py +253 -0
  24. benchmarks/adapters/native/mem0_native.py +219 -0
  25. benchmarks/adapters/native/store_vector_b0.py +792 -0
  26. benchmarks/adapters/native/store_vector_b2.py +808 -0
  27. benchmarks/agent_adapter_stdio_acceptance.py +86 -0
  28. benchmarks/bootstrap_cross_stack.py +218 -0
  29. benchmarks/canonical_profile_instrumentation.py +167 -0
  30. benchmarks/canonical_public_baseline.py +288 -0
  31. benchmarks/cli.py +525 -0
  32. benchmarks/collector_public30_offline.py +1653 -0
  33. benchmarks/datasets/__init__.py +45 -0
  34. benchmarks/datasets/conversational_generator.py +238 -0
  35. benchmarks/datasets/generator.py +283 -0
  36. benchmarks/datasets/holdout_topology_generator.py +290 -0
  37. benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
  38. benchmarks/datasets/multi_world.py +1643 -0
  39. benchmarks/datasets/scale_generator.py +289 -0
  40. benchmarks/datasets/schema.py +57 -0
  41. benchmarks/deep_collision_stress.py +199 -0
  42. benchmarks/evaluators/__init__.py +12 -0
  43. benchmarks/evaluators/manifest.py +142 -0
  44. benchmarks/evaluators/metrics.py +277 -0
  45. benchmarks/evaluators/runner.py +258 -0
  46. benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
  47. benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
  48. benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
  49. benchmarks/f1_step0/build_ledger.py +143 -0
  50. benchmarks/f1_step0/build_ledger_v2.py +296 -0
  51. benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
  52. benchmarks/f1_step0/locator_public_smoke.py +172 -0
  53. benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
  54. benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
  55. benchmarks/f1_step0/public_identity_probe.py +191 -0
  56. benchmarks/f1_step0/public_response_mapper.py +101 -0
  57. benchmarks/f1_step0/real_source_public_preflight.py +244 -0
  58. benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
  59. benchmarks/f1_step0/scorer.py +110 -0
  60. benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
  61. benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
  62. benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
  63. benchmarks/f1_step0/test_public_response_mapper.py +110 -0
  64. benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
  65. benchmarks/f1_step0/test_scorer.py +158 -0
  66. benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
  67. benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
  68. benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
  69. benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
  70. benchmarks/forensic_c4_1_dev.py +258 -0
  71. benchmarks/forensic_c4_dev.py +255 -0
  72. benchmarks/forensic_deep25_c4_1.py +478 -0
  73. benchmarks/legacy_migration_acceptance.py +29 -0
  74. benchmarks/longmemeval_actor.py +497 -0
  75. benchmarks/longmemeval_agent_judge.py +593 -0
  76. benchmarks/longmemeval_budget.py +139 -0
  77. benchmarks/longmemeval_full.py +410 -0
  78. benchmarks/longmemeval_judge.py +327 -0
  79. benchmarks/longmemeval_pilot.py +316 -0
  80. benchmarks/longmemeval_s.py +318 -0
  81. benchmarks/memory_bench.py +248 -0
  82. benchmarks/memory_formation/__init__.py +1 -0
  83. benchmarks/memory_formation/audit.py +801 -0
  84. benchmarks/memory_formation/real_audit.py +688 -0
  85. benchmarks/memory_session_acceptance.py +279 -0
  86. benchmarks/native_codex_host.py +513 -0
  87. benchmarks/owner_dense_acceptance.py +126 -0
  88. benchmarks/pilot_seeker_vault.py +266 -0
  89. benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
  90. benchmarks/profile_canonical.py +373 -0
  91. benchmarks/profile_canonical_lanes.py +405 -0
  92. benchmarks/profile_search_chunks.py +351 -0
  93. benchmarks/public_ranges_faq_proof.py +121 -0
  94. benchmarks/public_repos/build_history_scope_v3.py +330 -0
  95. benchmarks/public_repos/build_oracle.py +108 -0
  96. benchmarks/public_repos/build_oracle_v2.py +169 -0
  97. benchmarks/public_repos/build_oracle_v3.py +176 -0
  98. benchmarks/public_repos/map_public_v2.py +296 -0
  99. benchmarks/public_repos/project_prompts_v2.py +31 -0
  100. benchmarks/public_repos/project_prompts_v3.py +29 -0
  101. benchmarks/public_repos/ready_admin_v2.py +214 -0
  102. benchmarks/public_repos/score_public_v2.py +258 -0
  103. benchmarks/public_repos/test_map_public_v2.py +171 -0
  104. benchmarks/public_repos/test_public_repo_v2.py +44 -0
  105. benchmarks/public_repos/test_public_repo_v3.py +46 -0
  106. benchmarks/public_repos/test_ready_admin_v2.py +117 -0
  107. benchmarks/public_repos/test_score_public_v2.py +163 -0
  108. benchmarks/public_repos/test_wire_v2.py +292 -0
  109. benchmarks/public_repos/wire_v2.py +603 -0
  110. benchmarks/real_blocker_fix_acceptance.py +155 -0
  111. benchmarks/real_scan_rescan_acceptance.py +41 -0
  112. benchmarks/recall_v2_20k_smoke.py +112 -0
  113. benchmarks/recall_v2_e1_retrieve.py +70 -0
  114. benchmarks/recall_v2_e1_score.py +271 -0
  115. benchmarks/recall_v2_e2_prepare.py +69 -0
  116. benchmarks/recall_v2_e2_public_score.py +91 -0
  117. benchmarks/recall_v2_e2_score.py +108 -0
  118. benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
  119. benchmarks/recall_v2_reader_capture.py +118 -0
  120. benchmarks/recall_v2_reader_replay.py +136 -0
  121. benchmarks/run_blind_h10_holdout.py +284 -0
  122. benchmarks/run_blind_h9_holdout.py +275 -0
  123. benchmarks/run_c4_2_nightly_scale.py +446 -0
  124. benchmarks/run_context_scale_simulation.py +189 -0
  125. benchmarks/run_conversational_bench.py +170 -0
  126. benchmarks/run_h11_evaluation.py +316 -0
  127. benchmarks/run_h11_v2_evaluation.py +431 -0
  128. benchmarks/run_native_fixed12.py +469 -0
  129. benchmarks/run_native_preflight.py +280 -0
  130. benchmarks/run_scale_stress.py +225 -0
  131. benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
  132. benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
  133. benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
  134. benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
  135. benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
  136. benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
  137. benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
  138. benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
  139. benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
  140. benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
  141. benchmarks/statistics/__init__.py +15 -0
  142. benchmarks/statistics/significance.py +326 -0
  143. benchmarks/test_native_mcp_preflight.py +229 -0
  144. benchmarks/test_prepare_public30_range_owner.py +35 -0
  145. benchmarks/test_public30_source_reference.py +698 -0
  146. benchmarks/test_recall_v2_e1_reader_input.py +89 -0
  147. benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
  148. benchmarks/test_recall_v2_reader_replay.py +41 -0
  149. dandelion/__init__.py +3 -0
  150. dandelion/__main__.py +11 -0
  151. dandelion/agent_adapter.py +2369 -0
  152. dandelion/bootstrap/__init__.py +872 -0
  153. dandelion/bootstrap/alias_policy.py +50 -0
  154. dandelion/bootstrap/authority.py +225 -0
  155. dandelion/bootstrap/canonical_history.py +412 -0
  156. dandelion/bootstrap/classifier.py +403 -0
  157. dandelion/bootstrap/component_graph.py +280 -0
  158. dandelion/bootstrap/discovery.py +279 -0
  159. dandelion/bootstrap/domain_induction.py +180 -0
  160. dandelion/bootstrap/domain_topology.py +673 -0
  161. dandelion/bootstrap/evidence_envelope.py +132 -0
  162. dandelion/bootstrap/extractors/base.py +171 -0
  163. dandelion/bootstrap/extractors/exact.py +1234 -0
  164. dandelion/bootstrap/extractors/generic.py +100 -0
  165. dandelion/bootstrap/extractors/git_temporal.py +111 -0
  166. dandelion/bootstrap/extractors/registry.py +345 -0
  167. dandelion/bootstrap/extractors/relations.py +89 -0
  168. dandelion/bootstrap/extractors/semantic.py +424 -0
  169. dandelion/bootstrap/extractors/status.py +218 -0
  170. dandelion/bootstrap/git_sweeper.py +135 -0
  171. dandelion/bootstrap/manifest.py +246 -0
  172. dandelion/bootstrap/quarantine.py +264 -0
  173. dandelion/bootstrap/report.py +259 -0
  174. dandelion/bootstrap/root_resolver.py +221 -0
  175. dandelion/bootstrap/tech_profiler.py +125 -0
  176. dandelion/bootstrap/topology.py +75 -0
  177. dandelion/bootstrap/topology_index.py +110 -0
  178. dandelion/bootstrap/transaction.py +315 -0
  179. dandelion/canonical_bootstrap.py +257 -0
  180. dandelion/canonical_memory.py +4130 -0
  181. dandelion/captured_input.py +295 -0
  182. dandelion/cli.py +767 -0
  183. dandelion/config.py +40 -0
  184. dandelion/derived_index.py +745 -0
  185. dandelion/domains.py +706 -0
  186. dandelion/embedder.py +229 -0
  187. dandelion/event_evidence.py +571 -0
  188. dandelion/evidence.py +812 -0
  189. dandelion/evidence_response.py +453 -0
  190. dandelion/evidence_workspace.py +2360 -0
  191. dandelion/exact_context.py +129 -0
  192. dandelion/hydrator.py +124 -0
  193. dandelion/inspector.py +442 -0
  194. dandelion/inspector_assets/dandelion-mark.svg +33 -0
  195. dandelion/inspector_assets/dandelion_graph.js +361 -0
  196. dandelion/inspector_assets/inspector.css +223 -0
  197. dandelion/inspector_assets/inspector.html +135 -0
  198. dandelion/inspector_assets/inspector.js +1059 -0
  199. dandelion/legacy_import.py +431 -0
  200. dandelion/live_source_grants.py +133 -0
  201. dandelion/local_owner.py +114 -0
  202. dandelion/memory_client.py +252 -0
  203. dandelion/memory_contract.py +107 -0
  204. dandelion/memory_owner.py +560 -0
  205. dandelion/reader_providers.py +118 -0
  206. dandelion/recall_v2.py +767 -0
  207. dandelion/reconciler.py +219 -0
  208. dandelion/request_budget.py +125 -0
  209. dandelion/retained_basis.py +223 -0
  210. dandelion/retrieval.py +485 -0
  211. dandelion/router.py +405 -0
  212. dandelion/search_terms.py +53 -0
  213. dandelion/security.py +124 -0
  214. dandelion/server.py +810 -0
  215. dandelion/source_content_fts.py +219 -0
  216. dandelion/source_delivery.py +89 -0
  217. dandelion/source_history.py +241 -0
  218. dandelion/source_identity.py +88 -0
  219. dandelion/source_navigator.py +5 -0
  220. dandelion/source_reader.py +444 -0
  221. dandelion/source_references.py +379 -0
  222. dandelion/source_sections.py +265 -0
  223. dandelion/store_kv.py +684 -0
  224. dandelion/store_vector.py +1289 -0
  225. dandelion/studio.py +325 -0
  226. dandelion/wal.py +227 -0
  227. dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
  228. dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
  229. dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
  230. dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
  231. dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
  232. dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
@@ -0,0 +1,403 @@
1
+ """Universal Content-First File Classifier (BOOTSTRAP-02).
2
+
3
+ Classifies any workspace file into a normalized FileRole using multi-signal evidence:
4
+ 1. Path context and suppression rules (vendor, cache, generated)
5
+ 2. Binary vs text detection (null bytes, utf-8 decoding)
6
+ 3. Shebang / magic byte inspection (e.g. #!/usr/bin/env python)
7
+ 4. Structured content format detection (JSON, YAML, TOML, INI, SQL, XML, Markdown)
8
+ 5. Extension and filename conventions
9
+ 6. Security / sensitive data heuristics
10
+
11
+ Principle: Unknown format != invisible information.
12
+ Unknown text files degrade gracefully to UNKNOWN_TEXT.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import os
18
+ import re
19
+ from enum import Enum
20
+ from pathlib import Path
21
+ from typing import Optional
22
+
23
+
24
+ class FileRole(str, Enum):
25
+ """Universal classification of files in any software repository."""
26
+ SOURCE_CODE = "source_code"
27
+ STRUCTURED_CONFIG = "structured_config"
28
+ PROJECT_DESCRIPTOR = "project_descriptor"
29
+ BUILD_CONFIG = "build_config"
30
+ SCHEMA = "schema"
31
+ MIGRATION = "migration"
32
+ DOCUMENTATION = "documentation"
33
+ TEST = "test"
34
+ CI_CD = "ci_cd"
35
+ DEPLOYMENT = "deployment"
36
+ DATA = "data"
37
+ NOTEBOOK = "notebook"
38
+ ASSET = "asset"
39
+ BINARY = "binary"
40
+ GENERATED = "generated"
41
+ VENDOR = "vendor"
42
+ CACHE = "cache"
43
+ SECRET_SENSITIVE = "secret_sensitive"
44
+ UNKNOWN_TEXT = "unknown_text"
45
+ UNKNOWN_BINARY = "unknown_binary"
46
+
47
+
48
+ # Standard ignore directories across ecosystems
49
+ VENDOR_CACHE_DIRS: set[str] = {
50
+ ".git", ".hg", ".svn", "node_modules", "vendor", "__pycache__",
51
+ ".venv", "venv", "env", "dist", "build", "target", "out",
52
+ ".dandelion", ".pytest_cache", ".mypy_cache", ".ruff_cache",
53
+ ".idea", ".vscode", ".gradle", ".cargo", ".nuget", "obj",
54
+ "bin", ".next", ".nuxt", ".output", "coverage", ".turbo",
55
+ }
56
+
57
+ # Known binary asset extensions
58
+ ASSET_EXTENSIONS: set[str] = {
59
+ ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".svg",
60
+ ".mp3", ".wav", ".ogg", ".flac", ".mp4", ".webm", ".mov",
61
+ ".ttf", ".otf", ".woff", ".woff2", ".blend", ".fbx", ".obj",
62
+ ".dae", ".glb", ".gltf", ".tscn", ".scn", ".res", ".tres",
63
+ }
64
+
65
+ # Known compiled binary extensions
66
+ BINARY_EXTENSIONS: set[str] = {
67
+ ".exe", ".dll", ".so", ".dylib", ".bin", ".pyc", ".pyd",
68
+ ".class", ".jar", ".war", ".ear", ".a", ".o", ".lib",
69
+ ".iso", ".apk", ".aab", ".ipa", ".zip", ".tar", ".gz",
70
+ ".7z", ".rar", ".bz2", ".xz", ".wasm",
71
+ }
72
+
73
+ # Known source code extensions
74
+ SOURCE_EXTENSIONS: dict[str, str] = {
75
+ ".py": "python",
76
+ ".gd": "gdscript",
77
+ ".kt": "kotlin",
78
+ ".kts": "kotlin_dsl",
79
+ ".java": "java",
80
+ ".js": "javascript",
81
+ ".jsx": "javascript_react",
82
+ ".ts": "typescript",
83
+ ".tsx": "typescript_react",
84
+ ".rs": "rust",
85
+ ".go": "go",
86
+ ".c": "c",
87
+ ".cpp": "cpp",
88
+ ".h": "c_header",
89
+ ".hpp": "cpp_header",
90
+ ".cs": "csharp",
91
+ ".rb": "ruby",
92
+ ".php": "php",
93
+ ".swift": "swift",
94
+ ".scala": "scala",
95
+ ".sh": "shell",
96
+ ".bash": "bash",
97
+ ".zsh": "zsh",
98
+ ".ps1": "powershell",
99
+ ".lua": "lua",
100
+ ".r": "r",
101
+ ".dart": "dart",
102
+ ".ex": "elixir",
103
+ ".exs": "elixir",
104
+ ".erl": "erlang",
105
+ ".clj": "clojure",
106
+ }
107
+
108
+ # Known project descriptors and build manifests
109
+ PROJECT_DESCRIPTORS: dict[str, str] = {
110
+ "project.godot": "godot_project",
111
+ "export_presets.cfg": "godot_export",
112
+ "package.json": "npm_package",
113
+ "pyproject.toml": "pyproject",
114
+ "setup.py": "setuptools",
115
+ "setup.cfg": "setuptools_cfg",
116
+ "cargo.toml": "cargo_manifest",
117
+ "go.mod": "go_module",
118
+ "pom.xml": "maven_pom",
119
+ "build.gradle": "gradle_groovy",
120
+ "build.gradle.kts": "gradle_kotlin",
121
+ "settings.gradle": "gradle_settings",
122
+ "settings.gradle.kts": "gradle_settings_kotlin",
123
+ "gradle.properties": "gradle_properties",
124
+ "makefile": "makefile",
125
+ "cmakelists.txt": "cmake",
126
+ "gemfile": "bundler_gemfile",
127
+ "composer.json": "composer_package",
128
+ "mix.exs": "mix_project",
129
+ "pubspec.yaml": "pub_package",
130
+ }
131
+
132
+ # Known documentation extensions
133
+ DOC_EXTENSIONS: set[str] = {
134
+ ".md", ".markdown", ".mdown", ".rst", ".adoc", ".asciidoc", ".txt",
135
+ }
136
+
137
+ # Known schema and migration extensions
138
+ SCHEMA_EXTENSIONS: dict[str, str] = {
139
+ ".sql": "sql",
140
+ ".prisma": "prisma",
141
+ ".proto": "protobuf",
142
+ ".graphql": "graphql",
143
+ ".gql": "graphql",
144
+ ".avro": "avro",
145
+ }
146
+
147
+ # Known structured config extensions
148
+ CONFIG_EXTENSIONS: set[str] = {
149
+ ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
150
+ ".properties", ".env",
151
+ }
152
+
153
+
154
+ def is_binary_buffer(buf: bytes) -> bool:
155
+ """Detect if raw buffer is binary (contains null bytes or invalid utf-8)."""
156
+ if b"\x00" in buf:
157
+ return True
158
+ try:
159
+ buf.decode("utf-8")
160
+ return False
161
+ except UnicodeDecodeError:
162
+ # Check if non-text byte ratio is high
163
+ text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
164
+ non_text = sum(1 for byte in buf if byte not in text_chars)
165
+ return (non_text / max(1, len(buf))) > 0.30
166
+
167
+
168
+ def check_shebang(buf: bytes) -> Optional[tuple[FileRole, str]]:
169
+ """Inspect shebang line (e.g. #!/usr/bin/env python) for source classification."""
170
+ if not buf.startswith(b"#!"):
171
+ return None
172
+ try:
173
+ first_line = buf[:256].decode("utf-8", errors="ignore").split("\n")[0].lower()
174
+ if "python" in first_line:
175
+ return FileRole.SOURCE_CODE, "python"
176
+ if "bash" in first_line or "sh" in first_line:
177
+ return FileRole.SOURCE_CODE, "shell"
178
+ if "node" in first_line:
179
+ return FileRole.SOURCE_CODE, "javascript"
180
+ if "ruby" in first_line:
181
+ return FileRole.SOURCE_CODE, "ruby"
182
+ if "perl" in first_line:
183
+ return FileRole.SOURCE_CODE, "perl"
184
+ if "php" in first_line:
185
+ return FileRole.SOURCE_CODE, "php"
186
+ except Exception:
187
+ pass
188
+ return None
189
+
190
+
191
+ def is_sensitive_path(rel_norm: str, name: str) -> bool:
192
+ """Heuristic check for sensitive or secret-bearing configuration paths."""
193
+ if name.startswith(".env") and not name.endswith((".example", ".sample", ".template")):
194
+ return True
195
+ if any(k in name for k in ("id_rsa", "id_ed25519", "private_key", ".pem", ".pfx", ".keystore", ".p12")):
196
+ return True
197
+ if any(k in name for k in ("secret", "credential", "password", "auth_token")) and not name.endswith((".example", ".sample", ".md")):
198
+ return True
199
+ return False
200
+
201
+
202
+ def is_test_path(rel_norm: str, name: str) -> bool:
203
+ """Heuristic check if file belongs to testing suite."""
204
+ parts = rel_norm.split("/")
205
+ if any(p in ("tests", "test", "__tests__", "spec", "specs") for p in parts[:-1]):
206
+ return True
207
+ if name.startswith("test_") or name.endswith(("_test.py", "_test.go", "_test.rs", ".test.js", ".test.ts", ".spec.js", ".spec.ts", "test.gd")):
208
+ return True
209
+ return False
210
+
211
+
212
+ def is_ci_cd_path(rel_norm: str, name: str) -> bool:
213
+ """Heuristic check if file is CI/CD or workflow configuration."""
214
+ if ".github/workflows" in rel_norm or ".gitlab-ci" in rel_norm or "azure-pipelines" in rel_norm:
215
+ return True
216
+ if name in ("jenkinsfile", ".travis.yml", "circle.yml", "bitbucket-pipelines.yml"):
217
+ return True
218
+ return False
219
+
220
+
221
+ def is_deployment_path(rel_norm: str, name: str) -> bool:
222
+ """Heuristic check for deployment, container, or infra descriptors."""
223
+ if name.startswith("dockerfile") or name.startswith("containerfile"):
224
+ return True
225
+ if "docker-compose" in name:
226
+ return True
227
+ if any(k in rel_norm for k in ("/k8s/", "/kubernetes/", "/helm/", "/terraform/", "/deploy/", "/charts/")):
228
+ return True
229
+ if name.endswith((".tf", ".tfvars")):
230
+ return True
231
+ return False
232
+
233
+
234
+ class ClassificationResult:
235
+ """Rich classification result for a discovered file."""
236
+ def __init__(
237
+ self,
238
+ role: FileRole,
239
+ detected_format: str,
240
+ confidence: float,
241
+ is_text: bool,
242
+ is_suppressed: bool = False,
243
+ signals: Optional[list[str]] = None,
244
+ ):
245
+ self.role = role
246
+ self.detected_format = detected_format
247
+ self.confidence = confidence
248
+ self.is_text = is_text
249
+ self.is_suppressed = is_suppressed
250
+ self.signals = signals or []
251
+
252
+ def to_dict(self) -> dict:
253
+ return {
254
+ "role": self.role.value,
255
+ "format": self.detected_format,
256
+ "confidence": round(self.confidence, 2),
257
+ "is_text": self.is_text,
258
+ "is_suppressed": self.is_suppressed,
259
+ "signals": self.signals,
260
+ }
261
+
262
+
263
+ def classify_content(rel_path: str, abs_path: Path) -> ClassificationResult:
264
+ """Multi-signal universal file classification.
265
+
266
+ Evaluates path, content snippet, magic bytes, shebang, and extension.
267
+ Guarantees that unknown files degrade gracefully (never vanish).
268
+ """
269
+ name = abs_path.name.lower()
270
+ rel_norm = rel_path.replace("\\", "/").lower()
271
+ ext = abs_path.suffix.lower()
272
+ signals: list[str] = []
273
+
274
+ # 1. Check for vendor/cache/generated suppression
275
+ parts = set(rel_norm.split("/")[:-1])
276
+ if parts & VENDOR_CACHE_DIRS:
277
+ matched = list(parts & VENDOR_CACHE_DIRS)
278
+ return ClassificationResult(
279
+ role=FileRole.VENDOR if any(v in matched for v in ("node_modules", "vendor")) else FileRole.CACHE,
280
+ detected_format=ext.lstrip(".") or "binary",
281
+ confidence=1.0,
282
+ is_text=False,
283
+ is_suppressed=True,
284
+ signals=[f"suppressed_directory:{matched[0]}"],
285
+ )
286
+
287
+ # 2. Check secret / sensitive files early
288
+ if is_sensitive_path(rel_norm, name):
289
+ signals.append("sensitive_path_pattern")
290
+ return ClassificationResult(
291
+ role=FileRole.SECRET_SENSITIVE,
292
+ detected_format=ext.lstrip(".") or "secret",
293
+ confidence=0.95,
294
+ is_text=True,
295
+ is_suppressed=False,
296
+ signals=signals,
297
+ )
298
+
299
+ if name.startswith(".env"):
300
+ signals.append("env_file")
301
+ return ClassificationResult(
302
+ role=FileRole.STRUCTURED_CONFIG,
303
+ detected_format="env",
304
+ confidence=0.95,
305
+ is_text=True,
306
+ signals=signals,
307
+ )
308
+
309
+ # 3. Read header bytes if file exists
310
+ header_bytes = b""
311
+ if abs_path.exists() and abs_path.is_file():
312
+ try:
313
+ with open(abs_path, "rb") as f:
314
+ header_bytes = f.read(8192)
315
+ except Exception:
316
+ pass
317
+
318
+ is_bin = is_binary_buffer(header_bytes) if header_bytes else (ext in BINARY_EXTENSIONS or ext in ASSET_EXTENSIONS)
319
+
320
+ # 4. Check binary assets & binaries
321
+ if is_bin:
322
+ if ext in ASSET_EXTENSIONS:
323
+ return ClassificationResult(FileRole.ASSET, ext.lstrip("."), 0.95, is_text=False, signals=["asset_extension"])
324
+ if ext in BINARY_EXTENSIONS:
325
+ return ClassificationResult(FileRole.BINARY, ext.lstrip("."), 0.95, is_text=False, signals=["binary_extension"])
326
+ return ClassificationResult(FileRole.UNKNOWN_BINARY, ext.lstrip(".") or "bin", 0.50, is_text=False, signals=["binary_buffer"])
327
+
328
+ # ── Textual Path from here ──
329
+
330
+ # 5. Shebang inspection
331
+ shebang_match = check_shebang(header_bytes)
332
+ if shebang_match:
333
+ role, fmt = shebang_match
334
+ signals.append(f"shebang:{fmt}")
335
+ return ClassificationResult(role, fmt, 0.95, is_text=True, signals=signals)
336
+
337
+ # 6. CI/CD & Deployment
338
+ if is_ci_cd_path(rel_norm, name):
339
+ signals.append("ci_cd_path")
340
+ return ClassificationResult(FileRole.CI_CD, ext.lstrip(".") or "yaml", 0.90, is_text=True, signals=signals)
341
+
342
+ if is_deployment_path(rel_norm, name):
343
+ signals.append("deployment_path")
344
+ fmt = "docker" if "dockerfile" in name else (ext.lstrip(".") or "yaml")
345
+ return ClassificationResult(FileRole.DEPLOYMENT, fmt, 0.90, is_text=True, signals=signals)
346
+
347
+ # 7. Project Descriptors & Build Manifests
348
+ if name in PROJECT_DESCRIPTORS:
349
+ fmt = PROJECT_DESCRIPTORS[name]
350
+ role = FileRole.PROJECT_DESCRIPTOR if "project" in fmt or "package" in fmt or "manifest" in fmt else FileRole.BUILD_CONFIG
351
+ signals.append(f"exact_project_descriptor:{name}")
352
+ return ClassificationResult(role, fmt, 1.0, is_text=True, signals=signals)
353
+
354
+ # 8. Tests
355
+ if is_test_path(rel_norm, name):
356
+ signals.append("test_path_pattern")
357
+ fmt = SOURCE_EXTENSIONS.get(ext, ext.lstrip(".") or "test")
358
+ return ClassificationResult(FileRole.TEST, fmt, 0.90, is_text=True, signals=signals)
359
+
360
+ # 9. Source Code
361
+ if ext in SOURCE_EXTENSIONS:
362
+ fmt = SOURCE_EXTENSIONS[ext]
363
+ signals.append(f"source_extension:{ext}")
364
+ return ClassificationResult(FileRole.SOURCE_CODE, fmt, 0.95, is_text=True, signals=signals)
365
+
366
+ # 10. Schemas and Migrations
367
+ if ext in SCHEMA_EXTENSIONS or "/migrations/" in rel_norm or "/schema/" in rel_norm:
368
+ fmt = SCHEMA_EXTENSIONS.get(ext, "sql")
369
+ role = FileRole.MIGRATION if "/migration" in rel_norm else FileRole.SCHEMA
370
+ signals.append(f"schema_match:{fmt}")
371
+ return ClassificationResult(role, fmt, 0.90, is_text=True, signals=signals)
372
+
373
+ # 11. Documentation
374
+ if ext in DOC_EXTENSIONS:
375
+ fmt = "markdown" if ext in (".md", ".markdown") else ext.lstrip(".")
376
+ signals.append(f"doc_extension:{ext}")
377
+ return ClassificationResult(FileRole.DOCUMENTATION, fmt, 0.95, is_text=True, signals=signals)
378
+
379
+ # 12. Structured Configuration
380
+ if ext in CONFIG_EXTENSIONS:
381
+ fmt = ext.lstrip(".")
382
+ signals.append(f"config_extension:{ext}")
383
+ return ClassificationResult(FileRole.STRUCTURED_CONFIG, fmt, 0.85, is_text=True, signals=signals)
384
+
385
+ # 13. Data files (JSON/CSV/TSV/Parquet)
386
+ if ext in (".csv", ".tsv", ".jsonl"):
387
+ signals.append(f"data_extension:{ext}")
388
+ return ClassificationResult(FileRole.DATA, ext.lstrip("."), 0.85, is_text=True, signals=signals)
389
+
390
+ # 14. Jupyter Notebooks
391
+ if ext == ".ipynb":
392
+ signals.append("notebook_extension")
393
+ return ClassificationResult(FileRole.NOTEBOOK, "ipynb", 0.95, is_text=True, signals=signals)
394
+
395
+ # 15. Mandatory Graceful Degradation: Unknown Text
396
+ signals.append("generic_text_fallback")
397
+ return ClassificationResult(
398
+ role=FileRole.UNKNOWN_TEXT,
399
+ detected_format=ext.lstrip(".") or "text",
400
+ confidence=0.50,
401
+ is_text=True,
402
+ signals=signals,
403
+ )
@@ -0,0 +1,280 @@
1
+ """Evidence-backed component graph construction for BOOTSTRAP-02."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from collections import defaultdict
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+
11
+ from dandelion.bootstrap.discovery import DiscoveredFile, ProjectDiscoveryReport
12
+ from dandelion.bootstrap.root_resolver import RootResolutionResult
13
+ from dandelion.bootstrap.tech_profiler import TechnologyProfile
14
+
15
+
16
+ _DESCRIPTOR_ROLES = {"project_descriptor", "build_config"}
17
+ _CONFIG_ROLES = _DESCRIPTOR_ROLES | {"structured_config", "ci_cd", "deployment"}
18
+ _IMPORT_RE = re.compile(
19
+ r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+)|use\s+([\w:]+)|(?:require|include)\s*\(?[\"']([^\"']+))",
20
+ re.MULTILINE,
21
+ )
22
+
23
+
24
+ @dataclass
25
+ class ComponentNode:
26
+ """A structural node inferred from files, descriptors, and boundaries."""
27
+ id: str
28
+ name: str
29
+ kind: str
30
+ rel_path: str
31
+ files_count: int = 0
32
+ source_files_count: int = 0
33
+ descriptors: list[str] = field(default_factory=list)
34
+ technologies: list[str] = field(default_factory=list)
35
+ in_degree: int = 0
36
+ out_degree: int = 0
37
+ is_root: bool = False
38
+ resolution: str = "RESOLVED"
39
+ is_workspace_boundary: bool = False
40
+
41
+ def to_dict(self) -> dict:
42
+ return {
43
+ "id": self.id,
44
+ "name": self.name,
45
+ "kind": self.kind,
46
+ "rel_path": self.rel_path,
47
+ "files_count": self.files_count,
48
+ "source_files_count": self.source_files_count,
49
+ "descriptors": self.descriptors,
50
+ "technologies": self.technologies,
51
+ "in_degree": self.in_degree,
52
+ "out_degree": self.out_degree,
53
+ "is_root": self.is_root,
54
+ "resolution": self.resolution,
55
+ "is_workspace_boundary": self.is_workspace_boundary,
56
+ }
57
+
58
+
59
+ @dataclass
60
+ class ComponentEdge:
61
+ source_id: str
62
+ target_id: str
63
+ relation: str
64
+ confidence: float = 1.0
65
+
66
+ def to_dict(self) -> dict:
67
+ return {
68
+ "source": self.source_id,
69
+ "target": self.target_id,
70
+ "relation": self.relation,
71
+ "confidence": self.confidence,
72
+ }
73
+
74
+
75
+ @dataclass
76
+ class ComponentGraph:
77
+ root_id: str
78
+ nodes: dict[str, ComponentNode] = field(default_factory=dict)
79
+ edges: list[ComponentEdge] = field(default_factory=list)
80
+
81
+ def to_dict(self) -> dict:
82
+ return {
83
+ "root_id": self.root_id,
84
+ "nodes_count": len(self.nodes),
85
+ "edges_count": len(self.edges),
86
+ "nodes": {key: node.to_dict() for key, node in self.nodes.items()},
87
+ "edges": [edge.to_dict() for edge in self.edges],
88
+ }
89
+
90
+
91
+ def _norm(path: str) -> str:
92
+ return path.replace("\\", "/").strip("/")
93
+
94
+
95
+ def _component_path_for_file(file: DiscoveredFile, descriptor_dirs: set[str]) -> str:
96
+ parts = _norm(file.rel_path).split("/")
97
+ if len(parts) == 1:
98
+ return "root"
99
+ parent = _norm(str(Path(_norm(file.rel_path)).parent))
100
+ for descriptor_dir in sorted(descriptor_dirs, key=lambda value: value.count("/"), reverse=True):
101
+ if parent == descriptor_dir or parent.startswith(descriptor_dir + "/"):
102
+ return descriptor_dir
103
+ return "/".join(parts[:2]) if len(parts) > 2 else parts[0]
104
+
105
+
106
+ def _kind(files: list[DiscoveredFile], descriptors: list[DiscoveredFile]) -> tuple[str, str]:
107
+ roles = {file.role.value for file in files}
108
+ if descriptors:
109
+ return "package", "RESOLVED"
110
+ if roles <= {"documentation"}:
111
+ return "doc_set", "RESOLVED"
112
+ if roles <= {"test"}:
113
+ return "test_suite", "RESOLVED"
114
+ if "source_code" in roles:
115
+ return "subsystem", "RESOLVED"
116
+ return "unresolved_component", "UNRESOLVED_COMPONENT"
117
+
118
+
119
+ def _add_edge(graph: ComponentGraph, source_id: str, target_id: str, relation: str, confidence: float) -> None:
120
+ if source_id == target_id or source_id not in graph.nodes or target_id not in graph.nodes:
121
+ return
122
+ if any(edge.source_id == source_id and edge.target_id == target_id and edge.relation == relation for edge in graph.edges):
123
+ return
124
+ graph.edges.append(ComponentEdge(source_id, target_id, relation, confidence))
125
+ graph.nodes[source_id].out_degree += 1
126
+ graph.nodes[target_id].in_degree += 1
127
+
128
+
129
+ def _read_text(file: DiscoveredFile) -> str:
130
+ if not file.is_text or file.size_bytes > 131072:
131
+ return ""
132
+ try:
133
+ return file.abs_path.read_text(encoding="utf-8", errors="ignore")
134
+ except OSError:
135
+ return ""
136
+
137
+
138
+ def _target_for_reference(reference: str, graph: ComponentGraph, package_names: dict[str, str]) -> str | None:
139
+ cleaned = reference.strip().strip("'\"").split(":", 1)[0]
140
+ if cleaned in package_names:
141
+ return package_names[cleaned]
142
+ normalized = cleaned.replace(".", "/").strip("/")
143
+ for node in sorted(graph.nodes.values(), key=lambda item: len(item.rel_path), reverse=True):
144
+ if not node.rel_path or node.kind == "workspace_boundary":
145
+ continue
146
+ if normalized == node.rel_path or normalized.startswith(node.rel_path + "/"):
147
+ return node.id
148
+ return None
149
+
150
+
151
+ def _package_name_index(files: list[DiscoveredFile], owners: dict[str, str]) -> dict[str, str]:
152
+ result: dict[str, str] = {}
153
+ for file in files:
154
+ if file.format != "npm_package":
155
+ continue
156
+ try:
157
+ name = json.loads(_read_text(file)).get("name")
158
+ except (TypeError, ValueError):
159
+ continue
160
+ if isinstance(name, str) and file.rel_path in owners:
161
+ result[name] = owners[file.rel_path]
162
+ return result
163
+
164
+
165
+ def build_component_graph(
166
+ report: ProjectDiscoveryReport,
167
+ root_res: RootResolutionResult,
168
+ tech_profile: TechnologyProfile,
169
+ ) -> ComponentGraph:
170
+ """Build a bounded graph only from explicit file and manifest evidence."""
171
+ root_id = root_res.canonical_id
172
+ graph = ComponentGraph(root_id=root_id)
173
+ graph.nodes[root_id] = ComponentNode(
174
+ id=root_id,
175
+ name=root_res.project_name,
176
+ kind="application" if root_res.workspace_type == "single_project" else "workspace",
177
+ rel_path="",
178
+ is_root=True,
179
+ technologies=tech_profile.frameworks + [tech_profile.primary_language],
180
+ )
181
+
182
+ descriptor_dirs = {
183
+ _norm(str(Path(_norm(file.rel_path)).parent))
184
+ for file in report.all_discovered
185
+ if file.role.value in _DESCRIPTOR_ROLES and "/" in _norm(file.rel_path)
186
+ }
187
+ files_by_component: dict[str, list[DiscoveredFile]] = defaultdict(list)
188
+ descriptors_by_component: dict[str, list[DiscoveredFile]] = defaultdict(list)
189
+ owners: dict[str, str] = {}
190
+ for file in report.all_discovered:
191
+ component_path = _component_path_for_file(file, descriptor_dirs)
192
+ files_by_component[component_path].append(file)
193
+ if file.role.value in _DESCRIPTOR_ROLES:
194
+ descriptors_by_component[component_path].append(file)
195
+
196
+ for component_path, files in files_by_component.items():
197
+ if component_path == "root":
198
+ root = graph.nodes[root_id]
199
+ root.files_count = len(files)
200
+ root.source_files_count = sum(file.role.value == "source_code" for file in files)
201
+ root.descriptors = [file.rel_path for file in descriptors_by_component[component_path]]
202
+ for file in files:
203
+ owners[file.rel_path] = root_id
204
+ continue
205
+ component_id = component_path.replace("/", ".")
206
+ descriptors = descriptors_by_component[component_path]
207
+ kind, resolution = _kind(files, descriptors)
208
+ node = ComponentNode(
209
+ id=component_id,
210
+ name=component_path.rsplit("/", 1)[-1],
211
+ kind=kind,
212
+ rel_path=component_path,
213
+ files_count=len(files),
214
+ source_files_count=sum(file.role.value == "source_code" for file in files),
215
+ descriptors=[file.rel_path for file in descriptors],
216
+ technologies=tech_profile.component_technologies.get(component_path.split("/")[0], []),
217
+ resolution=resolution,
218
+ is_workspace_boundary=component_path in root_res.workspace_boundaries,
219
+ )
220
+ graph.nodes[component_id] = node
221
+ _add_edge(graph, root_id, component_id, "contains", 1.0)
222
+ for file in files:
223
+ owners[file.rel_path] = component_id
224
+
225
+ for boundary in root_res.workspace_boundaries:
226
+ if any(node.rel_path == boundary for node in graph.nodes.values()):
227
+ continue
228
+ boundary_id = f"workspace.{boundary.replace('/', '.')}"
229
+ graph.nodes[boundary_id] = ComponentNode(
230
+ id=boundary_id,
231
+ name=boundary.rsplit("/", 1)[-1],
232
+ kind="workspace_boundary",
233
+ rel_path=boundary,
234
+ is_workspace_boundary=True,
235
+ )
236
+ _add_edge(graph, root_id, boundary_id, "contains", 1.0)
237
+
238
+ package_names = _package_name_index(report.all_discovered, owners)
239
+ for file in report.all_discovered:
240
+ owner = owners.get(file.rel_path)
241
+ if owner is None:
242
+ continue
243
+ text = _read_text(file)
244
+ if not text:
245
+ continue
246
+ if file.role.value == "source_code":
247
+ for match in _IMPORT_RE.finditer(text):
248
+ target = _target_for_reference(next(value for value in match.groups() if value), graph, package_names)
249
+ if target:
250
+ _add_edge(graph, owner, target, "imports", 0.95)
251
+ if file.format == "npm_package":
252
+ try:
253
+ payload = json.loads(text)
254
+ except ValueError:
255
+ payload = {}
256
+ for section in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
257
+ dependencies = payload.get(section, {})
258
+ if not isinstance(dependencies, dict):
259
+ continue
260
+ for dependency in dependencies:
261
+ target = package_names.get(dependency)
262
+ if target:
263
+ _add_edge(graph, owner, target, "depends_on", 1.0)
264
+ if file.role.value in _CONFIG_ROLES:
265
+ for node in graph.nodes.values():
266
+ if not node.rel_path or node.id == owner or "/" not in node.rel_path:
267
+ continue
268
+ if node.rel_path in text:
269
+ relation = "builds" if file.role.value in _DESCRIPTOR_ROLES else "configures"
270
+ _add_edge(graph, owner, node.id, relation, 0.85)
271
+
272
+ for node in list(graph.nodes.values()):
273
+ if node.id == root_id:
274
+ continue
275
+ if node.kind == "test_suite":
276
+ _add_edge(graph, node.id, root_id, "tests", 0.75)
277
+ elif node.kind == "doc_set":
278
+ _add_edge(graph, node.id, root_id, "documents", 0.75)
279
+
280
+ return graph