dandelion-memory 1.0.0rc15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchmarks/__init__.py +3 -0
- benchmarks/adapters/__init__.py +29 -0
- benchmarks/adapters/ablations.py +406 -0
- benchmarks/adapters/base.py +72 -0
- benchmarks/adapters/controlled/__init__.py +13 -0
- benchmarks/adapters/controlled/core_archival.py +115 -0
- benchmarks/adapters/controlled/extract_resolve.py +130 -0
- benchmarks/adapters/controlled/hybrid_rag.py +120 -0
- benchmarks/adapters/controlled/no_late_binding.py +113 -0
- benchmarks/adapters/dandelion.py +94 -0
- benchmarks/adapters/native/__init__.py +31 -0
- benchmarks/adapters/native/base.py +45 -0
- benchmarks/adapters/native/dandelion_native.py +164 -0
- benchmarks/adapters/native/dandelion_native_b0.py +165 -0
- benchmarks/adapters/native/dandelion_native_b1.py +170 -0
- benchmarks/adapters/native/dandelion_native_b2.py +169 -0
- benchmarks/adapters/native/dandelion_native_b3.py +171 -0
- benchmarks/adapters/native/dandelion_native_c4.py +261 -0
- benchmarks/adapters/native/dandelion_native_c4_1.py +281 -0
- benchmarks/adapters/native/dandelion_native_c4_2.py +435 -0
- benchmarks/adapters/native/event_evidence_b2.py +616 -0
- benchmarks/adapters/native/evidence_b2.py +609 -0
- benchmarks/adapters/native/letta_native.py +253 -0
- benchmarks/adapters/native/mem0_native.py +219 -0
- benchmarks/adapters/native/store_vector_b0.py +792 -0
- benchmarks/adapters/native/store_vector_b2.py +808 -0
- benchmarks/agent_adapter_stdio_acceptance.py +86 -0
- benchmarks/bootstrap_cross_stack.py +218 -0
- benchmarks/canonical_profile_instrumentation.py +167 -0
- benchmarks/canonical_public_baseline.py +288 -0
- benchmarks/cli.py +525 -0
- benchmarks/collector_public30_offline.py +1653 -0
- benchmarks/datasets/__init__.py +45 -0
- benchmarks/datasets/conversational_generator.py +238 -0
- benchmarks/datasets/generator.py +283 -0
- benchmarks/datasets/holdout_topology_generator.py +290 -0
- benchmarks/datasets/holdout_topology_generator_v2.py +294 -0
- benchmarks/datasets/multi_world.py +1643 -0
- benchmarks/datasets/scale_generator.py +289 -0
- benchmarks/datasets/schema.py +57 -0
- benchmarks/deep_collision_stress.py +199 -0
- benchmarks/evaluators/__init__.py +12 -0
- benchmarks/evaluators/manifest.py +142 -0
- benchmarks/evaluators/metrics.py +277 -0
- benchmarks/evaluators/runner.py +258 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v1.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v2.py +127 -0
- benchmarks/f1_step0/build_dev_measurement_manifest_v3.py +146 -0
- benchmarks/f1_step0/build_ledger.py +143 -0
- benchmarks/f1_step0/build_ledger_v2.py +296 -0
- benchmarks/f1_step0/compare_dev52_readiness_v1.py +106 -0
- benchmarks/f1_step0/locator_public_smoke.py +172 -0
- benchmarks/f1_step0/prepare_dev52_owner_v1.py +133 -0
- benchmarks/f1_step0/prepare_dev52_owner_v2.py +133 -0
- benchmarks/f1_step0/public_identity_probe.py +191 -0
- benchmarks/f1_step0/public_response_mapper.py +101 -0
- benchmarks/f1_step0/real_source_public_preflight.py +244 -0
- benchmarks/f1_step0/run_exact_fusion_dev_experiment_v1.py +83 -0
- benchmarks/f1_step0/scorer.py +110 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v1.py +627 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v2.py +659 -0
- benchmarks/f1_step0/source_aware_dev_measurement_v3.py +776 -0
- benchmarks/f1_step0/test_public_response_mapper.py +110 -0
- benchmarks/f1_step0/test_real_source_public_preflight.py +25 -0
- benchmarks/f1_step0/test_scorer.py +158 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement.py +189 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v2.py +263 -0
- benchmarks/f1_step0/test_source_aware_dev_measurement_v3.py +334 -0
- benchmarks/f1_step0/typed_readback_public_smoke.py +181 -0
- benchmarks/forensic_c4_1_dev.py +258 -0
- benchmarks/forensic_c4_dev.py +255 -0
- benchmarks/forensic_deep25_c4_1.py +478 -0
- benchmarks/legacy_migration_acceptance.py +29 -0
- benchmarks/longmemeval_actor.py +497 -0
- benchmarks/longmemeval_agent_judge.py +593 -0
- benchmarks/longmemeval_budget.py +139 -0
- benchmarks/longmemeval_full.py +410 -0
- benchmarks/longmemeval_judge.py +327 -0
- benchmarks/longmemeval_pilot.py +316 -0
- benchmarks/longmemeval_s.py +318 -0
- benchmarks/memory_bench.py +248 -0
- benchmarks/memory_formation/__init__.py +1 -0
- benchmarks/memory_formation/audit.py +801 -0
- benchmarks/memory_formation/real_audit.py +688 -0
- benchmarks/memory_session_acceptance.py +279 -0
- benchmarks/native_codex_host.py +513 -0
- benchmarks/owner_dense_acceptance.py +126 -0
- benchmarks/pilot_seeker_vault.py +266 -0
- benchmarks/prepare_public30_ripgrep_range_owner.py +115 -0
- benchmarks/profile_canonical.py +373 -0
- benchmarks/profile_canonical_lanes.py +405 -0
- benchmarks/profile_search_chunks.py +351 -0
- benchmarks/public_ranges_faq_proof.py +121 -0
- benchmarks/public_repos/build_history_scope_v3.py +330 -0
- benchmarks/public_repos/build_oracle.py +108 -0
- benchmarks/public_repos/build_oracle_v2.py +169 -0
- benchmarks/public_repos/build_oracle_v3.py +176 -0
- benchmarks/public_repos/map_public_v2.py +296 -0
- benchmarks/public_repos/project_prompts_v2.py +31 -0
- benchmarks/public_repos/project_prompts_v3.py +29 -0
- benchmarks/public_repos/ready_admin_v2.py +214 -0
- benchmarks/public_repos/score_public_v2.py +258 -0
- benchmarks/public_repos/test_map_public_v2.py +171 -0
- benchmarks/public_repos/test_public_repo_v2.py +44 -0
- benchmarks/public_repos/test_public_repo_v3.py +46 -0
- benchmarks/public_repos/test_ready_admin_v2.py +117 -0
- benchmarks/public_repos/test_score_public_v2.py +163 -0
- benchmarks/public_repos/test_wire_v2.py +292 -0
- benchmarks/public_repos/wire_v2.py +603 -0
- benchmarks/real_blocker_fix_acceptance.py +155 -0
- benchmarks/real_scan_rescan_acceptance.py +41 -0
- benchmarks/recall_v2_20k_smoke.py +112 -0
- benchmarks/recall_v2_e1_retrieve.py +70 -0
- benchmarks/recall_v2_e1_score.py +271 -0
- benchmarks/recall_v2_e2_prepare.py +69 -0
- benchmarks/recall_v2_e2_public_score.py +91 -0
- benchmarks/recall_v2_e2_score.py +108 -0
- benchmarks/recall_v2_navigation_mcp_replay.py +308 -0
- benchmarks/recall_v2_reader_capture.py +118 -0
- benchmarks/recall_v2_reader_replay.py +136 -0
- benchmarks/run_blind_h10_holdout.py +284 -0
- benchmarks/run_blind_h9_holdout.py +275 -0
- benchmarks/run_c4_2_nightly_scale.py +446 -0
- benchmarks/run_context_scale_simulation.py +189 -0
- benchmarks/run_conversational_bench.py +170 -0
- benchmarks/run_h11_evaluation.py +316 -0
- benchmarks/run_h11_v2_evaluation.py +431 -0
- benchmarks/run_native_fixed12.py +469 -0
- benchmarks/run_native_preflight.py +280 -0
- benchmarks/run_scale_stress.py +225 -0
- benchmarks/solanazation_brownfield_01/build_oracle.py +970 -0
- benchmarks/solanazation_brownfield_01/check_level_a.py +78 -0
- benchmarks/solanazation_brownfield_01/debug_scope.py +13 -0
- benchmarks/solanazation_brownfield_01/evaluate_bootstrap_02.py +570 -0
- benchmarks/solanazation_brownfield_01/evaluate_first_pass.py +306 -0
- benchmarks/solanazation_brownfield_01/generate_dryrun_report.py +222 -0
- benchmarks/solanazation_brownfield_01/generate_manifests.py +146 -0
- benchmarks/solanazation_brownfield_01/inspect_misses.py +29 -0
- benchmarks/solanazation_brownfield_01/run_miss_forensic.py +206 -0
- benchmarks/solanazation_brownfield_01/test_facts.py +15 -0
- benchmarks/statistics/__init__.py +15 -0
- benchmarks/statistics/significance.py +326 -0
- benchmarks/test_native_mcp_preflight.py +229 -0
- benchmarks/test_prepare_public30_range_owner.py +35 -0
- benchmarks/test_public30_source_reference.py +698 -0
- benchmarks/test_recall_v2_e1_reader_input.py +89 -0
- benchmarks/test_recall_v2_navigation_mcp_replay.py +155 -0
- benchmarks/test_recall_v2_reader_replay.py +41 -0
- dandelion/__init__.py +3 -0
- dandelion/__main__.py +11 -0
- dandelion/agent_adapter.py +2369 -0
- dandelion/bootstrap/__init__.py +872 -0
- dandelion/bootstrap/alias_policy.py +50 -0
- dandelion/bootstrap/authority.py +225 -0
- dandelion/bootstrap/canonical_history.py +412 -0
- dandelion/bootstrap/classifier.py +403 -0
- dandelion/bootstrap/component_graph.py +280 -0
- dandelion/bootstrap/discovery.py +279 -0
- dandelion/bootstrap/domain_induction.py +180 -0
- dandelion/bootstrap/domain_topology.py +673 -0
- dandelion/bootstrap/evidence_envelope.py +132 -0
- dandelion/bootstrap/extractors/base.py +171 -0
- dandelion/bootstrap/extractors/exact.py +1234 -0
- dandelion/bootstrap/extractors/generic.py +100 -0
- dandelion/bootstrap/extractors/git_temporal.py +111 -0
- dandelion/bootstrap/extractors/registry.py +345 -0
- dandelion/bootstrap/extractors/relations.py +89 -0
- dandelion/bootstrap/extractors/semantic.py +424 -0
- dandelion/bootstrap/extractors/status.py +218 -0
- dandelion/bootstrap/git_sweeper.py +135 -0
- dandelion/bootstrap/manifest.py +246 -0
- dandelion/bootstrap/quarantine.py +264 -0
- dandelion/bootstrap/report.py +259 -0
- dandelion/bootstrap/root_resolver.py +221 -0
- dandelion/bootstrap/tech_profiler.py +125 -0
- dandelion/bootstrap/topology.py +75 -0
- dandelion/bootstrap/topology_index.py +110 -0
- dandelion/bootstrap/transaction.py +315 -0
- dandelion/canonical_bootstrap.py +257 -0
- dandelion/canonical_memory.py +4130 -0
- dandelion/captured_input.py +295 -0
- dandelion/cli.py +767 -0
- dandelion/config.py +40 -0
- dandelion/derived_index.py +745 -0
- dandelion/domains.py +706 -0
- dandelion/embedder.py +229 -0
- dandelion/event_evidence.py +571 -0
- dandelion/evidence.py +812 -0
- dandelion/evidence_response.py +453 -0
- dandelion/evidence_workspace.py +2360 -0
- dandelion/exact_context.py +129 -0
- dandelion/hydrator.py +124 -0
- dandelion/inspector.py +442 -0
- dandelion/inspector_assets/dandelion-mark.svg +33 -0
- dandelion/inspector_assets/dandelion_graph.js +361 -0
- dandelion/inspector_assets/inspector.css +223 -0
- dandelion/inspector_assets/inspector.html +135 -0
- dandelion/inspector_assets/inspector.js +1059 -0
- dandelion/legacy_import.py +431 -0
- dandelion/live_source_grants.py +133 -0
- dandelion/local_owner.py +114 -0
- dandelion/memory_client.py +252 -0
- dandelion/memory_contract.py +107 -0
- dandelion/memory_owner.py +560 -0
- dandelion/reader_providers.py +118 -0
- dandelion/recall_v2.py +767 -0
- dandelion/reconciler.py +219 -0
- dandelion/request_budget.py +125 -0
- dandelion/retained_basis.py +223 -0
- dandelion/retrieval.py +485 -0
- dandelion/router.py +405 -0
- dandelion/search_terms.py +53 -0
- dandelion/security.py +124 -0
- dandelion/server.py +810 -0
- dandelion/source_content_fts.py +219 -0
- dandelion/source_delivery.py +89 -0
- dandelion/source_history.py +241 -0
- dandelion/source_identity.py +88 -0
- dandelion/source_navigator.py +5 -0
- dandelion/source_reader.py +444 -0
- dandelion/source_references.py +379 -0
- dandelion/source_sections.py +265 -0
- dandelion/store_kv.py +684 -0
- dandelion/store_vector.py +1289 -0
- dandelion/studio.py +325 -0
- dandelion/wal.py +227 -0
- dandelion_memory-1.0.0rc15.dist-info/METADATA +48 -0
- dandelion_memory-1.0.0rc15.dist-info/RECORD +232 -0
- dandelion_memory-1.0.0rc15.dist-info/WHEEL +5 -0
- dandelion_memory-1.0.0rc15.dist-info/entry_points.txt +5 -0
- dandelion_memory-1.0.0rc15.dist-info/licenses/LICENSE +21 -0
- dandelion_memory-1.0.0rc15.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"""Universal Content-First File Classifier (BOOTSTRAP-02).
|
|
2
|
+
|
|
3
|
+
Classifies any workspace file into a normalized FileRole using multi-signal evidence:
|
|
4
|
+
1. Path context and suppression rules (vendor, cache, generated)
|
|
5
|
+
2. Binary vs text detection (null bytes, utf-8 decoding)
|
|
6
|
+
3. Shebang / magic byte inspection (e.g. #!/usr/bin/env python)
|
|
7
|
+
4. Structured content format detection (JSON, YAML, TOML, INI, SQL, XML, Markdown)
|
|
8
|
+
5. Extension and filename conventions
|
|
9
|
+
6. Security / sensitive data heuristics
|
|
10
|
+
|
|
11
|
+
Principle: Unknown format != invisible information.
|
|
12
|
+
Unknown text files degrade gracefully to UNKNOWN_TEXT.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
from enum import Enum
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Optional
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class FileRole(str, Enum):
|
|
25
|
+
"""Universal classification of files in any software repository."""
|
|
26
|
+
SOURCE_CODE = "source_code"
|
|
27
|
+
STRUCTURED_CONFIG = "structured_config"
|
|
28
|
+
PROJECT_DESCRIPTOR = "project_descriptor"
|
|
29
|
+
BUILD_CONFIG = "build_config"
|
|
30
|
+
SCHEMA = "schema"
|
|
31
|
+
MIGRATION = "migration"
|
|
32
|
+
DOCUMENTATION = "documentation"
|
|
33
|
+
TEST = "test"
|
|
34
|
+
CI_CD = "ci_cd"
|
|
35
|
+
DEPLOYMENT = "deployment"
|
|
36
|
+
DATA = "data"
|
|
37
|
+
NOTEBOOK = "notebook"
|
|
38
|
+
ASSET = "asset"
|
|
39
|
+
BINARY = "binary"
|
|
40
|
+
GENERATED = "generated"
|
|
41
|
+
VENDOR = "vendor"
|
|
42
|
+
CACHE = "cache"
|
|
43
|
+
SECRET_SENSITIVE = "secret_sensitive"
|
|
44
|
+
UNKNOWN_TEXT = "unknown_text"
|
|
45
|
+
UNKNOWN_BINARY = "unknown_binary"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# Standard ignore directories across ecosystems
|
|
49
|
+
VENDOR_CACHE_DIRS: set[str] = {
|
|
50
|
+
".git", ".hg", ".svn", "node_modules", "vendor", "__pycache__",
|
|
51
|
+
".venv", "venv", "env", "dist", "build", "target", "out",
|
|
52
|
+
".dandelion", ".pytest_cache", ".mypy_cache", ".ruff_cache",
|
|
53
|
+
".idea", ".vscode", ".gradle", ".cargo", ".nuget", "obj",
|
|
54
|
+
"bin", ".next", ".nuxt", ".output", "coverage", ".turbo",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
# Known binary asset extensions
|
|
58
|
+
ASSET_EXTENSIONS: set[str] = {
|
|
59
|
+
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".svg",
|
|
60
|
+
".mp3", ".wav", ".ogg", ".flac", ".mp4", ".webm", ".mov",
|
|
61
|
+
".ttf", ".otf", ".woff", ".woff2", ".blend", ".fbx", ".obj",
|
|
62
|
+
".dae", ".glb", ".gltf", ".tscn", ".scn", ".res", ".tres",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# Known compiled binary extensions
|
|
66
|
+
BINARY_EXTENSIONS: set[str] = {
|
|
67
|
+
".exe", ".dll", ".so", ".dylib", ".bin", ".pyc", ".pyd",
|
|
68
|
+
".class", ".jar", ".war", ".ear", ".a", ".o", ".lib",
|
|
69
|
+
".iso", ".apk", ".aab", ".ipa", ".zip", ".tar", ".gz",
|
|
70
|
+
".7z", ".rar", ".bz2", ".xz", ".wasm",
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
# Known source code extensions
|
|
74
|
+
SOURCE_EXTENSIONS: dict[str, str] = {
|
|
75
|
+
".py": "python",
|
|
76
|
+
".gd": "gdscript",
|
|
77
|
+
".kt": "kotlin",
|
|
78
|
+
".kts": "kotlin_dsl",
|
|
79
|
+
".java": "java",
|
|
80
|
+
".js": "javascript",
|
|
81
|
+
".jsx": "javascript_react",
|
|
82
|
+
".ts": "typescript",
|
|
83
|
+
".tsx": "typescript_react",
|
|
84
|
+
".rs": "rust",
|
|
85
|
+
".go": "go",
|
|
86
|
+
".c": "c",
|
|
87
|
+
".cpp": "cpp",
|
|
88
|
+
".h": "c_header",
|
|
89
|
+
".hpp": "cpp_header",
|
|
90
|
+
".cs": "csharp",
|
|
91
|
+
".rb": "ruby",
|
|
92
|
+
".php": "php",
|
|
93
|
+
".swift": "swift",
|
|
94
|
+
".scala": "scala",
|
|
95
|
+
".sh": "shell",
|
|
96
|
+
".bash": "bash",
|
|
97
|
+
".zsh": "zsh",
|
|
98
|
+
".ps1": "powershell",
|
|
99
|
+
".lua": "lua",
|
|
100
|
+
".r": "r",
|
|
101
|
+
".dart": "dart",
|
|
102
|
+
".ex": "elixir",
|
|
103
|
+
".exs": "elixir",
|
|
104
|
+
".erl": "erlang",
|
|
105
|
+
".clj": "clojure",
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
# Known project descriptors and build manifests
|
|
109
|
+
PROJECT_DESCRIPTORS: dict[str, str] = {
|
|
110
|
+
"project.godot": "godot_project",
|
|
111
|
+
"export_presets.cfg": "godot_export",
|
|
112
|
+
"package.json": "npm_package",
|
|
113
|
+
"pyproject.toml": "pyproject",
|
|
114
|
+
"setup.py": "setuptools",
|
|
115
|
+
"setup.cfg": "setuptools_cfg",
|
|
116
|
+
"cargo.toml": "cargo_manifest",
|
|
117
|
+
"go.mod": "go_module",
|
|
118
|
+
"pom.xml": "maven_pom",
|
|
119
|
+
"build.gradle": "gradle_groovy",
|
|
120
|
+
"build.gradle.kts": "gradle_kotlin",
|
|
121
|
+
"settings.gradle": "gradle_settings",
|
|
122
|
+
"settings.gradle.kts": "gradle_settings_kotlin",
|
|
123
|
+
"gradle.properties": "gradle_properties",
|
|
124
|
+
"makefile": "makefile",
|
|
125
|
+
"cmakelists.txt": "cmake",
|
|
126
|
+
"gemfile": "bundler_gemfile",
|
|
127
|
+
"composer.json": "composer_package",
|
|
128
|
+
"mix.exs": "mix_project",
|
|
129
|
+
"pubspec.yaml": "pub_package",
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
# Known documentation extensions
|
|
133
|
+
DOC_EXTENSIONS: set[str] = {
|
|
134
|
+
".md", ".markdown", ".mdown", ".rst", ".adoc", ".asciidoc", ".txt",
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
# Known schema and migration extensions
|
|
138
|
+
SCHEMA_EXTENSIONS: dict[str, str] = {
|
|
139
|
+
".sql": "sql",
|
|
140
|
+
".prisma": "prisma",
|
|
141
|
+
".proto": "protobuf",
|
|
142
|
+
".graphql": "graphql",
|
|
143
|
+
".gql": "graphql",
|
|
144
|
+
".avro": "avro",
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
# Known structured config extensions
|
|
148
|
+
CONFIG_EXTENSIONS: set[str] = {
|
|
149
|
+
".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
|
|
150
|
+
".properties", ".env",
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def is_binary_buffer(buf: bytes) -> bool:
|
|
155
|
+
"""Detect if raw buffer is binary (contains null bytes or invalid utf-8)."""
|
|
156
|
+
if b"\x00" in buf:
|
|
157
|
+
return True
|
|
158
|
+
try:
|
|
159
|
+
buf.decode("utf-8")
|
|
160
|
+
return False
|
|
161
|
+
except UnicodeDecodeError:
|
|
162
|
+
# Check if non-text byte ratio is high
|
|
163
|
+
text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
|
|
164
|
+
non_text = sum(1 for byte in buf if byte not in text_chars)
|
|
165
|
+
return (non_text / max(1, len(buf))) > 0.30
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def check_shebang(buf: bytes) -> Optional[tuple[FileRole, str]]:
|
|
169
|
+
"""Inspect shebang line (e.g. #!/usr/bin/env python) for source classification."""
|
|
170
|
+
if not buf.startswith(b"#!"):
|
|
171
|
+
return None
|
|
172
|
+
try:
|
|
173
|
+
first_line = buf[:256].decode("utf-8", errors="ignore").split("\n")[0].lower()
|
|
174
|
+
if "python" in first_line:
|
|
175
|
+
return FileRole.SOURCE_CODE, "python"
|
|
176
|
+
if "bash" in first_line or "sh" in first_line:
|
|
177
|
+
return FileRole.SOURCE_CODE, "shell"
|
|
178
|
+
if "node" in first_line:
|
|
179
|
+
return FileRole.SOURCE_CODE, "javascript"
|
|
180
|
+
if "ruby" in first_line:
|
|
181
|
+
return FileRole.SOURCE_CODE, "ruby"
|
|
182
|
+
if "perl" in first_line:
|
|
183
|
+
return FileRole.SOURCE_CODE, "perl"
|
|
184
|
+
if "php" in first_line:
|
|
185
|
+
return FileRole.SOURCE_CODE, "php"
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def is_sensitive_path(rel_norm: str, name: str) -> bool:
|
|
192
|
+
"""Heuristic check for sensitive or secret-bearing configuration paths."""
|
|
193
|
+
if name.startswith(".env") and not name.endswith((".example", ".sample", ".template")):
|
|
194
|
+
return True
|
|
195
|
+
if any(k in name for k in ("id_rsa", "id_ed25519", "private_key", ".pem", ".pfx", ".keystore", ".p12")):
|
|
196
|
+
return True
|
|
197
|
+
if any(k in name for k in ("secret", "credential", "password", "auth_token")) and not name.endswith((".example", ".sample", ".md")):
|
|
198
|
+
return True
|
|
199
|
+
return False
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def is_test_path(rel_norm: str, name: str) -> bool:
|
|
203
|
+
"""Heuristic check if file belongs to testing suite."""
|
|
204
|
+
parts = rel_norm.split("/")
|
|
205
|
+
if any(p in ("tests", "test", "__tests__", "spec", "specs") for p in parts[:-1]):
|
|
206
|
+
return True
|
|
207
|
+
if name.startswith("test_") or name.endswith(("_test.py", "_test.go", "_test.rs", ".test.js", ".test.ts", ".spec.js", ".spec.ts", "test.gd")):
|
|
208
|
+
return True
|
|
209
|
+
return False
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def is_ci_cd_path(rel_norm: str, name: str) -> bool:
|
|
213
|
+
"""Heuristic check if file is CI/CD or workflow configuration."""
|
|
214
|
+
if ".github/workflows" in rel_norm or ".gitlab-ci" in rel_norm or "azure-pipelines" in rel_norm:
|
|
215
|
+
return True
|
|
216
|
+
if name in ("jenkinsfile", ".travis.yml", "circle.yml", "bitbucket-pipelines.yml"):
|
|
217
|
+
return True
|
|
218
|
+
return False
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def is_deployment_path(rel_norm: str, name: str) -> bool:
|
|
222
|
+
"""Heuristic check for deployment, container, or infra descriptors."""
|
|
223
|
+
if name.startswith("dockerfile") or name.startswith("containerfile"):
|
|
224
|
+
return True
|
|
225
|
+
if "docker-compose" in name:
|
|
226
|
+
return True
|
|
227
|
+
if any(k in rel_norm for k in ("/k8s/", "/kubernetes/", "/helm/", "/terraform/", "/deploy/", "/charts/")):
|
|
228
|
+
return True
|
|
229
|
+
if name.endswith((".tf", ".tfvars")):
|
|
230
|
+
return True
|
|
231
|
+
return False
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class ClassificationResult:
|
|
235
|
+
"""Rich classification result for a discovered file."""
|
|
236
|
+
def __init__(
|
|
237
|
+
self,
|
|
238
|
+
role: FileRole,
|
|
239
|
+
detected_format: str,
|
|
240
|
+
confidence: float,
|
|
241
|
+
is_text: bool,
|
|
242
|
+
is_suppressed: bool = False,
|
|
243
|
+
signals: Optional[list[str]] = None,
|
|
244
|
+
):
|
|
245
|
+
self.role = role
|
|
246
|
+
self.detected_format = detected_format
|
|
247
|
+
self.confidence = confidence
|
|
248
|
+
self.is_text = is_text
|
|
249
|
+
self.is_suppressed = is_suppressed
|
|
250
|
+
self.signals = signals or []
|
|
251
|
+
|
|
252
|
+
def to_dict(self) -> dict:
|
|
253
|
+
return {
|
|
254
|
+
"role": self.role.value,
|
|
255
|
+
"format": self.detected_format,
|
|
256
|
+
"confidence": round(self.confidence, 2),
|
|
257
|
+
"is_text": self.is_text,
|
|
258
|
+
"is_suppressed": self.is_suppressed,
|
|
259
|
+
"signals": self.signals,
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def classify_content(rel_path: str, abs_path: Path) -> ClassificationResult:
|
|
264
|
+
"""Multi-signal universal file classification.
|
|
265
|
+
|
|
266
|
+
Evaluates path, content snippet, magic bytes, shebang, and extension.
|
|
267
|
+
Guarantees that unknown files degrade gracefully (never vanish).
|
|
268
|
+
"""
|
|
269
|
+
name = abs_path.name.lower()
|
|
270
|
+
rel_norm = rel_path.replace("\\", "/").lower()
|
|
271
|
+
ext = abs_path.suffix.lower()
|
|
272
|
+
signals: list[str] = []
|
|
273
|
+
|
|
274
|
+
# 1. Check for vendor/cache/generated suppression
|
|
275
|
+
parts = set(rel_norm.split("/")[:-1])
|
|
276
|
+
if parts & VENDOR_CACHE_DIRS:
|
|
277
|
+
matched = list(parts & VENDOR_CACHE_DIRS)
|
|
278
|
+
return ClassificationResult(
|
|
279
|
+
role=FileRole.VENDOR if any(v in matched for v in ("node_modules", "vendor")) else FileRole.CACHE,
|
|
280
|
+
detected_format=ext.lstrip(".") or "binary",
|
|
281
|
+
confidence=1.0,
|
|
282
|
+
is_text=False,
|
|
283
|
+
is_suppressed=True,
|
|
284
|
+
signals=[f"suppressed_directory:{matched[0]}"],
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
# 2. Check secret / sensitive files early
|
|
288
|
+
if is_sensitive_path(rel_norm, name):
|
|
289
|
+
signals.append("sensitive_path_pattern")
|
|
290
|
+
return ClassificationResult(
|
|
291
|
+
role=FileRole.SECRET_SENSITIVE,
|
|
292
|
+
detected_format=ext.lstrip(".") or "secret",
|
|
293
|
+
confidence=0.95,
|
|
294
|
+
is_text=True,
|
|
295
|
+
is_suppressed=False,
|
|
296
|
+
signals=signals,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
if name.startswith(".env"):
|
|
300
|
+
signals.append("env_file")
|
|
301
|
+
return ClassificationResult(
|
|
302
|
+
role=FileRole.STRUCTURED_CONFIG,
|
|
303
|
+
detected_format="env",
|
|
304
|
+
confidence=0.95,
|
|
305
|
+
is_text=True,
|
|
306
|
+
signals=signals,
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
# 3. Read header bytes if file exists
|
|
310
|
+
header_bytes = b""
|
|
311
|
+
if abs_path.exists() and abs_path.is_file():
|
|
312
|
+
try:
|
|
313
|
+
with open(abs_path, "rb") as f:
|
|
314
|
+
header_bytes = f.read(8192)
|
|
315
|
+
except Exception:
|
|
316
|
+
pass
|
|
317
|
+
|
|
318
|
+
is_bin = is_binary_buffer(header_bytes) if header_bytes else (ext in BINARY_EXTENSIONS or ext in ASSET_EXTENSIONS)
|
|
319
|
+
|
|
320
|
+
# 4. Check binary assets & binaries
|
|
321
|
+
if is_bin:
|
|
322
|
+
if ext in ASSET_EXTENSIONS:
|
|
323
|
+
return ClassificationResult(FileRole.ASSET, ext.lstrip("."), 0.95, is_text=False, signals=["asset_extension"])
|
|
324
|
+
if ext in BINARY_EXTENSIONS:
|
|
325
|
+
return ClassificationResult(FileRole.BINARY, ext.lstrip("."), 0.95, is_text=False, signals=["binary_extension"])
|
|
326
|
+
return ClassificationResult(FileRole.UNKNOWN_BINARY, ext.lstrip(".") or "bin", 0.50, is_text=False, signals=["binary_buffer"])
|
|
327
|
+
|
|
328
|
+
# ── Textual Path from here ──
|
|
329
|
+
|
|
330
|
+
# 5. Shebang inspection
|
|
331
|
+
shebang_match = check_shebang(header_bytes)
|
|
332
|
+
if shebang_match:
|
|
333
|
+
role, fmt = shebang_match
|
|
334
|
+
signals.append(f"shebang:{fmt}")
|
|
335
|
+
return ClassificationResult(role, fmt, 0.95, is_text=True, signals=signals)
|
|
336
|
+
|
|
337
|
+
# 6. CI/CD & Deployment
|
|
338
|
+
if is_ci_cd_path(rel_norm, name):
|
|
339
|
+
signals.append("ci_cd_path")
|
|
340
|
+
return ClassificationResult(FileRole.CI_CD, ext.lstrip(".") or "yaml", 0.90, is_text=True, signals=signals)
|
|
341
|
+
|
|
342
|
+
if is_deployment_path(rel_norm, name):
|
|
343
|
+
signals.append("deployment_path")
|
|
344
|
+
fmt = "docker" if "dockerfile" in name else (ext.lstrip(".") or "yaml")
|
|
345
|
+
return ClassificationResult(FileRole.DEPLOYMENT, fmt, 0.90, is_text=True, signals=signals)
|
|
346
|
+
|
|
347
|
+
# 7. Project Descriptors & Build Manifests
|
|
348
|
+
if name in PROJECT_DESCRIPTORS:
|
|
349
|
+
fmt = PROJECT_DESCRIPTORS[name]
|
|
350
|
+
role = FileRole.PROJECT_DESCRIPTOR if "project" in fmt or "package" in fmt or "manifest" in fmt else FileRole.BUILD_CONFIG
|
|
351
|
+
signals.append(f"exact_project_descriptor:{name}")
|
|
352
|
+
return ClassificationResult(role, fmt, 1.0, is_text=True, signals=signals)
|
|
353
|
+
|
|
354
|
+
# 8. Tests
|
|
355
|
+
if is_test_path(rel_norm, name):
|
|
356
|
+
signals.append("test_path_pattern")
|
|
357
|
+
fmt = SOURCE_EXTENSIONS.get(ext, ext.lstrip(".") or "test")
|
|
358
|
+
return ClassificationResult(FileRole.TEST, fmt, 0.90, is_text=True, signals=signals)
|
|
359
|
+
|
|
360
|
+
# 9. Source Code
|
|
361
|
+
if ext in SOURCE_EXTENSIONS:
|
|
362
|
+
fmt = SOURCE_EXTENSIONS[ext]
|
|
363
|
+
signals.append(f"source_extension:{ext}")
|
|
364
|
+
return ClassificationResult(FileRole.SOURCE_CODE, fmt, 0.95, is_text=True, signals=signals)
|
|
365
|
+
|
|
366
|
+
# 10. Schemas and Migrations
|
|
367
|
+
if ext in SCHEMA_EXTENSIONS or "/migrations/" in rel_norm or "/schema/" in rel_norm:
|
|
368
|
+
fmt = SCHEMA_EXTENSIONS.get(ext, "sql")
|
|
369
|
+
role = FileRole.MIGRATION if "/migration" in rel_norm else FileRole.SCHEMA
|
|
370
|
+
signals.append(f"schema_match:{fmt}")
|
|
371
|
+
return ClassificationResult(role, fmt, 0.90, is_text=True, signals=signals)
|
|
372
|
+
|
|
373
|
+
# 11. Documentation
|
|
374
|
+
if ext in DOC_EXTENSIONS:
|
|
375
|
+
fmt = "markdown" if ext in (".md", ".markdown") else ext.lstrip(".")
|
|
376
|
+
signals.append(f"doc_extension:{ext}")
|
|
377
|
+
return ClassificationResult(FileRole.DOCUMENTATION, fmt, 0.95, is_text=True, signals=signals)
|
|
378
|
+
|
|
379
|
+
# 12. Structured Configuration
|
|
380
|
+
if ext in CONFIG_EXTENSIONS:
|
|
381
|
+
fmt = ext.lstrip(".")
|
|
382
|
+
signals.append(f"config_extension:{ext}")
|
|
383
|
+
return ClassificationResult(FileRole.STRUCTURED_CONFIG, fmt, 0.85, is_text=True, signals=signals)
|
|
384
|
+
|
|
385
|
+
# 13. Data files (JSON/CSV/TSV/Parquet)
|
|
386
|
+
if ext in (".csv", ".tsv", ".jsonl"):
|
|
387
|
+
signals.append(f"data_extension:{ext}")
|
|
388
|
+
return ClassificationResult(FileRole.DATA, ext.lstrip("."), 0.85, is_text=True, signals=signals)
|
|
389
|
+
|
|
390
|
+
# 14. Jupyter Notebooks
|
|
391
|
+
if ext == ".ipynb":
|
|
392
|
+
signals.append("notebook_extension")
|
|
393
|
+
return ClassificationResult(FileRole.NOTEBOOK, "ipynb", 0.95, is_text=True, signals=signals)
|
|
394
|
+
|
|
395
|
+
# 15. Mandatory Graceful Degradation: Unknown Text
|
|
396
|
+
signals.append("generic_text_fallback")
|
|
397
|
+
return ClassificationResult(
|
|
398
|
+
role=FileRole.UNKNOWN_TEXT,
|
|
399
|
+
detected_format=ext.lstrip(".") or "text",
|
|
400
|
+
confidence=0.50,
|
|
401
|
+
is_text=True,
|
|
402
|
+
signals=signals,
|
|
403
|
+
)
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Evidence-backed component graph construction for BOOTSTRAP-02."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from dandelion.bootstrap.discovery import DiscoveredFile, ProjectDiscoveryReport
|
|
12
|
+
from dandelion.bootstrap.root_resolver import RootResolutionResult
|
|
13
|
+
from dandelion.bootstrap.tech_profiler import TechnologyProfile
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_DESCRIPTOR_ROLES = {"project_descriptor", "build_config"}
|
|
17
|
+
_CONFIG_ROLES = _DESCRIPTOR_ROLES | {"structured_config", "ci_cd", "deployment"}
|
|
18
|
+
_IMPORT_RE = re.compile(
|
|
19
|
+
r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+)|use\s+([\w:]+)|(?:require|include)\s*\(?[\"']([^\"']+))",
|
|
20
|
+
re.MULTILINE,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class ComponentNode:
|
|
26
|
+
"""A structural node inferred from files, descriptors, and boundaries."""
|
|
27
|
+
id: str
|
|
28
|
+
name: str
|
|
29
|
+
kind: str
|
|
30
|
+
rel_path: str
|
|
31
|
+
files_count: int = 0
|
|
32
|
+
source_files_count: int = 0
|
|
33
|
+
descriptors: list[str] = field(default_factory=list)
|
|
34
|
+
technologies: list[str] = field(default_factory=list)
|
|
35
|
+
in_degree: int = 0
|
|
36
|
+
out_degree: int = 0
|
|
37
|
+
is_root: bool = False
|
|
38
|
+
resolution: str = "RESOLVED"
|
|
39
|
+
is_workspace_boundary: bool = False
|
|
40
|
+
|
|
41
|
+
def to_dict(self) -> dict:
|
|
42
|
+
return {
|
|
43
|
+
"id": self.id,
|
|
44
|
+
"name": self.name,
|
|
45
|
+
"kind": self.kind,
|
|
46
|
+
"rel_path": self.rel_path,
|
|
47
|
+
"files_count": self.files_count,
|
|
48
|
+
"source_files_count": self.source_files_count,
|
|
49
|
+
"descriptors": self.descriptors,
|
|
50
|
+
"technologies": self.technologies,
|
|
51
|
+
"in_degree": self.in_degree,
|
|
52
|
+
"out_degree": self.out_degree,
|
|
53
|
+
"is_root": self.is_root,
|
|
54
|
+
"resolution": self.resolution,
|
|
55
|
+
"is_workspace_boundary": self.is_workspace_boundary,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class ComponentEdge:
|
|
61
|
+
source_id: str
|
|
62
|
+
target_id: str
|
|
63
|
+
relation: str
|
|
64
|
+
confidence: float = 1.0
|
|
65
|
+
|
|
66
|
+
def to_dict(self) -> dict:
|
|
67
|
+
return {
|
|
68
|
+
"source": self.source_id,
|
|
69
|
+
"target": self.target_id,
|
|
70
|
+
"relation": self.relation,
|
|
71
|
+
"confidence": self.confidence,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class ComponentGraph:
|
|
77
|
+
root_id: str
|
|
78
|
+
nodes: dict[str, ComponentNode] = field(default_factory=dict)
|
|
79
|
+
edges: list[ComponentEdge] = field(default_factory=list)
|
|
80
|
+
|
|
81
|
+
def to_dict(self) -> dict:
|
|
82
|
+
return {
|
|
83
|
+
"root_id": self.root_id,
|
|
84
|
+
"nodes_count": len(self.nodes),
|
|
85
|
+
"edges_count": len(self.edges),
|
|
86
|
+
"nodes": {key: node.to_dict() for key, node in self.nodes.items()},
|
|
87
|
+
"edges": [edge.to_dict() for edge in self.edges],
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _norm(path: str) -> str:
|
|
92
|
+
return path.replace("\\", "/").strip("/")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _component_path_for_file(file: DiscoveredFile, descriptor_dirs: set[str]) -> str:
|
|
96
|
+
parts = _norm(file.rel_path).split("/")
|
|
97
|
+
if len(parts) == 1:
|
|
98
|
+
return "root"
|
|
99
|
+
parent = _norm(str(Path(_norm(file.rel_path)).parent))
|
|
100
|
+
for descriptor_dir in sorted(descriptor_dirs, key=lambda value: value.count("/"), reverse=True):
|
|
101
|
+
if parent == descriptor_dir or parent.startswith(descriptor_dir + "/"):
|
|
102
|
+
return descriptor_dir
|
|
103
|
+
return "/".join(parts[:2]) if len(parts) > 2 else parts[0]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _kind(files: list[DiscoveredFile], descriptors: list[DiscoveredFile]) -> tuple[str, str]:
|
|
107
|
+
roles = {file.role.value for file in files}
|
|
108
|
+
if descriptors:
|
|
109
|
+
return "package", "RESOLVED"
|
|
110
|
+
if roles <= {"documentation"}:
|
|
111
|
+
return "doc_set", "RESOLVED"
|
|
112
|
+
if roles <= {"test"}:
|
|
113
|
+
return "test_suite", "RESOLVED"
|
|
114
|
+
if "source_code" in roles:
|
|
115
|
+
return "subsystem", "RESOLVED"
|
|
116
|
+
return "unresolved_component", "UNRESOLVED_COMPONENT"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _add_edge(graph: ComponentGraph, source_id: str, target_id: str, relation: str, confidence: float) -> None:
|
|
120
|
+
if source_id == target_id or source_id not in graph.nodes or target_id not in graph.nodes:
|
|
121
|
+
return
|
|
122
|
+
if any(edge.source_id == source_id and edge.target_id == target_id and edge.relation == relation for edge in graph.edges):
|
|
123
|
+
return
|
|
124
|
+
graph.edges.append(ComponentEdge(source_id, target_id, relation, confidence))
|
|
125
|
+
graph.nodes[source_id].out_degree += 1
|
|
126
|
+
graph.nodes[target_id].in_degree += 1
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _read_text(file: DiscoveredFile) -> str:
|
|
130
|
+
if not file.is_text or file.size_bytes > 131072:
|
|
131
|
+
return ""
|
|
132
|
+
try:
|
|
133
|
+
return file.abs_path.read_text(encoding="utf-8", errors="ignore")
|
|
134
|
+
except OSError:
|
|
135
|
+
return ""
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _target_for_reference(reference: str, graph: ComponentGraph, package_names: dict[str, str]) -> str | None:
|
|
139
|
+
cleaned = reference.strip().strip("'\"").split(":", 1)[0]
|
|
140
|
+
if cleaned in package_names:
|
|
141
|
+
return package_names[cleaned]
|
|
142
|
+
normalized = cleaned.replace(".", "/").strip("/")
|
|
143
|
+
for node in sorted(graph.nodes.values(), key=lambda item: len(item.rel_path), reverse=True):
|
|
144
|
+
if not node.rel_path or node.kind == "workspace_boundary":
|
|
145
|
+
continue
|
|
146
|
+
if normalized == node.rel_path or normalized.startswith(node.rel_path + "/"):
|
|
147
|
+
return node.id
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _package_name_index(files: list[DiscoveredFile], owners: dict[str, str]) -> dict[str, str]:
|
|
152
|
+
result: dict[str, str] = {}
|
|
153
|
+
for file in files:
|
|
154
|
+
if file.format != "npm_package":
|
|
155
|
+
continue
|
|
156
|
+
try:
|
|
157
|
+
name = json.loads(_read_text(file)).get("name")
|
|
158
|
+
except (TypeError, ValueError):
|
|
159
|
+
continue
|
|
160
|
+
if isinstance(name, str) and file.rel_path in owners:
|
|
161
|
+
result[name] = owners[file.rel_path]
|
|
162
|
+
return result
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def build_component_graph(
|
|
166
|
+
report: ProjectDiscoveryReport,
|
|
167
|
+
root_res: RootResolutionResult,
|
|
168
|
+
tech_profile: TechnologyProfile,
|
|
169
|
+
) -> ComponentGraph:
|
|
170
|
+
"""Build a bounded graph only from explicit file and manifest evidence."""
|
|
171
|
+
root_id = root_res.canonical_id
|
|
172
|
+
graph = ComponentGraph(root_id=root_id)
|
|
173
|
+
graph.nodes[root_id] = ComponentNode(
|
|
174
|
+
id=root_id,
|
|
175
|
+
name=root_res.project_name,
|
|
176
|
+
kind="application" if root_res.workspace_type == "single_project" else "workspace",
|
|
177
|
+
rel_path="",
|
|
178
|
+
is_root=True,
|
|
179
|
+
technologies=tech_profile.frameworks + [tech_profile.primary_language],
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
descriptor_dirs = {
|
|
183
|
+
_norm(str(Path(_norm(file.rel_path)).parent))
|
|
184
|
+
for file in report.all_discovered
|
|
185
|
+
if file.role.value in _DESCRIPTOR_ROLES and "/" in _norm(file.rel_path)
|
|
186
|
+
}
|
|
187
|
+
files_by_component: dict[str, list[DiscoveredFile]] = defaultdict(list)
|
|
188
|
+
descriptors_by_component: dict[str, list[DiscoveredFile]] = defaultdict(list)
|
|
189
|
+
owners: dict[str, str] = {}
|
|
190
|
+
for file in report.all_discovered:
|
|
191
|
+
component_path = _component_path_for_file(file, descriptor_dirs)
|
|
192
|
+
files_by_component[component_path].append(file)
|
|
193
|
+
if file.role.value in _DESCRIPTOR_ROLES:
|
|
194
|
+
descriptors_by_component[component_path].append(file)
|
|
195
|
+
|
|
196
|
+
for component_path, files in files_by_component.items():
|
|
197
|
+
if component_path == "root":
|
|
198
|
+
root = graph.nodes[root_id]
|
|
199
|
+
root.files_count = len(files)
|
|
200
|
+
root.source_files_count = sum(file.role.value == "source_code" for file in files)
|
|
201
|
+
root.descriptors = [file.rel_path for file in descriptors_by_component[component_path]]
|
|
202
|
+
for file in files:
|
|
203
|
+
owners[file.rel_path] = root_id
|
|
204
|
+
continue
|
|
205
|
+
component_id = component_path.replace("/", ".")
|
|
206
|
+
descriptors = descriptors_by_component[component_path]
|
|
207
|
+
kind, resolution = _kind(files, descriptors)
|
|
208
|
+
node = ComponentNode(
|
|
209
|
+
id=component_id,
|
|
210
|
+
name=component_path.rsplit("/", 1)[-1],
|
|
211
|
+
kind=kind,
|
|
212
|
+
rel_path=component_path,
|
|
213
|
+
files_count=len(files),
|
|
214
|
+
source_files_count=sum(file.role.value == "source_code" for file in files),
|
|
215
|
+
descriptors=[file.rel_path for file in descriptors],
|
|
216
|
+
technologies=tech_profile.component_technologies.get(component_path.split("/")[0], []),
|
|
217
|
+
resolution=resolution,
|
|
218
|
+
is_workspace_boundary=component_path in root_res.workspace_boundaries,
|
|
219
|
+
)
|
|
220
|
+
graph.nodes[component_id] = node
|
|
221
|
+
_add_edge(graph, root_id, component_id, "contains", 1.0)
|
|
222
|
+
for file in files:
|
|
223
|
+
owners[file.rel_path] = component_id
|
|
224
|
+
|
|
225
|
+
for boundary in root_res.workspace_boundaries:
|
|
226
|
+
if any(node.rel_path == boundary for node in graph.nodes.values()):
|
|
227
|
+
continue
|
|
228
|
+
boundary_id = f"workspace.{boundary.replace('/', '.')}"
|
|
229
|
+
graph.nodes[boundary_id] = ComponentNode(
|
|
230
|
+
id=boundary_id,
|
|
231
|
+
name=boundary.rsplit("/", 1)[-1],
|
|
232
|
+
kind="workspace_boundary",
|
|
233
|
+
rel_path=boundary,
|
|
234
|
+
is_workspace_boundary=True,
|
|
235
|
+
)
|
|
236
|
+
_add_edge(graph, root_id, boundary_id, "contains", 1.0)
|
|
237
|
+
|
|
238
|
+
package_names = _package_name_index(report.all_discovered, owners)
|
|
239
|
+
for file in report.all_discovered:
|
|
240
|
+
owner = owners.get(file.rel_path)
|
|
241
|
+
if owner is None:
|
|
242
|
+
continue
|
|
243
|
+
text = _read_text(file)
|
|
244
|
+
if not text:
|
|
245
|
+
continue
|
|
246
|
+
if file.role.value == "source_code":
|
|
247
|
+
for match in _IMPORT_RE.finditer(text):
|
|
248
|
+
target = _target_for_reference(next(value for value in match.groups() if value), graph, package_names)
|
|
249
|
+
if target:
|
|
250
|
+
_add_edge(graph, owner, target, "imports", 0.95)
|
|
251
|
+
if file.format == "npm_package":
|
|
252
|
+
try:
|
|
253
|
+
payload = json.loads(text)
|
|
254
|
+
except ValueError:
|
|
255
|
+
payload = {}
|
|
256
|
+
for section in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
|
257
|
+
dependencies = payload.get(section, {})
|
|
258
|
+
if not isinstance(dependencies, dict):
|
|
259
|
+
continue
|
|
260
|
+
for dependency in dependencies:
|
|
261
|
+
target = package_names.get(dependency)
|
|
262
|
+
if target:
|
|
263
|
+
_add_edge(graph, owner, target, "depends_on", 1.0)
|
|
264
|
+
if file.role.value in _CONFIG_ROLES:
|
|
265
|
+
for node in graph.nodes.values():
|
|
266
|
+
if not node.rel_path or node.id == owner or "/" not in node.rel_path:
|
|
267
|
+
continue
|
|
268
|
+
if node.rel_path in text:
|
|
269
|
+
relation = "builds" if file.role.value in _DESCRIPTOR_ROLES else "configures"
|
|
270
|
+
_add_edge(graph, owner, node.id, relation, 0.85)
|
|
271
|
+
|
|
272
|
+
for node in list(graph.nodes.values()):
|
|
273
|
+
if node.id == root_id:
|
|
274
|
+
continue
|
|
275
|
+
if node.kind == "test_suite":
|
|
276
|
+
_add_edge(graph, node.id, root_id, "tests", 0.75)
|
|
277
|
+
elif node.kind == "doc_set":
|
|
278
|
+
_add_edge(graph, node.id, root_id, "documents", 0.75)
|
|
279
|
+
|
|
280
|
+
return graph
|