reponeedle 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reponeedle/__init__.py +42 -0
- reponeedle/cli.py +125 -0
- reponeedle/core/__init__.py +23 -0
- reponeedle/core/abstention.py +132 -0
- reponeedle/core/arm101.py +301 -0
- reponeedle/core/attenuation.py +74 -0
- reponeedle/core/budget.py +25 -0
- reponeedle/core/cascade.py +295 -0
- reponeedle/core/change_impact.py +246 -0
- reponeedle/core/function_graph.py +507 -0
- reponeedle/core/graph.py +860 -0
- reponeedle/core/lexical.py +170 -0
- reponeedle/core/packer.py +509 -0
- reponeedle/core/router.py +108 -0
- reponeedle/core/semantic.py +217 -0
- reponeedle/core/tokenizer.py +133 -0
- reponeedle/engine.py +580 -0
- reponeedle/models.py +287 -0
- reponeedle-1.1.0.dist-info/METADATA +406 -0
- reponeedle-1.1.0.dist-info/RECORD +24 -0
- reponeedle-1.1.0.dist-info/WHEEL +5 -0
- reponeedle-1.1.0.dist-info/entry_points.txt +2 -0
- reponeedle-1.1.0.dist-info/licenses/LICENSE +22 -0
- reponeedle-1.1.0.dist-info/top_level.txt +1 -0
reponeedle/__init__.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RepoNeedle: Token-Budget-Aware, Workflow-Adaptive Repository Context Retrieval for AI Coding Agents.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from reponeedle.models import (
|
|
6
|
+
FileDoc,
|
|
7
|
+
Query,
|
|
8
|
+
RetrievalResult,
|
|
9
|
+
ScoredFile,
|
|
10
|
+
TaskType,
|
|
11
|
+
RankerType,
|
|
12
|
+
BudgetConfig,
|
|
13
|
+
ContextTier,
|
|
14
|
+
)
|
|
15
|
+
from reponeedle.engine import RepoNeedleEngine
|
|
16
|
+
from reponeedle.core.change_impact import ChangeImpactRanker, ChangeImpactConfig
|
|
17
|
+
from reponeedle.core.packer import (
|
|
18
|
+
CUDPackOptimizer,
|
|
19
|
+
BudgetOptimizer,
|
|
20
|
+
ExactTokenizer,
|
|
21
|
+
CodeSpanExtractor,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__version__ = "1.1.0"
|
|
25
|
+
__all__ = [
|
|
26
|
+
"RepoNeedleEngine",
|
|
27
|
+
"FileDoc",
|
|
28
|
+
"Query",
|
|
29
|
+
"RetrievalResult",
|
|
30
|
+
"ScoredFile",
|
|
31
|
+
"TaskType",
|
|
32
|
+
"RankerType",
|
|
33
|
+
"BudgetConfig",
|
|
34
|
+
"ContextTier",
|
|
35
|
+
"ChangeImpactRanker",
|
|
36
|
+
"ChangeImpactConfig",
|
|
37
|
+
"CUDPackOptimizer",
|
|
38
|
+
"BudgetOptimizer",
|
|
39
|
+
"ExactTokenizer",
|
|
40
|
+
"CodeSpanExtractor",
|
|
41
|
+
]
|
|
42
|
+
|
reponeedle/cli.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for RepoNeedle.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import argparse
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
|
|
11
|
+
from reponeedle.engine import RepoNeedleEngine
|
|
12
|
+
from reponeedle.models import TaskType, BudgetConfig
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
17
|
+
try:
|
|
18
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
19
|
+
except Exception:
|
|
20
|
+
pass
|
|
21
|
+
parser = argparse.ArgumentParser(
|
|
22
|
+
prog="reponeedle",
|
|
23
|
+
description="Token-Budget-Aware, Workflow-Adaptive Repository Context Retrieval for AI Coding Agents.",
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument("query", type=str, help="Coding task, issue description, or error traceback.")
|
|
26
|
+
parser.add_argument("--repo", "-r", type=str, default=".", help="Path to repository root (default: current directory).")
|
|
27
|
+
parser.add_argument("--budget", "-b", type=int, default=4000, help="Maximum context token budget (default: 4000).")
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"--task",
|
|
30
|
+
"-t",
|
|
31
|
+
type=str,
|
|
32
|
+
default="auto",
|
|
33
|
+
choices=["auto", "code2test", "comment2context", "trace2code", "edit2ripple", "selective"],
|
|
34
|
+
help="Explicit workflow task type (default: auto).",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--mode",
|
|
38
|
+
"-m",
|
|
39
|
+
type=str,
|
|
40
|
+
default="auto",
|
|
41
|
+
choices=["auto", "compact", "complete"],
|
|
42
|
+
help="CUD-Pack context delivery mode: compact (high-density spans) or complete (whole files). Default: auto.",
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--serialize",
|
|
46
|
+
"-s",
|
|
47
|
+
action="store_true",
|
|
48
|
+
help="Print the full packaged markdown prompt to stdout.",
|
|
49
|
+
)
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"--output-file",
|
|
52
|
+
"-o",
|
|
53
|
+
type=str,
|
|
54
|
+
default=None,
|
|
55
|
+
help="Save the serialized markdown prompt context to a specified file.",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument("--no-skeletons", action="store_true", help="Disallow structural skeleton mode; require full files.")
|
|
58
|
+
parser.add_argument(
|
|
59
|
+
"--ranker",
|
|
60
|
+
"--ranking-engine",
|
|
61
|
+
dest="ranker",
|
|
62
|
+
type=str,
|
|
63
|
+
default="legacy",
|
|
64
|
+
choices=["legacy", "bm25", "arm101", "cascade", "change_impact"],
|
|
65
|
+
help="Retrieval ranker architecture (default: legacy).",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
args = parser.parse_args()
|
|
69
|
+
console = Console()
|
|
70
|
+
|
|
71
|
+
console.print(Panel(f"[bold cyan]RepoNeedle Context Retrieval[/bold cyan]\nTarget Repository: [green]{args.repo}[/green]\nQuery: [yellow]\"{args.query}\"[/yellow]\nToken Budget: [magenta]{args.budget}[/magenta]\nRanker: [cyan]{args.ranker}[/cyan]", expand=False))
|
|
72
|
+
|
|
73
|
+
config = BudgetConfig(
|
|
74
|
+
max_tokens=args.budget,
|
|
75
|
+
allow_skeletons=not args.no_skeletons,
|
|
76
|
+
mode=args.mode,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
with console.status("[bold green]Indexing repository..."):
|
|
80
|
+
engine = RepoNeedleEngine.from_directory(args.repo, budget_config=config)
|
|
81
|
+
|
|
82
|
+
task_type = TaskType(args.task) if args.task != "auto" else None
|
|
83
|
+
result = engine.retrieve(query=args.query, budget_tokens=args.budget, task_type=task_type, ranker=args.ranker)
|
|
84
|
+
|
|
85
|
+
if result.abstained:
|
|
86
|
+
console.print(Panel(f"[bold red]System Abstained[/bold red]\n{result.explanation}\nConfidence Score: {result.confidence_score:.3f}\nExecution Time: {result.execution_time_ms:.1f}ms", border_style="red"))
|
|
87
|
+
return
|
|
88
|
+
|
|
89
|
+
table = Table(title=f"Retrieved Context ({result.total_tokens}/{result.budget_tokens} tokens) - {result.execution_time_ms:.1f}ms")
|
|
90
|
+
table.add_column("Rank", justify="center", style="cyan")
|
|
91
|
+
table.add_column("File Path", style="green")
|
|
92
|
+
table.add_column("Tier", justify="center", style="magenta")
|
|
93
|
+
table.add_column("Tokens", justify="right", style="yellow")
|
|
94
|
+
table.add_column("Score", justify="right", style="blue")
|
|
95
|
+
table.add_column("Explanation", style="white")
|
|
96
|
+
|
|
97
|
+
for rank, sf in enumerate(result.selected_files, start=1):
|
|
98
|
+
table.add_row(
|
|
99
|
+
str(rank),
|
|
100
|
+
sf.path,
|
|
101
|
+
sf.tier.value.upper(),
|
|
102
|
+
str(sf.token_count),
|
|
103
|
+
f"{sf.score:.3f}",
|
|
104
|
+
sf.explanation,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
console.print(table)
|
|
108
|
+
|
|
109
|
+
if args.output_file:
|
|
110
|
+
with open(args.output_file, "w", encoding="utf-8") as f:
|
|
111
|
+
f.write(result.rendered_context)
|
|
112
|
+
console.print(f"[bold green]Serialized prompt saved to:[/bold green] {args.output_file}")
|
|
113
|
+
|
|
114
|
+
if args.serialize:
|
|
115
|
+
console.print("\n[bold cyan]=== Serialized Context Prompt ===[/bold cyan]\n")
|
|
116
|
+
try:
|
|
117
|
+
print(result.rendered_context)
|
|
118
|
+
except UnicodeEncodeError:
|
|
119
|
+
enc = sys.stdout.encoding or "utf-8"
|
|
120
|
+
print(result.rendered_context.encode(enc, errors="replace").decode(enc))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
main()
|
|
125
|
+
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core algorithms and sub-modules for RepoNeedle.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from reponeedle.core.tokenizer import CodeTokenizer, estimate_tokens
|
|
6
|
+
from reponeedle.core.lexical import BM25Index
|
|
7
|
+
from reponeedle.core.semantic import SubwordSemanticIndex
|
|
8
|
+
from reponeedle.core.graph import RepoGraph
|
|
9
|
+
from reponeedle.core.router import WorkflowRouter
|
|
10
|
+
from reponeedle.core.abstention import SelectiveAbstainer
|
|
11
|
+
from reponeedle.core.budget import BudgetOptimizer
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CodeTokenizer",
|
|
15
|
+
"estimate_tokens",
|
|
16
|
+
"BM25Index",
|
|
17
|
+
"SubwordSemanticIndex",
|
|
18
|
+
"RepoGraph",
|
|
19
|
+
"WorkflowRouter",
|
|
20
|
+
"SelectiveAbstainer",
|
|
21
|
+
"BudgetOptimizer",
|
|
22
|
+
]
|
|
23
|
+
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Calibrated selective abstention engine for no-gold and out-of-repository queries.
|
|
3
|
+
Addresses the selective retrieval calibration gap identified in Agent Retrieval Bench.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import List, Dict, Set, Tuple
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from reponeedle.models import FileDoc
|
|
10
|
+
from reponeedle.core.tokenizer import CodeTokenizer
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SelectiveAbstainer:
|
|
14
|
+
"""
|
|
15
|
+
Evaluates whether a query genuinely corresponds to evidence in the repository,
|
|
16
|
+
or whether the retriever should abstain to avoid polluting the agent context window.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, confidence_threshold: float = 0.38):
|
|
20
|
+
self.confidence_threshold = confidence_threshold
|
|
21
|
+
self.tokenizer = CodeTokenizer()
|
|
22
|
+
self.repo_vocabulary: Set[str] = set()
|
|
23
|
+
self._doc_vocab_cache: Dict[str, Tuple[str, Set[str]]] = {}
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def _stem(token: str) -> str:
|
|
27
|
+
w = token.lower()
|
|
28
|
+
for suf in ("ing", "ed", "es", "s"):
|
|
29
|
+
if len(w) > len(suf) + 2 and w.endswith(suf):
|
|
30
|
+
return w[:-len(suf)]
|
|
31
|
+
return w
|
|
32
|
+
|
|
33
|
+
def fit(self, docs: List[FileDoc]) -> "SelectiveAbstainer":
|
|
34
|
+
"""
|
|
35
|
+
Builds repository vocabulary from paths, symbols, and contents.
|
|
36
|
+
"""
|
|
37
|
+
self.repo_vocabulary = set()
|
|
38
|
+
for doc in docs:
|
|
39
|
+
cache_key = doc.path
|
|
40
|
+
sym_str = ",".join(doc.symbols) if doc.symbols else ""
|
|
41
|
+
content_sig = f"{doc.path}::{len(doc.content)}::{hash(doc.content)}::{sym_str}"
|
|
42
|
+
if cache_key in self._doc_vocab_cache and self._doc_vocab_cache[cache_key][0] == content_sig:
|
|
43
|
+
doc_vocab = self._doc_vocab_cache[cache_key][1]
|
|
44
|
+
else:
|
|
45
|
+
doc_vocab = set()
|
|
46
|
+
# Add path tokens
|
|
47
|
+
for t in self.tokenizer.tokenize_path(doc.path):
|
|
48
|
+
tl = t.lower()
|
|
49
|
+
doc_vocab.add(tl)
|
|
50
|
+
doc_vocab.add(self._stem(tl))
|
|
51
|
+
# Add symbol tokens
|
|
52
|
+
for s in doc.symbols:
|
|
53
|
+
for t in self.tokenizer.tokenize_text(s):
|
|
54
|
+
tl = t.lower()
|
|
55
|
+
doc_vocab.add(tl)
|
|
56
|
+
doc_vocab.add(self._stem(tl))
|
|
57
|
+
# Add content identifiers if doc.content is present
|
|
58
|
+
if doc.content:
|
|
59
|
+
for t in self.tokenizer.tokenize_text(doc.content[:3000]):
|
|
60
|
+
if len(t) > 2:
|
|
61
|
+
tl = t.lower()
|
|
62
|
+
doc_vocab.add(tl)
|
|
63
|
+
doc_vocab.add(self._stem(tl))
|
|
64
|
+
self._doc_vocab_cache[cache_key] = (content_sig, doc_vocab)
|
|
65
|
+
|
|
66
|
+
self.repo_vocabulary.update(doc_vocab)
|
|
67
|
+
|
|
68
|
+
return self
|
|
69
|
+
|
|
70
|
+
def evaluate_abstention(
|
|
71
|
+
self,
|
|
72
|
+
query: str,
|
|
73
|
+
fused_scores: np.ndarray,
|
|
74
|
+
) -> Tuple[bool, float, str]:
|
|
75
|
+
"""
|
|
76
|
+
Evaluates abstention criteria.
|
|
77
|
+
Returns:
|
|
78
|
+
abstained (bool): True if system should abstain.
|
|
79
|
+
confidence (float): Calibrated confidence score in [0, 1].
|
|
80
|
+
reason (str): Diagnostic explanation.
|
|
81
|
+
"""
|
|
82
|
+
if len(fused_scores) == 0:
|
|
83
|
+
return True, 0.0, "Empty repository candidate set."
|
|
84
|
+
|
|
85
|
+
q_tokens = self.tokenizer.tokenize_text(query)
|
|
86
|
+
if not q_tokens:
|
|
87
|
+
return True, 0.0, "Empty or stop-word only query."
|
|
88
|
+
|
|
89
|
+
# 1. Lexical Overlap Ratio against repository symbol vocabulary
|
|
90
|
+
meaningful_tokens = [t for t in q_tokens if len(t) > 2]
|
|
91
|
+
if not meaningful_tokens:
|
|
92
|
+
overlap_ratio = 0.5
|
|
93
|
+
else:
|
|
94
|
+
in_repo_count = sum(
|
|
95
|
+
1 for t in meaningful_tokens
|
|
96
|
+
if t.lower() in self.repo_vocabulary or self._stem(t.lower()) in self.repo_vocabulary
|
|
97
|
+
)
|
|
98
|
+
overlap_ratio = in_repo_count / len(meaningful_tokens)
|
|
99
|
+
|
|
100
|
+
# 2. Peak-to-Noise Ratio (Z-score of top-1 file)
|
|
101
|
+
mean_score = float(np.mean(fused_scores))
|
|
102
|
+
std_score = float(np.std(fused_scores))
|
|
103
|
+
max_score = float(np.max(fused_scores))
|
|
104
|
+
|
|
105
|
+
if std_score > 1e-6:
|
|
106
|
+
z_score = (max_score - mean_score) / std_score
|
|
107
|
+
else:
|
|
108
|
+
z_score = 0.0
|
|
109
|
+
|
|
110
|
+
# Normalized Z-score component: z_score of 3.0 or higher maps to ~1.0
|
|
111
|
+
z_component = min(1.0, max(0.0, z_score / 3.0))
|
|
112
|
+
|
|
113
|
+
# 3. Score margin between top-1 and 50th percentile
|
|
114
|
+
median_score = float(np.median(fused_scores))
|
|
115
|
+
margin = max(0.0, max_score - median_score)
|
|
116
|
+
|
|
117
|
+
# Calibrated composite confidence score
|
|
118
|
+
confidence = (
|
|
119
|
+
0.30 * z_component +
|
|
120
|
+
0.45 * overlap_ratio +
|
|
121
|
+
0.25 * margin
|
|
122
|
+
)
|
|
123
|
+
confidence = float(np.clip(confidence, 0.0, 1.0))
|
|
124
|
+
|
|
125
|
+
# Decision
|
|
126
|
+
if overlap_ratio <= 0.10:
|
|
127
|
+
return True, confidence, f"Negligible query symbol overlap with repository vocabulary (overlap: {overlap_ratio:.2f}, z: {z_score:.2f})."
|
|
128
|
+
|
|
129
|
+
if confidence < self.confidence_threshold:
|
|
130
|
+
return True, confidence, f"Confidence {confidence:.3f} below threshold {self.confidence_threshold:.3f} (z: {z_score:.2f}, overlap: {overlap_ratio:.2f})."
|
|
131
|
+
|
|
132
|
+
return False, confidence, f"Confident repository match (confidence: {confidence:.3f}, z: {z_score:.2f}, overlap: {overlap_ratio:.2f})."
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ARM-101 Ranker: Soft Lexical Damping with Verified Function-Call Graph Expansion.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import math
|
|
9
|
+
from typing import List, Dict, Optional, Any
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from reponeedle.models import FileDoc
|
|
13
|
+
from reponeedle.core.attenuation import compute_hub_attenuation
|
|
14
|
+
from reponeedle.core.function_graph import FunctionGraphEngine
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ARM101Ranker:
|
|
18
|
+
"""
|
|
19
|
+
Controlled production ranker combining decoupled Okapi BM25, subword TF-IDF,
|
|
20
|
+
verified function-call graph diffusion, and Soft Lexical Damping (tau = 0.25).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
policy: str = "unattenuated",
|
|
26
|
+
soft_damping_tau: float = 0.25,
|
|
27
|
+
forward_weight: float = 0.25,
|
|
28
|
+
reverse_weight: float = 0.20,
|
|
29
|
+
graph_weight: float = 0.25,
|
|
30
|
+
class_to_test_weight: float = 0.35,
|
|
31
|
+
doc_filter_penalty: float = 0.50,
|
|
32
|
+
):
|
|
33
|
+
self.policy = policy
|
|
34
|
+
self.soft_damping_tau = soft_damping_tau
|
|
35
|
+
self.forward_weight = forward_weight
|
|
36
|
+
self.reverse_weight = reverse_weight
|
|
37
|
+
self.graph_weight = graph_weight
|
|
38
|
+
self.class_to_test_weight = class_to_test_weight
|
|
39
|
+
self.doc_filter_penalty = doc_filter_penalty
|
|
40
|
+
|
|
41
|
+
def _is_pkg_compat(self, dir_a: str, dir_b: str) -> bool:
|
|
42
|
+
return dir_a == dir_b or dir_a.startswith(dir_b) or dir_b.startswith(dir_a)
|
|
43
|
+
|
|
44
|
+
def rank_with_scores(
|
|
45
|
+
self,
|
|
46
|
+
docs: List[FileDoc],
|
|
47
|
+
bm25_scores: np.ndarray,
|
|
48
|
+
semantic_scores: Optional[np.ndarray] = None,
|
|
49
|
+
fg: Optional[FunctionGraphEngine] = None,
|
|
50
|
+
repo_graph: Optional[Any] = None,
|
|
51
|
+
) -> Tuple[List[int], Dict[int, float]]:
|
|
52
|
+
n_docs = len(docs)
|
|
53
|
+
if n_docs <= 1:
|
|
54
|
+
fused = {i: float(bm25_scores[i]) if len(bm25_scores) > i else 0.0 for i in range(n_docs)}
|
|
55
|
+
return list(range(n_docs)), fused
|
|
56
|
+
|
|
57
|
+
# 1. Deterministic BM25 sort order with path tie-breaking
|
|
58
|
+
bm25_order = sorted(range(n_docs), key=lambda i: (-float(bm25_scores[i]), docs[i].path))
|
|
59
|
+
seeds = bm25_order[:3]
|
|
60
|
+
graph_boost = np.zeros(n_docs, dtype=np.float32)
|
|
61
|
+
|
|
62
|
+
# 2. Base RepoGraph propagation
|
|
63
|
+
if repo_graph is not None and getattr(repo_graph, "graph", None) is not None and len(repo_graph.graph) > 0:
|
|
64
|
+
self._propagate_repograph(seeds, docs, bm25_scores, repo_graph, graph_boost)
|
|
65
|
+
self._propagate_class_to_test(seeds, docs, bm25_scores, repo_graph, graph_boost)
|
|
66
|
+
|
|
67
|
+
# 3. Function Graph propagation
|
|
68
|
+
if fg is not None and getattr(fg, "graph", None) is not None and len(fg.graph) > 0:
|
|
69
|
+
self._propagate_function_graph(seeds, docs, bm25_scores, fg, graph_boost)
|
|
70
|
+
|
|
71
|
+
# 4. Fused score combination
|
|
72
|
+
sem = semantic_scores if semantic_scores is not None else np.zeros(n_docs, dtype=np.float32)
|
|
73
|
+
fused = {}
|
|
74
|
+
for idx in range(n_docs):
|
|
75
|
+
score = 0.60 * float(bm25_scores[idx]) + 0.15 * float(sem[idx]) + self.graph_weight * float(graph_boost[idx])
|
|
76
|
+
p_lower = docs[idx].path.lower().replace("\\", "/")
|
|
77
|
+
if any(seg in p_lower for seg in ["/doc/", "/docs/", "doc/", "docs/", "documentation/", "guide/", "tutorial/", ".github/"]) or p_lower.endswith((".md", ".rst", ".txt", ".adoc", ".html")):
|
|
78
|
+
score *= self.doc_filter_penalty
|
|
79
|
+
fused[idx] = float(score)
|
|
80
|
+
|
|
81
|
+
sorted_by_fused = sorted(range(n_docs), key=lambda idx: (-fused[idx], docs[idx].path))
|
|
82
|
+
|
|
83
|
+
# 5. Soft Lexical Damping (Rank 1 Guard)
|
|
84
|
+
s1 = bm25_order[0]
|
|
85
|
+
s2 = bm25_order[1] if n_docs > 1 else s1
|
|
86
|
+
bm25_margin = max(0.0, float(bm25_scores[s1]) - float(bm25_scores[s2]))
|
|
87
|
+
threshold = self.soft_damping_tau * bm25_margin
|
|
88
|
+
|
|
89
|
+
best = sorted_by_fused[0]
|
|
90
|
+
if best == s1 or (fused[best] - fused[s1] >= threshold):
|
|
91
|
+
return sorted_by_fused, fused
|
|
92
|
+
else:
|
|
93
|
+
fused[s1] = fused[best] + 1e-4
|
|
94
|
+
return [s1] + [idx for idx in sorted_by_fused if idx != s1], fused
|
|
95
|
+
|
|
96
|
+
def rank(
|
|
97
|
+
self,
|
|
98
|
+
docs: List[FileDoc],
|
|
99
|
+
bm25_scores: np.ndarray,
|
|
100
|
+
semantic_scores: Optional[np.ndarray] = None,
|
|
101
|
+
fg: Optional[FunctionGraphEngine] = None,
|
|
102
|
+
repo_graph: Optional[Any] = None,
|
|
103
|
+
) -> List[int]:
|
|
104
|
+
ranked_indices, _ = self.rank_with_scores(docs, bm25_scores, semantic_scores, fg, repo_graph)
|
|
105
|
+
return ranked_indices
|
|
106
|
+
|
|
107
|
+
def _propagate_function_graph(
|
|
108
|
+
self,
|
|
109
|
+
seeds: List[int],
|
|
110
|
+
docs: List[FileDoc],
|
|
111
|
+
bm25_scores: np.ndarray,
|
|
112
|
+
fg: FunctionGraphEngine,
|
|
113
|
+
graph_boost: np.ndarray,
|
|
114
|
+
):
|
|
115
|
+
path_to_idx = fg.path_to_idx
|
|
116
|
+
allowed = {"verified_call", "unique_global_call", "intra_file_call"}
|
|
117
|
+
|
|
118
|
+
for s_idx in seeds:
|
|
119
|
+
seed_p = docs[s_idx].path
|
|
120
|
+
seed_score = float(bm25_scores[s_idx])
|
|
121
|
+
if seed_score <= 0.0 or seed_p not in path_to_idx:
|
|
122
|
+
continue
|
|
123
|
+
|
|
124
|
+
seed_dir = os.path.dirname(seed_p.replace("\\", "/"))
|
|
125
|
+
|
|
126
|
+
# Forward propagation (callees)
|
|
127
|
+
callees = fg.callees_map.get(seed_p, {})
|
|
128
|
+
out_deg = max(1, len(callees))
|
|
129
|
+
for callee_p, conf in callees.items():
|
|
130
|
+
if callee_p in path_to_idx:
|
|
131
|
+
c_idx = path_to_idx[callee_p]
|
|
132
|
+
edges = fg.file_to_file_edges.get((seed_p, callee_p), [])
|
|
133
|
+
matching = [e for e in edges if e.edge_type in allowed]
|
|
134
|
+
if not matching:
|
|
135
|
+
continue
|
|
136
|
+
edge_conf = max(e.confidence for e in matching)
|
|
137
|
+
|
|
138
|
+
callee_dir = os.path.dirname(callee_p.replace("\\", "/"))
|
|
139
|
+
is_test = docs[s_idx].is_test or docs[c_idx].is_test
|
|
140
|
+
if not is_test and not self._is_pkg_compat(seed_dir, callee_dir):
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
in_deg = max(1, len(fg.callers_map.get(callee_p, {})))
|
|
144
|
+
callee_out_deg = max(1, len(fg.callees_map.get(callee_p, {})))
|
|
145
|
+
norm = 1.0 / math.sqrt(out_deg * in_deg)
|
|
146
|
+
boost = seed_score * self.forward_weight * edge_conf * norm
|
|
147
|
+
|
|
148
|
+
w_hub = compute_hub_attenuation(callee_p, in_deg, callee_out_deg, self.policy)
|
|
149
|
+
boost *= w_hub
|
|
150
|
+
graph_boost[c_idx] = max(graph_boost[c_idx], float(boost))
|
|
151
|
+
|
|
152
|
+
# Reverse propagation (callers)
|
|
153
|
+
callers = fg.callers_map.get(seed_p, {})
|
|
154
|
+
in_deg_seed = max(1, len(callers))
|
|
155
|
+
for caller_p, conf in callers.items():
|
|
156
|
+
if caller_p in path_to_idx:
|
|
157
|
+
c_idx = path_to_idx[caller_p]
|
|
158
|
+
edges = fg.file_to_file_edges.get((caller_p, seed_p), [])
|
|
159
|
+
matching = [e for e in edges if e.edge_type in allowed]
|
|
160
|
+
if not matching:
|
|
161
|
+
continue
|
|
162
|
+
edge_conf = max(e.confidence for e in matching)
|
|
163
|
+
|
|
164
|
+
caller_dir = os.path.dirname(caller_p.replace("\\", "/"))
|
|
165
|
+
is_test = docs[s_idx].is_test or docs[c_idx].is_test
|
|
166
|
+
if not is_test and not self._is_pkg_compat(seed_dir, caller_dir):
|
|
167
|
+
continue
|
|
168
|
+
|
|
169
|
+
caller_in_deg = max(1, len(fg.callers_map.get(caller_p, {})))
|
|
170
|
+
caller_out_deg = max(1, len(fg.callees_map.get(caller_p, {})))
|
|
171
|
+
norm = 1.0 / math.sqrt(in_deg_seed * caller_out_deg)
|
|
172
|
+
boost = seed_score * self.reverse_weight * edge_conf * norm
|
|
173
|
+
|
|
174
|
+
w_hub = compute_hub_attenuation(caller_p, caller_in_deg, caller_out_deg, self.policy)
|
|
175
|
+
boost *= w_hub
|
|
176
|
+
graph_boost[c_idx] = max(graph_boost[c_idx], float(boost))
|
|
177
|
+
|
|
178
|
+
def _propagate_repograph(
|
|
179
|
+
self,
|
|
180
|
+
seeds: List[int],
|
|
181
|
+
docs: List[FileDoc],
|
|
182
|
+
bm25_scores: np.ndarray,
|
|
183
|
+
repo_graph: Any,
|
|
184
|
+
graph_boost: np.ndarray,
|
|
185
|
+
):
|
|
186
|
+
g = repo_graph.graph
|
|
187
|
+
path_to_idx = repo_graph.path_to_idx
|
|
188
|
+
|
|
189
|
+
for s_idx in seeds:
|
|
190
|
+
seed_p = docs[s_idx].path
|
|
191
|
+
seed_score = float(bm25_scores[s_idx])
|
|
192
|
+
if seed_score <= 0.0 or seed_p not in g:
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
seed_dir = os.path.dirname(seed_p.replace("\\", "/"))
|
|
196
|
+
out_deg = max(1, g.out_degree(seed_p))
|
|
197
|
+
in_deg_seed = max(1, g.in_degree(seed_p))
|
|
198
|
+
|
|
199
|
+
for succ in g.successors(seed_p):
|
|
200
|
+
if succ in path_to_idx:
|
|
201
|
+
n_idx = path_to_idx[succ]
|
|
202
|
+
succ_dir = os.path.dirname(succ.replace("\\", "/"))
|
|
203
|
+
if not self._is_pkg_compat(seed_dir, succ_dir):
|
|
204
|
+
continue
|
|
205
|
+
in_deg_succ = max(1, g.in_degree(succ))
|
|
206
|
+
norm = 1.0 / math.sqrt(out_deg * in_deg_succ)
|
|
207
|
+
boost = seed_score * 0.25 * norm
|
|
208
|
+
if self.policy == "uniform_log2":
|
|
209
|
+
boost *= (1.0 / math.log2(2.0 + in_deg_succ))
|
|
210
|
+
graph_boost[n_idx] = max(graph_boost[n_idx], float(boost))
|
|
211
|
+
|
|
212
|
+
for pred in g.predecessors(seed_p):
|
|
213
|
+
if pred in path_to_idx:
|
|
214
|
+
n_idx = path_to_idx[pred]
|
|
215
|
+
pred_dir = os.path.dirname(pred.replace("\\", "/"))
|
|
216
|
+
if not self._is_pkg_compat(seed_dir, pred_dir):
|
|
217
|
+
continue
|
|
218
|
+
out_deg_pred = max(1, g.out_degree(pred))
|
|
219
|
+
in_deg_pred = max(1, g.in_degree(pred))
|
|
220
|
+
norm = 1.0 / math.sqrt(in_deg_seed * out_deg_pred)
|
|
221
|
+
boost = seed_score * 0.10 * norm
|
|
222
|
+
if self.policy == "uniform_log2":
|
|
223
|
+
boost *= (1.0 / math.log2(2.0 + in_deg_pred))
|
|
224
|
+
graph_boost[n_idx] = max(graph_boost[n_idx], float(boost))
|
|
225
|
+
|
|
226
|
+
self._apply_test_discovery(seed_p, seed_score, path_to_idx, graph_boost)
|
|
227
|
+
|
|
228
|
+
def _apply_test_discovery(
|
|
229
|
+
self,
|
|
230
|
+
seed_path: str,
|
|
231
|
+
seed_score: float,
|
|
232
|
+
path_to_idx: Dict[str, int],
|
|
233
|
+
graph_boost: np.ndarray,
|
|
234
|
+
):
|
|
235
|
+
norm_p = seed_path.replace("\\", "/")
|
|
236
|
+
stem = os.path.splitext(os.path.basename(norm_p))[0]
|
|
237
|
+
ext = os.path.splitext(norm_p)[1].lower()
|
|
238
|
+
cur_dir = os.path.dirname(norm_p)
|
|
239
|
+
tc_list = []
|
|
240
|
+
|
|
241
|
+
if ext == ".go":
|
|
242
|
+
tc_list.append(f"{cur_dir}/{stem}_test.go" if cur_dir else f"{stem}_test.go")
|
|
243
|
+
elif ext in (".py", ".pyi"):
|
|
244
|
+
tc_list.append(f"{cur_dir}/test_{stem}.py" if cur_dir else f"test_{stem}.py")
|
|
245
|
+
tc_list.append(f"tests/test_{stem}.py")
|
|
246
|
+
tc_list.append(f"tests/{stem}_test.py")
|
|
247
|
+
elif ext == ".rs":
|
|
248
|
+
tc_list.append(f"tests/{stem}.rs")
|
|
249
|
+
tc_list.append(f"tests/test_{stem}.rs")
|
|
250
|
+
elif ext in (".ts", ".tsx", ".js", ".jsx"):
|
|
251
|
+
tc_list.append(f"{cur_dir}/{stem}.test{ext}" if cur_dir else f"{stem}.test{ext}")
|
|
252
|
+
tc_list.append(f"{cur_dir}/__tests__/{stem}.test{ext}" if cur_dir else f"__tests__/{stem}.test{ext}")
|
|
253
|
+
|
|
254
|
+
for tc in tc_list:
|
|
255
|
+
if tc in path_to_idx:
|
|
256
|
+
idx = path_to_idx[tc]
|
|
257
|
+
graph_boost[idx] = max(graph_boost[idx], float(seed_score * 0.35))
|
|
258
|
+
|
|
259
|
+
def _propagate_class_to_test(
|
|
260
|
+
self,
|
|
261
|
+
seeds: List[int],
|
|
262
|
+
docs: List[FileDoc],
|
|
263
|
+
bm25_scores: np.ndarray,
|
|
264
|
+
repo_graph: Any,
|
|
265
|
+
graph_boost: np.ndarray,
|
|
266
|
+
):
|
|
267
|
+
prod_to_tests = getattr(repo_graph, "prod_to_test_invocations", {})
|
|
268
|
+
test_to_prods = getattr(repo_graph, "test_to_prod_invocations", {})
|
|
269
|
+
path_to_idx = repo_graph.path_to_idx
|
|
270
|
+
|
|
271
|
+
for s_idx in seeds:
|
|
272
|
+
seed_p = docs[s_idx].path
|
|
273
|
+
seed_score = float(bm25_scores[s_idx])
|
|
274
|
+
if seed_score <= 0.0:
|
|
275
|
+
continue
|
|
276
|
+
|
|
277
|
+
if seed_p in prod_to_tests:
|
|
278
|
+
target_tests = prod_to_tests[seed_p]
|
|
279
|
+
out_deg = max(1, len(target_tests))
|
|
280
|
+
for test_p in target_tests:
|
|
281
|
+
if test_p in path_to_idx:
|
|
282
|
+
t_idx = path_to_idx[test_p]
|
|
283
|
+
in_deg = max(1, len(test_to_prods.get(test_p, {seed_p})))
|
|
284
|
+
norm = 1.0 / math.sqrt(out_deg * in_deg)
|
|
285
|
+
boost = seed_score * self.class_to_test_weight * norm
|
|
286
|
+
if self.policy == "uniform_log2":
|
|
287
|
+
boost *= (1.0 / math.log2(2.0 + in_deg))
|
|
288
|
+
graph_boost[t_idx] = max(graph_boost[t_idx], float(boost))
|
|
289
|
+
|
|
290
|
+
if seed_p in test_to_prods:
|
|
291
|
+
target_prods = test_to_prods[seed_p]
|
|
292
|
+
out_deg = max(1, len(target_prods))
|
|
293
|
+
for prod_p in target_prods:
|
|
294
|
+
if prod_p in path_to_idx:
|
|
295
|
+
p_idx = path_to_idx[prod_p]
|
|
296
|
+
in_deg = max(1, len(prod_to_tests.get(prod_p, {seed_p})))
|
|
297
|
+
norm = 1.0 / math.sqrt(out_deg * in_deg)
|
|
298
|
+
boost = seed_score * self.class_to_test_weight * norm
|
|
299
|
+
if self.policy == "uniform_log2":
|
|
300
|
+
boost *= (1.0 / math.log2(2.0 + in_deg))
|
|
301
|
+
graph_boost[p_idx] = max(graph_boost[p_idx], float(boost))
|