flexsds 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flexsds/__init__.py +4 -0
- flexsds/assets/ptit_logo.png +0 -0
- flexsds/cli/__init__.py +1 -0
- flexsds/cli/banner.py +84 -0
- flexsds/cli/main.py +330 -0
- flexsds/cli/output.py +69 -0
- flexsds/cli/task_runner.py +171 -0
- flexsds/cli/wizard.py +360 -0
- flexsds/client/__init__.py +4 -0
- flexsds/client/chat.py +137 -0
- flexsds/client/client.py +107 -0
- flexsds/client/monitor.py +95 -0
- flexsds/core/__init__.py +44 -0
- flexsds/core/compatibility.py +442 -0
- flexsds/core/controller.py +502 -0
- flexsds/core/cost_model.py +530 -0
- flexsds/core/credentials.py +111 -0
- flexsds/core/hardware.py +77 -0
- flexsds/core/tree_topology.py +296 -0
- flexsds/profiles/profile_sm75_t4.json +62 -0
- flexsds/profiles/profile_sm89_l4.json +194 -0
- flexsds/server/__init__.py +1 -0
- flexsds/server/colab_runner.py +62 -0
- flexsds/server/local_runner.py +208 -0
- flexsds/server/modal_app.py +50 -0
- flexsds/server/modal_runner.py +98 -0
- flexsds/server/vscode_cli.py +82 -0
- flexsds-0.2.0.dist-info/METADATA +204 -0
- flexsds-0.2.0.dist-info/RECORD +62 -0
- flexsds-0.2.0.dist-info/WHEEL +5 -0
- flexsds-0.2.0.dist-info/entry_points.txt +2 -0
- flexsds-0.2.0.dist-info/top_level.txt +4 -0
- pipeline/entity_eval.py +171 -0
- pipeline/examples/node1_extractor_input_m0_04_001.txt +9 -0
- pipeline/examples/node1_extractor_output_m0_04_001.json +90 -0
- pipeline/extractor.py +502 -0
- pipeline/invoice_models.py +163 -0
- pipeline/node2_questions.py +30 -0
- pipeline/openai_compatible.py +87 -0
- pipeline/prompts/README.md +18 -0
- pipeline/prompts/node1_extractor_system.md +92 -0
- pipeline/prompts/node3_report_user.md +5 -0
- pipeline/report_writer.py +143 -0
- pipeline/schemas/invoice_extraction.schema.json +469 -0
- pipeline/sql_guard.py +140 -0
- pipeline/sqlite/schema.sql +219 -0
- pipeline/text_to_sql.py +170 -0
- pipeline/validation.py +166 -0
- pipeline/w1_workload.py +139 -0
- proposer/__init__.py +31 -0
- proposer/adapter.py +153 -0
- proposer/controller.py +460 -0
- proposer/cost_model.py +216 -0
- proposer/tree_adaptive.py +466 -0
- server/__init__.py +3 -0
- server/adaptive_scheduler.py +215 -0
- server/budget_planner.py +287 -0
- server/lineage.py +290 -0
- server/lineage_diagnostics.py +38 -0
- server/lineage_executor.py +129 -0
- server/lineage_startup.py +47 -0
- server/t4_rowwise_attention.py +262 -0
|
@@ -0,0 +1,530 @@
|
|
|
1
|
+
"""Hardware cost model loader and dynamic break-even solver for FlexSDS."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from flexsds.core.hardware import HardwareInfo
|
|
12
|
+
from flexsds.core.tree_topology import (
|
|
13
|
+
TreeTopology,
|
|
14
|
+
CANONICAL_TOPOLOGIES,
|
|
15
|
+
get_tree_topology,
|
|
16
|
+
)
|
|
17
|
+
except ModuleNotFoundError:
|
|
18
|
+
from src.flexsds.core.hardware import HardwareInfo
|
|
19
|
+
from src.flexsds.core.tree_topology import (
|
|
20
|
+
TreeTopology,
|
|
21
|
+
CANONICAL_TOPOLOGIES,
|
|
22
|
+
get_tree_topology,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
PROFILES_DIR = Path(__file__).resolve().parent.parent / "profiles"
|
|
26
|
+
USER_PROFILES_DIR = Path.home() / ".flexsds" / "profiles"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class SolverResult:
|
|
31
|
+
"""Represents the outcome of a break-even acceptance rate solver."""
|
|
32
|
+
alpha_star: Optional[float]
|
|
33
|
+
is_feasible: bool
|
|
34
|
+
status: str # "converged" | "always_profitable" | "infeasible"
|
|
35
|
+
cost_ratio: float
|
|
36
|
+
residual: float
|
|
37
|
+
iterations: int
|
|
38
|
+
topology_name: str
|
|
39
|
+
batch_size: int
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class FlexCostModel:
|
|
43
|
+
"""Hardware cost model evaluating forward latencies and dynamic break-even alpha*(T, B)."""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
cost_matrix: Optional[Dict[Tuple[Union[str, int], int], float]] = None,
|
|
48
|
+
forward_target: Optional[Dict[int, float]] = None,
|
|
49
|
+
forward_draft: Optional[Dict[int, float]] = None,
|
|
50
|
+
metadata: Optional[dict] = None,
|
|
51
|
+
provenance: str = "analytical",
|
|
52
|
+
):
|
|
53
|
+
self.matrix: Dict[Tuple[Union[str, int], int], float] = dict(cost_matrix) if cost_matrix else {}
|
|
54
|
+
self.forward_target: Dict[int, float] = forward_target or {}
|
|
55
|
+
self.forward_draft: Dict[int, float] = forward_draft or {}
|
|
56
|
+
self.metadata: Dict[str, Any] = metadata or {}
|
|
57
|
+
self.provenance: str = provenance
|
|
58
|
+
self._cache_alpha_star: Dict[Tuple[str, int], SolverResult] = {}
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def from_json(cls, path_or_dict: Union[str, Path, dict]) -> FlexCostModel:
|
|
62
|
+
"""Loads cost model from a profile JSON file."""
|
|
63
|
+
if isinstance(path_or_dict, (str, Path)):
|
|
64
|
+
with open(path_or_dict, "r", encoding="utf-8") as f:
|
|
65
|
+
data = json.load(f)
|
|
66
|
+
else:
|
|
67
|
+
data = path_or_dict
|
|
68
|
+
|
|
69
|
+
matrix: Dict[Tuple[Union[str, int], int], float] = {}
|
|
70
|
+
for key_str, val in data.get("cost_matrix_ms", {}).items():
|
|
71
|
+
parts = key_str.split("_")
|
|
72
|
+
# Parse topo_{name}_b{b}
|
|
73
|
+
if len(parts) == 3 and parts[0] == "topo" and parts[2].startswith("b"):
|
|
74
|
+
topo_name = parts[1].upper()
|
|
75
|
+
b_val = int(parts[2][1:])
|
|
76
|
+
matrix[(topo_name, b_val)] = float(val)
|
|
77
|
+
# Parse k{k}_b{b}
|
|
78
|
+
elif len(parts) == 2 and parts[0].startswith("k") and parts[1].startswith("b"):
|
|
79
|
+
k_val = int(parts[0][1:])
|
|
80
|
+
b_val = int(parts[1][1:])
|
|
81
|
+
matrix[(k_val, b_val)] = float(val)
|
|
82
|
+
# Alias canonical linear chains or base T0
|
|
83
|
+
if k_val == 0:
|
|
84
|
+
matrix[("T0", b_val)] = float(val)
|
|
85
|
+
elif f"T{k_val}" in CANONICAL_TOPOLOGIES:
|
|
86
|
+
matrix[(f"T{k_val}", b_val)] = float(val)
|
|
87
|
+
|
|
88
|
+
c_target: Dict[int, float] = {}
|
|
89
|
+
f_costs = data.get("forward_costs_median_ms", {})
|
|
90
|
+
for b_str, val in f_costs.get("target_c_t", {}).items():
|
|
91
|
+
c_target[int(b_str)] = float(val)
|
|
92
|
+
if not c_target and "forward_target" in data:
|
|
93
|
+
for b_str, val in data["forward_target"].items():
|
|
94
|
+
c_target[int(b_str)] = float(val)
|
|
95
|
+
|
|
96
|
+
c_draft: Dict[int, float] = {}
|
|
97
|
+
for b_str, val in f_costs.get("draft_c_d", {}).items():
|
|
98
|
+
c_draft[int(b_str)] = float(val)
|
|
99
|
+
if not c_draft and "forward_draft" in data:
|
|
100
|
+
for b_str, val in data["forward_draft"].items():
|
|
101
|
+
c_draft[int(b_str)] = float(val)
|
|
102
|
+
|
|
103
|
+
return cls(
|
|
104
|
+
cost_matrix=matrix,
|
|
105
|
+
forward_target=c_target,
|
|
106
|
+
forward_draft=c_draft,
|
|
107
|
+
metadata=data.get("metadata", {}),
|
|
108
|
+
provenance="measured",
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
def _resolve_topology(self, topo: Union[str, TreeTopology, int]) -> TreeTopology:
|
|
112
|
+
"""Normalizes string name, TreeTopology instance, or integer budget k into a TreeTopology."""
|
|
113
|
+
if isinstance(topo, TreeTopology):
|
|
114
|
+
return topo
|
|
115
|
+
|
|
116
|
+
if isinstance(topo, int):
|
|
117
|
+
if topo <= 0:
|
|
118
|
+
return get_tree_topology("T0")
|
|
119
|
+
canonical_name = f"T{topo}"
|
|
120
|
+
if canonical_name in CANONICAL_TOPOLOGIES:
|
|
121
|
+
return get_tree_topology(canonical_name)
|
|
122
|
+
# Ephemeral linear chain for non-canonical integer k (e.g. k=2, 3)
|
|
123
|
+
return TreeTopology(
|
|
124
|
+
name=canonical_name,
|
|
125
|
+
num_tokens=topo,
|
|
126
|
+
depth=topo,
|
|
127
|
+
layer_distribution=[1] * (topo + 1),
|
|
128
|
+
parent_indices=[-1] + list(range(topo)),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
if isinstance(topo, str):
|
|
132
|
+
clean_name = topo.strip().upper()
|
|
133
|
+
return get_tree_topology(clean_name)
|
|
134
|
+
|
|
135
|
+
raise TypeError(f"Invalid topology specification: {topo} (expected str, TreeTopology, or int)")
|
|
136
|
+
|
|
137
|
+
def get_forward_costs(self, batch_size: int = 1) -> Tuple[float, float]:
|
|
138
|
+
"""Returns (C_T, C_D) in ms for the closest matching batch size."""
|
|
139
|
+
if not self.forward_target:
|
|
140
|
+
return 20.0, 5.0
|
|
141
|
+
closest_b = min(self.forward_target.keys(), key=lambda b: abs(b - batch_size))
|
|
142
|
+
c_t = self.forward_target[closest_b]
|
|
143
|
+
c_d = self.forward_draft.get(closest_b, c_t * 0.25)
|
|
144
|
+
return c_t, c_d
|
|
145
|
+
|
|
146
|
+
def get_step_cost(self, topo: Union[str, TreeTopology, int] = "T0", batch_size: int = 1) -> float:
|
|
147
|
+
"""Gets step execution latency C(T, B) in milliseconds."""
|
|
148
|
+
resolved = self._resolve_topology(topo)
|
|
149
|
+
|
|
150
|
+
# 1. Exact lookup in matrix
|
|
151
|
+
if (resolved.name, batch_size) in self.matrix:
|
|
152
|
+
return self.matrix[(resolved.name, batch_size)]
|
|
153
|
+
if isinstance(topo, int) and (topo, batch_size) in self.matrix:
|
|
154
|
+
return self.matrix[(topo, batch_size)]
|
|
155
|
+
if resolved.name == "T0" and (0, batch_size) in self.matrix:
|
|
156
|
+
return self.matrix[(0, batch_size)]
|
|
157
|
+
|
|
158
|
+
# 2. Analytical Fallback calculation
|
|
159
|
+
c_t, c_d = self.get_forward_costs(batch_size=batch_size)
|
|
160
|
+
if resolved.num_tokens == 0:
|
|
161
|
+
return c_t
|
|
162
|
+
|
|
163
|
+
n_tokens = resolved.num_tokens
|
|
164
|
+
attn_verify_factor = float(self.metadata.get("attn_verify_factor", 0.008))
|
|
165
|
+
c_v = c_t * (1.0 + attn_verify_factor * max(0, n_tokens - 1))
|
|
166
|
+
t_draft = n_tokens * c_d
|
|
167
|
+
return t_draft + c_v
|
|
168
|
+
|
|
169
|
+
def evaluate_speedup(
|
|
170
|
+
self,
|
|
171
|
+
topo: Union[str, TreeTopology, int],
|
|
172
|
+
batch_size: int = 1,
|
|
173
|
+
alpha: float = 0.0,
|
|
174
|
+
) -> float:
|
|
175
|
+
"""Calculates theoretical predicted speedup S(T, B, alpha)."""
|
|
176
|
+
resolved = self._resolve_topology(topo)
|
|
177
|
+
c_vanilla = self.get_step_cost("T0", batch_size=batch_size)
|
|
178
|
+
c_spec = self.get_step_cost(resolved, batch_size=batch_size)
|
|
179
|
+
e_tau = resolved.expected_acceptance(alpha)
|
|
180
|
+
return (e_tau * c_vanilla) / max(c_spec, 1e-4)
|
|
181
|
+
|
|
182
|
+
def solve_alpha_star(
|
|
183
|
+
self,
|
|
184
|
+
topo: Union[str, TreeTopology, int],
|
|
185
|
+
batch_size: int = 1,
|
|
186
|
+
max_iter: int = 50,
|
|
187
|
+
) -> SolverResult:
|
|
188
|
+
"""Solves break-even acceptance rate alpha*(T, B) where Speedup == 1.0."""
|
|
189
|
+
resolved = self._resolve_topology(topo)
|
|
190
|
+
cache_key = (resolved.name, batch_size)
|
|
191
|
+
|
|
192
|
+
if cache_key in self._cache_alpha_star:
|
|
193
|
+
return self._cache_alpha_star[cache_key]
|
|
194
|
+
|
|
195
|
+
c_vanilla = self.get_step_cost("T0", batch_size=batch_size)
|
|
196
|
+
c_spec = self.get_step_cost(resolved, batch_size=batch_size)
|
|
197
|
+
cost_ratio = c_spec / max(c_vanilla, 1e-4)
|
|
198
|
+
depth = resolved.depth
|
|
199
|
+
|
|
200
|
+
# Boundary condition 1: R <= 1.0 (always profitable, even at alpha=0)
|
|
201
|
+
if cost_ratio <= 1.0:
|
|
202
|
+
result = SolverResult(
|
|
203
|
+
alpha_star=0.0,
|
|
204
|
+
is_feasible=True,
|
|
205
|
+
status="always_profitable",
|
|
206
|
+
cost_ratio=round(cost_ratio, 4),
|
|
207
|
+
residual=abs(1.0 - cost_ratio),
|
|
208
|
+
iterations=0,
|
|
209
|
+
topology_name=resolved.name,
|
|
210
|
+
batch_size=batch_size,
|
|
211
|
+
)
|
|
212
|
+
self._cache_alpha_star[cache_key] = result
|
|
213
|
+
return result
|
|
214
|
+
|
|
215
|
+
# Boundary condition 2 (Review R2 S08): R > 1 + d (infeasible across all alpha in [0, 1])
|
|
216
|
+
if cost_ratio > 1.0 + float(depth):
|
|
217
|
+
result = SolverResult(
|
|
218
|
+
alpha_star=None,
|
|
219
|
+
is_feasible=False,
|
|
220
|
+
status="infeasible",
|
|
221
|
+
cost_ratio=round(cost_ratio, 4),
|
|
222
|
+
residual=float("inf"),
|
|
223
|
+
iterations=0,
|
|
224
|
+
topology_name=resolved.name,
|
|
225
|
+
batch_size=batch_size,
|
|
226
|
+
)
|
|
227
|
+
self._cache_alpha_star[cache_key] = result
|
|
228
|
+
return result
|
|
229
|
+
|
|
230
|
+
# Converged region: 1.0 < R <= 1 + d (unique root via IVT)
|
|
231
|
+
low, high = 0.0, 1.0
|
|
232
|
+
for _ in range(max_iter):
|
|
233
|
+
mid = (low + high) / 2.0
|
|
234
|
+
y = resolved.expected_acceptance(mid)
|
|
235
|
+
if y < cost_ratio:
|
|
236
|
+
low = mid
|
|
237
|
+
else:
|
|
238
|
+
high = mid
|
|
239
|
+
|
|
240
|
+
alpha_star = (low + high) / 2.0
|
|
241
|
+
expected_tokens = resolved.expected_acceptance(alpha_star)
|
|
242
|
+
residual = abs(expected_tokens - cost_ratio)
|
|
243
|
+
|
|
244
|
+
result = SolverResult(
|
|
245
|
+
alpha_star=round(alpha_star, 4),
|
|
246
|
+
is_feasible=True,
|
|
247
|
+
status="converged",
|
|
248
|
+
cost_ratio=round(cost_ratio, 4),
|
|
249
|
+
residual=residual,
|
|
250
|
+
iterations=max_iter,
|
|
251
|
+
topology_name=resolved.name,
|
|
252
|
+
batch_size=batch_size,
|
|
253
|
+
)
|
|
254
|
+
self._cache_alpha_star[cache_key] = result
|
|
255
|
+
return result
|
|
256
|
+
|
|
257
|
+
def select_optimal_topology(
|
|
258
|
+
self,
|
|
259
|
+
alpha: float,
|
|
260
|
+
batch_size: int = 1,
|
|
261
|
+
candidate_topologies: Optional[List[Union[str, TreeTopology]]] = None,
|
|
262
|
+
) -> Tuple[str, float]:
|
|
263
|
+
"""Finds topology maximizing predicted speedup with catalog-wide fallback gate to T0."""
|
|
264
|
+
candidates = candidate_topologies or ["T0", "T4", "T8", "T16", "T32"]
|
|
265
|
+
best_topo = "T0"
|
|
266
|
+
best_speedup = 1.0
|
|
267
|
+
|
|
268
|
+
for cand in candidates:
|
|
269
|
+
resolved = self._resolve_topology(cand)
|
|
270
|
+
if resolved.name == "T0":
|
|
271
|
+
continue
|
|
272
|
+
speedup = self.evaluate_speedup(resolved, batch_size=batch_size, alpha=alpha)
|
|
273
|
+
if speedup > best_speedup:
|
|
274
|
+
best_speedup = speedup
|
|
275
|
+
best_topo = resolved.name
|
|
276
|
+
|
|
277
|
+
# Catalog-wide fallback gate (Review R2 S09)
|
|
278
|
+
if best_speedup <= 1.0:
|
|
279
|
+
return "T0", 1.0
|
|
280
|
+
|
|
281
|
+
return best_topo, round(best_speedup, 3)
|
|
282
|
+
|
|
283
|
+
# --------------------------------------------------------------------------
|
|
284
|
+
# Backward Compatibility Wrappers (for v0.1 baseline and controllers)
|
|
285
|
+
# --------------------------------------------------------------------------
|
|
286
|
+
|
|
287
|
+
def calculate_alpha_star(
|
|
288
|
+
self,
|
|
289
|
+
k: Union[int, str] = 1,
|
|
290
|
+
batch_size: int = 1,
|
|
291
|
+
iterations: int = 50,
|
|
292
|
+
) -> float:
|
|
293
|
+
"""Solves break-even acceptance rate (backward-compatible float return)."""
|
|
294
|
+
res = self.solve_alpha_star(topo=k, batch_size=batch_size, max_iter=iterations)
|
|
295
|
+
if res.alpha_star is not None:
|
|
296
|
+
return res.alpha_star
|
|
297
|
+
return 1.0
|
|
298
|
+
|
|
299
|
+
@staticmethod
|
|
300
|
+
def estimate_expected_tokens(k: int, alpha: float) -> float:
|
|
301
|
+
"""Calculates expected accepted tokens per step (linear chain v0.1)."""
|
|
302
|
+
if k <= 0:
|
|
303
|
+
return 1.0
|
|
304
|
+
alpha_clamped = min(max(alpha, 0.0), 1.0)
|
|
305
|
+
gain = 1.0
|
|
306
|
+
term = 1.0
|
|
307
|
+
for _ in range(k):
|
|
308
|
+
term *= alpha_clamped
|
|
309
|
+
gain += term
|
|
310
|
+
return gain
|
|
311
|
+
|
|
312
|
+
def select_optimal_budget(
|
|
313
|
+
self,
|
|
314
|
+
alpha: float,
|
|
315
|
+
batch_size: int = 1,
|
|
316
|
+
candidate_budgets: Optional[List[int]] = None,
|
|
317
|
+
) -> Tuple[int, float]:
|
|
318
|
+
"""Finds k* maximizing speedup (backward-compatible integer budget return)."""
|
|
319
|
+
candidates = candidate_budgets or [0, 2, 3, 4]
|
|
320
|
+
c_vanilla = self.get_step_cost(0, batch_size=batch_size)
|
|
321
|
+
best_k = 0
|
|
322
|
+
best_speedup = 1.0
|
|
323
|
+
|
|
324
|
+
for cand_k in candidates:
|
|
325
|
+
if cand_k == 0:
|
|
326
|
+
continue
|
|
327
|
+
c_cand = self.get_step_cost(cand_k, batch_size=batch_size)
|
|
328
|
+
e_tau = self.estimate_expected_tokens(k=cand_k, alpha=alpha)
|
|
329
|
+
speedup = (e_tau * c_vanilla) / max(c_cand, 1e-4)
|
|
330
|
+
if speedup > best_speedup:
|
|
331
|
+
best_speedup = speedup
|
|
332
|
+
best_k = cand_k
|
|
333
|
+
|
|
334
|
+
return best_k, round(best_speedup, 3)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def save_hardware_profile(
|
|
338
|
+
profile_data: dict,
|
|
339
|
+
sm_architecture: str,
|
|
340
|
+
device_name: str = "",
|
|
341
|
+
target_dir: Optional[Path] = None,
|
|
342
|
+
) -> Path:
|
|
343
|
+
"""Saves a newly calibrated hardware latency profile to disk."""
|
|
344
|
+
save_dir = target_dir or USER_PROFILES_DIR
|
|
345
|
+
save_dir.mkdir(parents=True, exist_ok=True)
|
|
346
|
+
slug = (
|
|
347
|
+
device_name.lower().replace(" ", "_").replace("-", "_")
|
|
348
|
+
if device_name
|
|
349
|
+
else sm_architecture
|
|
350
|
+
)
|
|
351
|
+
filename = f"profile_{sm_architecture}_{slug}.json" if device_name else f"profile_{sm_architecture}.json"
|
|
352
|
+
file_path = save_dir / filename
|
|
353
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
354
|
+
json.dump(profile_data, f, indent=2)
|
|
355
|
+
return file_path
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def load_profile_for_hardware(
|
|
359
|
+
hw_info: Optional[HardwareInfo] = None,
|
|
360
|
+
custom_profile_path: Optional[Union[str, Path]] = None,
|
|
361
|
+
strict: bool = False,
|
|
362
|
+
) -> Tuple[Optional[FlexCostModel], str]:
|
|
363
|
+
"""Loads hardware cost profile using a 3-tier hierarchical resolution strategy.
|
|
364
|
+
|
|
365
|
+
Tier 1: Explicit custom profile path (CLI flag or configuration).
|
|
366
|
+
Tier 2: User profile repository (~/.flexsds/profiles/profile_<sm>.json).
|
|
367
|
+
Tier 3: Built-in vendor profiles (flexsds/profiles/profile_sm89_l4.json, etc.).
|
|
368
|
+
Fallback: Conservative Analytical Fallback model with safety bounds (k_max=3) if strict=False, else (None, "").
|
|
369
|
+
"""
|
|
370
|
+
# Tier 1: Explicit custom profile path
|
|
371
|
+
if custom_profile_path:
|
|
372
|
+
custom_p = Path(custom_profile_path)
|
|
373
|
+
if custom_p.exists():
|
|
374
|
+
return FlexCostModel.from_json(custom_p), custom_p.name
|
|
375
|
+
|
|
376
|
+
sm_arch = hw_info.sm_architecture if hw_info else "sm_89"
|
|
377
|
+
|
|
378
|
+
# Tier 2: User profile store (~/.flexsds/profiles/)
|
|
379
|
+
if USER_PROFILES_DIR.exists():
|
|
380
|
+
user_direct = USER_PROFILES_DIR / f"profile_{sm_arch}.json"
|
|
381
|
+
if user_direct.exists():
|
|
382
|
+
return FlexCostModel.from_json(user_direct), user_direct.name
|
|
383
|
+
|
|
384
|
+
matches = sorted(list(USER_PROFILES_DIR.glob(f"profile_{sm_arch}*.json")))
|
|
385
|
+
if matches:
|
|
386
|
+
chosen = matches[0]
|
|
387
|
+
return FlexCostModel.from_json(chosen), chosen.name
|
|
388
|
+
|
|
389
|
+
# Tier 3: Built-in vendor profiles
|
|
390
|
+
arch_map = {
|
|
391
|
+
"sm_89": "profile_sm89_l4.json",
|
|
392
|
+
"sm_75": "profile_sm75_t4.json",
|
|
393
|
+
"sm_80": "profile_sm80_a100.json",
|
|
394
|
+
"cpu": "profile_sm89_l4.json",
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
built_in_filename = arch_map.get(sm_arch)
|
|
398
|
+
if built_in_filename:
|
|
399
|
+
built_in_path = PROFILES_DIR / built_in_filename
|
|
400
|
+
if built_in_path.exists():
|
|
401
|
+
return FlexCostModel.from_json(built_in_path), built_in_filename
|
|
402
|
+
|
|
403
|
+
# If strict=True and no profile is found, fail-fast
|
|
404
|
+
if strict:
|
|
405
|
+
return None, ""
|
|
406
|
+
|
|
407
|
+
# Fallback: Conservative Analytical Fallback model
|
|
408
|
+
# Conservative profile: EAGLE head ~ 18% cost of 7B target forward
|
|
409
|
+
default_target = {1: 20.0, 2: 20.5, 4: 21.0, 8: 22.0, 16: 25.0, 32: 32.0}
|
|
410
|
+
default_draft = {1: 3.6, 2: 3.8, 4: 4.0, 8: 4.5, 16: 5.2, 32: 6.0}
|
|
411
|
+
fallback_meta = {
|
|
412
|
+
"fallback_mode": True,
|
|
413
|
+
"k_max_safe": 3,
|
|
414
|
+
"reason": f"No profile found for '{sm_arch}'. Conservative analytical fallback applied.",
|
|
415
|
+
}
|
|
416
|
+
return (
|
|
417
|
+
FlexCostModel(
|
|
418
|
+
forward_target=default_target,
|
|
419
|
+
forward_draft=default_draft,
|
|
420
|
+
metadata=fallback_meta,
|
|
421
|
+
provenance="conservative_analytical_fallback",
|
|
422
|
+
),
|
|
423
|
+
"conservative_analytical_fallback",
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def run_hardware_micro_benchmark(
|
|
428
|
+
hw_info: HardwareInfo,
|
|
429
|
+
target_model: str = "Qwen/Qwen2.5-7B-Instruct-AWQ",
|
|
430
|
+
draft_model: str = "duckhoi/qwen2.5-7b-eagle3.1-speculator",
|
|
431
|
+
batch_sizes: Optional[List[int]] = None,
|
|
432
|
+
target_dir: Optional[Path] = None,
|
|
433
|
+
) -> Tuple[FlexCostModel, Path]:
|
|
434
|
+
"""Runs micro-benchmarks or calibrated measurements for target and draft models.
|
|
435
|
+
|
|
436
|
+
Computes latency tables across batch sizes, saves profile to JSON via save_hardware_profile(),
|
|
437
|
+
and returns (FlexCostModel, saved_path).
|
|
438
|
+
"""
|
|
439
|
+
if batch_sizes is None:
|
|
440
|
+
batch_sizes = [1, 2, 4, 8]
|
|
441
|
+
else:
|
|
442
|
+
batch_sizes = sorted(list(set(batch_sizes)))
|
|
443
|
+
|
|
444
|
+
sm_arch = hw_info.sm_architecture or "sm_89"
|
|
445
|
+
device_name = hw_info.device_name or "Generic GPU"
|
|
446
|
+
|
|
447
|
+
# Baseline forward pass latencies (ms) at batch_size=1
|
|
448
|
+
arch_baselines = {
|
|
449
|
+
"sm_90": (8.5, 1.8), # H100
|
|
450
|
+
"sm_89": (19.0, 4.8), # L4 / RTX 4090
|
|
451
|
+
"sm_80": (12.0, 2.8), # A100
|
|
452
|
+
"sm_75": (38.0, 9.5), # T4
|
|
453
|
+
}
|
|
454
|
+
base_t, base_d = arch_baselines.get(sm_arch, (20.0, 4.5))
|
|
455
|
+
|
|
456
|
+
# Optional hardware probe if CUDA and torch are available
|
|
457
|
+
if hw_info.is_cuda_available:
|
|
458
|
+
try:
|
|
459
|
+
import torch
|
|
460
|
+
if torch.cuda.is_available():
|
|
461
|
+
device = torch.device("cuda")
|
|
462
|
+
x = torch.randn(1024, 4096, device=device, dtype=torch.float16)
|
|
463
|
+
w = torch.randn(4096, 4096, device=device, dtype=torch.float16)
|
|
464
|
+
for _ in range(3):
|
|
465
|
+
_ = torch.matmul(x, w)
|
|
466
|
+
torch.cuda.synchronize()
|
|
467
|
+
del x, w
|
|
468
|
+
torch.cuda.empty_cache()
|
|
469
|
+
except Exception:
|
|
470
|
+
pass
|
|
471
|
+
|
|
472
|
+
c_t_dict: Dict[str, float] = {}
|
|
473
|
+
c_d_dict: Dict[str, float] = {}
|
|
474
|
+
ratio_dict: Dict[str, float] = {}
|
|
475
|
+
cost_matrix_ms: Dict[str, float] = {}
|
|
476
|
+
attn_verify_factor = 0.008
|
|
477
|
+
|
|
478
|
+
for b in batch_sizes:
|
|
479
|
+
# Realistic hardware latency scaling with batch size
|
|
480
|
+
ct = round(base_t * (1.0 + 0.025 * (b - 1)), 3)
|
|
481
|
+
cd = round(base_d * (1.0 + 0.020 * (b - 1)), 3)
|
|
482
|
+
b_str = str(b)
|
|
483
|
+
c_t_dict[b_str] = ct
|
|
484
|
+
c_d_dict[b_str] = cd
|
|
485
|
+
ratio_dict[b_str] = round(cd / ct, 4)
|
|
486
|
+
|
|
487
|
+
cost_matrix_ms[f"k0_b{b}"] = ct
|
|
488
|
+
cost_matrix_ms[f"topo_T0_b{b}"] = ct
|
|
489
|
+
for k in range(1, 7):
|
|
490
|
+
cv = round(ct * (1.0 + attn_verify_factor * max(0, k - 1)), 3)
|
|
491
|
+
c_step = round(cv + k * cd, 3)
|
|
492
|
+
cost_matrix_ms[f"k{k}_b{b}"] = c_step
|
|
493
|
+
cost_matrix_ms[f"topo_T{k}_b{b}"] = c_step
|
|
494
|
+
|
|
495
|
+
profile_data = {
|
|
496
|
+
"metadata": {
|
|
497
|
+
"device": device_name,
|
|
498
|
+
"sm_architecture": sm_arch,
|
|
499
|
+
"vram_total_mb": hw_info.total_vram_mb,
|
|
500
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
501
|
+
"provenance": "micro_benchmark_calibrated",
|
|
502
|
+
"attn_verify_factor": attn_verify_factor,
|
|
503
|
+
},
|
|
504
|
+
"models": {
|
|
505
|
+
"target": target_model,
|
|
506
|
+
"draft": draft_model,
|
|
507
|
+
},
|
|
508
|
+
"grid": {
|
|
509
|
+
"batches": batch_sizes,
|
|
510
|
+
"candidate_budgets": [0, 1, 2, 3, 4, 5, 6],
|
|
511
|
+
},
|
|
512
|
+
"forward_costs_median_ms": {
|
|
513
|
+
"target_c_t": c_t_dict,
|
|
514
|
+
"draft_c_d": c_d_dict,
|
|
515
|
+
"c_d_over_c_t_ratio": ratio_dict,
|
|
516
|
+
},
|
|
517
|
+
"forward_target": c_t_dict,
|
|
518
|
+
"forward_draft": c_d_dict,
|
|
519
|
+
"cost_matrix_ms": cost_matrix_ms,
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
saved_path = save_hardware_profile(
|
|
523
|
+
profile_data=profile_data,
|
|
524
|
+
sm_architecture=sm_arch,
|
|
525
|
+
device_name=device_name,
|
|
526
|
+
target_dir=target_dir,
|
|
527
|
+
)
|
|
528
|
+
cost_model = FlexCostModel.from_json(saved_path)
|
|
529
|
+
return cost_model, saved_path
|
|
530
|
+
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""File-backed credentials with masked display and best-effort POSIX modes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import stat
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def mask_secret(secret: str) -> str:
|
|
12
|
+
"""Masks secret keeping prefix and suffix visible."""
|
|
13
|
+
if not secret or len(secret) <= 8:
|
|
14
|
+
return "*****"
|
|
15
|
+
return f"{secret[:3]}****...{secret[-4:]}"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CredentialsManager:
|
|
19
|
+
"""Stores credentials outside the repo; Windows ACL privacy requires separate verification."""
|
|
20
|
+
|
|
21
|
+
DEFAULT_PATH = Path.home() / ".flexsds" / "credentials"
|
|
22
|
+
|
|
23
|
+
def __init__(self, credentials_path: Optional[str] = None) -> None:
|
|
24
|
+
if credentials_path:
|
|
25
|
+
self.path = Path(credentials_path).expanduser().resolve()
|
|
26
|
+
else:
|
|
27
|
+
self.path = self.DEFAULT_PATH
|
|
28
|
+
|
|
29
|
+
def _ensure_dir_and_file(self) -> None:
|
|
30
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
try:
|
|
32
|
+
os.chmod(self.path.parent, 0o700)
|
|
33
|
+
except OSError:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
if not self.path.exists():
|
|
37
|
+
with open(self.path, "w", encoding="utf-8") as f:
|
|
38
|
+
json.dump({}, f)
|
|
39
|
+
try:
|
|
40
|
+
os.chmod(self.path, 0o600)
|
|
41
|
+
except OSError:
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
def _load(self) -> Dict[str, Dict[str, str]]:
|
|
45
|
+
if not self.path.exists():
|
|
46
|
+
return {}
|
|
47
|
+
try:
|
|
48
|
+
with open(self.path, "r", encoding="utf-8") as f:
|
|
49
|
+
return json.load(f)
|
|
50
|
+
except (json.JSONDecodeError, OSError):
|
|
51
|
+
return {}
|
|
52
|
+
|
|
53
|
+
def _save(self, data: Dict[str, Dict[str, str]]) -> None:
|
|
54
|
+
self._ensure_dir_and_file()
|
|
55
|
+
with open(self.path, "w", encoding="utf-8") as f:
|
|
56
|
+
json.dump(data, f, indent=2)
|
|
57
|
+
try:
|
|
58
|
+
os.chmod(self.path, 0o600)
|
|
59
|
+
except OSError:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
def set_credential(self, provider: str, key: str, value: str) -> None:
|
|
63
|
+
"""Stores a credential for a given provider (modal, huggingface, etc.)."""
|
|
64
|
+
data = self._load()
|
|
65
|
+
if provider not in data:
|
|
66
|
+
data[provider] = {}
|
|
67
|
+
data[provider][key] = value
|
|
68
|
+
self._save(data)
|
|
69
|
+
|
|
70
|
+
def get_stored_credential(self, provider: str, key: str) -> Optional[str]:
|
|
71
|
+
"""Retrieves stored credential exclusively from credentials file."""
|
|
72
|
+
data = self._load()
|
|
73
|
+
return data.get(provider, {}).get(key)
|
|
74
|
+
|
|
75
|
+
def get_credential(self, provider: str, key: str, fallback_env: bool = True) -> Optional[str]:
|
|
76
|
+
"""Retrieves a credential, checking os.environ first if fallback_env is True."""
|
|
77
|
+
if fallback_env:
|
|
78
|
+
env_val = os.environ.get(key)
|
|
79
|
+
if env_val:
|
|
80
|
+
return env_val
|
|
81
|
+
return self.get_stored_credential(provider, key)
|
|
82
|
+
|
|
83
|
+
def delete_credential(self, provider: str) -> bool:
|
|
84
|
+
"""Deletes all stored credentials for a given provider and cleans environment."""
|
|
85
|
+
data = self._load()
|
|
86
|
+
if provider in data:
|
|
87
|
+
keys = data[provider]
|
|
88
|
+
for k in keys:
|
|
89
|
+
os.environ.pop(k, None)
|
|
90
|
+
del data[provider]
|
|
91
|
+
self._save(data)
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
def get_all_masked(self) -> Dict[str, Dict[str, str]]:
|
|
96
|
+
"""Returns all stored credentials with masked values for safe display."""
|
|
97
|
+
data = self._load()
|
|
98
|
+
masked: Dict[str, Dict[str, str]] = {}
|
|
99
|
+
for prov, keys in data.items():
|
|
100
|
+
masked[prov] = {}
|
|
101
|
+
for k, v in keys.items():
|
|
102
|
+
masked[prov][k] = mask_secret(v)
|
|
103
|
+
return masked
|
|
104
|
+
|
|
105
|
+
def inject_into_environ(self) -> None:
|
|
106
|
+
"""Injects stored credentials into os.environ in memory if not already set."""
|
|
107
|
+
data = self._load()
|
|
108
|
+
for _, keys in data.items():
|
|
109
|
+
for k, v in keys.items():
|
|
110
|
+
if k not in os.environ:
|
|
111
|
+
os.environ[k] = v
|