flexsds 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flexsds/__init__.py +4 -0
- flexsds/assets/ptit_logo.png +0 -0
- flexsds/cli/__init__.py +1 -0
- flexsds/cli/banner.py +84 -0
- flexsds/cli/main.py +330 -0
- flexsds/cli/output.py +69 -0
- flexsds/cli/task_runner.py +171 -0
- flexsds/cli/wizard.py +360 -0
- flexsds/client/__init__.py +4 -0
- flexsds/client/chat.py +137 -0
- flexsds/client/client.py +107 -0
- flexsds/client/monitor.py +95 -0
- flexsds/core/__init__.py +44 -0
- flexsds/core/compatibility.py +442 -0
- flexsds/core/controller.py +502 -0
- flexsds/core/cost_model.py +530 -0
- flexsds/core/credentials.py +111 -0
- flexsds/core/hardware.py +77 -0
- flexsds/core/tree_topology.py +296 -0
- flexsds/profiles/profile_sm75_t4.json +62 -0
- flexsds/profiles/profile_sm89_l4.json +194 -0
- flexsds/server/__init__.py +1 -0
- flexsds/server/colab_runner.py +62 -0
- flexsds/server/local_runner.py +208 -0
- flexsds/server/modal_app.py +50 -0
- flexsds/server/modal_runner.py +98 -0
- flexsds/server/vscode_cli.py +82 -0
- flexsds-0.2.0.dist-info/METADATA +204 -0
- flexsds-0.2.0.dist-info/RECORD +62 -0
- flexsds-0.2.0.dist-info/WHEEL +5 -0
- flexsds-0.2.0.dist-info/entry_points.txt +2 -0
- flexsds-0.2.0.dist-info/top_level.txt +4 -0
- pipeline/entity_eval.py +171 -0
- pipeline/examples/node1_extractor_input_m0_04_001.txt +9 -0
- pipeline/examples/node1_extractor_output_m0_04_001.json +90 -0
- pipeline/extractor.py +502 -0
- pipeline/invoice_models.py +163 -0
- pipeline/node2_questions.py +30 -0
- pipeline/openai_compatible.py +87 -0
- pipeline/prompts/README.md +18 -0
- pipeline/prompts/node1_extractor_system.md +92 -0
- pipeline/prompts/node3_report_user.md +5 -0
- pipeline/report_writer.py +143 -0
- pipeline/schemas/invoice_extraction.schema.json +469 -0
- pipeline/sql_guard.py +140 -0
- pipeline/sqlite/schema.sql +219 -0
- pipeline/text_to_sql.py +170 -0
- pipeline/validation.py +166 -0
- pipeline/w1_workload.py +139 -0
- proposer/__init__.py +31 -0
- proposer/adapter.py +153 -0
- proposer/controller.py +460 -0
- proposer/cost_model.py +216 -0
- proposer/tree_adaptive.py +466 -0
- server/__init__.py +3 -0
- server/adaptive_scheduler.py +215 -0
- server/budget_planner.py +287 -0
- server/lineage.py +290 -0
- server/lineage_diagnostics.py +38 -0
- server/lineage_executor.py +129 -0
- server/lineage_startup.py +47 -0
- server/t4_rowwise_attention.py +262 -0
flexsds/core/hardware.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Hardware detection module for FlexSDS."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class HardwareInfo:
|
|
10
|
+
"""Hardware specification details for adaptive serving."""
|
|
11
|
+
device_name: str
|
|
12
|
+
sm_architecture: str
|
|
13
|
+
total_vram_mb: float
|
|
14
|
+
free_vram_mb: float
|
|
15
|
+
cuda_version: str
|
|
16
|
+
is_cuda_available: bool
|
|
17
|
+
device_count: int = 0
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def free_vram_gb(self) -> float:
|
|
21
|
+
return round(self.free_vram_mb / 1024.0, 2)
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def total_vram_gb(self) -> float:
|
|
25
|
+
return round(self.total_vram_mb / 1024.0, 2)
|
|
26
|
+
|
|
27
|
+
def summary(self) -> str:
|
|
28
|
+
if not self.is_cuda_available:
|
|
29
|
+
return f"CPU Only · RAM Free: {self.free_vram_gb} GB"
|
|
30
|
+
return f"{self.device_name} ({self.sm_architecture}) · VRAM: {self.free_vram_gb}/{self.total_vram_gb} GB Free"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def detect_hardware(device_index: int = 0) -> HardwareInfo:
|
|
34
|
+
"""Detects active GPU hardware details without initializing heavy CUDA context."""
|
|
35
|
+
try:
|
|
36
|
+
import torch
|
|
37
|
+
|
|
38
|
+
if torch.cuda.is_available() and torch.cuda.device_count() > 0:
|
|
39
|
+
dev_idx = min(device_index, torch.cuda.device_count() - 1)
|
|
40
|
+
name = torch.cuda.get_device_name(dev_idx)
|
|
41
|
+
major, minor = torch.cuda.get_device_capability(dev_idx)
|
|
42
|
+
sm_arch = f"sm_{major}{minor}"
|
|
43
|
+
|
|
44
|
+
free_bytes, total_bytes = torch.cuda.mem_get_info(dev_idx)
|
|
45
|
+
free_mb = free_bytes / (1024.0 * 1024.0)
|
|
46
|
+
total_mb = total_bytes / (1024.0 * 1024.0)
|
|
47
|
+
cuda_ver = torch.version.cuda or "N/A"
|
|
48
|
+
|
|
49
|
+
return HardwareInfo(
|
|
50
|
+
device_name=name,
|
|
51
|
+
sm_architecture=sm_arch,
|
|
52
|
+
total_vram_mb=round(total_mb, 1),
|
|
53
|
+
free_vram_mb=round(free_mb, 1),
|
|
54
|
+
cuda_version=cuda_ver,
|
|
55
|
+
is_cuda_available=True,
|
|
56
|
+
device_count=torch.cuda.device_count(),
|
|
57
|
+
)
|
|
58
|
+
except Exception:
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
# CPU Fallback
|
|
62
|
+
ram_mb = 16000.0
|
|
63
|
+
try:
|
|
64
|
+
import psutil
|
|
65
|
+
ram_mb = psutil.virtual_memory().available / (1024.0 * 1024.0)
|
|
66
|
+
except Exception:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
return HardwareInfo(
|
|
70
|
+
device_name="CPU",
|
|
71
|
+
sm_architecture="cpu",
|
|
72
|
+
total_vram_mb=round(ram_mb, 1),
|
|
73
|
+
free_vram_mb=round(ram_mb, 1),
|
|
74
|
+
cuda_version="N/A",
|
|
75
|
+
is_cuda_available=False,
|
|
76
|
+
device_count=0,
|
|
77
|
+
)
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Tree topology representations, causal attention masks, and expected yield models for FlexSDS.
|
|
2
|
+
|
|
3
|
+
Implements discrete canonical tree configurations (T0, T4, T8, T16, T32) based on
|
|
4
|
+
OPT-Tree (Wang et al., 2025, TACL) and speculative decoding theory.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
10
|
+
import math
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class TreeTopology:
|
|
15
|
+
"""Represents a speculative draft tree structure.
|
|
16
|
+
|
|
17
|
+
Attributes:
|
|
18
|
+
name: Short topology identifier (e.g. 'T0', 'T4', 'T8', 'T16', 'T32').
|
|
19
|
+
num_tokens: Number of non-root draft tokens N.
|
|
20
|
+
depth: Maximum depth of the tree d (root has depth 0).
|
|
21
|
+
parent_indices: List of length N+1 where parent_indices[0] == -1 and
|
|
22
|
+
parent_indices[i] is the index of the parent node for token i (0 <= parent < i).
|
|
23
|
+
layer_distribution: Number of nodes at each depth level [l_0, l_1, ..., l_d].
|
|
24
|
+
node_depths: Computed depth for each node in 0..N.
|
|
25
|
+
position_ids: Relative position index in tree (matches node_depths).
|
|
26
|
+
tree_mask: 2D causal attention mask of shape (N+1, N+1) where
|
|
27
|
+
mask[i][j] is True iff j is an ancestor of i (or j == i).
|
|
28
|
+
nodes_at_depth: Mapping from depth level to list of node indices at that depth.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
name: str
|
|
32
|
+
num_tokens: int
|
|
33
|
+
depth: int
|
|
34
|
+
parent_indices: List[int]
|
|
35
|
+
layer_distribution: List[int]
|
|
36
|
+
node_depths: List[int] = field(default_factory=list)
|
|
37
|
+
position_ids: List[int] = field(default_factory=list)
|
|
38
|
+
tree_mask: List[List[bool]] = field(default_factory=list)
|
|
39
|
+
nodes_at_depth: Dict[int, List[int]] = field(default_factory=dict)
|
|
40
|
+
_lookup_table: List[float] = field(default_factory=list, repr=False)
|
|
41
|
+
|
|
42
|
+
def __post_init__(self) -> None:
|
|
43
|
+
"""Validates invariants and computes derived properties if not already set."""
|
|
44
|
+
self._validate_basic_invariants()
|
|
45
|
+
if not self.node_depths:
|
|
46
|
+
self.node_depths = self._compute_depths()
|
|
47
|
+
if not self.position_ids:
|
|
48
|
+
self.position_ids = list(self.node_depths)
|
|
49
|
+
if not self.nodes_at_depth:
|
|
50
|
+
self.nodes_at_depth = self._build_nodes_at_depth()
|
|
51
|
+
if not self.tree_mask:
|
|
52
|
+
self.tree_mask = self._compute_causal_mask()
|
|
53
|
+
self._validate_derived_invariants()
|
|
54
|
+
if not self._lookup_table:
|
|
55
|
+
self._lookup_table = self._build_lookup_table()
|
|
56
|
+
|
|
57
|
+
def _validate_basic_invariants(self) -> None:
|
|
58
|
+
"""Validates fundamental structural invariants."""
|
|
59
|
+
expected_len = self.num_tokens + 1
|
|
60
|
+
actual_len = len(self.parent_indices)
|
|
61
|
+
if actual_len != expected_len:
|
|
62
|
+
raise ValueError(
|
|
63
|
+
f"Parent array length mismatch for {self.name}: "
|
|
64
|
+
f"expected {expected_len} (N+1), got {actual_len}"
|
|
65
|
+
)
|
|
66
|
+
if self.parent_indices[0] != -1:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"Root parent index must be -1, got {self.parent_indices[0]}"
|
|
69
|
+
)
|
|
70
|
+
for i in range(1, actual_len):
|
|
71
|
+
p = self.parent_indices[i]
|
|
72
|
+
if p < 0 or p >= i:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"Acyclic/topological ordering violation at node {i}: parent={p} (must be in 0..{i-1})"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def _compute_depths(self) -> List[int]:
|
|
78
|
+
"""Computes depth for each node from root 0."""
|
|
79
|
+
n = len(self.parent_indices)
|
|
80
|
+
depths = [0] * n
|
|
81
|
+
for i in range(1, n):
|
|
82
|
+
p = self.parent_indices[i]
|
|
83
|
+
depths[i] = depths[p] + 1
|
|
84
|
+
return depths
|
|
85
|
+
|
|
86
|
+
def _build_nodes_at_depth(self) -> Dict[int, List[int]]:
|
|
87
|
+
"""Groups node indices by depth."""
|
|
88
|
+
mapping: Dict[int, List[int]] = {}
|
|
89
|
+
for idx, d in enumerate(self.node_depths):
|
|
90
|
+
mapping.setdefault(d, []).append(idx)
|
|
91
|
+
return mapping
|
|
92
|
+
|
|
93
|
+
def _compute_causal_mask(self) -> List[List[bool]]:
|
|
94
|
+
"""Constructs 2D causal attention mask where node i can attend to j iff j is an ancestor of i."""
|
|
95
|
+
n = len(self.parent_indices)
|
|
96
|
+
mask = [[False] * n for _ in range(n)]
|
|
97
|
+
for i in range(n):
|
|
98
|
+
curr = i
|
|
99
|
+
while curr != -1:
|
|
100
|
+
mask[i][curr] = True
|
|
101
|
+
curr = self.parent_indices[curr]
|
|
102
|
+
return mask
|
|
103
|
+
|
|
104
|
+
def _validate_derived_invariants(self) -> None:
|
|
105
|
+
"""Validates depth, layer distribution, and mask properties."""
|
|
106
|
+
actual_max_depth = max(self.node_depths) if self.node_depths else 0
|
|
107
|
+
if actual_max_depth != self.depth:
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"Maximum depth mismatch for {self.name}: "
|
|
110
|
+
f"computed max depth {actual_max_depth} != declared depth {self.depth}"
|
|
111
|
+
)
|
|
112
|
+
actual_dist = [len(self.nodes_at_depth.get(d, [])) for d in range(self.depth + 1)]
|
|
113
|
+
if actual_dist != self.layer_distribution:
|
|
114
|
+
raise ValueError(
|
|
115
|
+
f"Layer distribution mismatch for {self.name}: "
|
|
116
|
+
f"actual {actual_dist} != declared {self.layer_distribution}"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def _build_lookup_table(self, num_points: int = 1001) -> List[float]:
|
|
120
|
+
"""Precomputes lookup table of E[tau(alpha)] for alpha in [0.0, 1.0] in 1001 steps."""
|
|
121
|
+
table = []
|
|
122
|
+
for i in range(num_points):
|
|
123
|
+
alpha = i / (num_points - 1)
|
|
124
|
+
table.append(self.expected_acceptance(alpha))
|
|
125
|
+
return table
|
|
126
|
+
|
|
127
|
+
def expected_acceptance(self, alpha: float, method: str = "branch_aware") -> float:
|
|
128
|
+
"""Computes theoretical expected accepted token length E[tau(T, alpha)].
|
|
129
|
+
|
|
130
|
+
Follows OPT-Tree expectation formula:
|
|
131
|
+
E[tau(T)] = 1 + sum_{v in T \\ {r}} q(v)
|
|
132
|
+
|
|
133
|
+
Under greedy decoding against target model:
|
|
134
|
+
At any depth level l in 1..d, at most ONE token can be accepted because the target
|
|
135
|
+
model generates a unique greedy token at each decoding position.
|
|
136
|
+
|
|
137
|
+
Methods:
|
|
138
|
+
- 'homogeneous': Linear expectation along the longest chain:
|
|
139
|
+
E[tau] = 1 + sum_{l=1}^d alpha^l
|
|
140
|
+
- 'branch_aware' (default): Multi-branch coverage at each layer:
|
|
141
|
+
With n_l candidate branches at depth l, the step acceptance probability is:
|
|
142
|
+
P_step(l, alpha) = 1 - (1 - alpha)^{n_l}
|
|
143
|
+
Cumulative path acceptance:
|
|
144
|
+
P_path(l, alpha) = prod_{k=1}^l P_step(k, alpha)
|
|
145
|
+
E[tau] = 1 + sum_{l=1}^d P_path(l, alpha)
|
|
146
|
+
|
|
147
|
+
Properties guaranteed:
|
|
148
|
+
- Monotonically non-decreasing in alpha in [0.0, 1.0].
|
|
149
|
+
- E[tau(alpha=0.0)] == 1.0 (only the root/verified token is emitted).
|
|
150
|
+
- E[tau(alpha=1.0)] == 1.0 + depth (longest path bound).
|
|
151
|
+
- Strict upper bound: E[tau] <= 1 + depth.
|
|
152
|
+
"""
|
|
153
|
+
if self.depth == 0 or self.num_tokens == 0:
|
|
154
|
+
return 1.0
|
|
155
|
+
|
|
156
|
+
clamped_alpha = max(0.0, min(1.0, float(alpha)))
|
|
157
|
+
if clamped_alpha <= 0.0:
|
|
158
|
+
return 1.0
|
|
159
|
+
if clamped_alpha >= 1.0:
|
|
160
|
+
return 1.0 + float(self.depth)
|
|
161
|
+
|
|
162
|
+
if method == "homogeneous":
|
|
163
|
+
# Geometric progression along max depth d
|
|
164
|
+
# E[tau] = 1 + alpha + alpha^2 + ... + alpha^d
|
|
165
|
+
val = 1.0 + sum(clamped_alpha ** l for l in range(1, self.depth + 1))
|
|
166
|
+
return min(1.0 + float(self.depth), val)
|
|
167
|
+
|
|
168
|
+
# Default: branch-aware multi-candidate coverage
|
|
169
|
+
expected = 1.0
|
|
170
|
+
cum_prob = 1.0
|
|
171
|
+
for l in range(1, self.depth + 1):
|
|
172
|
+
n_l = self.layer_distribution[l] if l < len(self.layer_distribution) else 1
|
|
173
|
+
# Step acceptance probability at depth l given parent accepted
|
|
174
|
+
step_prob = 1.0 - math.pow(1.0 - clamped_alpha, n_l)
|
|
175
|
+
cum_prob *= step_prob
|
|
176
|
+
expected += cum_prob
|
|
177
|
+
|
|
178
|
+
# Strict bound enforcement
|
|
179
|
+
return max(1.0, min(1.0 + float(self.depth), expected))
|
|
180
|
+
|
|
181
|
+
def lookup_expected_acceptance(self, alpha: float) -> float:
|
|
182
|
+
"""Fast O(1) lookup of precomputed E[tau(alpha)] (< 0.005 ms)."""
|
|
183
|
+
clamped_alpha = max(0.0, min(1.0, float(alpha)))
|
|
184
|
+
idx = int(round(clamped_alpha * (len(self._lookup_table) - 1)))
|
|
185
|
+
return self._lookup_table[idx]
|
|
186
|
+
|
|
187
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
188
|
+
"""Serializes topology to dictionary."""
|
|
189
|
+
return {
|
|
190
|
+
"name": self.name,
|
|
191
|
+
"num_tokens": self.num_tokens,
|
|
192
|
+
"depth": self.depth,
|
|
193
|
+
"parent_indices": self.parent_indices,
|
|
194
|
+
"layer_distribution": self.layer_distribution,
|
|
195
|
+
"node_depths": self.node_depths,
|
|
196
|
+
"position_ids": self.position_ids,
|
|
197
|
+
"tree_mask": self.tree_mask,
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def from_dict(cls, data: Dict[str, Any]) -> TreeTopology:
|
|
202
|
+
"""Deserializes topology from dictionary."""
|
|
203
|
+
return cls(
|
|
204
|
+
name=data["name"],
|
|
205
|
+
num_tokens=data["num_tokens"],
|
|
206
|
+
depth=data["depth"],
|
|
207
|
+
parent_indices=data["parent_indices"],
|
|
208
|
+
layer_distribution=data["layer_distribution"],
|
|
209
|
+
node_depths=data.get("node_depths", []),
|
|
210
|
+
position_ids=data.get("position_ids", []),
|
|
211
|
+
tree_mask=data.get("tree_mask", []),
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# ==============================================================================
|
|
216
|
+
# CANONICAL CATALOG DEFINITIONS (T0, T4, T8, T16, T32)
|
|
217
|
+
# ==============================================================================
|
|
218
|
+
|
|
219
|
+
CANONICAL_TOPOLOGIES: Dict[str, TreeTopology] = {
|
|
220
|
+
# T0: Baseline autoregressive decoding (no speculative draft tokens)
|
|
221
|
+
"T0": TreeTopology(
|
|
222
|
+
name="T0",
|
|
223
|
+
num_tokens=0,
|
|
224
|
+
depth=0,
|
|
225
|
+
layer_distribution=[1],
|
|
226
|
+
parent_indices=[-1],
|
|
227
|
+
),
|
|
228
|
+
# T4: Linear chain (N=4, depth=4)
|
|
229
|
+
"T4": TreeTopology(
|
|
230
|
+
name="T4",
|
|
231
|
+
num_tokens=4,
|
|
232
|
+
depth=4,
|
|
233
|
+
layer_distribution=[1, 1, 1, 1, 1],
|
|
234
|
+
parent_indices=[-1, 0, 1, 2, 3],
|
|
235
|
+
),
|
|
236
|
+
# T8: Narrow branching tree (N=8, depth=3)
|
|
237
|
+
"T8": TreeTopology(
|
|
238
|
+
name="T8",
|
|
239
|
+
num_tokens=8,
|
|
240
|
+
depth=3,
|
|
241
|
+
layer_distribution=[1, 2, 4, 2],
|
|
242
|
+
parent_indices=[-1, 0, 0, 1, 1, 2, 2, 3, 5],
|
|
243
|
+
),
|
|
244
|
+
# T16: Moderate branching tree (N=16, depth=4)
|
|
245
|
+
# Fixed invariant: node 16's parent is node 13 (depth 3) to strictly enforce max depth 4
|
|
246
|
+
"T16": TreeTopology(
|
|
247
|
+
name="T16",
|
|
248
|
+
num_tokens=16,
|
|
249
|
+
depth=4,
|
|
250
|
+
layer_distribution=[1, 3, 6, 4, 3],
|
|
251
|
+
parent_indices=[
|
|
252
|
+
-1,
|
|
253
|
+
0, 0, 0,
|
|
254
|
+
1, 1, 2, 2, 3, 3,
|
|
255
|
+
4, 4, 6, 8,
|
|
256
|
+
10, 12, 13,
|
|
257
|
+
],
|
|
258
|
+
),
|
|
259
|
+
# T32: Wide branching tree (N=32, depth=5)
|
|
260
|
+
# Deterministic 33-element parent array strictly adhering to [1, 4, 8, 8, 6, 6]
|
|
261
|
+
"T32": TreeTopology(
|
|
262
|
+
name="T32",
|
|
263
|
+
num_tokens=32,
|
|
264
|
+
depth=5,
|
|
265
|
+
layer_distribution=[1, 4, 8, 8, 6, 6],
|
|
266
|
+
parent_indices=[
|
|
267
|
+
-1,
|
|
268
|
+
# Depth 1 (4 nodes: 1..4, parent 0)
|
|
269
|
+
0, 0, 0, 0,
|
|
270
|
+
# Depth 2 (8 nodes: 5..12, parents 1, 1, 2, 2, 3, 3, 4, 4)
|
|
271
|
+
1, 1, 2, 2, 3, 3, 4, 4,
|
|
272
|
+
# Depth 3 (8 nodes: 13..20, parents 5..12)
|
|
273
|
+
5, 6, 7, 8, 9, 10, 11, 12,
|
|
274
|
+
# Depth 4 (6 nodes: 21..26, parents 13..18)
|
|
275
|
+
13, 14, 15, 16, 17, 18,
|
|
276
|
+
# Depth 5 (6 nodes: 27..32, parents 21..26)
|
|
277
|
+
21, 22, 23, 24, 25, 26,
|
|
278
|
+
],
|
|
279
|
+
),
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def get_tree_topology(name: str) -> TreeTopology:
|
|
284
|
+
"""Retrieves a canonical tree topology by name."""
|
|
285
|
+
clean_name = name.strip().upper()
|
|
286
|
+
if clean_name not in CANONICAL_TOPOLOGIES:
|
|
287
|
+
raise KeyError(
|
|
288
|
+
f"Unknown topology '{name}'. Available canonical topologies: "
|
|
289
|
+
f"{list(CANONICAL_TOPOLOGIES.keys())}"
|
|
290
|
+
)
|
|
291
|
+
return CANONICAL_TOPOLOGIES[clean_name]
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def list_available_topologies() -> List[str]:
|
|
295
|
+
"""Returns list of canonical topology identifiers."""
|
|
296
|
+
return list(CANONICAL_TOPOLOGIES.keys())
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"metadata": {
|
|
3
|
+
"device": "Tesla T4",
|
|
4
|
+
"sm_architecture": "sm_75",
|
|
5
|
+
"vram_total_mb": 15360.0,
|
|
6
|
+
"vllm_version": "0.30.0",
|
|
7
|
+
"source": "t4-m1-m2-measurements",
|
|
8
|
+
"attn_verify_factor": 0.012
|
|
9
|
+
},
|
|
10
|
+
"models": {
|
|
11
|
+
"target": "Qwen/Qwen2.5-7B-Instruct-AWQ",
|
|
12
|
+
"draft": "Qwen/Qwen2.5-0.5B-Instruct"
|
|
13
|
+
},
|
|
14
|
+
"grid": {
|
|
15
|
+
"batches": [1, 2, 4, 8, 16, 32],
|
|
16
|
+
"candidate_budgets": [0, 1, 2, 3, 4]
|
|
17
|
+
},
|
|
18
|
+
"forward_costs_median_ms": {
|
|
19
|
+
"target_c_t": {
|
|
20
|
+
"1": 36.5,
|
|
21
|
+
"2": 37.8,
|
|
22
|
+
"4": 39.2,
|
|
23
|
+
"8": 42.1,
|
|
24
|
+
"16": 48.5,
|
|
25
|
+
"32": 62.0
|
|
26
|
+
},
|
|
27
|
+
"draft_c_d": {
|
|
28
|
+
"1": 9.2,
|
|
29
|
+
"2": 9.8,
|
|
30
|
+
"4": 10.4,
|
|
31
|
+
"8": 11.2,
|
|
32
|
+
"16": 12.8,
|
|
33
|
+
"32": 15.5
|
|
34
|
+
},
|
|
35
|
+
"c_d_over_c_t_ratio": {
|
|
36
|
+
"1": 0.252,
|
|
37
|
+
"2": 0.259,
|
|
38
|
+
"4": 0.265,
|
|
39
|
+
"8": 0.266,
|
|
40
|
+
"16": 0.264,
|
|
41
|
+
"32": 0.250
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"verification_costs_ms": {
|
|
45
|
+
"k1_b1": 36.5,
|
|
46
|
+
"k2_b1": 36.9,
|
|
47
|
+
"k3_b1": 37.4,
|
|
48
|
+
"k4_b1": 37.8
|
|
49
|
+
},
|
|
50
|
+
"break_even_alpha": {
|
|
51
|
+
"k0_b1": 0.0,
|
|
52
|
+
"k1_b1": 0.252,
|
|
53
|
+
"k2_b1": 0.380,
|
|
54
|
+
"k3_b1": 0.470,
|
|
55
|
+
"k4_b1": 0.540,
|
|
56
|
+
"k0_b32": 0.0,
|
|
57
|
+
"k1_b32": 0.250,
|
|
58
|
+
"k2_b32": 0.385,
|
|
59
|
+
"k3_b32": 0.485,
|
|
60
|
+
"k4_b32": 0.560
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
{
|
|
2
|
+
"metadata": {
|
|
3
|
+
"device": "NVIDIA L4",
|
|
4
|
+
"sm_architecture": "sm_89",
|
|
5
|
+
"vram_total_mb": 23034.0,
|
|
6
|
+
"vllm_version": "0.30.0",
|
|
7
|
+
"source_run_id": "20261025-m302-l4-v2",
|
|
8
|
+
"source_commit": "98c92cf3dd9043d803953be4e5bf385bf8667601",
|
|
9
|
+
"timestamp_utc": "2026-10-25T05:00:00.292569+00:00",
|
|
10
|
+
"attn_verify_factor": 0.008
|
|
11
|
+
},
|
|
12
|
+
"models": {
|
|
13
|
+
"target": "Qwen/Qwen2.5-7B-Instruct-AWQ",
|
|
14
|
+
"draft": "Qwen/Qwen2.5-0.5B-Instruct"
|
|
15
|
+
},
|
|
16
|
+
"grid": {
|
|
17
|
+
"batches": [
|
|
18
|
+
1,
|
|
19
|
+
2,
|
|
20
|
+
4,
|
|
21
|
+
8,
|
|
22
|
+
16,
|
|
23
|
+
32
|
|
24
|
+
],
|
|
25
|
+
"contexts": [
|
|
26
|
+
256,
|
|
27
|
+
512,
|
|
28
|
+
1024,
|
|
29
|
+
2048,
|
|
30
|
+
3072
|
|
31
|
+
],
|
|
32
|
+
"candidate_budgets": [
|
|
33
|
+
0,
|
|
34
|
+
1,
|
|
35
|
+
2,
|
|
36
|
+
3,
|
|
37
|
+
4,
|
|
38
|
+
5,
|
|
39
|
+
6
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
"forward_costs_median_ms": {
|
|
43
|
+
"target_c_t": {
|
|
44
|
+
"1": 19.095,
|
|
45
|
+
"2": 19.606,
|
|
46
|
+
"4": 20.082,
|
|
47
|
+
"8": 20.809,
|
|
48
|
+
"16": 23.273,
|
|
49
|
+
"32": 30.043
|
|
50
|
+
},
|
|
51
|
+
"draft_c_d": {
|
|
52
|
+
"1": 4.885,
|
|
53
|
+
"2": 5.23,
|
|
54
|
+
"4": 5.266,
|
|
55
|
+
"8": 5.415,
|
|
56
|
+
"16": 6.27,
|
|
57
|
+
"32": 6.697
|
|
58
|
+
},
|
|
59
|
+
"c_d_over_c_t_ratio": {
|
|
60
|
+
"1": 0.2558,
|
|
61
|
+
"2": 0.2668,
|
|
62
|
+
"4": 0.2622,
|
|
63
|
+
"8": 0.2602,
|
|
64
|
+
"16": 0.2694,
|
|
65
|
+
"32": 0.2229
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"verification_costs_ms": {
|
|
69
|
+
"k1_b1": 19.095,
|
|
70
|
+
"k2_b1": 19.248,
|
|
71
|
+
"k3_b1": 19.401,
|
|
72
|
+
"k4_b1": 19.553,
|
|
73
|
+
"k5_b1": 19.706,
|
|
74
|
+
"k6_b1": 19.859,
|
|
75
|
+
"k1_b2": 19.606,
|
|
76
|
+
"k2_b2": 19.763,
|
|
77
|
+
"k3_b2": 19.92,
|
|
78
|
+
"k4_b2": 20.077,
|
|
79
|
+
"k5_b2": 20.233,
|
|
80
|
+
"k6_b2": 20.39,
|
|
81
|
+
"k1_b4": 20.082,
|
|
82
|
+
"k2_b4": 20.243,
|
|
83
|
+
"k3_b4": 20.403,
|
|
84
|
+
"k4_b4": 20.564,
|
|
85
|
+
"k5_b4": 20.725,
|
|
86
|
+
"k6_b4": 20.885,
|
|
87
|
+
"k1_b8": 20.809,
|
|
88
|
+
"k2_b8": 20.975,
|
|
89
|
+
"k3_b8": 21.142,
|
|
90
|
+
"k4_b8": 21.308,
|
|
91
|
+
"k5_b8": 21.475,
|
|
92
|
+
"k6_b8": 21.641,
|
|
93
|
+
"k1_b16": 23.273,
|
|
94
|
+
"k2_b16": 23.459,
|
|
95
|
+
"k3_b16": 23.645,
|
|
96
|
+
"k4_b16": 23.832,
|
|
97
|
+
"k5_b16": 24.018,
|
|
98
|
+
"k6_b16": 24.204,
|
|
99
|
+
"k1_b32": 30.043,
|
|
100
|
+
"k2_b32": 30.283,
|
|
101
|
+
"k3_b32": 30.524,
|
|
102
|
+
"k4_b32": 30.764,
|
|
103
|
+
"k5_b32": 31.004,
|
|
104
|
+
"k6_b32": 31.245
|
|
105
|
+
},
|
|
106
|
+
"cost_matrix_ms": {
|
|
107
|
+
"k0_b1": 19.095,
|
|
108
|
+
"k1_b1": 23.98,
|
|
109
|
+
"k2_b1": 29.018,
|
|
110
|
+
"k3_b1": 34.056,
|
|
111
|
+
"k4_b1": 39.093,
|
|
112
|
+
"k5_b1": 44.131,
|
|
113
|
+
"k6_b1": 49.169,
|
|
114
|
+
"k0_b2": 19.606,
|
|
115
|
+
"k1_b2": 24.836,
|
|
116
|
+
"k2_b2": 30.223,
|
|
117
|
+
"k3_b2": 35.61,
|
|
118
|
+
"k4_b2": 40.997,
|
|
119
|
+
"k5_b2": 46.383,
|
|
120
|
+
"k6_b2": 51.77,
|
|
121
|
+
"k0_b4": 20.082,
|
|
122
|
+
"k1_b4": 25.348,
|
|
123
|
+
"k2_b4": 30.775,
|
|
124
|
+
"k3_b4": 36.201,
|
|
125
|
+
"k4_b4": 41.628,
|
|
126
|
+
"k5_b4": 47.055,
|
|
127
|
+
"k6_b4": 52.481,
|
|
128
|
+
"k0_b8": 20.809,
|
|
129
|
+
"k1_b8": 26.224,
|
|
130
|
+
"k2_b8": 31.805,
|
|
131
|
+
"k3_b8": 37.387,
|
|
132
|
+
"k4_b8": 42.968,
|
|
133
|
+
"k5_b8": 48.55,
|
|
134
|
+
"k6_b8": 54.131,
|
|
135
|
+
"k0_b16": 23.273,
|
|
136
|
+
"k1_b16": 29.543,
|
|
137
|
+
"k2_b16": 35.999,
|
|
138
|
+
"k3_b16": 42.455,
|
|
139
|
+
"k4_b16": 48.912,
|
|
140
|
+
"k5_b16": 55.368,
|
|
141
|
+
"k6_b16": 61.824,
|
|
142
|
+
"k0_b32": 30.043,
|
|
143
|
+
"k1_b32": 36.74,
|
|
144
|
+
"k2_b32": 43.677,
|
|
145
|
+
"k3_b32": 50.615,
|
|
146
|
+
"k4_b32": 57.552,
|
|
147
|
+
"k5_b32": 64.489,
|
|
148
|
+
"k6_b32": 71.427
|
|
149
|
+
},
|
|
150
|
+
"break_even_alpha": {
|
|
151
|
+
"k0_b1": 0.0,
|
|
152
|
+
"k1_b1": 0.2558,
|
|
153
|
+
"k2_b1": 0.3773,
|
|
154
|
+
"k3_b1": 0.4657,
|
|
155
|
+
"k4_b1": 0.5325,
|
|
156
|
+
"k5_b1": 0.5846,
|
|
157
|
+
"k6_b1": 0.6263,
|
|
158
|
+
"k0_b2": 0.0,
|
|
159
|
+
"k1_b2": 0.2668,
|
|
160
|
+
"k2_b2": 0.3897,
|
|
161
|
+
"k3_b2": 0.4782,
|
|
162
|
+
"k4_b2": 0.5447,
|
|
163
|
+
"k5_b2": 0.5963,
|
|
164
|
+
"k6_b2": 0.6375,
|
|
165
|
+
"k0_b4": 0.0,
|
|
166
|
+
"k1_b4": 0.2622,
|
|
167
|
+
"k2_b4": 0.3846,
|
|
168
|
+
"k3_b4": 0.473,
|
|
169
|
+
"k4_b4": 0.5397,
|
|
170
|
+
"k5_b4": 0.5915,
|
|
171
|
+
"k6_b4": 0.6329,
|
|
172
|
+
"k0_b8": 0.0,
|
|
173
|
+
"k1_b8": 0.2602,
|
|
174
|
+
"k2_b8": 0.3823,
|
|
175
|
+
"k3_b8": 0.4707,
|
|
176
|
+
"k4_b8": 0.5374,
|
|
177
|
+
"k5_b8": 0.5893,
|
|
178
|
+
"k6_b8": 0.6309,
|
|
179
|
+
"k0_b16": 0.0,
|
|
180
|
+
"k1_b16": 0.2694,
|
|
181
|
+
"k2_b16": 0.3926,
|
|
182
|
+
"k3_b16": 0.4812,
|
|
183
|
+
"k4_b16": 0.5476,
|
|
184
|
+
"k5_b16": 0.5991,
|
|
185
|
+
"k6_b16": 0.6401,
|
|
186
|
+
"k0_b32": 0.0,
|
|
187
|
+
"k1_b32": 0.2229,
|
|
188
|
+
"k2_b32": 0.3389,
|
|
189
|
+
"k3_b32": 0.426,
|
|
190
|
+
"k4_b32": 0.4932,
|
|
191
|
+
"k5_b32": 0.5466,
|
|
192
|
+
"k6_b32": 0.5898
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""FlexSDS server launcher package."""
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Google Colab serving runner and Cloudflare tunnel integration for FlexSDS."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import urllib.error
|
|
5
|
+
import urllib.request
|
|
6
|
+
from typing import Any, Dict
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ColabServingRunner:
|
|
10
|
+
"""Quản lý thông tin khởi chạy FlexSDS trên Google Colab qua Cloudflare Tunnel."""
|
|
11
|
+
|
|
12
|
+
COLAB_NOTEBOOK_URL = (
|
|
13
|
+
"https://colab.research.google.com/github/theflyingkhui04/do-an-ptit-d22/blob/develop/notebooks/flexsds_colab_server.ipynb"
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
def get_colab_instructions(self, target_gpu: str = "T4") -> Dict[str, Any]:
|
|
17
|
+
"""Trả về đường dẫn notebook và hướng dẫn kết nối máy chủ Google Colab."""
|
|
18
|
+
gpu_upper = target_gpu.upper()
|
|
19
|
+
if "T4" in gpu_upper:
|
|
20
|
+
profile = "profile_sm75_t4.json"
|
|
21
|
+
runtime_desc = "T4 GPU (Miễn phí trên Google Colab)"
|
|
22
|
+
elif "L4" in gpu_upper:
|
|
23
|
+
profile = "profile_sm89_l4.json"
|
|
24
|
+
runtime_desc = "L4 GPU (Colab Pro)"
|
|
25
|
+
elif "A100" in gpu_upper:
|
|
26
|
+
profile = "profile_sm80_a100.json"
|
|
27
|
+
runtime_desc = "A100 GPU (Colab Pro+)"
|
|
28
|
+
else:
|
|
29
|
+
profile = "profile_sm75_t4.json"
|
|
30
|
+
runtime_desc = f"{target_gpu} GPU"
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
"notebook_url": self.COLAB_NOTEBOOK_URL,
|
|
34
|
+
"recommended_runtime": runtime_desc,
|
|
35
|
+
"cost_profile": profile,
|
|
36
|
+
"tunnel_method": "cloudflared",
|
|
37
|
+
"steps": [
|
|
38
|
+
"1. Nhấp vào liên kết Google Colab để mở notebook FlexSDS Server.",
|
|
39
|
+
"2. Chọn Runtime -> Change runtime type -> Chọn T4 hoặc GPU mong muốn.",
|
|
40
|
+
"3. Nhấp 'Run all' để tiến hành cài đặt môi trường và khởi chạy vLLM + Cloudflare tunnel.",
|
|
41
|
+
"4. Sao chép đường dẫn công khai kết thúc bằng .trycloudflare.com và dán vào CLI này.",
|
|
42
|
+
],
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
def _probe_endpoint(self, url: str, timeout: int = 5) -> bool:
|
|
46
|
+
"""Gửi yêu cầu thăm dò tình trạng hoạt động (health check) của endpoint."""
|
|
47
|
+
health_url = f"{url.rstrip('/')}/health"
|
|
48
|
+
req = urllib.request.Request(
|
|
49
|
+
health_url,
|
|
50
|
+
headers={"User-Agent": "FlexSDS-TunnelVerifier/0.1.0"},
|
|
51
|
+
)
|
|
52
|
+
try:
|
|
53
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
54
|
+
return resp.status == 200
|
|
55
|
+
except Exception:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
def verify_tunnel_health(self, public_url: str, timeout: int = 5) -> bool:
|
|
59
|
+
"""Kiểm tra xem đường hầm Cloudflare từ Google Colab đã sẵn sàng hay chưa."""
|
|
60
|
+
if not public_url or not public_url.startswith("http"):
|
|
61
|
+
return False
|
|
62
|
+
return self._probe_endpoint(public_url, timeout=timeout)
|