flexsds 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. flexsds/__init__.py +4 -0
  2. flexsds/assets/ptit_logo.png +0 -0
  3. flexsds/cli/__init__.py +1 -0
  4. flexsds/cli/banner.py +84 -0
  5. flexsds/cli/main.py +330 -0
  6. flexsds/cli/output.py +69 -0
  7. flexsds/cli/task_runner.py +171 -0
  8. flexsds/cli/wizard.py +360 -0
  9. flexsds/client/__init__.py +4 -0
  10. flexsds/client/chat.py +137 -0
  11. flexsds/client/client.py +107 -0
  12. flexsds/client/monitor.py +95 -0
  13. flexsds/core/__init__.py +44 -0
  14. flexsds/core/compatibility.py +442 -0
  15. flexsds/core/controller.py +502 -0
  16. flexsds/core/cost_model.py +530 -0
  17. flexsds/core/credentials.py +111 -0
  18. flexsds/core/hardware.py +77 -0
  19. flexsds/core/tree_topology.py +296 -0
  20. flexsds/profiles/profile_sm75_t4.json +62 -0
  21. flexsds/profiles/profile_sm89_l4.json +194 -0
  22. flexsds/server/__init__.py +1 -0
  23. flexsds/server/colab_runner.py +62 -0
  24. flexsds/server/local_runner.py +208 -0
  25. flexsds/server/modal_app.py +50 -0
  26. flexsds/server/modal_runner.py +98 -0
  27. flexsds/server/vscode_cli.py +82 -0
  28. flexsds-0.2.0.dist-info/METADATA +204 -0
  29. flexsds-0.2.0.dist-info/RECORD +62 -0
  30. flexsds-0.2.0.dist-info/WHEEL +5 -0
  31. flexsds-0.2.0.dist-info/entry_points.txt +2 -0
  32. flexsds-0.2.0.dist-info/top_level.txt +4 -0
  33. pipeline/entity_eval.py +171 -0
  34. pipeline/examples/node1_extractor_input_m0_04_001.txt +9 -0
  35. pipeline/examples/node1_extractor_output_m0_04_001.json +90 -0
  36. pipeline/extractor.py +502 -0
  37. pipeline/invoice_models.py +163 -0
  38. pipeline/node2_questions.py +30 -0
  39. pipeline/openai_compatible.py +87 -0
  40. pipeline/prompts/README.md +18 -0
  41. pipeline/prompts/node1_extractor_system.md +92 -0
  42. pipeline/prompts/node3_report_user.md +5 -0
  43. pipeline/report_writer.py +143 -0
  44. pipeline/schemas/invoice_extraction.schema.json +469 -0
  45. pipeline/sql_guard.py +140 -0
  46. pipeline/sqlite/schema.sql +219 -0
  47. pipeline/text_to_sql.py +170 -0
  48. pipeline/validation.py +166 -0
  49. pipeline/w1_workload.py +139 -0
  50. proposer/__init__.py +31 -0
  51. proposer/adapter.py +153 -0
  52. proposer/controller.py +460 -0
  53. proposer/cost_model.py +216 -0
  54. proposer/tree_adaptive.py +466 -0
  55. server/__init__.py +3 -0
  56. server/adaptive_scheduler.py +215 -0
  57. server/budget_planner.py +287 -0
  58. server/lineage.py +290 -0
  59. server/lineage_diagnostics.py +38 -0
  60. server/lineage_executor.py +129 -0
  61. server/lineage_startup.py +47 -0
  62. server/t4_rowwise_attention.py +262 -0
@@ -0,0 +1,77 @@
1
+ """Hardware detection module for FlexSDS."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ from dataclasses import dataclass
6
+
7
+
8
+ @dataclass
9
+ class HardwareInfo:
10
+ """Hardware specification details for adaptive serving."""
11
+ device_name: str
12
+ sm_architecture: str
13
+ total_vram_mb: float
14
+ free_vram_mb: float
15
+ cuda_version: str
16
+ is_cuda_available: bool
17
+ device_count: int = 0
18
+
19
+ @property
20
+ def free_vram_gb(self) -> float:
21
+ return round(self.free_vram_mb / 1024.0, 2)
22
+
23
+ @property
24
+ def total_vram_gb(self) -> float:
25
+ return round(self.total_vram_mb / 1024.0, 2)
26
+
27
+ def summary(self) -> str:
28
+ if not self.is_cuda_available:
29
+ return f"CPU Only · RAM Free: {self.free_vram_gb} GB"
30
+ return f"{self.device_name} ({self.sm_architecture}) · VRAM: {self.free_vram_gb}/{self.total_vram_gb} GB Free"
31
+
32
+
33
+ def detect_hardware(device_index: int = 0) -> HardwareInfo:
34
+ """Detects active GPU hardware details without initializing heavy CUDA context."""
35
+ try:
36
+ import torch
37
+
38
+ if torch.cuda.is_available() and torch.cuda.device_count() > 0:
39
+ dev_idx = min(device_index, torch.cuda.device_count() - 1)
40
+ name = torch.cuda.get_device_name(dev_idx)
41
+ major, minor = torch.cuda.get_device_capability(dev_idx)
42
+ sm_arch = f"sm_{major}{minor}"
43
+
44
+ free_bytes, total_bytes = torch.cuda.mem_get_info(dev_idx)
45
+ free_mb = free_bytes / (1024.0 * 1024.0)
46
+ total_mb = total_bytes / (1024.0 * 1024.0)
47
+ cuda_ver = torch.version.cuda or "N/A"
48
+
49
+ return HardwareInfo(
50
+ device_name=name,
51
+ sm_architecture=sm_arch,
52
+ total_vram_mb=round(total_mb, 1),
53
+ free_vram_mb=round(free_mb, 1),
54
+ cuda_version=cuda_ver,
55
+ is_cuda_available=True,
56
+ device_count=torch.cuda.device_count(),
57
+ )
58
+ except Exception:
59
+ pass
60
+
61
+ # CPU Fallback
62
+ ram_mb = 16000.0
63
+ try:
64
+ import psutil
65
+ ram_mb = psutil.virtual_memory().available / (1024.0 * 1024.0)
66
+ except Exception:
67
+ pass
68
+
69
+ return HardwareInfo(
70
+ device_name="CPU",
71
+ sm_architecture="cpu",
72
+ total_vram_mb=round(ram_mb, 1),
73
+ free_vram_mb=round(ram_mb, 1),
74
+ cuda_version="N/A",
75
+ is_cuda_available=False,
76
+ device_count=0,
77
+ )
@@ -0,0 +1,296 @@
1
+ """Tree topology representations, causal attention masks, and expected yield models for FlexSDS.
2
+
3
+ Implements discrete canonical tree configurations (T0, T4, T8, T16, T32) based on
4
+ OPT-Tree (Wang et al., 2025, TACL) and speculative decoding theory.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ from dataclasses import dataclass, field
9
+ from typing import Any, Dict, List, Optional, Tuple, Union
10
+ import math
11
+
12
+
13
+ @dataclass
14
+ class TreeTopology:
15
+ """Represents a speculative draft tree structure.
16
+
17
+ Attributes:
18
+ name: Short topology identifier (e.g. 'T0', 'T4', 'T8', 'T16', 'T32').
19
+ num_tokens: Number of non-root draft tokens N.
20
+ depth: Maximum depth of the tree d (root has depth 0).
21
+ parent_indices: List of length N+1 where parent_indices[0] == -1 and
22
+ parent_indices[i] is the index of the parent node for token i (0 <= parent < i).
23
+ layer_distribution: Number of nodes at each depth level [l_0, l_1, ..., l_d].
24
+ node_depths: Computed depth for each node in 0..N.
25
+ position_ids: Relative position index in tree (matches node_depths).
26
+ tree_mask: 2D causal attention mask of shape (N+1, N+1) where
27
+ mask[i][j] is True iff j is an ancestor of i (or j == i).
28
+ nodes_at_depth: Mapping from depth level to list of node indices at that depth.
29
+ """
30
+
31
+ name: str
32
+ num_tokens: int
33
+ depth: int
34
+ parent_indices: List[int]
35
+ layer_distribution: List[int]
36
+ node_depths: List[int] = field(default_factory=list)
37
+ position_ids: List[int] = field(default_factory=list)
38
+ tree_mask: List[List[bool]] = field(default_factory=list)
39
+ nodes_at_depth: Dict[int, List[int]] = field(default_factory=dict)
40
+ _lookup_table: List[float] = field(default_factory=list, repr=False)
41
+
42
+ def __post_init__(self) -> None:
43
+ """Validates invariants and computes derived properties if not already set."""
44
+ self._validate_basic_invariants()
45
+ if not self.node_depths:
46
+ self.node_depths = self._compute_depths()
47
+ if not self.position_ids:
48
+ self.position_ids = list(self.node_depths)
49
+ if not self.nodes_at_depth:
50
+ self.nodes_at_depth = self._build_nodes_at_depth()
51
+ if not self.tree_mask:
52
+ self.tree_mask = self._compute_causal_mask()
53
+ self._validate_derived_invariants()
54
+ if not self._lookup_table:
55
+ self._lookup_table = self._build_lookup_table()
56
+
57
+ def _validate_basic_invariants(self) -> None:
58
+ """Validates fundamental structural invariants."""
59
+ expected_len = self.num_tokens + 1
60
+ actual_len = len(self.parent_indices)
61
+ if actual_len != expected_len:
62
+ raise ValueError(
63
+ f"Parent array length mismatch for {self.name}: "
64
+ f"expected {expected_len} (N+1), got {actual_len}"
65
+ )
66
+ if self.parent_indices[0] != -1:
67
+ raise ValueError(
68
+ f"Root parent index must be -1, got {self.parent_indices[0]}"
69
+ )
70
+ for i in range(1, actual_len):
71
+ p = self.parent_indices[i]
72
+ if p < 0 or p >= i:
73
+ raise ValueError(
74
+ f"Acyclic/topological ordering violation at node {i}: parent={p} (must be in 0..{i-1})"
75
+ )
76
+
77
+ def _compute_depths(self) -> List[int]:
78
+ """Computes depth for each node from root 0."""
79
+ n = len(self.parent_indices)
80
+ depths = [0] * n
81
+ for i in range(1, n):
82
+ p = self.parent_indices[i]
83
+ depths[i] = depths[p] + 1
84
+ return depths
85
+
86
+ def _build_nodes_at_depth(self) -> Dict[int, List[int]]:
87
+ """Groups node indices by depth."""
88
+ mapping: Dict[int, List[int]] = {}
89
+ for idx, d in enumerate(self.node_depths):
90
+ mapping.setdefault(d, []).append(idx)
91
+ return mapping
92
+
93
+ def _compute_causal_mask(self) -> List[List[bool]]:
94
+ """Constructs 2D causal attention mask where node i can attend to j iff j is an ancestor of i."""
95
+ n = len(self.parent_indices)
96
+ mask = [[False] * n for _ in range(n)]
97
+ for i in range(n):
98
+ curr = i
99
+ while curr != -1:
100
+ mask[i][curr] = True
101
+ curr = self.parent_indices[curr]
102
+ return mask
103
+
104
+ def _validate_derived_invariants(self) -> None:
105
+ """Validates depth, layer distribution, and mask properties."""
106
+ actual_max_depth = max(self.node_depths) if self.node_depths else 0
107
+ if actual_max_depth != self.depth:
108
+ raise ValueError(
109
+ f"Maximum depth mismatch for {self.name}: "
110
+ f"computed max depth {actual_max_depth} != declared depth {self.depth}"
111
+ )
112
+ actual_dist = [len(self.nodes_at_depth.get(d, [])) for d in range(self.depth + 1)]
113
+ if actual_dist != self.layer_distribution:
114
+ raise ValueError(
115
+ f"Layer distribution mismatch for {self.name}: "
116
+ f"actual {actual_dist} != declared {self.layer_distribution}"
117
+ )
118
+
119
+ def _build_lookup_table(self, num_points: int = 1001) -> List[float]:
120
+ """Precomputes lookup table of E[tau(alpha)] for alpha in [0.0, 1.0] in 1001 steps."""
121
+ table = []
122
+ for i in range(num_points):
123
+ alpha = i / (num_points - 1)
124
+ table.append(self.expected_acceptance(alpha))
125
+ return table
126
+
127
+ def expected_acceptance(self, alpha: float, method: str = "branch_aware") -> float:
128
+ """Computes theoretical expected accepted token length E[tau(T, alpha)].
129
+
130
+ Follows OPT-Tree expectation formula:
131
+ E[tau(T)] = 1 + sum_{v in T \\ {r}} q(v)
132
+
133
+ Under greedy decoding against target model:
134
+ At any depth level l in 1..d, at most ONE token can be accepted because the target
135
+ model generates a unique greedy token at each decoding position.
136
+
137
+ Methods:
138
+ - 'homogeneous': Linear expectation along the longest chain:
139
+ E[tau] = 1 + sum_{l=1}^d alpha^l
140
+ - 'branch_aware' (default): Multi-branch coverage at each layer:
141
+ With n_l candidate branches at depth l, the step acceptance probability is:
142
+ P_step(l, alpha) = 1 - (1 - alpha)^{n_l}
143
+ Cumulative path acceptance:
144
+ P_path(l, alpha) = prod_{k=1}^l P_step(k, alpha)
145
+ E[tau] = 1 + sum_{l=1}^d P_path(l, alpha)
146
+
147
+ Properties guaranteed:
148
+ - Monotonically non-decreasing in alpha in [0.0, 1.0].
149
+ - E[tau(alpha=0.0)] == 1.0 (only the root/verified token is emitted).
150
+ - E[tau(alpha=1.0)] == 1.0 + depth (longest path bound).
151
+ - Strict upper bound: E[tau] <= 1 + depth.
152
+ """
153
+ if self.depth == 0 or self.num_tokens == 0:
154
+ return 1.0
155
+
156
+ clamped_alpha = max(0.0, min(1.0, float(alpha)))
157
+ if clamped_alpha <= 0.0:
158
+ return 1.0
159
+ if clamped_alpha >= 1.0:
160
+ return 1.0 + float(self.depth)
161
+
162
+ if method == "homogeneous":
163
+ # Geometric progression along max depth d
164
+ # E[tau] = 1 + alpha + alpha^2 + ... + alpha^d
165
+ val = 1.0 + sum(clamped_alpha ** l for l in range(1, self.depth + 1))
166
+ return min(1.0 + float(self.depth), val)
167
+
168
+ # Default: branch-aware multi-candidate coverage
169
+ expected = 1.0
170
+ cum_prob = 1.0
171
+ for l in range(1, self.depth + 1):
172
+ n_l = self.layer_distribution[l] if l < len(self.layer_distribution) else 1
173
+ # Step acceptance probability at depth l given parent accepted
174
+ step_prob = 1.0 - math.pow(1.0 - clamped_alpha, n_l)
175
+ cum_prob *= step_prob
176
+ expected += cum_prob
177
+
178
+ # Strict bound enforcement
179
+ return max(1.0, min(1.0 + float(self.depth), expected))
180
+
181
+ def lookup_expected_acceptance(self, alpha: float) -> float:
182
+ """Fast O(1) lookup of precomputed E[tau(alpha)] (< 0.005 ms)."""
183
+ clamped_alpha = max(0.0, min(1.0, float(alpha)))
184
+ idx = int(round(clamped_alpha * (len(self._lookup_table) - 1)))
185
+ return self._lookup_table[idx]
186
+
187
+ def to_dict(self) -> Dict[str, Any]:
188
+ """Serializes topology to dictionary."""
189
+ return {
190
+ "name": self.name,
191
+ "num_tokens": self.num_tokens,
192
+ "depth": self.depth,
193
+ "parent_indices": self.parent_indices,
194
+ "layer_distribution": self.layer_distribution,
195
+ "node_depths": self.node_depths,
196
+ "position_ids": self.position_ids,
197
+ "tree_mask": self.tree_mask,
198
+ }
199
+
200
+ @classmethod
201
+ def from_dict(cls, data: Dict[str, Any]) -> TreeTopology:
202
+ """Deserializes topology from dictionary."""
203
+ return cls(
204
+ name=data["name"],
205
+ num_tokens=data["num_tokens"],
206
+ depth=data["depth"],
207
+ parent_indices=data["parent_indices"],
208
+ layer_distribution=data["layer_distribution"],
209
+ node_depths=data.get("node_depths", []),
210
+ position_ids=data.get("position_ids", []),
211
+ tree_mask=data.get("tree_mask", []),
212
+ )
213
+
214
+
215
+ # ==============================================================================
216
+ # CANONICAL CATALOG DEFINITIONS (T0, T4, T8, T16, T32)
217
+ # ==============================================================================
218
+
219
+ CANONICAL_TOPOLOGIES: Dict[str, TreeTopology] = {
220
+ # T0: Baseline autoregressive decoding (no speculative draft tokens)
221
+ "T0": TreeTopology(
222
+ name="T0",
223
+ num_tokens=0,
224
+ depth=0,
225
+ layer_distribution=[1],
226
+ parent_indices=[-1],
227
+ ),
228
+ # T4: Linear chain (N=4, depth=4)
229
+ "T4": TreeTopology(
230
+ name="T4",
231
+ num_tokens=4,
232
+ depth=4,
233
+ layer_distribution=[1, 1, 1, 1, 1],
234
+ parent_indices=[-1, 0, 1, 2, 3],
235
+ ),
236
+ # T8: Narrow branching tree (N=8, depth=3)
237
+ "T8": TreeTopology(
238
+ name="T8",
239
+ num_tokens=8,
240
+ depth=3,
241
+ layer_distribution=[1, 2, 4, 2],
242
+ parent_indices=[-1, 0, 0, 1, 1, 2, 2, 3, 5],
243
+ ),
244
+ # T16: Moderate branching tree (N=16, depth=4)
245
+ # Fixed invariant: node 16's parent is node 13 (depth 3) to strictly enforce max depth 4
246
+ "T16": TreeTopology(
247
+ name="T16",
248
+ num_tokens=16,
249
+ depth=4,
250
+ layer_distribution=[1, 3, 6, 4, 3],
251
+ parent_indices=[
252
+ -1,
253
+ 0, 0, 0,
254
+ 1, 1, 2, 2, 3, 3,
255
+ 4, 4, 6, 8,
256
+ 10, 12, 13,
257
+ ],
258
+ ),
259
+ # T32: Wide branching tree (N=32, depth=5)
260
+ # Deterministic 33-element parent array strictly adhering to [1, 4, 8, 8, 6, 6]
261
+ "T32": TreeTopology(
262
+ name="T32",
263
+ num_tokens=32,
264
+ depth=5,
265
+ layer_distribution=[1, 4, 8, 8, 6, 6],
266
+ parent_indices=[
267
+ -1,
268
+ # Depth 1 (4 nodes: 1..4, parent 0)
269
+ 0, 0, 0, 0,
270
+ # Depth 2 (8 nodes: 5..12, parents 1, 1, 2, 2, 3, 3, 4, 4)
271
+ 1, 1, 2, 2, 3, 3, 4, 4,
272
+ # Depth 3 (8 nodes: 13..20, parents 5..12)
273
+ 5, 6, 7, 8, 9, 10, 11, 12,
274
+ # Depth 4 (6 nodes: 21..26, parents 13..18)
275
+ 13, 14, 15, 16, 17, 18,
276
+ # Depth 5 (6 nodes: 27..32, parents 21..26)
277
+ 21, 22, 23, 24, 25, 26,
278
+ ],
279
+ ),
280
+ }
281
+
282
+
283
+ def get_tree_topology(name: str) -> TreeTopology:
284
+ """Retrieves a canonical tree topology by name."""
285
+ clean_name = name.strip().upper()
286
+ if clean_name not in CANONICAL_TOPOLOGIES:
287
+ raise KeyError(
288
+ f"Unknown topology '{name}'. Available canonical topologies: "
289
+ f"{list(CANONICAL_TOPOLOGIES.keys())}"
290
+ )
291
+ return CANONICAL_TOPOLOGIES[clean_name]
292
+
293
+
294
+ def list_available_topologies() -> List[str]:
295
+ """Returns list of canonical topology identifiers."""
296
+ return list(CANONICAL_TOPOLOGIES.keys())
@@ -0,0 +1,62 @@
1
+ {
2
+ "metadata": {
3
+ "device": "Tesla T4",
4
+ "sm_architecture": "sm_75",
5
+ "vram_total_mb": 15360.0,
6
+ "vllm_version": "0.30.0",
7
+ "source": "t4-m1-m2-measurements",
8
+ "attn_verify_factor": 0.012
9
+ },
10
+ "models": {
11
+ "target": "Qwen/Qwen2.5-7B-Instruct-AWQ",
12
+ "draft": "Qwen/Qwen2.5-0.5B-Instruct"
13
+ },
14
+ "grid": {
15
+ "batches": [1, 2, 4, 8, 16, 32],
16
+ "candidate_budgets": [0, 1, 2, 3, 4]
17
+ },
18
+ "forward_costs_median_ms": {
19
+ "target_c_t": {
20
+ "1": 36.5,
21
+ "2": 37.8,
22
+ "4": 39.2,
23
+ "8": 42.1,
24
+ "16": 48.5,
25
+ "32": 62.0
26
+ },
27
+ "draft_c_d": {
28
+ "1": 9.2,
29
+ "2": 9.8,
30
+ "4": 10.4,
31
+ "8": 11.2,
32
+ "16": 12.8,
33
+ "32": 15.5
34
+ },
35
+ "c_d_over_c_t_ratio": {
36
+ "1": 0.252,
37
+ "2": 0.259,
38
+ "4": 0.265,
39
+ "8": 0.266,
40
+ "16": 0.264,
41
+ "32": 0.250
42
+ }
43
+ },
44
+ "verification_costs_ms": {
45
+ "k1_b1": 36.5,
46
+ "k2_b1": 36.9,
47
+ "k3_b1": 37.4,
48
+ "k4_b1": 37.8
49
+ },
50
+ "break_even_alpha": {
51
+ "k0_b1": 0.0,
52
+ "k1_b1": 0.252,
53
+ "k2_b1": 0.380,
54
+ "k3_b1": 0.470,
55
+ "k4_b1": 0.540,
56
+ "k0_b32": 0.0,
57
+ "k1_b32": 0.250,
58
+ "k2_b32": 0.385,
59
+ "k3_b32": 0.485,
60
+ "k4_b32": 0.560
61
+ }
62
+ }
@@ -0,0 +1,194 @@
1
+ {
2
+ "metadata": {
3
+ "device": "NVIDIA L4",
4
+ "sm_architecture": "sm_89",
5
+ "vram_total_mb": 23034.0,
6
+ "vllm_version": "0.30.0",
7
+ "source_run_id": "20261025-m302-l4-v2",
8
+ "source_commit": "98c92cf3dd9043d803953be4e5bf385bf8667601",
9
+ "timestamp_utc": "2026-10-25T05:00:00.292569+00:00",
10
+ "attn_verify_factor": 0.008
11
+ },
12
+ "models": {
13
+ "target": "Qwen/Qwen2.5-7B-Instruct-AWQ",
14
+ "draft": "Qwen/Qwen2.5-0.5B-Instruct"
15
+ },
16
+ "grid": {
17
+ "batches": [
18
+ 1,
19
+ 2,
20
+ 4,
21
+ 8,
22
+ 16,
23
+ 32
24
+ ],
25
+ "contexts": [
26
+ 256,
27
+ 512,
28
+ 1024,
29
+ 2048,
30
+ 3072
31
+ ],
32
+ "candidate_budgets": [
33
+ 0,
34
+ 1,
35
+ 2,
36
+ 3,
37
+ 4,
38
+ 5,
39
+ 6
40
+ ]
41
+ },
42
+ "forward_costs_median_ms": {
43
+ "target_c_t": {
44
+ "1": 19.095,
45
+ "2": 19.606,
46
+ "4": 20.082,
47
+ "8": 20.809,
48
+ "16": 23.273,
49
+ "32": 30.043
50
+ },
51
+ "draft_c_d": {
52
+ "1": 4.885,
53
+ "2": 5.23,
54
+ "4": 5.266,
55
+ "8": 5.415,
56
+ "16": 6.27,
57
+ "32": 6.697
58
+ },
59
+ "c_d_over_c_t_ratio": {
60
+ "1": 0.2558,
61
+ "2": 0.2668,
62
+ "4": 0.2622,
63
+ "8": 0.2602,
64
+ "16": 0.2694,
65
+ "32": 0.2229
66
+ }
67
+ },
68
+ "verification_costs_ms": {
69
+ "k1_b1": 19.095,
70
+ "k2_b1": 19.248,
71
+ "k3_b1": 19.401,
72
+ "k4_b1": 19.553,
73
+ "k5_b1": 19.706,
74
+ "k6_b1": 19.859,
75
+ "k1_b2": 19.606,
76
+ "k2_b2": 19.763,
77
+ "k3_b2": 19.92,
78
+ "k4_b2": 20.077,
79
+ "k5_b2": 20.233,
80
+ "k6_b2": 20.39,
81
+ "k1_b4": 20.082,
82
+ "k2_b4": 20.243,
83
+ "k3_b4": 20.403,
84
+ "k4_b4": 20.564,
85
+ "k5_b4": 20.725,
86
+ "k6_b4": 20.885,
87
+ "k1_b8": 20.809,
88
+ "k2_b8": 20.975,
89
+ "k3_b8": 21.142,
90
+ "k4_b8": 21.308,
91
+ "k5_b8": 21.475,
92
+ "k6_b8": 21.641,
93
+ "k1_b16": 23.273,
94
+ "k2_b16": 23.459,
95
+ "k3_b16": 23.645,
96
+ "k4_b16": 23.832,
97
+ "k5_b16": 24.018,
98
+ "k6_b16": 24.204,
99
+ "k1_b32": 30.043,
100
+ "k2_b32": 30.283,
101
+ "k3_b32": 30.524,
102
+ "k4_b32": 30.764,
103
+ "k5_b32": 31.004,
104
+ "k6_b32": 31.245
105
+ },
106
+ "cost_matrix_ms": {
107
+ "k0_b1": 19.095,
108
+ "k1_b1": 23.98,
109
+ "k2_b1": 29.018,
110
+ "k3_b1": 34.056,
111
+ "k4_b1": 39.093,
112
+ "k5_b1": 44.131,
113
+ "k6_b1": 49.169,
114
+ "k0_b2": 19.606,
115
+ "k1_b2": 24.836,
116
+ "k2_b2": 30.223,
117
+ "k3_b2": 35.61,
118
+ "k4_b2": 40.997,
119
+ "k5_b2": 46.383,
120
+ "k6_b2": 51.77,
121
+ "k0_b4": 20.082,
122
+ "k1_b4": 25.348,
123
+ "k2_b4": 30.775,
124
+ "k3_b4": 36.201,
125
+ "k4_b4": 41.628,
126
+ "k5_b4": 47.055,
127
+ "k6_b4": 52.481,
128
+ "k0_b8": 20.809,
129
+ "k1_b8": 26.224,
130
+ "k2_b8": 31.805,
131
+ "k3_b8": 37.387,
132
+ "k4_b8": 42.968,
133
+ "k5_b8": 48.55,
134
+ "k6_b8": 54.131,
135
+ "k0_b16": 23.273,
136
+ "k1_b16": 29.543,
137
+ "k2_b16": 35.999,
138
+ "k3_b16": 42.455,
139
+ "k4_b16": 48.912,
140
+ "k5_b16": 55.368,
141
+ "k6_b16": 61.824,
142
+ "k0_b32": 30.043,
143
+ "k1_b32": 36.74,
144
+ "k2_b32": 43.677,
145
+ "k3_b32": 50.615,
146
+ "k4_b32": 57.552,
147
+ "k5_b32": 64.489,
148
+ "k6_b32": 71.427
149
+ },
150
+ "break_even_alpha": {
151
+ "k0_b1": 0.0,
152
+ "k1_b1": 0.2558,
153
+ "k2_b1": 0.3773,
154
+ "k3_b1": 0.4657,
155
+ "k4_b1": 0.5325,
156
+ "k5_b1": 0.5846,
157
+ "k6_b1": 0.6263,
158
+ "k0_b2": 0.0,
159
+ "k1_b2": 0.2668,
160
+ "k2_b2": 0.3897,
161
+ "k3_b2": 0.4782,
162
+ "k4_b2": 0.5447,
163
+ "k5_b2": 0.5963,
164
+ "k6_b2": 0.6375,
165
+ "k0_b4": 0.0,
166
+ "k1_b4": 0.2622,
167
+ "k2_b4": 0.3846,
168
+ "k3_b4": 0.473,
169
+ "k4_b4": 0.5397,
170
+ "k5_b4": 0.5915,
171
+ "k6_b4": 0.6329,
172
+ "k0_b8": 0.0,
173
+ "k1_b8": 0.2602,
174
+ "k2_b8": 0.3823,
175
+ "k3_b8": 0.4707,
176
+ "k4_b8": 0.5374,
177
+ "k5_b8": 0.5893,
178
+ "k6_b8": 0.6309,
179
+ "k0_b16": 0.0,
180
+ "k1_b16": 0.2694,
181
+ "k2_b16": 0.3926,
182
+ "k3_b16": 0.4812,
183
+ "k4_b16": 0.5476,
184
+ "k5_b16": 0.5991,
185
+ "k6_b16": 0.6401,
186
+ "k0_b32": 0.0,
187
+ "k1_b32": 0.2229,
188
+ "k2_b32": 0.3389,
189
+ "k3_b32": 0.426,
190
+ "k4_b32": 0.4932,
191
+ "k5_b32": 0.5466,
192
+ "k6_b32": 0.5898
193
+ }
194
+ }
@@ -0,0 +1 @@
1
+ """FlexSDS server launcher package."""
@@ -0,0 +1,62 @@
1
+ """Google Colab serving runner and Cloudflare tunnel integration for FlexSDS."""
2
+ from __future__ import annotations
3
+
4
+ import urllib.error
5
+ import urllib.request
6
+ from typing import Any, Dict
7
+
8
+
9
+ class ColabServingRunner:
10
+ """Quản lý thông tin khởi chạy FlexSDS trên Google Colab qua Cloudflare Tunnel."""
11
+
12
+ COLAB_NOTEBOOK_URL = (
13
+ "https://colab.research.google.com/github/theflyingkhui04/do-an-ptit-d22/blob/develop/notebooks/flexsds_colab_server.ipynb"
14
+ )
15
+
16
+ def get_colab_instructions(self, target_gpu: str = "T4") -> Dict[str, Any]:
17
+ """Trả về đường dẫn notebook và hướng dẫn kết nối máy chủ Google Colab."""
18
+ gpu_upper = target_gpu.upper()
19
+ if "T4" in gpu_upper:
20
+ profile = "profile_sm75_t4.json"
21
+ runtime_desc = "T4 GPU (Miễn phí trên Google Colab)"
22
+ elif "L4" in gpu_upper:
23
+ profile = "profile_sm89_l4.json"
24
+ runtime_desc = "L4 GPU (Colab Pro)"
25
+ elif "A100" in gpu_upper:
26
+ profile = "profile_sm80_a100.json"
27
+ runtime_desc = "A100 GPU (Colab Pro+)"
28
+ else:
29
+ profile = "profile_sm75_t4.json"
30
+ runtime_desc = f"{target_gpu} GPU"
31
+
32
+ return {
33
+ "notebook_url": self.COLAB_NOTEBOOK_URL,
34
+ "recommended_runtime": runtime_desc,
35
+ "cost_profile": profile,
36
+ "tunnel_method": "cloudflared",
37
+ "steps": [
38
+ "1. Nhấp vào liên kết Google Colab để mở notebook FlexSDS Server.",
39
+ "2. Chọn Runtime -> Change runtime type -> Chọn T4 hoặc GPU mong muốn.",
40
+ "3. Nhấp 'Run all' để tiến hành cài đặt môi trường và khởi chạy vLLM + Cloudflare tunnel.",
41
+ "4. Sao chép đường dẫn công khai kết thúc bằng .trycloudflare.com và dán vào CLI này.",
42
+ ],
43
+ }
44
+
45
+ def _probe_endpoint(self, url: str, timeout: int = 5) -> bool:
46
+ """Gửi yêu cầu thăm dò tình trạng hoạt động (health check) của endpoint."""
47
+ health_url = f"{url.rstrip('/')}/health"
48
+ req = urllib.request.Request(
49
+ health_url,
50
+ headers={"User-Agent": "FlexSDS-TunnelVerifier/0.1.0"},
51
+ )
52
+ try:
53
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
54
+ return resp.status == 200
55
+ except Exception:
56
+ return False
57
+
58
+ def verify_tunnel_health(self, public_url: str, timeout: int = 5) -> bool:
59
+ """Kiểm tra xem đường hầm Cloudflare từ Google Colab đã sẵn sàng hay chưa."""
60
+ if not public_url or not public_url.startswith("http"):
61
+ return False
62
+ return self._probe_endpoint(public_url, timeout=timeout)