blockyos-client 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
blocky.py ADDED
@@ -0,0 +1,258 @@
1
+ """blocky: a small Python client for a Blocky box.
2
+
3
+ import blocky
4
+ b = blocky.Client() # http://localhost:8000 by default
5
+ exp = b.experiment("quantization study") # by (part of the) name, or by id
6
+ df = exp.results() # one row per run, every metric as a column (pandas, if installed)
7
+ run = exp.branch(exp.run("Q4_K_M"), kv="q4_0") # same setup as that run + one change, queued
8
+ exp.wait() # until the queue is empty
9
+ exp.sweep({"kv": ["f16", "q8_0", "q4_0"], "ctx": [4096, 16384]})
10
+
11
+ b.models() # GGUF files on the box
12
+ b.variant("Qwen3.5-9B-Q8_0.gguf", "Q3_K_M", imatrix=True, overrides={"attn_v": "q8_0"}).wait()
13
+ b.scan("Qwen3.5-0.8B-Q8_0.gguf", mode="quantize", qtype="q2_k").wait()
14
+ b.build_engine("https://github.com/you/llama.cpp", "my-branch", name="my-kernel").wait()
15
+
16
+ Only the standard library is required; pandas is used for tables when it's installed. The full REST API is
17
+ documented at http://<box>:8000/docs.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import time
23
+ import urllib.error
24
+ import urllib.parse
25
+ import urllib.request
26
+
27
+ __all__ = ["Client", "Experiment", "Run", "BlockyError"]
28
+ __version__ = "0.1"
29
+
30
+
31
+ class BlockyError(RuntimeError):
32
+ pass
33
+
34
+
35
+ def _flat(d: dict, prefix: str = "") -> dict:
36
+ out = {}
37
+ for k, v in (d or {}).items():
38
+ key = f"{prefix}{k}"
39
+ if isinstance(v, dict):
40
+ out.update(_flat(v, key + "."))
41
+ elif not isinstance(v, list) and k != "answers":
42
+ out[key] = v
43
+ return out
44
+
45
+
46
+ class Client:
47
+ def __init__(self, url: str = "http://localhost:8000", timeout: float = 120):
48
+ self.url = url.rstrip("/")
49
+ self.timeout = timeout
50
+
51
+ # ---- HTTP ----
52
+ def _req(self, method: str, path: str, body=None, params: dict | None = None):
53
+ q = f"?{urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})}" if params else ""
54
+ data = json.dumps(body).encode() if body is not None else None
55
+ r = urllib.request.Request(self.url + path + q, data, {"Content-Type": "application/json"}, method=method)
56
+ try:
57
+ with urllib.request.urlopen(r, timeout=self.timeout) as resp:
58
+ raw = resp.read()
59
+ except urllib.error.HTTPError as e:
60
+ try:
61
+ detail = json.loads(e.read()).get("detail")
62
+ except ValueError:
63
+ detail = None
64
+ raise BlockyError(f"{method} {path}: {e.code} {detail or e.reason}") from None
65
+ return json.loads(raw) if raw else None
66
+
67
+ def get(self, path, **params):
68
+ return self._req("GET", path, params=params)
69
+
70
+ def post(self, path, body=None):
71
+ return self._req("POST", path, body if body is not None else {})
72
+
73
+ # ---- Box ----
74
+ def status(self) -> dict:
75
+ return self.get("/api/status")
76
+
77
+ def hardware(self) -> dict:
78
+ return self.get("/api/hardware/limits")
79
+
80
+ def models(self) -> list[dict]:
81
+ return self.get("/api/models")["models"]
82
+
83
+ def engines(self) -> list[dict]:
84
+ return self.get("/api/engines")
85
+
86
+ # ---- Experiments ----
87
+ def experiments(self) -> list[dict]:
88
+ return self.get("/api/experiments")
89
+
90
+ def experiment(self, key: int | str) -> "Experiment":
91
+ if isinstance(key, int):
92
+ return Experiment(self, key)
93
+ exps = self.experiments()
94
+ exact = [e for e in exps if e["name"].lower() == str(key).lower()]
95
+ hits = exact or [e for e in exps if str(key).lower() in e["name"].lower()]
96
+ if not hits:
97
+ raise BlockyError(f"No experiment matching {key!r}")
98
+ if len(hits) > 1:
99
+ raise BlockyError(f"{key!r} matches several experiments: " + ", ".join(f"{e['id']}: {e['name']}" for e in hits) + ". Use the id or the full name.")
100
+ return Experiment(self, hits[0]["id"])
101
+
102
+ def new_experiment(self, model: str, name: str | None = None) -> "Experiment":
103
+ return Experiment(self, self.post("/api/experiments", {"model": model, "name": name or model.removesuffix(".gguf")})["id"])
104
+
105
+ # ---- Weights, scans, engines ----
106
+ def variant(self, source: str, qtype: str, *, imatrix: bool = False, overrides: dict | None = None, output_type: str | None = None,
107
+ embedding_type: str | None = None, prune: list[int] | None = None, name: str | None = None) -> "Job":
108
+ jid = self.post("/api/weights/variants", {"source": source, "qtype": qtype, "imatrix": imatrix, "overrides": overrides or {},
109
+ "output_type": output_type, "embedding_type": embedding_type, "prune": prune or [], "name": name})["id"]
110
+ return Job(lambda: next((j for j in self.get("/api/weights/jobs") if j["id"] == jid), {}), ("queued", "running"))
111
+
112
+ def scan(self, model: str, mode: str = "quantize", qtype: str = "q2_k") -> "Job":
113
+ sid = self.post("/api/scans", {"model": model, "mode": mode, "qtype": qtype})["id"]
114
+ return Job(lambda: next((s for s in self.get("/api/scans") if s["id"] == sid), {}), ("queued", "running"))
115
+
116
+ def build_engine(self, repo: str, ref: str = "master", name: str | None = None) -> "Job":
117
+ eid = self.post("/api/engines", {"repo": repo, "ref": ref, "name": name})["id"]
118
+ return Job(lambda: next((e for e in self.engines() if e["id"] == eid), {}), ("queued", "building"))
119
+
120
+
121
+ class Job:
122
+ """Something running on the box (a weights build, a scan, an engine build)."""
123
+
124
+ def __init__(self, fetch, active: tuple):
125
+ self._fetch, self._active = fetch, active
126
+
127
+ def info(self) -> dict:
128
+ return self._fetch()
129
+
130
+ def wait(self, poll: float = 10, verbose: bool = True) -> dict:
131
+ last = None
132
+ while True:
133
+ i = self.info()
134
+ st = (i.get("status"), i.get("step"))
135
+ if verbose and st != last:
136
+ print(time.strftime("%H:%M:%S"), *[x for x in st if x])
137
+ last = st
138
+ if i.get("status") not in self._active:
139
+ return i
140
+ time.sleep(poll)
141
+
142
+
143
+ class Run(dict):
144
+ """One profiled configuration. A dict (the raw API record) with conveniences."""
145
+
146
+ @property
147
+ def metrics(self) -> dict:
148
+ return _flat({k: v for k, v in self.get("results", {}).items() if k not in ("series", "config", "commands")})
149
+
150
+
151
+ class Experiment:
152
+ def __init__(self, client: Client, exp_id: int):
153
+ self.c, self.id = client, exp_id
154
+
155
+ def info(self) -> dict:
156
+ return self.c.get(f"/api/experiments/{self.id}")
157
+
158
+ def runs(self) -> list[Run]:
159
+ return [Run(r) for r in self.info()["runs_list"]]
160
+
161
+ def run(self, key: int | str) -> Run:
162
+ for r in self.runs():
163
+ if r["id"] == key or (isinstance(key, str) and r["label"] == key):
164
+ return r
165
+ raise BlockyError(f"No run {key!r}")
166
+
167
+ def baseline(self) -> Run:
168
+ return next(r for r in self.runs() if r["status"] == "done")
169
+
170
+ def results(self, only_done: bool = True):
171
+ """All runs × all metrics. A pandas DataFrame when pandas is installed, else a list of dicts."""
172
+ rows = []
173
+ for r in self.runs():
174
+ if only_done and r["status"] != "done":
175
+ continue
176
+ label = r["label"] if r["label"] != "Baseline" else f"Baseline · {r['config']['model'].removesuffix('.gguf')}"
177
+ rows.append({"run_id": r["id"], "label": label, "parent_id": r.get("parent_id"), "status": r["status"],
178
+ **_flat(r["config"], "config."), **r.metrics, **_flat(r.get("stats") or {}, "vs_baseline.")})
179
+ try:
180
+ import pandas as pd
181
+ return pd.DataFrame(rows).set_index("run_id")
182
+ except ImportError:
183
+ return rows
184
+
185
+ def branch(self, parent: Run | int | None = None, label: str = "", **changes) -> int:
186
+ """Queue a run: the parent's exact setup + `changes` (model=, engine=, kv=, ctx=, vram_limit_mb=, ngl=, fa=, threads=)."""
187
+ p = parent if isinstance(parent, dict) else (self.run(parent) if parent is not None else self.baseline())
188
+ return self.c.post(f"/api/experiments/{self.id}/runs", {"label": label, "config": {**p["config"], **changes}, "parent_id": p["id"]})["id"]
189
+
190
+ def sweep(self, grid: dict, parent: Run | int | None = None, repeats: int = 1) -> list[int]:
191
+ p = parent if isinstance(parent, dict) else (self.run(parent) if parent is not None else None)
192
+ return self.c.post(f"/api/experiments/{self.id}/sweep", {"grid": grid, "repeats": repeats, "parent_id": p["id"] if p else None})["ids"]
193
+
194
+ def wait(self, poll: float = 15, verbose: bool = True) -> None:
195
+ last = None
196
+ while True:
197
+ rs = self.runs()
198
+ busy = [r for r in rs if r["status"] in ("queued", "running")]
199
+ now = next((f"{r['label']}: {r.get('phase')}" for r in busy if r["status"] == "running"), None)
200
+ if verbose and (now, len(busy)) != last:
201
+ print(time.strftime("%H:%M:%S"), now or "", f"({len(busy)} left)" if busy else "done")
202
+ last = (now, len(busy))
203
+ if not busy:
204
+ return
205
+ time.sleep(poll)
206
+
207
+ # ---- Experiment trackers ----
208
+ def _tracker_rows(self):
209
+ import re
210
+ clean = lambda k: re.sub(r"[^A-Za-z0-9_.\- /]", "_", k)[:250] # noqa: E731
211
+ info = self.info()
212
+ for r in info["runs_list"]:
213
+ if r["status"] != "done":
214
+ continue
215
+ metrics = {clean(k): float(v) for k, v in Run(r).metrics.items()
216
+ if isinstance(v, (int, float)) and not isinstance(v, bool) and not k.startswith(("provenance.", "hardware.", "imported_from."))}
217
+ params = {clean(k): str(v) for k, v in _flat(r["config"]).items()}
218
+ prov = {clean(f"blocky.{k}"): str(v)[:500] for k, v in _flat({"provenance": r["results"].get("provenance") or {},
219
+ "hardware": r["results"].get("hardware") or {}}).items() if v is not None}
220
+ tags = {"blocky.run_id": str(r["id"]), "blocky.parent_id": str(r.get("parent_id") or ""), "blocky.experiment": info["name"], **prov}
221
+ yield r, metrics, params, tags
222
+
223
+ def to_mlflow(self, tracking_uri: str | None = None, experiment_name: str | None = None) -> list[str]:
224
+ """Log every finished run to MLflow: config as params, every number as metrics, provenance + hardware as tags.
225
+
226
+ Default: a local SQLite file `./mlflow.db` (open it with `mlflow ui --backend-store-uri sqlite:///mlflow.db`).
227
+ Pass a tracking server URI to log there instead. Needs `pip install mlflow` (or mlflow-skinny + sqlalchemy + alembic).
228
+ """
229
+ import mlflow
230
+ mlflow.set_tracking_uri(tracking_uri or "sqlite:///mlflow.db")
231
+ exp = mlflow.set_experiment(experiment_name or f"blocky · {self.info()['name']}")
232
+ ids = []
233
+ for r, metrics, params, tags in self._tracker_rows():
234
+ with mlflow.start_run(experiment_id=exp.experiment_id, run_name=r["label"]) as run:
235
+ mlflow.log_params(params)
236
+ mlflow.log_metrics(metrics)
237
+ mlflow.set_tags(tags)
238
+ mlflow.log_dict({k: v for k, v in r["results"].items() if k != "series"}, "blocky_results.json")
239
+ ids.append(run.info.run_id)
240
+ return ids
241
+
242
+ def to_wandb(self, project: str = "blocky", mode: str = "offline", entity: str | None = None) -> int:
243
+ """Log every finished run to Weights & Biases. Offline by default: nothing leaves this machine until you
244
+ run `wandb sync` yourself. Needs `pip install wandb`."""
245
+ import wandb
246
+ name = self.info()["name"]
247
+ n = 0
248
+ for r, metrics, params, tags in self._tracker_rows():
249
+ run = wandb.init(project=project, entity=entity, name=r["label"], group=name, config={**params, **tags}, mode=mode, reinit="finish_previous")
250
+ run.summary.update(metrics)
251
+ run.finish()
252
+ n += 1
253
+ return n
254
+
255
+ def export_csv(self, path: str) -> str:
256
+ with urllib.request.urlopen(f"{self.c.url}/api/experiments/{self.id}/export.csv", timeout=self.c.timeout) as r, open(path, "wb") as f:
257
+ f.write(r.read())
258
+ return path
blocky_research.py ADDED
@@ -0,0 +1,66 @@
1
+ """Dependency-free reporting SDK for Blocky research jobs. No network calls.
2
+
3
+ Runs receive BLOCKY_RUN_DIR and BLOCKY_CONFIG from the runner. Reports are JSONL on disk.
4
+ """
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import os
9
+ import time
10
+ from pathlib import Path
11
+
12
+
13
+ def fingerprint(path):
14
+ h = hashlib.sha256()
15
+ with Path(path).open("rb") as f:
16
+ for chunk in iter(lambda: f.read(1024 * 1024), b""):
17
+ h.update(chunk)
18
+ return h.hexdigest()
19
+
20
+
21
+ class Run:
22
+ def __init__(self):
23
+ self.directory = Path(os.environ["BLOCKY_RUN_DIR"]).resolve()
24
+ self.config = json.loads(os.environ.get("BLOCKY_CONFIG", "{}"))
25
+
26
+ def _event(self, event):
27
+ event["time"] = time.time()
28
+ encoded = json.dumps(event, allow_nan=False)
29
+ if len(encoded.encode()) > 256_000:
30
+ raise ValueError("Report event exceeds 256 KB; save large data as an artifact")
31
+ with (self.directory / "events.jsonl").open("a", encoding="utf-8") as f:
32
+ f.write(encoded + "\n")
33
+
34
+ def metric(self, name, value, *, unit, protocol, task, dataset_sha256, scope,
35
+ samples=None, kind="measured", better=None, note="", tokenizer_sha256=None):
36
+ """Describe an evaluator's output. Matching contracts permit comparison, not automatic significance.
37
+
38
+ protocol identifies the measurement procedure; task identifies what is scored; scope describes timing/
39
+ inclusion boundaries. Cross-tokenizer comparisons should use bytes or common task scores, not tokens.
40
+ """
41
+ if kind not in ("measured", "estimated", "simulated") or better not in (None, "higher", "lower"):
42
+ raise ValueError("Invalid metric kind/direction")
43
+ if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value):
44
+ raise ValueError("Metrics must be finite numbers; omit unavailable measurements")
45
+ if not all(isinstance(x, str) and x for x in (name, unit, protocol, task, dataset_sha256, scope)):
46
+ raise ValueError("Metric requires a unit, protocol, task, dataset fingerprint and measurement scope")
47
+ if "token" in unit.lower() and not tokenizer_sha256:
48
+ raise ValueError("Token-based metrics require a tokenizer fingerprint; use byte units for cross-tokenizer comparisons")
49
+ samples = list(samples) if samples is not None else None
50
+ if samples is not None and any(not isinstance(x, (int, float)) or isinstance(x, bool) or not math.isfinite(x) for x in samples):
51
+ raise ValueError("Samples must be finite numbers")
52
+ self._event({"type": "metric", "name": name, "value": value, "unit": unit, "protocol": protocol,
53
+ "task": task, "dataset_sha256": dataset_sha256, "scope": scope, "samples": samples,
54
+ "kind": kind, "better": better, "note": note, "tokenizer_sha256": tokenizer_sha256})
55
+
56
+ def artifact(self, path, description=""):
57
+ if Path(path).is_symlink():
58
+ raise ValueError("Artifact symlinks are not supported")
59
+ path = Path(path).resolve()
60
+ if not path.is_relative_to(self.directory) or not path.is_file() or path.is_symlink():
61
+ raise ValueError("Artifacts must be files within the run directory")
62
+ self._event({"type": "artifact", "path": str(path.relative_to(self.directory)), "description": description,
63
+ "sha256": fingerprint(path), "bytes": path.stat().st_size})
64
+
65
+ def progress(self, message, step=None):
66
+ self._event({"type": "progress", "message": str(message)[:2000], "step": step})
@@ -0,0 +1,41 @@
1
+ Metadata-Version: 2.4
2
+ Name: blockyos-client
3
+ Version: 0.2.0
4
+ Summary: Python client for blockyOS: queue experiments, read results as tables, report metrics from research programs
5
+ Author: The blockyOS authors
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/matco01/blockyOS
8
+ Project-URL: Documentation, https://github.com/matco01/blockyOS/blob/main/docs/PROTOCOL.md
9
+ Keywords: llm,benchmarking,quantization,llama.cpp,reproducibility
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
13
+ Requires-Python: >=3.10
14
+ Description-Content-Type: text/markdown
15
+ Provides-Extra: tables
16
+ Requires-Dist: pandas; extra == "tables"
17
+ Provides-Extra: mlflow
18
+ Requires-Dist: mlflow; extra == "mlflow"
19
+ Provides-Extra: wandb
20
+ Requires-Dist: wandb; extra == "wandb"
21
+
22
+ # blockyos-client
23
+
24
+ A small, dependency-free Python client for a [blockyOS](https://github.com/matco01/blockyOS) device.
25
+
26
+ ```python
27
+ import blocky
28
+
29
+ b = blocky.Client("http://localhost:8000")
30
+ exp = b.new_experiment("Qwen3-4B-Q8_0.gguf", name="KV cache types")
31
+ base = exp.branch() # the baseline
32
+ exp.sweep({"kv": ["f16", "q8_0", "q4_0"]}, parent=base)
33
+ exp.wait()
34
+ df = exp.results() # one row per run (pandas, with the `tables` extra)
35
+ ```
36
+
37
+ `blocky_research` is the reporting SDK for custom research programs run by Blocky (see the repository's
38
+ docs/RESEARCH_PROJECTS.md).
39
+
40
+ Install: `pip install blockyos-client` (add `[tables]` for pandas results, `[mlflow]` or `[wandb]` for trackers).
41
+ Licensed under Apache-2.0.
@@ -0,0 +1,6 @@
1
+ blocky.py,sha256=pLvk9ww0DMlxq07j_XBVn20gCRzEkat71ivvIWKmfO0,12336
2
+ blocky_research.py,sha256=o7WbCvUaFIP2M0e0TWifwIlwon8hyfeEU-sUOhWM1g8,3581
3
+ blockyos_client-0.2.0.dist-info/METADATA,sha256=o13MWgzyY1BmJ11gsxDsv8qUmdOCqFlvsbJVa373KY8,1669
4
+ blockyos_client-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
5
+ blockyos_client-0.2.0.dist-info/top_level.txt,sha256=GamBUXxe5uXt8T2C0eZLV0jTCz7iF6JMOl3zNAJP9WM,23
6
+ blockyos_client-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ blocky
2
+ blocky_research