robotensor-attune 0.2.1__py3-none-any.whl

robotensor/miner.py

@@ -0,0 +1,260 @@
+"""`attune miner`: check a `vector_v1.1` weights file, submit it and commit it on chain.
+
+    check    model.safetensors, the only file a submission holds, against the architecture the
+             validator serves, as the validator checks it
+    upload   model.safetensors (and an optional README) to your Hugging Face model repository;
+             prints the commit sha to commit
+    commit   write your repository, its revision and your weights' sha256 as your hotkey's
+             commitment on the subnet (`vector:<repo>@<commit>.<digest>`)
+    status   your hotkey's commitment on chain, and what the Hub shows for it
+
+Order matters for copy protection: upload to a PRIVATE repository, commit the sha on chain, then
+make the repository public - at once: the validator queues a commitment as soon as it reads it and
+looks the repository up on the Hub when its turn comes, and a repository it cannot see then (still
+private, gone, or the Hub not answering) is dropped as missing and the next entry duels instead. Weights byte-identical to an earlier
+commitment's are refused as a duplicate, and the earlier commitment keeps them, so whoever commits
+first owns the weights.
+
+Each hotkey makes one submission: once a commitment of yours is queued, anything else the hotkey
+commits is refused, before its duel and after it - even if its repository is then skipped. Check the
+weights (`check`) before you commit, and keep the repository up: a king whose repository is gone is
+dethroned, and its share of the emission is burned.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from . import hub
+from .protocol import commitment as commitment_
+
+DEFAULT_NETWORK = "test"
+
+
+def weights_digest(path: Path) -> str:
+    """The sha256 of the weights file, as the commitment names it."""
+    digest = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for block in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    from vector_runtime import check
+
+    report = check.check_weights(
+        Path(args.dir) / hub.WEIGHTS_FILE if Path(args.dir).is_dir() else Path(args.dir)
+    )
+    print(
+        json.dumps(
+            {
+                "ok": report.ok,
+                "errors": list(report.errors),
+                "weights_sha256": report.weights_sha256,
+            },
+            indent=1,
+        )
+    )
+    return 0 if report.ok else 1
+
+
+def cmd_upload(args: argparse.Namespace) -> int:
+    weights = Path(args.dir) / hub.WEIGHTS_FILE if Path(args.dir).is_dir() else Path(args.dir)
+    if not weights.is_file():
+        print(f"error: no {hub.WEIGHTS_FILE} at {weights}", file=sys.stderr)
+        return 2
+    readme = Path(args.readme).read_text(encoding="utf-8") if args.readme else None
+    sha = hub.upload_weights(
+        args.repo,
+        str(weights),
+        token=os.environ.get("HF_TOKEN"),
+        private=args.private,
+        readme=readme,
+    )
+    digest = weights_digest(weights)
+    print(
+        json.dumps(
+            {
+                "repo": args.repo,
+                "revision": sha,
+                "digest": digest,
+                "commitment": commitment_.encode(args.repo, sha, digest),
+            }
+        )
+    )
+    return 0
+
+
+def cmd_commit(args: argparse.Namespace) -> int:
+    from . import chain as chain_
+
+    data = commitment_.encode(args.repo, args.revision, args.digest)
+    wallet = chain_.wallet(args.wallet_name, args.wallet_hotkey, args.wallet_path)
+    chain = chain_.Chain(args.network, args.netuid)
+    try:
+        chain.commit(wallet, data)
+        block = chain.block()
+    finally:
+        chain.close()
+    print(json.dumps({"committed": data, "hotkey": wallet.hotkey.ss58_address, "block": block}))
+    return 0
+
+
+def cmd_submit(args: argparse.Namespace) -> int:
+    """check -> upload private -> commit on chain -> publish, in the order that keeps your
+    weights yours: the earliest commitment of a set of weights keeps them, so nobody may see the
+    file until the chain says it is yours."""
+    from . import chain as chain_
+
+    if cmd_check(args) != 0:
+        print("error: the weights did not pass the check the validator runs", file=sys.stderr)
+        return 1
+    weights = Path(args.dir) / hub.WEIGHTS_FILE if Path(args.dir).is_dir() else Path(args.dir)
+    readme = Path(args.readme).read_text(encoding="utf-8") if args.readme else None
+    token = os.environ.get("HF_TOKEN")
+    sha = hub.upload_weights(args.repo, str(weights), token=token, private=True, readme=readme)
+    data = commitment_.encode(args.repo, sha, weights_digest(weights))
+    wallet = chain_.wallet(args.wallet_name, args.wallet_hotkey, args.wallet_path)
+    chain = chain_.Chain(args.network, args.netuid)
+    try:
+        chain.commit(wallet, data)
+        block = chain.block()
+    finally:
+        chain.close()
+    published = False
+    if not args.keep_private:
+        from huggingface_hub import HfApi
+
+        HfApi(token=token).update_repo_settings(args.repo, private=False)
+        published = True
+    print(
+        json.dumps(
+            {
+                "repo": args.repo,
+                "revision": sha,
+                "committed": data,
+                "block": block,
+                "public": published,
+                "fee": "none: this subnet charges nothing to submit",
+            },
+            indent=1,
+        )
+    )
+    return 0
+
+
+def cmd_status(args: argparse.Namespace) -> int:
+    from . import chain as chain_
+
+    chain = chain_.Chain(args.network, args.netuid)
+    try:
+        mine = [c for c in chain.commitments() if c.hotkey == args.hotkey]
+    finally:
+        chain.close()
+    out = []
+    for c in mine:
+        entry = {"hotkey": c.hotkey, "block": c.block, "data": c.data}
+        try:
+            sub = commitment_.parse(c.data)
+            found = hub.inspect(
+                sub.repo,
+                sub.revision,
+                frozenset({hub.WEIGHTS_FILE, "README.md", ".gitattributes"}),
+                token=os.environ.get("HF_TOKEN"),
+            )
+            entry["weights_sha256"] = found.weights_sha256
+            entry["digest"] = sub.digest
+            entry["digest_matches"] = sub.digest == found.weights_sha256
+        except Exception as exc:  # noqa: BLE001 - reported, not raised
+            entry["problem"] = str(exc)
+        out.append(entry)
+    print(json.dumps(out, indent=1))
+    return 0
+
+
+def _chain_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--network", default=DEFAULT_NETWORK, help="finney, test, or ws://host:port"
+    )
+    parser.add_argument("--netuid", type=int, required=True)
+
+
+def add_subcommands(sub: Any) -> None:
+    """Hang a miner's verbs off `sub`, so `attune miner ...` and the standalone command are
+    the same parser rather than two that drift."""
+    check = sub.add_parser("check", help="check model.safetensors as the validator does")
+    check.add_argument(
+        "--dir", required=True, help="the directory holding model.safetensors, or the file"
+    )
+    check.set_defaults(func=cmd_check)
+
+    upload = sub.add_parser("upload", help="upload model.safetensors; prints the sha to commit")
+    upload.add_argument(
+        "--dir", required=True, help="the directory holding model.safetensors, or the file"
+    )
+    upload.add_argument(
+        "--repo", required=True, help="your Hugging Face model repository, owner/name"
+    )
+    upload.add_argument("--readme", default=None, help="a README.md to upload beside it")
+    upload.add_argument("--private", action="store_true", help="create the repository private")
+    upload.set_defaults(func=cmd_upload)
+
+    commit = sub.add_parser("commit", help="commit repo@sha on chain for your hotkey")
+    commit.add_argument("--repo", required=True)
+    commit.add_argument("--revision", required=True, help="the 40-hex commit sha upload printed")
+    commit.add_argument(
+        "--digest",
+        required=True,
+        help="the sha256 of model.safetensors (check and upload print it)",
+    )
+    _chain_args(commit)
+    commit.add_argument("--wallet.name", dest="wallet_name", required=True)
+    commit.add_argument("--wallet.hotkey", dest="wallet_hotkey", required=True)
+    commit.add_argument("--wallet.path", dest="wallet_path", default=None)
+    commit.set_defaults(func=cmd_commit)
+
+    submit = sub.add_parser(
+        "submit", help="check, upload private, commit on chain, then publish - in that order"
+    )
+    submit.add_argument(
+        "--dir", required=True, help="the directory holding model.safetensors, or the file"
+    )
+    submit.add_argument("--repo", required=True, help="your Hugging Face model repository")
+    submit.add_argument("--readme", default=None, help="a README.md to upload beside it")
+    submit.add_argument(
+        "--keep-private",
+        action="store_true",
+        help="leave the repository private; the validator drops it as missing at its turn",
+    )
+    _chain_args(submit)
+    submit.add_argument("--wallet.name", dest="wallet_name", required=True)
+    submit.add_argument("--wallet.hotkey", dest="wallet_hotkey", required=True)
+    submit.add_argument("--wallet.path", dest="wallet_path", default=None)
+    submit.set_defaults(func=cmd_submit)
+
+    status = sub.add_parser("status", help="your hotkey's commitment on chain")
+    status.add_argument("--hotkey", required=True, help="your hotkey's ss58 address")
+    _chain_args(status)
+    status.set_defaults(func=cmd_status)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="robotensor-miner", description=__doc__.split("\n")[0])
+    add_subcommands(parser.add_subparsers(dest="command", required=True))
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return int(args.func(args) or 0)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

robotensor/miner_cli.py

@@ -0,0 +1,51 @@
+"""`attune`, as a miner installs it (`pip install robotensor-attune`): a submission, start to end.
+
+    attune --version
+    attune init [DIR]
+    attune miner check | upload | commit | submit | status
+
+The same `init` and `miner` commands as the validator's install, which has the validator's
+besides. `--role miner` is accepted, so a command written for either one runs here.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    from . import init as init_
+
+    print(init_.miner(Path(args.directory).expanduser().resolve()))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    from . import miner as miner_
+
+    parser = argparse.ArgumentParser(prog="attune", description=__doc__.split("\n")[0])
+    parser.add_argument("--version", action="version", version=f"attune {__version__}")
+    sub = parser.add_subparsers(dest="part", required=True)
+
+    start = sub.add_parser("init", help="a working directory for a submission")
+    start.add_argument("directory", nargs="?", default=".", help="where (default: .)")
+    start.add_argument("--role", default="miner", choices=("miner",), help=argparse.SUPPRESS)
+    start.add_argument("--competition", default=None, help=argparse.SUPPRESS)
+    start.set_defaults(func=cmd_init)
+
+    miner = sub.add_parser("miner", help="check, upload and commit a submission")
+    miner_.add_subcommands(miner.add_subparsers(dest="command", required=True))
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return int(args.func(args) or 0)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

robotensor/protocol/commitment.py

@@ -0,0 +1,127 @@
+"""What a miner writes on chain: one short string naming its submission.
+
+    vector:<owner>/<name>@<commit>.<digest>
+    horizon:<owner>/<name>@<40-hex commit>
+
+A Vector commitment names its weights as well as where they are: `<commit>` is the repository
+revision's 20 bytes and `<digest>` the sha256 of its `model.safetensors`, both in unpadded base64url
+(27 and 43 characters), so that a repository id of up to 49 characters fits. The validator refuses
+a commitment whose digest is not what the Hub says that file at that revision hashes to, so a
+miner's commitment pins the exact weights it duels with from the block it was made at.
+
+The first word names the competition the submission enters, so one subnet reads both competitions'
+commitments from the same storage and a miner never has to say which netuid slot they meant. The
+string is stored as a raw commitment, which holds at most `MAX_BYTES` bytes, so a repository id can
+be at most about 80 characters. The revision is always a full commit sha: a branch would let a
+miner change what was committed after its block, and the block is what orders the queue.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import re
+from dataclasses import dataclass
+
+#: A raw commitment's limit on chain (`Raw` data, as `Subtensor.set_commitment` writes it).
+MAX_BYTES = 128
+#: Competition 1, action-grounded in-context learning: the only one live at launch.
+VECTOR = "vector"
+#: Competition 2, video-prompted world-action learning. Its commitments are read and taken in;
+#: what they enter does not pay yet (`config.SHARES`), and the round that scores them is
+#: staged in `lanes/horizon.py`.
+HORIZON = "horizon"
+LANES = (VECTOR, HORIZON)
+
+REPO_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*")
+SHA_RE = re.compile(r"[0-9a-f]{40}")
+DIGEST_RE = re.compile(r"[0-9a-f]{64}")
+#: Unpadded base64url of 20 and of 32 bytes.
+B64_RE = {20: re.compile(r"[A-Za-z0-9_-]{27}"), 32: re.compile(r"[A-Za-z0-9_-]{43}")}
+
+
+class CommitmentError(ValueError):
+    """A commitment that is not one this subnet reads."""
+
+
+@dataclass(frozen=True)
+class Submission:
+    """A parsed commitment: which lane, which repository, at which commit."""
+
+    lane: str
+    repo: str
+    revision: str
+    #: The sha256 of the weights, as 64 lowercase hex; None for a lane whose commitment has none.
+    digest: str | None = None
+
+    @property
+    def entry(self) -> str:
+        return f"{self.repo}@{self.revision}"
+
+
+def _b64(hex_text: str) -> str:
+    return base64.urlsafe_b64encode(bytes.fromhex(hex_text)).rstrip(b"=").decode("ascii")
+
+
+def _unb64(text: str, size: int) -> str | None:
+    """`text` as the hex of `size` bytes, or None when it is not their unpadded base64url."""
+    if not B64_RE[size].fullmatch(text):
+        return None
+    try:
+        raw = base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))
+    except (ValueError, binascii.Error):
+        return None
+    return raw.hex() if len(raw) == size and _b64(raw.hex()) == text else None
+
+
+def encode(repo: str, revision: str, digest: str | None = None, lane: str = VECTOR) -> str:
+    """The commitment string for `repo@revision` (and, for Vector, the weights' sha256 `digest`);
+    `CommitmentError` if it cannot be one."""
+    if lane not in LANES:
+        raise CommitmentError(f"unknown lane {lane!r}; this subnet reads {', '.join(LANES)}")
+    if not REPO_RE.fullmatch(repo):
+        raise CommitmentError(f"{repo!r} is not a Hugging Face repo id (owner/name)")
+    if not SHA_RE.fullmatch(revision):
+        raise CommitmentError(
+            f"{revision!r} is not a full 40-hex commit sha; commit the revision the upload printed"
+        )
+    if lane == VECTOR:
+        if digest is None or not DIGEST_RE.fullmatch(digest):
+            raise CommitmentError(
+                f"{digest!r} is not the 64-hex sha256 of model.safetensors; `check` prints it"
+            )
+        data = f"{lane}:{repo}@{_b64(revision)}.{_b64(digest)}"
+    elif digest is not None:
+        raise CommitmentError(f"a {lane} commitment names no digest")
+    else:
+        data = f"{lane}:{repo}@{revision}"
+    if len(data.encode()) > MAX_BYTES:
+        raise CommitmentError(
+            f"the commitment is {len(data.encode())} bytes, over the chain's {MAX_BYTES}; use a "
+            "shorter repository name"
+        )
+    return data
+
+
+def parse(data: str) -> Submission:
+    """A commitment string as a `Submission`; `CommitmentError` for anything else."""
+    if not isinstance(data, str) or len(data.encode()) > MAX_BYTES:
+        raise CommitmentError("not a commitment string of at most 128 bytes")
+    lane, sep, rest = data.strip().partition(":")
+    if not sep or lane not in LANES:
+        raise CommitmentError(f"no known lane prefix in {data[:40]!r}")
+    repo, at, pinned = rest.rpartition("@")
+    if not at or not REPO_RE.fullmatch(repo):
+        raise CommitmentError(f"{data[:80]!r} names no <owner>/<name>@...")
+    if lane != VECTOR:
+        if not SHA_RE.fullmatch(pinned):
+            raise CommitmentError(f"{data[:80]!r} is not {lane}:<owner>/<name>@<40-hex sha>")
+        return Submission(lane=lane, repo=repo, revision=pinned)
+    commit, dot, weights = pinned.partition(".")
+    revision, digest = _unb64(commit, 20), _unb64(weights, 32)
+    if not dot or revision is None or digest is None:
+        raise CommitmentError(
+            f"{data[:80]!r} is not vector:<owner>/<name>@<commit>.<digest>: a Vector commitment "
+            "names the weights' sha256 too (`attune miner commit --digest`)"
+        )
+    return Submission(lane=lane, repo=repo, revision=revision, digest=digest)

robotensor_attune-0.2.1.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.5
+Name: robotensor-attune
+Version: 0.2.1
+Summary: The miner's command for Attune, Robotensor's Bittensor subnet: check, upload and commit a submission.
+Project-URL: Homepage, https://attune.robotensor.ai
+Project-URL: Documentation, https://attune.robotensor.ai/docs/vector
+License: Apache-2.0
+License-File: LICENSE
+Keywords: attune,bittensor,robot-learning,robotics
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.11
+Requires-Dist: bittensor==10.5.0
+Requires-Dist: huggingface-hub>=1.0

robotensor_attune-0.2.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+robotensor/__init__.py,sha256=4sWsYF8tlgBv7XT2Iad5wwO1EAkCWcNUaXAAiEE2oyc,124
+robotensor/chain.py,sha256=7ndiQUWp92gfbKqbvuIQzylnOtWf-nzrLfUWO_Foqvk,6357
+robotensor/hub.py,sha256=-yez2cbXi6MMKnOUAUEoVw1FkgZDLUW21LGkAfzSgWo,10590
+robotensor/init.py,sha256=d5NVn4JHrDJJ0UXs2NOUMJ8Pu0X1F_9muv7ZJ9C1pGE,2937
+robotensor/miner.py,sha256=liYI_TCKzmlqhVMr_h5Lb3e8z3Zd0g6duaFnKC4t0P0,10017
+robotensor/miner_cli.py,sha256=qMnjdwc_-EXRmdjxuZ66SpG0qQ5K_iBe75QNDos6hKA,1720
+robotensor/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+robotensor/protocol/commitment.py,sha256=2adI8yNfLDg7Moq4J0E0KFACJ6AHQHgBXBpMYX9j0kQ,5639
+vector_runtime/__init__.py,sha256=LGrY1t_lu9bN2eNIKImm_y_jCV0dgvHYm4kLb8IXUfw,391
+vector_runtime/check.py,sha256=_9x6IqfuP033ZHmiqDo1LEk98ILZXeUg9rmZtWnBGDc,6163
+vector_runtime/header.py,sha256=9yfk1-KePWxQGDsyQamu1PrHZCm8G218TwcZqj2YGYE,7172
+vector_runtime/vector_v1.1.json,sha256=1lLhxED_qQ4KBjIObXK_DVky6bRoi6Jvp-7-8hYevqQ,72329
+robotensor_attune-0.2.1.dist-info/METADATA,sha256=SxCY0gEljDilnsbv3Cq8jbAd9F6R8IIImInN7Op3q4g,692
+robotensor_attune-0.2.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+robotensor_attune-0.2.1.dist-info/entry_points.txt,sha256=g3gTPMl0X-sk3-NL1XDnNVKhFcupBQT8pIZrd16w81A,53
+robotensor_attune-0.2.1.dist-info/licenses/LICENSE,sha256=o1OyBT8s55NsyDfGidngR7fgXAIvQEp6IKDoVRMNJr4,11353
+robotensor_attune-0.2.1.dist-info/RECORD,,

robotensor_attune-0.2.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

robotensor_attune-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+attune = robotensor.miner_cli:main