robotensor-attune 0.2.1__py3-none-any.whl

robotensor/__init__.py

@@ -0,0 +1,3 @@
+"""Attune, Robotensor's Bittensor subnet: chain commitments in, lane engines' duels, weights out."""
+
+__version__ = "0.2.1"

robotensor/chain.py

@@ -0,0 +1,158 @@
+"""The chain, and the only module that imports bittensor.
+
+Pinned to bittensor 10.5 (`pyproject.toml`): version 11 replaced the SDK's API, and the subnets this
+one follows (SN3, SN80, SN98) all still pin 10.x. Moving to 11 changes this file and nothing else.
+
+Everything here is a thin call to `Subtensor`:
+
+- `commitments()`: every hotkey's current commitment on the subnet with the block it was made at,
+  from one query of the `Commitments.CommitmentOf` storage map. The chain keeps one commitment per
+  hotkey; a new one replaces it and resets its block.
+- `block()`, `block_hash(n)`, `finalized()`, `uids()`: the head, a block's hash, the finalized
+  head and its hash, and the metagraph's hotkeys.
+- `commit(wallet, data)` and `set_weights(wallet, weights)`: the two writes.
+
+`Subtensor` holds a websocket that is not safe to share between threads, so a `Chain` opens its own
+and a thread that needs the chain makes its own `Chain`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Commitment:
+    hotkey: str
+    block: int
+    data: str
+
+
+class ChainError(RuntimeError):
+    """The chain refused a write, or could not be read."""
+
+
+class RateLimited(ChainError):
+    """Weights were set too recently: `blocks` more must pass (the subnet's weights rate limit)."""
+
+    def __init__(self, blocks: int) -> None:
+        self.blocks = blocks
+        super().__init__(f"weights were set too recently; {blocks} more block(s) must pass")
+
+
+class Chain:
+    def __init__(self, network: str, netuid: int) -> None:
+        import bittensor as bt
+
+        self.network = network
+        self.netuid = int(netuid)
+        self.subtensor = bt.Subtensor(network=network)
+
+    # -- reads ------------------------------------------------------------------------------
+
+    def block(self) -> int:
+        return int(self.subtensor.get_current_block())
+
+    def block_hash(self, block: int) -> str:
+        value = self.subtensor.get_block_hash(int(block))
+        if not value:
+            raise ChainError(f"the chain has no hash for block {block}")
+        return str(value)
+
+    def finalized(self) -> tuple[int, str]:
+        """The finalized head: its number and its hash, read together so they name one block."""
+        substrate = self.subtensor.substrate
+        value = substrate.get_chain_finalised_head()
+        if not value:
+            raise ChainError("the chain reported no finalized head")
+        return int(substrate.get_block_number(value)), str(value)
+
+    def commitments(self) -> list[Commitment]:
+        """Every hotkey's commitment on the subnet, with the block it was made at, oldest first."""
+        from bittensor.core.chain_data.utils import decode_metadata
+
+        out = []
+        for hotkey, value in self.subtensor.query_map(
+            module="Commitments", name="CommitmentOf", params=[self.netuid]
+        ):
+            record = getattr(value, "value", value)
+            try:
+                data = decode_metadata(record)
+                block = int(record["block"])
+            except Exception:  # noqa: BLE001 - a commitment we cannot read is no submission
+                continue
+            key = getattr(hotkey, "value", hotkey)
+            out.append(Commitment(hotkey=str(key), block=block, data=data))
+        return sorted(out, key=lambda c: (c.block, c.hotkey))
+
+    def uids(self) -> dict[str, int]:
+        """The metagraph's hotkeys, each with its UID now. UIDs are recycled: map at the end."""
+        metagraph = self.subtensor.metagraph(self.netuid, lite=True)
+        return {
+            str(hk): int(uid) for uid, hk in zip(metagraph.uids, metagraph.hotkeys, strict=True)
+        }
+
+    def owner_hotkey(self) -> str | None:
+        return self.subtensor.get_subnet_owner_hotkey(self.netuid)
+
+    def weights_rate_limit(self) -> int | None:
+        return self.subtensor.weights_rate_limit(self.netuid)
+
+    # -- writes -----------------------------------------------------------------------------
+
+    def commit(self, wallet: Any, data: str) -> None:
+        response = self.subtensor.set_commitment(
+            wallet=wallet,
+            netuid=self.netuid,
+            data=data,
+            wait_for_inclusion=True,
+            wait_for_finalization=True,
+        )
+        if not getattr(response, "success", False):
+            raise ChainError(f"the chain refused the commitment: {_message(response)}")
+
+    def set_weights(self, wallet: Any, weights: Mapping[int, float]) -> None:
+        """Set `weights` for `wallet`'s hotkey. `RateLimited` while the subnet's weights rate limit
+        has not passed since its last update: the SDK then returns a bare failure without trying,
+        so the limit is read here first and said plainly."""
+        uid = self.subtensor.get_uid_for_hotkey_on_subnet(wallet.hotkey.ss58_address, self.netuid)
+        if uid is None:
+            raise ChainError(f"{wallet.hotkey.ss58_address} is not registered on {self.netuid}")
+        since = self.subtensor.blocks_since_last_update(self.netuid, uid)
+        limit = self.subtensor.weights_rate_limit(self.netuid)
+        if since is not None and limit is not None and since <= limit:
+            raise RateLimited(int(limit) - int(since) + 1)
+        uids = sorted(weights)
+        response = self.subtensor.set_weights(
+            wallet=wallet,
+            netuid=self.netuid,
+            uids=uids,
+            weights=[float(weights[u]) for u in uids],
+            wait_for_inclusion=True,
+            wait_for_finalization=False,
+        )
+        if not getattr(response, "success", False):
+            raise ChainError(f"the chain refused the weights: {_message(response)}")
+
+    def close(self) -> None:
+        try:
+            self.subtensor.close()
+        except Exception:  # noqa: BLE001 - closing a socket that already went is fine
+            pass
+
+
+def wallet(name: str, hotkey: str, path: str | None = None) -> Any:
+    """A bittensor wallet by name, as `btcli` keeps them (`~/.bittensor/wallets`)."""
+    import bittensor as bt
+
+    return (
+        bt.Wallet(name=name, hotkey=hotkey, path=path)
+        if path
+        else bt.Wallet(name=name, hotkey=hotkey)
+    )
+
+
+def _message(response: Any) -> str:
+    return str(getattr(response, "message", None) or response)

robotensor/hub.py

@@ -0,0 +1,266 @@
+"""The Hugging Face Hub, as the subnet sees a submission there.
+
+Intake decides from the Hub's metadata alone and never downloads a model. Two competitions ask
+different things of a repository, so what each may hold and what makes two submissions the same
+model is a `Shape` the lane brings.
+
+**Why metadata is enough.** Git LFS stores a file under its own sha256, and the Hub reports it, so
+the validator can tell a copy from an original without moving a byte. Vector's submission is one
+`model.safetensors` and its content key is that file's sha256 - the value this subnet has always
+used. Horizon's is tens of gigabytes of shards beside a few kilobyte files; its key is the digest
+of the listing `sha256sum` would print for the files it covers, which is what its runtime computes
+on disk. The kilobyte files are not in LFS, so those - and only those - are fetched.
+
+A shard that was pushed as a plain git blob rather than through LFS has no sha256 on the Hub and
+is refused, with the remedy: track `*.safetensors` in `.gitattributes`.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+WEIGHTS_FILE = "model.safetensors"
+#: A hashed file the Hub cannot give a sha256 for is fetched when it is no larger than this.
+FETCH_MAX_BYTES = 1 << 20
+
+
+class NotVisible(RuntimeError):
+    """The Hub will not show the repository at that commit (missing, private, or a bad sha)."""
+
+
+class HubUnreachable(NotVisible):
+    """The Hub did not answer (a server error, a timeout, no connection): it said nothing about the
+    repository, so nothing may be decided from it. A `NotVisible` to a caller that only waits."""
+
+
+class NotASubmission(ValueError):
+    """The repository holds what this competition's submission may not, or lacks what it must."""
+
+
+@dataclass(frozen=True)
+class HubFile:
+    """One file of a repository, as the Hub describes it without downloading anything."""
+
+    name: str
+    size: int
+    #: Its own sha256, when Git LFS stores it; `None` for a plain git blob, whose id is a sha1.
+    lfs_sha256: str | None = None
+
+
+@dataclass(frozen=True)
+class Shape:
+    """What a competition's submission may hold, and what makes two of them the same model.
+
+    `allowed`, `required` and `hashed` are exact names or `fnmatch` globs. `key` is how the
+    content key is built from the hashed files:
+
+    - `"file"`: the one hashed file's own sha256. Vector's, and the value it has always had.
+    - `"listing"`: `sha256` of `"<sha256>  <name>\\n"` for each hashed file, ordered by the bytes
+      of its path - the digest `sha256sum <files> | sha256sum` gives, which is what Horizon's
+      runtime computes over the same files on disk.
+    """
+
+    #: Files a repository may hold; one matching none of these is refused.
+    allowed: tuple[str, ...]
+    #: Patterns at least one file must match, or the repository is not a submission at all.
+    required: tuple[str, ...]
+    #: The files the content key covers.
+    hashed: tuple[str, ...]
+    key: str = "file"
+    #: The repository's size cap, over the files that count; 0 for none.
+    max_bytes: int = 0
+    #: Patterns whose files must be in LFS, so their sha256 is known without downloading them.
+    lfs_only: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if self.key not in ("file", "listing"):
+            raise ValueError(f"{self.key!r} is not a content key rule (file, listing)")
+
+    def matches(self, name: str, patterns: Iterable[str]) -> bool:
+        return any(fnmatch.fnmatch(name, pattern) for pattern in patterns)
+
+
+#: Vector's submission: the weights of one architecture, and nothing else that runs.
+def vector_shape(allowed: Iterable[str], max_bytes: int = 0) -> Shape:
+    return Shape(
+        allowed=tuple(sorted(allowed)),
+        required=(WEIGHTS_FILE,),
+        hashed=(WEIGHTS_FILE,),
+        key="file",
+        max_bytes=max_bytes,
+        lfs_only=(WEIGHTS_FILE,),
+    )
+
+
+@dataclass(frozen=True)
+class HubSubmission:
+    repo: str
+    revision: str
+    #: What two submissions of the same bytes share, whatever the competition.
+    content_key: str
+    #: The bytes the hashed files come to.
+    weights_bytes: int
+    files: tuple[str, ...] = ()
+    #: Every file with what the Hub says about it, for a lane that wants more than the names.
+    manifest: tuple[HubFile, ...] = field(default_factory=tuple)
+
+    @property
+    def weights_sha256(self) -> str:
+        """What the content key was called when one file was all a submission could be."""
+        return self.content_key
+
+
+def content_key(
+    files: Sequence[HubFile],
+    shape: Shape,
+    *,
+    fetch: Callable[[str], bytes] | None = None,
+) -> tuple[str, int]:
+    """The submission's content key and the bytes it covers, from the Hub's metadata.
+
+    `fetch` reads one small file whose sha256 the Hub does not know (a config, a statistics file);
+    it is never called for anything matching `lfs_only`, and never for a large file.
+    """
+    covered = sorted(
+        (f for f in files if shape.matches(f.name, shape.hashed)), key=lambda f: f.name.encode()
+    )
+    if not covered:
+        raise NotASubmission(f"holds none of {', '.join(shape.hashed)}")
+    digests = []
+    for one in covered:
+        if one.lfs_sha256:
+            digests.append(one.lfs_sha256)
+            continue
+        if shape.matches(one.name, shape.lfs_only):
+            raise NotASubmission(
+                f"{one.name} is not stored in Git LFS, so the Hub cannot say what is in it: "
+                "track it in .gitattributes and push it again"
+            )
+        if fetch is None or one.size > FETCH_MAX_BYTES:
+            raise NotASubmission(
+                f"{one.name} is {one.size} bytes and not in Git LFS; intake reads no more than "
+                f"{FETCH_MAX_BYTES} bytes of a submission"
+            )
+        digests.append(hashlib.sha256(fetch(one.name)).hexdigest())
+    size = sum(f.size for f in covered)
+    if shape.key == "file":
+        if len(covered) != 1:
+            raise NotASubmission(
+                f"holds {len(covered)} of {', '.join(shape.hashed)}; this competition takes one"
+            )
+        return digests[0], size
+    listing = "".join(f"{d}  {f.name}\n" for d, f in zip(digests, covered, strict=True))
+    return hashlib.sha256(listing.encode()).hexdigest(), size
+
+
+def manifest_of(info: Any) -> tuple[HubFile, ...]:
+    """What the Hub said the repository holds, as plain files."""
+    out = []
+    for sibling in info.siblings or []:
+        lfs = getattr(sibling, "lfs", None)
+        out.append(
+            HubFile(
+                name=str(sibling.rfilename),
+                size=int(getattr(sibling, "size", 0) or 0),
+                lfs_sha256=str(getattr(lfs, "sha256", "")) or None if lfs is not None else None,
+            )
+        )
+    return tuple(sorted(out, key=lambda f: f.name))
+
+
+def inspect(
+    repo: str,
+    revision: str,
+    shape: Shape,
+    *,
+    api: Any = None,
+    token: str | None = None,
+    fetch: Callable[[str], bytes] | None = None,
+) -> HubSubmission:
+    """What `repo@revision` holds, from the Hub's metadata alone.
+
+    `fetch` reads one small file the Hub knows no sha256 for; the default downloads it. Nothing
+    large is ever fetched, and nothing at all for a competition whose key covers one LFS file.
+    """
+    from huggingface_hub import HfApi
+    from huggingface_hub.errors import (
+        GatedRepoError,
+        HfHubHTTPError,
+        RepositoryNotFoundError,
+        RevisionNotFoundError,
+    )
+
+    api = api or HfApi(token=token or None)  # an empty token is no token
+    try:
+        info = api.model_info(repo, revision=revision, files_metadata=True)
+    except (RepositoryNotFoundError, RevisionNotFoundError, GatedRepoError) as exc:
+        raise NotVisible(f"{repo}@{revision}: {type(exc).__name__}") from None
+    except HfHubHTTPError as exc:
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+        if status in (401, 403, 404):
+            raise NotVisible(f"{repo}@{revision}: {exc}") from None
+        raise HubUnreachable(f"{repo}@{revision}: {exc}") from None
+    except OSError as exc:  # requests' connection errors and timeouts are OSErrors
+        raise HubUnreachable(f"{repo}@{revision}: {type(exc).__name__}: {exc}") from None
+    if getattr(info, "sha", None) != revision:
+        raise NotVisible(f"{repo}@{revision}: the Hub resolved it to {getattr(info, 'sha', None)}")
+
+    manifest = manifest_of(info)
+    names = tuple(f.name for f in manifest)
+    extra = [name for name in names if not shape.matches(name, shape.allowed)]
+    if extra:
+        raise NotASubmission(
+            f"holds files this competition's submission may not: {', '.join(extra[:5])}"
+        )
+    for pattern in shape.required:
+        if not any(shape.matches(name, [pattern]) for name in names):
+            raise NotASubmission(f"holds no {pattern}")
+
+    def download(name: str) -> bytes:
+        from huggingface_hub import hf_hub_download
+
+        path = hf_hub_download(repo, name, revision=revision, token=token or None)
+        with open(path, "rb") as handle:
+            return handle.read()
+
+    key, size = content_key(manifest, shape, fetch=fetch or download)
+    if shape.max_bytes and size > shape.max_bytes:
+        raise NotASubmission(
+            f"its weights are {size} bytes, over this competition's {shape.max_bytes}"
+        )
+    return HubSubmission(
+        repo=repo,
+        revision=revision,
+        content_key=key,
+        weights_bytes=size,
+        files=names,
+        manifest=manifest,
+    )
+
+
+def upload_weights(
+    repo: str,
+    weights: str,
+    *,
+    token: str | None = None,
+    private: bool = False,
+    readme: str | None = None,
+    message: str = "Upload model.safetensors",
+) -> str:
+    """Upload `weights` as the repository's `model.safetensors` (and a README if given), creating
+    the repository if needed; the commit sha of the upload."""
+    from huggingface_hub import CommitOperationAdd, HfApi
+
+    api = HfApi(token=token or None)
+    api.create_repo(repo, repo_type="model", private=private, exist_ok=True)
+    operations = [CommitOperationAdd(path_in_repo=WEIGHTS_FILE, path_or_fileobj=weights)]
+    if readme is not None:
+        operations.append(
+            CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme.encode())
+        )
+    info = api.create_commit(repo, operations=operations, commit_message=message, repo_type="model")
+    return str(info.oid)

robotensor/init.py

@@ -0,0 +1,74 @@
+"""`attune init`: a working directory, and a config that only needs your wallet filled in.
+
+What is fiddly about running a validator is not the code, it is the paths around it. This writes a
+config holding only what differs between hosts, and the directories its data is kept in.
+
+What it will not do is guess a wallet or a netuid. Those are the two lines it leaves for you.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+CONFIG = "attune.toml"
+
+VALIDATOR_TOML = """\
+# Written by `attune init`. State, stores and runs are kept in var/ beside this file.
+# `finney` is mainnet, `test` the test network, or a ws:// address of your own.
+network = "{network}"
+netuid = {netuid}
+# The wallet this validator sets weights with. `btcli wallet list` shows what you have.
+wallet = {{ name = "{wallet_name}", hotkey = "{wallet_hotkey}" }}
+
+[vector]
+# The two environments a duel needs, and the RoboTwin-Vector checkout the simulator runs from; see
+# the README.
+policy_python = "{policy_python}"
+simulator_python = "{simulator_python}"
+simulator_root = "{simulator_root}"
+# A Hugging Face dataset the store is published to; empty to keep it local.
+mirror = ""
+"""
+
+MINER_NEXT = """\
+A submission is weights and nothing else. Three commands, in this order:
+
+    attune miner check  --dir {directory}/submission
+    attune miner upload --dir {directory}/submission --repo <you>/<name> --private
+    attune miner commit --repo <you>/<name> --revision <the sha upload printed> \\
+        --netuid <N> --network test --wallet.name <w> --wallet.hotkey <h>
+
+Then make the repository public. Committing while it is private is what keeps your weights from
+being copied and committed by somebody else first: the earliest commitment of a set of weights
+keeps them.
+
+This subnet charges nothing to submit.
+"""
+
+
+def validator(directory: Path, **values: str) -> Path:
+    """Write a validator's config and the directories it names; the config's path."""
+    fields = {
+        "network": "test",
+        "netuid": "0",
+        "wallet_name": "validator",
+        "wallet_hotkey": "default",
+        "policy_python": "/path/to/policy-env/bin/python",
+        "simulator_python": "/path/to/simulator-env/bin/python",
+        "simulator_root": "/path/to/RoboTwin-Vector",
+        **values,
+    }
+    directory.mkdir(parents=True, exist_ok=True)
+    for made in ("var/state", "var/vector/store", "var/vector/runs", "var/vector/cache"):
+        (directory / made).mkdir(parents=True, exist_ok=True)
+    path = directory / CONFIG
+    if path.exists():
+        raise FileExistsError(f"{path} is already there; init will not write over a config")
+    path.write_text(VALIDATOR_TOML.format(**fields), encoding="utf-8")
+    return path
+
+
+def miner(directory: Path) -> str:
+    """Make a miner's working directory; what to run next."""
+    (directory / "submission").mkdir(parents=True, exist_ok=True)
+    return MINER_NEXT.format(directory=directory)