chialoops 1.0.0__tar.gz → 2.0.0__tar.gz

{chialoops-1.0.0 → chialoops-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chialoops
-Version: 1.0.0
+Version: 2.0.0
 Summary: The Chia agentic AI flow for hardware
 Author: UC Berkeley Architecture Research
 License-Expression: BSD-3-Clause
@@ -50,7 +50,7 @@ Dynamic: license-file
 <p align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="docs/_static/chia-logo-inv.png">
-    <img alt="CHIA" src="docs/_static/chia-logo.png" width="420"/>
+    <img alt="CHIA" src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/chia-logo.png" width="420"/>
   </picture>
 </p>
 
@@ -77,9 +77,18 @@ CHIA requires **Python 3.10.19** (matching the Python in the Docker images). Wit
 ```bash
 conda create -n chia_env python=3.10.19
 conda activate chia_env
-pip install -e /path/to/chia
+pip install chialoops
 ```
 
+If you are actively developing CHIA, install from a clone in editable mode instead:
+
+```bash
+git clone https://github.com/ucb-bar/chia.git
+pip install -e ./chia
+```
+
+See the [installation guide](https://docs.chialoops.ai/en/latest/getting-started/installation.html) for optional extras.
+
 ## Learn more about CHIA
 
 - **[CHIA Basics](https://docs.chialoops.ai/en/latest/getting-started/chia-basics.html)** — the core ideas, start here.
@@ -97,7 +106,7 @@ User guides:
 ## Overview of CHIA and Early Case Studies
 
 <p align="center">
-  <img src="docs/_static/overviewfig.png" alt="CHIA overview" width="820"/>
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/overviewfig.png" alt="CHIA overview" width="820"/>
 </p>
 
 ## Integrations & Ecosystem
@@ -105,23 +114,23 @@ User guides:
 CHIA workflows compose the tools you already use across the hardware/software co-design stack:
 
 <p align="center">
-  <img src="docs/_static/project-logos/claude-code.svg" alt="Claude Code" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/openai-codex.svg" alt="OpenAI Codex" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/github-copilot.svg" alt="GitHub Copilot" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/google-antigravity.svg" alt="Google Antigravity" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/alphaevolve.svg" alt="AlphaEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/openevolve.png" alt="OpenEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/adaevolve.svg" alt="AdaEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/skydiscover.png" alt="SkyDiscover" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/chisel.svg" alt="Chisel" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/chipyard.svg" alt="Chipyard" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/circt.svg" alt="CIRCT" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/hammer.png" alt="Hammer" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/gem5.png" alt="gem5" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/firesim.png" alt="FireSim" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/verilator.png" alt="Verilator" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/spike.svg" alt="Spike" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/champsim.svg" alt="ChampSim" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/claude-code.svg" alt="Claude Code" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/openai-codex.svg" alt="OpenAI Codex" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/github-copilot.svg" alt="GitHub Copilot" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/google-antigravity.svg" alt="Google Antigravity" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/alphaevolve.svg" alt="AlphaEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/openevolve.png" alt="OpenEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/adaevolve.svg" alt="AdaEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/skydiscover.png" alt="SkyDiscover" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/chisel.svg" alt="Chisel" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/chipyard.svg" alt="Chipyard" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/circt.svg" alt="CIRCT" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/hammer.png" alt="Hammer" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/gem5.png" alt="gem5" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/firesim.png" alt="FireSim" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/verilator.png" alt="Verilator" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/spike.svg" alt="Spike" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/champsim.svg" alt="ChampSim" height="40"/> &nbsp;&nbsp;
 </p>
 
 <p align="center">
@@ -129,7 +138,7 @@ CHIA workflows compose the tools you already use across the hardware/software co
 </p>
 
 <p align="center">
-  <img src="docs/_static/project-logos/ray.png" alt="Ray" height="40"/>
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/ray.png" alt="Ray" height="40"/>
 </p>
 
 ## Attribution

{chialoops-1.0.0 → chialoops-2.0.0}/README.md

@@ -1,7 +1,7 @@
 <p align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="docs/_static/chia-logo-inv.png">
-    <img alt="CHIA" src="docs/_static/chia-logo.png" width="420"/>
+    <img alt="CHIA" src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/chia-logo.png" width="420"/>
   </picture>
 </p>
 
@@ -28,9 +28,18 @@ CHIA requires **Python 3.10.19** (matching the Python in the Docker images). Wit
 ```bash
 conda create -n chia_env python=3.10.19
 conda activate chia_env
-pip install -e /path/to/chia
+pip install chialoops
 ```
 
+If you are actively developing CHIA, install from a clone in editable mode instead:
+
+```bash
+git clone https://github.com/ucb-bar/chia.git
+pip install -e ./chia
+```
+
+See the [installation guide](https://docs.chialoops.ai/en/latest/getting-started/installation.html) for optional extras.
+
 ## Learn more about CHIA
 
 - **[CHIA Basics](https://docs.chialoops.ai/en/latest/getting-started/chia-basics.html)** — the core ideas, start here.
@@ -48,7 +57,7 @@ User guides:
 ## Overview of CHIA and Early Case Studies
 
 <p align="center">
-  <img src="docs/_static/overviewfig.png" alt="CHIA overview" width="820"/>
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/overviewfig.png" alt="CHIA overview" width="820"/>
 </p>
 
 ## Integrations & Ecosystem
@@ -56,23 +65,23 @@ User guides:
 CHIA workflows compose the tools you already use across the hardware/software co-design stack:
 
 <p align="center">
-  <img src="docs/_static/project-logos/claude-code.svg" alt="Claude Code" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/openai-codex.svg" alt="OpenAI Codex" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/github-copilot.svg" alt="GitHub Copilot" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/google-antigravity.svg" alt="Google Antigravity" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/alphaevolve.svg" alt="AlphaEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/openevolve.png" alt="OpenEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/adaevolve.svg" alt="AdaEvolve" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/skydiscover.png" alt="SkyDiscover" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/chisel.svg" alt="Chisel" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/chipyard.svg" alt="Chipyard" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/circt.svg" alt="CIRCT" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/hammer.png" alt="Hammer" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/gem5.png" alt="gem5" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/firesim.png" alt="FireSim" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/verilator.png" alt="Verilator" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/spike.svg" alt="Spike" height="40"/> &nbsp;&nbsp;
-  <img src="docs/_static/project-logos/champsim.svg" alt="ChampSim" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/claude-code.svg" alt="Claude Code" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/openai-codex.svg" alt="OpenAI Codex" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/github-copilot.svg" alt="GitHub Copilot" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/google-antigravity.svg" alt="Google Antigravity" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/alphaevolve.svg" alt="AlphaEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/openevolve.png" alt="OpenEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/adaevolve.svg" alt="AdaEvolve" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/skydiscover.png" alt="SkyDiscover" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/chisel.svg" alt="Chisel" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/chipyard.svg" alt="Chipyard" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/circt.svg" alt="CIRCT" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/hammer.png" alt="Hammer" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/gem5.png" alt="gem5" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/firesim.png" alt="FireSim" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/verilator.png" alt="Verilator" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/spike.svg" alt="Spike" height="40"/> &nbsp;&nbsp;
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/champsim.svg" alt="ChampSim" height="40"/> &nbsp;&nbsp;
 </p>
 
 <p align="center">
@@ -80,7 +89,7 @@ CHIA workflows compose the tools you already use across the hardware/software co
 </p>
 
 <p align="center">
-  <img src="docs/_static/project-logos/ray.png" alt="Ray" height="40"/>
+  <img src="https://raw.githubusercontent.com/ucb-bar/chia/main/docs/_static/project-logos/ray.png" alt="Ray" height="40"/>
 </p>
 
 ## Attribution

chialoops-2.0.0/chia/aws/__init__.py

@@ -0,0 +1,3 @@
+# Ray resource a worker advertises when it holds AWS credentials (e.g. an IAM
+# role); S3 uploads such as ``publish`` require it.
+AWS_CREDS_RESOURCE = "aws_creds"

chialoops-2.0.0/chia/aws/manager.py

@@ -0,0 +1,180 @@
+"""Bring a given EC2 node up as a Chia worker at runtime, and take it down.
+
+The node is given, not discovered: a ``(NodeTypeConfig, AWSNodeConfig)`` pair,
+the same classes a cluster file's node type and ``aws_nodes:`` entry parse
+into. Each step is the code ``chia up`` runs for AWS nodes:
+
+    provision_aws_nodes     launch the instances
+    run_aws_setup           install docker (and chia's other defaults), synchronously
+    add_nodes_to_cluster    ssh tunnel, container, ray start
+
+With ``connection: tunnel``, the tunnel carries every Ray connection between the
+worker and the head, so the head needs no inbound ports and may sit anywhere. Its
+ports are read from the running cluster rather than from the cluster file. With
+``connection: vpc``, the default, they use private IPs in the head's VPC instead.
+
+AWSManager runs as a Ray actor on the head (:func:`start_aws_manager`): the
+tunnels are processes on the head, and the head raylet's ports are read from
+its ``/proc``.
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+from dataclasses import dataclass, replace
+
+import ray
+
+from chia.aws.config import AWSConfig
+from chia.aws.ec2 import get_default_ami, terminate_ec2_instances
+from chia.base.ChiaFunction import chia_actor
+from chia.cluster.aws_nodes import AWSNodeConfig, provision_aws_nodes, run_aws_setup
+from chia.cluster.config import (ClusterConfig, NodeTypeConfig, SSHAuthConfig,
+                                 TunnelConfig, assign_nodes)
+from chia.cluster.log import get_logger, setup_logging
+from chia.cluster.node_setup import add_nodes_to_cluster
+
+logger = get_logger("aws.manager")
+
+AWSWorker = tuple[NodeTypeConfig, AWSNodeConfig]
+
+
+def worker_resources(worker: AWSWorker) -> dict[str, float]:
+    # The resources that a machine of this definition advertises when it joins the cluster.
+    return dict(worker[0].resources)
+
+
+@dataclass
+class Farm:
+    """The instances one :meth:`AWSManager.launch` brought up, and that manager."""
+    name: str
+    region: str
+    ips: list[str]
+    manager: "ray.actor.ActorHandle | None" = None
+
+    def teardown(self) -> None:
+        """Close the farm's tunnels and terminate its instances, through its manager."""
+        ray.get(self.manager.teardown.remote(self))
+
+
+class AWSManager:
+    """Adds a given EC2 node to a running cluster, and removes it."""
+
+    def __init__(self, cluster_config: ClusterConfig, aws_config: AWSConfig):
+        """
+        Args:
+            cluster_config: The running cluster the node joins.
+            aws_config: Account values the node definition leaves out: region,
+                EC2 key pair, and the ssh user and key for it.
+        """
+        setup_logging()        # an actor process has no log handler; chia's CLIs set one up
+        self.cluster_config = cluster_config
+        self.aws_config = aws_config
+        self._tunnels = {}     # farm ips -> the TunnelManager carrying their Ray traffic
+        if cluster_config.connection not in ("tunnel", "vpc"):
+            raise ValueError(f"AWSManager supports connection tunnel or vpc, not {cluster_config.connection}")
+        self._vpc_id = cluster_config.aws_config.vpc_id if cluster_config.aws_config else None
+        # The head's private IP with connection "vpc"; None means SSH tunnels.
+        self._vpc_head_ip = cluster_config.head_ip if cluster_config.connection == "vpc" else None
+
+    def launch(self, worker: AWSWorker, count: int = 1) -> Farm:
+        """Bring up ``count`` instances of ``worker`` and join them to the cluster."""
+        node_type, machine = self._configured(worker)
+        aws = self.aws_config
+        machine = replace(machine, count=count, KeyName=aws.key_name,
+                          ssh_user=aws.ssh_user, ssh_private_key=aws.ssh_private_key,
+                          ImageId=machine.ImageId or get_default_ami(aws.region))
+        nodes = {node_type.name: machine}
+
+        ips = provision_aws_nodes(self.cluster_config.cluster_name, nodes, aws.region,
+                                  self._vpc_id, self._vpc_head_ip)[node_type.name]
+        farm = Farm(node_type.name, aws.region, ips, ray.get_runtime_context().current_actor)
+        try:
+            # A copy: this launch's machines stay out of the manager's config.
+            config = copy.deepcopy(self.cluster_config)
+            tunnel = None if self._vpc_head_ip else self._head_tunnel_config()
+            for ip in ips:
+                config.auth_overrides[ip] = SSHAuthConfig(
+                    ssh_user=aws.ssh_user, ssh_private_key=aws.ssh_private_key,
+                    tunnel=tunnel)
+            run_aws_setup(nodes, {node_type.name: ips}, config.get_ssh_auth)
+
+            # add_nodes_to_cluster allocates tunnels across the whole config, keyed
+            # by assign_nodes' worker indexes, so the node joins the copy first
+            # and its assignments come from assign_nodes.
+            config.worker_ips = config.worker_ips + ips
+            # The container's AWS calls (F2_VIVADO's aws_create_afi) need a region.
+            docker = node_type.docker and replace(node_type.docker, run_options=[
+                *node_type.docker.run_options, "-e", f"AWS_DEFAULT_REGION={aws.region}"])
+            config.node_types[node_type.name] = replace(
+                node_type, num_workers=count, compatible_ips=ips, docker=docker)
+            self._tunnels[tuple(ips)] = add_nodes_to_cluster(
+                config, [a for a in assign_nodes(config) if a.ip in ips])
+        except Exception:
+            self.teardown(farm)
+            raise
+        logger.info(f"{len(ips)} '{node_type.name}' worker(s) joined")
+        return farm
+
+    def _configured(self, worker: AWSWorker) -> AWSWorker:
+        """``worker`` with the changes of its entry in the cluster file's ``aws: workers:``."""
+        node_type, machine = worker
+        aws = self.cluster_config.aws_config
+        changes = dict(aws.workers.get(node_type.name, {})) if aws else {}
+        if "image" in changes:
+            node_type = replace(node_type, docker=replace(node_type.docker, image=changes.pop("image")))
+        return node_type, replace(machine, **changes)
+
+    def teardown(self, farm: Farm) -> None:
+        """Close the farm's tunnels and terminate its instances."""
+        # TODO: tunnels are kept per launch, so a farm with part of a launch's machines
+        # (one FPGA taken down early) closes none; each exits when its machine goes.
+        tunnels = self._tunnels.pop(tuple(farm.ips), None)
+        if tunnels is not None:
+            tunnels.stop_all()
+        if farm.ips:
+            import boto3
+
+            # With connection "vpc", the farm's IPs are private ones.
+            address = "private-ip-address" if self._vpc_head_ip else "ip-address"
+            reservations = boto3.client("ec2", region_name=farm.region).describe_instances(
+                Filters=[{"Name": address, "Values": farm.ips}])["Reservations"]
+            ids = [i["InstanceId"] for r in reservations for i in r["Instances"]]
+            if ids:
+                logger.info(f"Terminating {len(ids)} '{farm.name}' instance(s)")
+                terminate_ec2_instances(ids, region=farm.region)
+
+    def _head_tunnel_config(self) -> TunnelConfig:
+        """Tunnel ports for the running head, all read from the running cluster:
+        GCS and raylet ports as Ray reports them, and the worker-port range from
+        the head raylet's arguments (this runs on the head)."""
+        gcs_port = int(ray.get_runtime_context().gcs_address.rsplit(":", 1)[1])
+        head = next(n for n in ray.nodes() if "node:__internal_head__" in n["Resources"])
+        # Sorted: the workers' iptables rule takes the two as one low:high range.
+        low, high = sorted((head["NodeManagerPort"], head["ObjectManagerPort"]))
+        args = next(a for a in (_cmdline(p) for p in os.listdir("/proc") if p.isdigit())
+                    if f"--node_id={head['NodeID']}" in a)
+        worker = {k: int(next(a.split("=", 1)[1] for a in args
+                              if a.startswith(f"--{k}_worker_port=")))
+                  for k in ("min", "max")}
+        return TunnelConfig(gcs_tunnel_port=gcs_port,
+                            head_node_manager_port=low, head_object_manager_port=high,
+                            head_worker_port_min=worker["min"],
+                            head_worker_port_max=worker["max"])
+
+
+def start_aws_manager(cluster_config: ClusterConfig, aws_config: AWSConfig):
+    """Start an :class:`AWSManager` actor on the head and return its handle."""
+    # TODO: read the cluster file that `chia up` used, so loops need not pass it.
+    actor = ray.remote(AWSManager).options(
+        num_cpus=0, resources={"node:__internal_head__": 0.001})
+    return chia_actor(actor.remote(cluster_config, aws_config))
+
+
+def _cmdline(pid: str) -> list[str]:
+    try:
+        with open(f"/proc/{pid}/cmdline", "rb") as f:
+            return f.read().decode().split("\0")
+    except OSError:
+        return []

{chialoops-1.0.0 → chialoops-2.0.0}/chia/aws/s3.py

@@ -21,6 +21,7 @@ drop down to the raw boto3 client via ``node._client``.
 
 from __future__ import annotations
 
+import io
 import logging
 import time
 from dataclasses import dataclass
@@ -206,7 +207,13 @@ class S3Node:
         )
 
     def put_bytes(self, key: str, data: bytes, content_type: str | None = None) -> None:
-        """Write ``data`` to ``key``."""
+        """Write ``data`` to ``key``, in parts above 1 GiB (one PUT takes 5 GB at most)."""
+        if len(data) > 2**30:
+            extra = {"ContentType": content_type} if content_type is not None else {}
+            self._call(f"upload_fileobj s3://{self.bucket}/{key}",
+                       lambda: self._client.upload_fileobj(io.BytesIO(data), self.bucket, key,
+                                                           ExtraArgs=extra))
+            return
         kwargs: dict[str, Any] = {"Bucket": self.bucket, "Key": key, "Body": data}
         if content_type is not None:
             kwargs["ContentType"] = content_type

{chialoops-1.0.0 → chialoops-2.0.0}/chia/base/ChiaFunction.py

@@ -384,6 +384,18 @@ def _try_proxy(func, opts, bypass_state, hooks, args, kwargs, display_name):
         func, full_opts, bypass_state, hooks, profile, args, kwargs)
 
 
+def _setup_worker_logging() -> None:
+    """Show a task's INFO logs: a Ray worker has no log handler, so Python drops
+    everything below WARNING. Adds one handler per worker."""
+    root = logging.getLogger()
+    if not root.handlers:
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.INFO)
+        handler.setFormatter(logging.Formatter("%(levelname)s [%(name)s] %(message)s"))
+        root.addHandler(handler)
+        root.setLevel(logging.INFO)
+
+
 def _chia_trampoline(func, _chia_bypass_state_, _chia_hooks_, *args, **kwargs):
     """Trampoline function used by ChiaFunction to dispatch remote calls.
 
@@ -393,6 +405,7 @@ def _chia_trampoline(func, _chia_bypass_state_, _chia_hooks_, *args, **kwargs):
     callables. Both are attached by chia_remote() and named with underscores to
     avoid collisions with user function parameters.
     """
+    _setup_worker_logging()
     _restore_bypass(_chia_bypass_state_)
     from chia.base.pid_registry import _pid_tracking_scope
     _run_chia_setup(_chia_hooks_)
@@ -410,6 +423,7 @@ def _chia_trampoline_profiled(func, call_id, dispatch_meta, _chia_bypass_state_,
     event at function end, both from the worker.  Returns a
     ``_ProfiledResult`` that ``get()`` unwraps transparently.
     """
+    _setup_worker_logging()
     _restore_bypass(_chia_bypass_state_)
     import time as _time
     from chia.trace.profiler import _ProfiledResult, get_profiler

chialoops-2.0.0/chia/base/chia_log.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from chia.base.ChiaFunction import ChiaFunction, get
+
+HEAD = "node:__internal_head__"
+
+
+@ChiaFunction(num_cpus=0)
+def _append(data: bytes, path: str) -> str:
+    if path.startswith("s3://"):
+        from chia.aws.s3 import S3Node
+        bucket, key = path.removeprefix("s3://").split("/", 1)
+        s3 = S3Node(bucket)
+        s3.put_bytes(key, (s3.get_bytes(key) if s3.exists(key) else b"") + data)
+    else:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "ab") as f:
+            f.write(data)
+    return path
+
+
+def chia_log(data: bytes | str, path: str, resource: str | None = None) -> str:
+    """Add ``data`` to the end of ``path`` on a machine with ``resource``, the head when
+    ``None``, and return ``path``. A path that starts with ``s3://bucket/key`` is an S3
+    object, written with that machine's AWS credentials.
+
+    Do not pass a resource that the calling task holds: the write would wait for that task
+    to end. Writing on the caller's own machine needs a local option, not supported yet.
+    """
+    if isinstance(data, str):
+        data = data.encode()
+    return get(_append.options(resources={resource or HEAD: 0.001}).chia_remote(data, path))

{chialoops-1.0.0 → chialoops-2.0.0}/chia/chipyard/chisel_build_node.py

@@ -4,10 +4,13 @@ import re
 import shlex
 import subprocess
 import logging
+import uuid
+from datetime import datetime
 from chia.chipyard.state_def import BuildArtifact, BuildTarget, SpikeBuildArtifact
 from chia.chipyard.spike_build_node import STATIC_ARCHIVES, stage_golden_model
 from chia.trace.profiler import get_profiler
 from chia.base.ChiaFunction import ChiaFunction
+from chia.base.chia_log import chia_log
 
 # Maximum number of WF_SCOPES paths supported by the Makefile and TestDriver.v
 # (the .v file has WF_SCOPE_1 ... WF_SCOPE_8 ifdef branches).
@@ -51,6 +54,8 @@ class ChiselBuildNode:
         wf_scopes: list[str] | tuple[str, ...] = (),
         clean_wf_stamp: bool = True,
         logging_level: int = logging.DEBUG,
+        log_dir: str | None = None,
+        logging_resource: str | None = None,
         name: str = "chipyard",
     ):
         """Configure a single Chipyard Verilator build.
@@ -122,6 +127,10 @@ class ChiselBuildNode:
                 re-applies the scope filter even when it would otherwise
                 consider nothing changed (mirrors ``run_wave.sh``).
             logging_level: Python logging level for this node's logger.
+            log_dir: A folder, or an ``s3://`` prefix, for this node's build
+                logs. ``None`` keeps none; the result has them either way.
+            logging_resource: The resource of the machine that writes into
+                ``log_dir``; ``None`` means the head.
             name: Label copied into the resulting ``BuildArtifact.name``;
                 purely for downstream identification/traceability.
         """
@@ -143,6 +152,8 @@ class ChiselBuildNode:
         self.wf_scopes = list(wf_scopes)
         self.clean_wf_stamp = clean_wf_stamp
         self.logging_level = logging_level
+        self.log_dir = log_dir
+        self.logging_resource = logging_resource
         self.name = name
         self.logger = logging.getLogger(self.logging_name)
         self.logger.setLevel(self.logging_level)
@@ -300,6 +311,24 @@ class ChiselBuildNode:
             needed.append((name, path if path and path != "not found" else None))
         return needed
 
+    def _log(self, cmd: str, sims_dir: str, returncode: int,
+             stdout: str, stderr: str) -> None:
+        """Write this build's output to its own file in ``log_dir``, when
+        ``log_dir`` is set. One file per call, so parallel builds never share
+        a file."""
+        if not self.log_dir:
+            return
+        name = f"{self.config}_{datetime.now().strftime('%Y%m%d-%H%M%S')}_{uuid.uuid4().hex[:8]}.log"
+        chia_log(
+            "=" * 80 + "\n"
+            f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {self.config} "
+            f"returncode={returncode}\n"
+            + "=" * 80 + "\n\n"
+            f"[Command] {cmd} (in {sims_dir})\n\n"
+            f"[stdout]\n{stdout}\n\n"
+            f"[stderr]\n{stderr}\n",
+            f"{self.log_dir}/{name}", self.logging_resource)
+
     @ChiaFunction(resources={"chipyard": 1})
     def build(self) -> BuildArtifact:
         """Run the Chipyard Make flow and return the compiled simulator.
@@ -437,6 +466,7 @@ class ChiselBuildNode:
             self.logger.info(f"Build timed out after {self.timeout_seconds} seconds")
             stdout = e.stdout.decode(errors="replace") if e.stdout else ""
             stderr = e.stderr.decode(errors="replace") if e.stderr else ""
+            self._log(cmd, sims_dir, -1, stdout, stderr + "\nBuild timed out")
             return BuildArtifact(
                 name=self.name,
                 simulator_binary_content=b"",
@@ -453,6 +483,7 @@ class ChiselBuildNode:
         if result.returncode != 0:
             self.logger.info(f"Build failed stderr: {result.stderr[-500:]}")
             self.logger.info(f"Build failed stdout: {result.stdout[-500:]}")
+            self._log(cmd, sims_dir, result.returncode, result.stdout, result.stderr)
             return BuildArtifact(
                 name=self.name,
                 simulator_binary_content=b"",
@@ -476,6 +507,7 @@ class ChiselBuildNode:
         runtime_libs = (self._collect_runtime_libs(binary_path)
                         if self.bundle_runtime_libs else [])
 
+        self._log(cmd, sims_dir, result.returncode, result.stdout, result.stderr)
         return BuildArtifact(
             name=self.name,
             simulator_binary_content=binary_content,

{chialoops-1.0.0 → chialoops-2.0.0}/chia/chipyard/firemarshal_node.py

@@ -144,6 +144,7 @@ class FireMarshalNode:
         work_dir: str = "/tmp/fm",
         config: "dict | None" = None,
         rootfs_size_mib: int = 0,
+        overlay_modes: "dict[str, int] | None" = None,
     ) -> FireMarshalArtifact:
         """Compose ``overlay_files`` onto the stored base ``base_name`` and return
         the workload image(s) + FireSim descriptor as sparse-tar bytes.
@@ -161,10 +162,13 @@ class FireMarshalNode:
             config: Workload options as a dict (the marshal-config run knobs):
                 ``command`` (str, baked as the boot run-script), ``jobs`` (list of
                 ``{"name","command","outputs"}`` — one image per job, emitted as the
-                FireSim descriptor's ``workloads``), ``outputs`` (default guest
+                FireSim descriptor's ``workloads``; a job's ``files``, keys of
+                ``overlay_files``, limit its image to them), ``outputs`` (default guest
                 paths), ``post_run_hook`` (str), ``simulation_outputs`` (default
                 ``["uartlog"]``).
             rootfs_size_mib: Grow the rootfs before composing (``0`` keeps base size).
+            overlay_modes: Permission bits of ``overlay_files``, same keys (e.g.
+                ``0o755`` for an executable). A file without one gets the default.
         """
         config = config or {}
         json_path = os.path.join(self._dir(name), f"{name}.json")
@@ -175,7 +179,8 @@ class FireMarshalNode:
                     b"", "", "", f"{name}.json", success=False, stdout="",
                     stderr=f"base '{base_name}' not found in {self.image_store}", returncode=1)
             returncode, stdout, stderr = self._compose_workload(
-                base_name, name, overlay_files, config, rootfs_size_mib, work_dir)
+                base_name, name, overlay_files, overlay_modes or {}, config,
+                rootfs_size_mib, work_dir)
 
         archive = b""
         if returncode == 0 and os.path.isfile(json_path):
@@ -259,12 +264,11 @@ class FireMarshalNode:
 
     # ---- compose internals --------------------------------------------------
 
-    def _compose_workload(self, base_name, name, overlay_files, config,
+    def _compose_workload(self, base_name, name, overlay_files, overlay_modes, config,
                           rootfs_size_mib, work_dir) -> "tuple[int, str, str]":
         os.makedirs(work_dir, exist_ok=True)
         work = os.path.join(work_dir, f".fm-{uuid.uuid4().hex[:8]}")
         overlay_dir = os.path.join(work, "overlay")
-        self._write_overlay(overlay_files, overlay_dir)
         dst = self._dir(name)
         os.makedirs(dst, exist_ok=True)
         base_img = os.path.join(self._dir(base_name), f"{base_name}.img")
@@ -277,7 +281,13 @@ class FireMarshalNode:
                    else [(name, config.get("command"), config.get("outputs", []))])
 
         stdout, stderr = "", ""
-        for target, command, _outs in targets:
+        written = None                     # the overlay files now in overlay_dir
+        for (target, command, _outs), job in zip(targets, jobs or [{}]):
+            files = job.get("files", list(overlay_files))
+            if files != written:
+                shutil.rmtree(overlay_dir, ignore_errors=True)
+                self._write_overlay({p: overlay_files[p] for p in files}, overlay_modes, overlay_dir)
+                written = files
             self._set_run_script(overlay_dir, command)
             self.logger.info(f"Composing {target} from base {base_name}")
             out, err, rc = self._run(
@@ -308,14 +318,18 @@ class FireMarshalNode:
             os.remove(path)
 
     @staticmethod
-    def _write_overlay(overlay_files: dict[str, bytes], overlay_dir: str) -> None:
-        """Materialize overlay_files into overlay_dir, mirroring the rootfs layout."""
+    def _write_overlay(overlay_files: dict[str, bytes], overlay_modes: dict[str, int],
+                       overlay_dir: str) -> None:
+        """Materialize overlay_files into overlay_dir, mirroring the rootfs layout,
+        with the permission bits in overlay_modes."""
         os.makedirs(overlay_dir, exist_ok=True)
         for rel_path, content in overlay_files.items():
             dest = os.path.join(overlay_dir, rel_path)
             os.makedirs(os.path.dirname(dest) or overlay_dir, exist_ok=True)
             with open(dest, "wb") as f:
                 f.write(content)
+            if rel_path in overlay_modes:
+                os.chmod(dest, overlay_modes[rel_path])
 
     @staticmethod
     def _write_descriptor(path: str, name: str, config: dict, targets: list) -> None: