letify 2.8.7__tar.gz → 2.10.0__tar.gz

{letify-2.8.7 → letify-2.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.8.7
+Version: 2.10.0
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs
@@ -492,6 +492,8 @@ letify setup tailcat          # install tailcat or eci from its publisher ahead
 letify providers              # who is declared, storage, channel kind
 letify devices                # what each one offers
 letify status                 # what is running right now
+letify sessions               # what each account holds, workers on a shell host
+letify stop --orphans         # end what a dead process left behind
 letify usage                  # what is left on each account
 letify utilization            # how busy each instance's GPU is
 letify check lab              # does this machine answer?

{letify-2.8.7 → letify-2.10.0}/PROJECT.md

@@ -190,6 +190,22 @@ cache.fetch_checkpoint("run-1", target)
 cache.resume(runtime, "run-1", path)   # put the newest one inside a session
 ```
 
+### Exit hooks
+
+```python
+letify.on_exit(fn)                    # inside a body: run fn if the worker ends before the body does
+```
+
+The hook runs on `SIGTERM`, when the lease deadline passes and when the channel closes during the call, newest first, within 120 s in total. Outside a body it raises `UnsupportedMode`. `letify runs`, `interrupt` and `exec` reach a running call from the command line; see [Controlling a running call](docs/SPEC.md#run-control).
+
+### Taking over a running call
+
+```python
+value = letify.takeover(run)         # become the owner of a running call, return its value
+```
+
+`letify.takeover(run)` and `letify takeover <run>` make the calling process the owner of a call another process started: it renews the lease, receives the output and the result, writes the call's outputs back under the project the call was sent from, and ends the session. The previous owner, if it is alive, is asked to let go and its waiting call raises `OwnershipMoved`. A provider whose runtime a second process cannot reach raises `UnsupportedMode` naming it; see [Taking over a running call](docs/SPEC.md#run-takeover).
+
 ### Handoffs
 
 ```python
@@ -225,6 +241,7 @@ Both work in the local process and inside a shipped function, on any provider. T
 | `ProtocolError` | The remote process died before producing a result | yes |
 | `RemoteError` | The shipped function raised; carries the remote traceback | no |
 | `UnsupportedMode` | The requested mode cannot work here | no |
+| `OwnershipMoved` | The call you were waiting on was taken over by another process, which now receives its result | no |
 | `InsufficientDevices` | The devices a call asks for cannot be allocated; names what holds them | no |
 
 ### Command line
@@ -237,6 +254,10 @@ letify devices                # accelerators each provider offers
 letify status                 # live sessions and what they are costing
 letify usage [alias]          # what is left on each account
 letify utilization [alias]    # how busy each instance's accelerator is
+letify runs                   # calls running from this machine, with their run ids
+letify interrupt <run>        # raise KeyboardInterrupt in a running call's body
+letify exec <run> CODE        # run source beside a running call (-f FILE, --timeout S)
+letify takeover <run>         # become the owner of a running call (--result PATH, --json)
 letify check <alias>          # confirm a machine answers
 letify efficiency 0.5 3 150   # expected share of a direct run
 letify stubs                  # write typings/letify_providers.pyi for editor completion

{letify-2.8.7 → letify-2.10.0}/README.md

@@ -467,6 +467,8 @@ letify setup tailcat          # install tailcat or eci from its publisher ahead
 letify providers              # who is declared, storage, channel kind
 letify devices                # what each one offers
 letify status                 # what is running right now
+letify sessions               # what each account holds, workers on a shell host
+letify stop --orphans         # end what a dead process left behind
 letify usage                  # what is left on each account
 letify utilization            # how busy each instance's GPU is
 letify check lab              # does this machine answer?

{letify-2.8.7 → letify-2.10.0}/docs/SPEC.md

@@ -320,7 +320,7 @@ letify creates one notebook per run, with `CreateKernelWithSettings`, and never
 
 `Kaggle.sessions()` returns the recorded refs that the account still has, checked against the official CLI, `kaggle kernels list --mine --csv`, authenticated by the account's API token in `KAGGLE_API_TOKEN` and never by the cookie, which keeps the boundary this section states. A ref the record names and the account no longer has was deleted elsewhere, and is dropped from the record rather than reported. This matters because a process that dies before `stop` leaves its notebook and its run behind, and that notebook holds weekly accelerator quota with nothing pointing at it: `letify sessions` is how it is found and `kaggle kernels delete` how it is removed. Discovery needs the API token and uv, and the listing is empty rather than failing when either is missing, when the command exits non-zero, or when its output does not parse, because reading what exists must not fail a command that is asking what exists. A notebook created before letify kept this record is not in it and so is not reported.
 
-Whether an account can be asked at all is the provider's own answer, `discovers_sessions`, rather than the launcher naming a provider kind. `colab` and `kaggle` answer yes; every other kind reports that discovery is not supported, which is what `letify sessions` prints for it.
+Whether an account can be asked at all is the provider's own answer, `discovers_sessions`, rather than the launcher naming a provider kind. `colab`, `kaggle` and `shell` answer yes, the last as [Workers on a shell host](#shell-workers) describes. `tunnel` and `elice` answer no, because reaching their machine runs the connection pipeline or allocates a machine, and listing must do neither; every other kind reports that discovery is not supported, which is what `letify sessions` prints for it.
 
 **Not making sessions faster than Kaggle allows.** Creating and ending interactive sessions in quick succession makes Kaggle refuse the next run, and nothing letify does recovers it except waiting. The refusal has no message: `CommitAndRun` answers without a `kernelRunId`, which letify reports as `Kaggle did not start a session run`.
 
@@ -364,6 +364,24 @@ The accelerators letify offers are the fixed set `discover` returns, `CPU` and `
 
 `Shell` and its subclasses default to ephemeral because a machine's disk policy is not knowable in advance. Assuming ephemeral costs time, since letify rebuilds the environment each runtime and the work still succeeds; assuming persistent fails outright when the disk turns out to be wiped. A configuration entry overrides it with `persistent = true`.
 
+### Provider session discovery <!-- id: provider-session-discovery -->
+
+> `letify sessions` lists what an account already holds, and `letify stop` ends what a dead process left behind, for every provider that can be asked.
+
+Whether a provider can be asked is its own answer, `discovers_sessions`. `sessions()` returns one readable line per session, `orphans()` returns the refs of those whose owner is gone, and `stop_orphan(ref)` ends one and reports whether it is gone. Kaggle and Colab answer through their accounts, as their sections describe. A provider that cannot be asked prints `session discovery is not supported`.
+
+#### Workers on a shell host <!-- id: shell-workers -->
+
+> A `Shell` machine answers by listing the letify workers running on it, because the machine has no account to ask.
+
+1. **The worker records itself.** After the environment is built and the worker has moved to the project interpreter, the client sends the op `register` with the workspace root, the runtime name, the pool key and its own host name and pid. The worker writes `<workspace root>/.letify/workers/<worker pid>.json` and removes it when it ends, including when it ends through the lease or `SIGTERM`. The directory has mode 0700 and the file mode 0600, and the file is replaced whole so a reader never sees half of it.
+2. **The record.** `pid`, `proc_start` (the process start time in clock ticks since boot, field 22 of `/proc/<pid>/stat`, or null where `/proc` is missing), `started` (UTC, ISO 8601), `runtime`, `key`, `devices` (the worker's `CUDA_VISIBLE_DEVICES`, or null when unset), `owner` (`host` and `pid` of the local process that opened the session), `run` (the run id of the call being served, or null) and `lease_deadline` (Unix seconds on the host, rewritten at each lease renewal, or null when no lease is armed). `run` is set when the second door opens and cleared when the call ends. A provider whose class sets `records_workers` to false writes no record. That is every provider but `Shell` and its subclasses reached over a persistent channel.
+3. **Listing.** `Shell.sessions()` runs one program on the host over the account's link. It reads the records, drops each whose pid is not a live worker and deletes that file, and prints the rest. A pid is a live worker when a process with that pid runs and its start time equals `proc_start`, which keeps a reused pid from reading as a worker; with no `proc_start` in the record, when the pid answers signal 0. Each line reads `pid <pid>  gpu <indices or none>  started <time>  runtime <name>  key <key>  run <id or none>  owner <host>:<pid> <alive|gone|remote>`. An unreachable host raises, as every other command on the account does, rather than listing nothing.
+4. **Owner state.** `alive` and `gone` are judged only for an owner whose host is this machine's host name: `gone` when that pid does not run. An owner on another host is `remote` and is never judged, because its liveness cannot be read from here.
+5. **Orphans.** A worker is an orphan when its `lease_deadline` has passed on the host, or when its owner is `gone` and no call is running (`run` is null). A worker whose owner is `gone` while a call runs and whose lease is in the future is not an orphan: it can be taken over, see [Taking over a running call](#run-takeover), and stopping it would end a training run that can still be saved. `letify stop --orphans` never touches a worker whose owner is `alive` or `remote` unless its lease has passed.
+6. **Stopping one.** `Shell.stop_orphan(pid)` and `letify stop --alias <alias> --worker <pid>` run one program on the host that reads the record for that pid, refuses a pid with no record or no live worker, sends `SIGTERM`, waits up to 30 s (`STOP_GRACE_S`) for the worker to end, then sends `SIGKILL`, and answers whether the pid is gone. A pid with no record is refused, so the command cannot end a process that is not a letify worker. `--worker` stops what is named whatever its owner state, so it asks for confirmation unless `-y` is given.
+7. **Colab keeps its own answers.** `Colab` inherits `Shell` for its link, but sets `records_workers` to false and lists, orphans and stops through its CLI as before.
+
 ### Remaining usage
 
 > Every provider is asked what is left on its account, reads the answer from the service itself, and says why when it cannot.
@@ -818,7 +836,7 @@ A sandbox that reaches that lifetime takes its worker with it, so the channel se
 
 Otherwise a Modal worker that stopped without replying is followed by one `poll` of its sandbox from a second adapter process, waiting at most 30 s for the exit code, and the failure names what the poll found instead of listing causes: `the Modal sandbox exited with code <n>` for a sandbox that ended, with `, which is SIGKILL, most often the container running out of memory` added for 137, and `the Modal sandbox is still running, so the worker process died inside it` for one still running. A poll that fails or times out leaves the message as it was. The message is what tells an out-of-memory kill from a preemption, which a training run that died twice in one day could not.
 
-A worker reached through a subprocess (`PersistentChannel`, for example `ssh` to a Colab or Kaggle machine) that exits before it says hello, or while a call is running, has its exit code named in the message text, so a user can tell a lost connection from a kill from a crash. A signal death is reported as the shell does, 128 plus the signal number. The message reads `the worker exited with code <n>` followed by a short meaning: `255` adds `: ssh could not reach or lost the machine`, `128+N` adds `: killed by signal N`, with `137` adding `, which is SIGKILL, most often out of memory`, and any other nonzero code adds nothing. A worker that never said hello reads `the worker exited before it was ready, with code <n>` followed by the same meaning. The last 2000 characters of the process's standard error follow under `--- last remote output ---`, and the startup failure also keeps the whole text as its `stderr` attribute. A channel whose provider can name a cause itself, such as Modal, keeps that cause. When the command's program is `ssh` or `ssh.exe`, `PersistentChannel` starts it as `ssh -E <file> ...`, with a fresh temporary file per channel placed right after the program. `ControlPersist` forks a master whose standard error is /dev/null, so its errors (`Timeout, server not responding`, `Connection reset`, `closed by remote host`, `Broken pipe`) reach only that file. A death message, both before hello and during a call, appends the last 2000 characters of the file under `--- ssh log ---` when it is not empty, after the standard error of the ssh client. Other commands are not changed. The file is deleted when the channel closes.
+A worker reached through a subprocess (`PersistentChannel`, for example `ssh` to a Colab or Kaggle machine) that exits before it says hello, or while a call is running, has its exit code named in the message text, so a user can tell a lost connection from a kill from a crash. A signal death is reported as the shell does, 128 plus the signal number. The message reads `the worker exited with code <n>` followed by a short meaning: `255` adds `: ssh could not reach or lost the machine`, `128+N` adds `: killed by signal N`, with `137` adding `, which is SIGKILL, most often out of memory`, and any other nonzero code adds nothing. A worker that never said hello reads `the worker exited before it was ready, with code <n>` followed by the same meaning. The last 2000 characters of the process's standard error follow under `--- last remote output ---`, and the startup failure also keeps the whole text as its `stderr` attribute. A channel whose provider can name a cause itself, such as Modal, keeps that cause. When the command's program is `ssh` or `ssh.exe`, `PersistentChannel` starts it as `ssh -E <file> ...`, with a fresh temporary file per channel placed right after the program. The ssh of a worker channel is never a client of a shared connection (see "SSH authentication"), so its own transport errors (`Connection to <host> closed by remote host.`, `Timeout, server not responding`, `Broken pipe`) reach its standard error and this file. A death message, both before hello and during a call, appends the last 2000 characters of the file under `--- ssh log ---` when it is not empty, after the standard error of the ssh client. Other commands are not changed. The file is deleted when the channel closes.
 
 ### Modal data channel <!-- id: modal-data-channel -->
 
@@ -1034,6 +1052,71 @@ Each runtime also reports `price_type`, as [Price type](#elice-price-type) descr
 
 `status()` describes this process only. A session started by a different process is not in it, since the pool lives in the process that owns it. What a machine itself is doing is a different question, answered by `letify utilization`.
 
+## Controlling a running call <!-- id: run-control -->
+
+> A running call can be interrupted, inspected and changed from outside, the way a local process can, without the declaration growing a handle.
+
+A call on a remote runtime is a program running somewhere else. On a laptop the person who started a training loop can press Ctrl-C, attach a debugger or read a variable; letify gives the same reach to a call, so a run that turns out to save no checkpoint can still be saved. None of this adds a value to user code: a call is addressed from the command line by a run id, the way a local process is addressed by its pid, and the session lifetime in [Lifetime](#lifetime) is unchanged.
+
+### Run ids and the registry
+
+Every call on a runtime gets a run id, 12 lowercase hex characters, when it is sent. The owning process writes `~/.letify/runs/<run id>.json` before the call is sent and removes it when the call returns or raises. The record holds `run`, `function` (the qualified name), `runtime`, `provider` (the account alias), `started_at` (Unix seconds), `pid` (the owning process) and `socket`, the owner's control socket. The directory is created with mode 0700 and each file with mode 0600.
+
+Each owning process listens on one Unix socket, `~/.letify/runs/<pid>.sock`, created on the first call and removed at exit. A command reaches a run by reading its record and connecting to that socket; the owner forwards the request to the runtime over the call's own channel. When `~/.letify/runs/<pid>.sock` would exceed 100 bytes, the limit of a Unix socket path, the socket is created as `run-<pid>.sock` in the SSH control directory of [Declared filesystem paths](#declared-paths) instead, and `socket` in the record names it. A record whose `pid` no longer exists is stale: it is removed when read and the run is reported as not found. On Windows there is no control socket, and the commands below say so.
+
+The request travels as one JSON object per line on the socket and the answer as one JSON object per line, so a tool that is not letify can drive it.
+
+### Commands
+
+Every command takes `--json`, which prints one JSON object instead of text. The exit status is 0 on success, 2 for a usage error, 3 when the run is not found, and 1 for any other failure.
+
+- `letify runs` lists the live runs of this machine: run id, function, runtime, provider, age and owner pid.
+- `letify interrupt <run>` raises `KeyboardInterrupt` in the call's body on the runtime, in the thread that runs it, as Ctrl-C does in a local process. The body's `except` and `finally` blocks run, so the call ends the way an interrupted local function ends: its outputs are written back as for any call that raises, and the caller sees the exception. A body that catches it keeps running.
+- `letify exec <run> CODE` or `letify exec <run> -f FILE` runs Python source inside the worker process, on a thread of its own, while the body keeps running. The source sees these names:
+  - `frame`, the innermost frame of the body's thread whose code is not in the standard library, site-packages or letify, so it is the user's own loop;
+  - `frames`, every such frame from the outermost to the innermost;
+  - `L`, a mapping over `frame.f_locals` that records the names the source assigns, so the answer can say whether each was applied, and `G`, `frame.f_globals`.
+
+  When the source is a single expression the answer carries the `repr` of its value. The answer always carries what the source printed on stdout and stderr, and a traceback when it raised. `--timeout S`, 60 by default, bounds the wait for the answer, not the thread. The body is not paused, so the source sees the body's objects as they are while it runs. Mutating an object, such as `L["config"].save_every = 100`, reaches the running loop on every Python. Assigning a name, such as `L["save_every"] = 100`, reaches the running frame only on Python 3.13 and later, where `f_locals` writes through; on an earlier Python the answer lists the assigned names and says that they were not applied. A file written by the source lives on the runtime: calling `letify.stash(name, path)` in the source moves it to the Drive store, as from the body, so a checkpoint can be rescued from a runtime that may not survive.
+
+A command against a run on a provider whose worker has no persistent channel, the one-shot channels, fails with `UnsupportedMode` naming the provider.
+
+### Taking over a running call <!-- id: run-takeover -->
+
+> `letify takeover <run>` makes the process that runs it the owner of a running call and its session, so a training run outlives a stuck or dying owner without anything outliving its owner.
+
+A call has exactly one owner at a time, and the session still ends with the call that needed it. Taking over moves that ownership to another process; it does not let a call run with no owner. The new owner renews the lease, receives the body's output and the call's result, writes the call's outputs back, and ends the session when the call ends, as the first owner would have.
+
+1. **The registry keeps an orphan.** The record also holds `project`, the project root the call was sent from, and `lease_deadline`, the Unix time at which the runtime's lease ends unless renewed, rewritten at each renewal. It also holds what a second process needs to reach the call and finish it: `door`, the path of the worker's socket below; `config`, the project `config.toml` that names the account; and `call_dir`, `blobs` and `outputs`, the runtime paths and the output locations to write back, each location stored relative to `project` when it lies inside it. No secret is among them. A record whose owner pid is gone is no longer removed when read while its `lease_deadline` is in the future: `letify runs` lists it with `owner` `gone` and the seconds left, and it can be taken over. After the deadline it is removed as stale.
+2. **The worker keeps a second door.** While a call runs, the worker listens on a Unix socket on the runtime, `<workspace root>/.letify/run-<run id>.sock`, mode 0600. When that path is longer than 100 bytes, the limit of a Unix socket address, the worker and the relay reach the socket through `/proc/self/fd/<n>`, a descriptor of its directory, so it stays at that path; where `/proc` is missing the worker reports that the run has no second door. A channel that closes while a call runs, with no shutdown frame before it, does not end the worker: the body keeps running until the lease deadline passes. A shutdown frame is different: it is the owner ending the run on purpose, as [A deliberate shutdown](#run-shutdown) describes. Output the body makes while no channel is open is kept, the last 256 KiB per stream, and is sent to the next owner first. A body that returns while no channel is open keeps its result for a new owner until the lease deadline; on a provider with no lease the worker ends then, as before.
+3. **The new owner connects.** `letify takeover <run>` reads the record, opens a channel to the same runtime through the record's provider and runtime name, and starts a relay to that socket instead of a new worker. It then sends `adopt` with the run id. The worker answers once with the call's state, which is `running` or `finished`, and from then on sends the call's output and its reply on the new channel, the reply as the last piece of the same answer. The old channel, if still open, is told `moved` on the call's stream and answers every later request on it with `moved`. A door takes one connection at a time and no request but `adopt` before it, and a second owner can adopt after the first one's connection has closed.
+4. **The old owner lets go.** When the old owner is alive, takeover first asks it through its control socket to release the run. Its waiting call raises `OwnershipMoved` naming the run and the new owner's pid, it stops renewing that runtime's lease, and it drops the runtime from its pool without stopping it. A `keep_alive` block in the old owner no longer holds that session. An old owner that does not answer within 5 s is treated as gone.
+5. **The call ends at the new owner.** `letify takeover` prints the body's output as it arrives. When the call returns it prints the `repr` of the value, or with `--result PATH` writes the value with pickle to `PATH`, and exits 0. When the body raises it prints the traceback and exits 1. With `--json` the last line is one object with `run`, `ok`, `value_repr` or `error`, and `result_path`. The call's outputs are written back under the record's `project`, not the current directory. The session is then ended.
+6. **The same from Python.** `letify.takeover(run)` does the same in the calling process and returns the value or raises the body's exception, as a call does. It holds no handle: it returns once the call has ended and the session is gone. A darkpyonix kernel uses this to adopt a run started elsewhere.
+
+A provider whose runtime cannot be reached by a second process refuses with `UnsupportedMode` naming it. On the `Local` provider the runtime is this machine, so the new owner connects to the socket directly, but the worker is a child of the owning process and dies with it: a `Local` run can be taken over only from an owner that is still alive.
+
+### A deliberate shutdown <!-- id: run-shutdown -->
+
+> A shutdown frame from the channel that owns a running call ends the body and the worker, because a lost owner can be replaced and an owner that left on purpose cannot.
+
+`Pool.shutdown`, a session ending and a process exiting normally close a persistent channel by sending a shutdown frame before closing its input. The worker tells that from a lost owner:
+
+1. **A shutdown frame on the owning channel while a body runs** runs the exit hooks with the reason "the owner shut down", then ends the process with status 128 + `SIGTERM`, as a received `SIGTERM` does. No lease wait and no takeover follow. Before it exits, the worker ends the processes the body started (data loader workers, subprocesses). It builds the tree of descendants of its own pid from `/proc/<pid>/stat`, sends each `SIGTERM`, waits up to 5 s (`CHILD_GRACE_S`) for them to end, and sends `SIGKILL` to the survivors. The tree is walked rather than the process group, because a worker started over SSH is usually not a group leader (the remote shell leads the session). Where `/proc` is missing, the worker sends `SIGTERM` to its process group only when it leads that group, and signals no one else.
+2. **End of input without a shutdown frame** keeps the behaviour of [Taking over a running call](#run-takeover): the body runs on until the lease deadline.
+3. **A shutdown frame while no body runs** ends the worker, as before.
+4. **A shutdown frame on a channel the call moved away from** does not end the call. The channel that gave the call up closes with `detach`, which sends no shutdown frame, and a shutdown that does arrive on it is ignored.
+
+### Ctrl-C in the owning process
+
+The first `KeyboardInterrupt` that reaches a thread waiting for a call's reply sends the same interrupt as `letify interrupt` and keeps waiting, so the body's handlers run and the outputs come back before the call raises. The waiting thread prints `letify: interrupting <run>; press Ctrl-C again to stop waiting`. A second `KeyboardInterrupt` during that wait stops waiting at once and propagates, as before this section existed. No signal handler is installed for this, so a notebook kernel's interrupt behaves the same way.
+
+### Exit hooks
+
+`letify.on_exit(fn)`, called inside a body, registers `fn` to run when the worker is about to end while that body is still running, for a reason other than the body returning or raising. The reasons are the worker receiving `SIGTERM` (a preempted session, a provider stopping it), the lease deadline passing, the channel closing while a call runs, and the owner shutting down. Hooks run in the reverse order of registration, each at most once, on the body's process, with 120 s in total; a hook that raises is reported on stderr and the next one runs. Hooks registered by a call are dropped when that call ends. `SIGKILL`, the out of memory killer and a power cut run no code at all, so no hook runs for them; checkpoints are what covers those.
+
+Outside a body, `letify.on_exit` raises `UnsupportedMode`, because a local process has `atexit` already.
+
 ## Storage
 
 > A volume is a content addressed blob store on whichever backend a provider has. Each provider holds one volume per project, filled automatically from what a call uses, so a declaration never names one.
@@ -1250,7 +1333,7 @@ connection, but a call's second large blob is split whenever splitting is faster
 
 A connection that costs something to open has to pay for itself, so a provider reports `transfer_connection_cost_s`, the seconds one more connection takes. It is 0 for a provider that dials an address, and about 10 for one reached by a hole punch, because that is what a punch measures. The streams are opened only when the transfer is expected to finish sooner with them: the estimate uses the probe's upload rate, assumes the streams scale and that their connections are opened at the same time, and keeps one stream when `size / (rate x streams) + cost` is not less than `size / rate`. A measured case: a Kaggle runtime at a 44.7 ms round trip carried 128 MiB at 21.0 MiB/s on one stream and 9.5 MiB/s on four, 2.2 times worse, because three more punches cost more than the transfer saved. The same account at a 193 ms round trip carried 1.57 MiB/s on one raw SSH connection and 6.10 MiB/s on four, so the gain is real where the link is slow enough and the blob large enough for it to be worth 10 s.
 
-The connections have to be real ones. Every SSH command letify builds carries `ControlMaster=auto`, which multiplexes further sessions onto the first connection, so opening more channels that way would add streams inside one TCP connection and gain nothing. A transfer channel therefore sets `ControlPath=none` and `ControlMaster=no`, and a punched link redials for each, so each transfer channel is its own kernel connection.
+The connections have to be real ones. Every SSH command letify builds carries `ControlMaster=auto`, which multiplexes further sessions onto the first connection, so opening more channels that way would add streams inside one TCP connection and gain nothing. A transfer channel therefore sets `ControlPath=none` and `ControlMaster=no`, and a punched link gives each its own socket, so each transfer channel is its own kernel connection. A punched link holds the spare sockets of its punch, `PUNCH_SOCKETS` of them in all, [Rendezvous](#rendezvous), and hands one to each SSH command before it redials. The worker channel and the first transfer channels therefore cost no rendezvous, and `transfer_connection_cost_s` applies only to the streams beyond the spares.
 
 The worker's `data_put` appends to one handle per digest and hashes as it goes, which assumes the pieces arrive in order on one stream. Parallel senders cannot use it, so they use two other requests:
 
@@ -1829,7 +1912,9 @@ The remote side's punch window is 150 s, or the request's `window` in seconds wh
 
 Every SSH command letify builds sets `HostKeyAlias=letify-<alias>`, `StrictHostKeyChecking=accept-new` and `UserKnownHostsFile=~/.letify/accounts/<alias>/known_hosts`. Building the command creates that account directory with mode 0700 when it is missing, because `ssh` creates only `~/.ssh` and otherwise cannot record the key: it prints `Failed to add the host to the list of known hosts` on every connection and never pins the key. A host key that differs from the recorded one still fails the connection.
 
-A punched connection first answers the probe, then on request is spliced to the remote machine's SSH server, and SSH runs over a local forwarding port. The splice copies with `os.splice` through a pipe where the platform has it, which is Linux, so the bytes move between the two sockets inside the kernel. Every byte of a punched session crosses it, and the machine on the other end is often a two core VM, so a read into Python and a write back out costs real throughput there. A platform without `os.splice`, or a call it refuses, falls back to the read and write loop. A second SSH connection over the same link punches again.
+A punched connection first answers the probe, then on request is spliced to the remote machine's SSH server, and SSH runs over a local forwarding port. The splice copies with `os.splice` through a pipe where the platform has it, which is Linux, so the bytes move between the two sockets inside the kernel. Every byte of a punched session crosses it, and the machine on the other end is often a two core VM, so a read into Python and a write back out costs real throughput there. A platform without `os.splice`, or a call it refuses, falls back to the read and write loop. A second SSH connection over the same link takes a spare socket from the same punch, and punches again only when the spares are used up. Next section.
+
+A punch opens several sockets in one rendezvous, because each extra rendezvous costs about 10 s of lead time and one TCP flow over a long path carries a fraction of the path. The request names `count`, 1 when it is absent and at most 8, `PUNCH_SOCKETS` (4) when the strategy asks. Each side binds `count` local ports, learns a STUN mapping for each, and the request and the answer carry them as `mappings`, in order, with `mapping` still holding the first. Pair `i` is the user's port `i` and the remote's port `i`. It uses the token `token + bytes([i])`, so a hello written for one pair is refused by every other, and all pairs punch in the same window, each on its own thread. A pair that does not connect is absent, and the punch fails only when no pair connected. The remote side serves every connected socket as it serves one, each on its own thread, and its detached process stays until the last of them has ended, because the first one, which carries the probe, often ends long before a spare in use does. Every connected socket has `SO_KEEPALIVE` with `TCP_KEEPIDLE` 20 s and `TCP_KEEPINTVL` 10 s where the platform has them, so the NAT mapping of a socket waiting for its turn is not dropped. The strategy logs `tcp_punch connected <k> of <n> sockets in <seconds> s`. [PR #75](https://github.com/DarkPyonix/letify/pull/75) records the measurements.
 
 A local port that letify binds for forwarding is chosen by the operating system, never fixed, because Windows reserves port ranges that vary by machine. The remote end of a reverse forward is chosen the same way, with `ssh -R 0:`.
 
@@ -2102,6 +2187,7 @@ Local filesystem modifications are consolidated under the single root `~/.letify
 | `~/.letify/cache/tools/<alias>/` | Redirected `HOME` and `USERPROFILE` for provider CLIs (`colab`, `kaggle`). Mode 0700. Keeps CLI temporary files, packages, and `.config/` out of the credential directory. The CLI finds its token via a symbolic link pointing to `~/.letify/accounts/<alias>/token.json` (or copy fallback on systems without symlink support) | `LETIFY_HOME` |
 | `~/.letify/tmp/` | Temporary directory for ephemeral keys during reverse SSH negotiation and adapter stderr files. Mode 0700 | `LETIFY_HOME` |
 | `~/.letify/ssh/` | OpenSSH multiplexing control sockets (`ControlPath`). Mode 0700 | `XDG_RUNTIME_DIR` (moved to `$XDG_RUNTIME_DIR/letify`), `LETIFY_HOME` (with `XDG_RUNTIME_DIR`: `$XDG_RUNTIME_DIR/letify-<h>`; without it: `$LETIFY_HOME/ssh`) |
+| `~/.letify/runs/` | One record `<run id>.json` per running call and one control socket `<pid>.sock` per owning process, as [Controlling a running call](#run-control) and [Taking over a running call](#run-takeover) describe. Mode 0700 directory, mode 0600 records | `LETIFY_HOME` |
 | `~/.letify/runtime/` | Local provider workspace root for calls and local volumes. Holds `data/blobs`, `data/calls`, `tmp`. Falls back to reading `~/.letify-runtime` if it exists | `LETIFY_HOME` |
 
 **Documented local exceptions:**
@@ -2127,6 +2213,7 @@ Every remote write is constrained to the designated workspace root (`~/.letify-r
 | `<workspace root>/blobs` | Argument blobs on persistent providers | account `workspace` setting |
 | `<workspace root>/modules` | Shipped module sources placed on the runtime and added to sys.path | account `workspace` setting |
 | `<workspace root>/tmp/letify-<hex>` | Ephemeral reverse SSH key directory on remote instance | account `workspace` setting |
+| `<workspace root>/.letify/run-<run id>.sock` | The second door of a running call, mode 0600 in a mode 0700 directory, removed when the call's reply has been delivered. [Taking over a running call](#run-takeover) | account `workspace` setting |
 
 **Documented remote exceptions:**
 
@@ -2247,7 +2334,7 @@ So `letify login shell` sets up key authentication and treats the password as a
 4. The connection is confirmed with `BatchMode=yes`, which proves the key works before the alias is declared rather than at the first call.
 5. The machine's GPUs are detected over that confirmed connection, as described under Recording devices at login.
 
-Every SSH command letify builds also carries `-o ControlMaster=auto`, `-o ControlPersist=60` and `-o ControlPath=<control directory>/<tag>-%C`, so the worker channel, the busy card check and every later command to the same machine share one authenticated connection, instead of paying about 0.24 s for a new one each. On Windows, where OpenSSH does not implement connection multiplexing, the three options are left out and each command opens its own connection.
+Every SSH command letify builds also carries `-o ControlMaster=auto`, `-o ControlPersist=60` and `-o ControlPath=<control directory>/<tag>-%C`, so the worker channel, the busy card check and every later command to the same machine share one authenticated connection, instead of paying about 0.24 s for a new one each. The worker channel's ssh and every transfer stream's ssh are the exception: `sshopts.unshared` removes the three options and puts `-o ControlMaster=no -o ControlPath=none` after the program, so each runs as its own connection. A detached master has no standard error and writes no `-E` file, so a client of it that loses the connection exits 255 and prints nothing; a connection of its own prints the cause. A command over a punched link (`PunchedLink`) carries none of the three options, because its local forwarding port is new for every command, so no two commands could share a connection and each would leave a master holding a punched socket for 60 s. On Windows, where OpenSSH does not implement connection multiplexing, the three options are left out and each command opens its own connection.
 
 Control sockets live in a designated directory checked before use, constrained by the Unix domain socket path limit of 108 bytes on Linux and 104 bytes on macOS, including the terminating byte:
 
@@ -2397,11 +2484,28 @@ The two `Tensor` methods are replaced on the class and the `torch.accelerator` f
 
 ### Compilation <!-- id: forwarding-compile -->
 
-> Under `host="local"`, `torch.compile` returns the function or module it is given, unchanged, and warns once that it runs eagerly, so a compiled training loop runs and computes what eager code computes.
+> Under `host="local"`, `torch.compile` returns the function or module it is given, unchanged. When step compilation is enabled it also asks the runtime to compile each captured step with Inductor, so the same Python code gets fused kernels without a CUDA driver in this process. Step compilation is off by default.
+
+**Switch.** Step compilation is enabled by the environment variable `LETIFY_COMPILE_STEPS` when it is set, else by `compile_steps = true` at the top level of `~/.letify/config.toml`, else it is off. For the variable, `1`, `true`, `yes` and `on` enable it and `0`, `false`, `no` and `off` disable it, in any case; any other value disables it. The client reads the switch once, when it is created, as it does `auto_fetch`. The default is off because it measured no gain: on a Colab T4, 15.0 ms per step eager against 16.0 ms compiled, because under `host="local"` the step time is bound by the client's dispatch, not by the runtime GPU.
+
+**Client.** Inductor cannot run here: before tracing it creates a CUDA tensor in this process to set up a device context, which needs a driver this process does not have. Dynamo with any backend also traces into `RemoteTensor` dispatch, which is letify's own Python. So while forwarding is active, `torch.compile(model, ...)` with any arguments returns `model` itself, and `torch.compile(...)` used as a decorator factory returns a decorator that returns the function itself. `Module.compile(...)` does nothing else. `torch.compile` and `Module.compile` are restored when forwarding ends and undone in a forked process, as the rest of the mapping is.
+
+While step compilation is off, `torch.compile` and `Module.compile` send nothing, the executor replays every step as [Step capture](#forwarding-step-capture) describes, and the first call in a process emits a `UserWarning` naming `host="local"`, saying that the model runs eagerly, and naming `LETIFY_COMPILE_STEPS=1` and `compile_steps = true` as the way to compile captured steps on the runtime. While it is on, the first call in a session that does not pass `disable=True` and has the default `backend` sends the request `letify.compile` with its `mode` and `options`, `None` and `{}` when absent. A later call in the session sends nothing. The first such call in a process also emits a `UserWarning` naming `host="local"` and saying that compilation happens on the runtime per captured step. Until the request arrives, and when nobody calls `torch.compile`, the runtime replays every step as [Step capture](#forwarding-step-capture) describes, unchanged.
+
+**Compiled segments.** Once compilation is enabled, the worker compiles a segment of a step: the operators `start` to `stop - 1` of one step entry. A segment is identified by `(step, start, stop)`, so a partial entry is its own segment. Its compiled function returns every tensor that some entry of the segment kept, and an entry stores only the ones in its own `keep`; an entry whose `keep` has an offset not yet returned is another variant. A segment that has any of these stays eager for the rest of the session: an operator whose name contains a random-number token (`rand`, `dropout`, `bernoulli`, `normal`, `multinomial`, `uniform`, `exponential`, `random`), because Inductor draws from its own generator; a compilation failure; or a fifth variant of its step.
+
+- **Sightings.** The first entry of a segment runs eagerly and records its scalars. On the second, a scalar slot whose value differs from the first and whose type is `int` or `float` is varying, and every other slot is constant. The second entry also starts the compilation, and runs eagerly.
+- **Function.** The compiled function takes, in this order, the distinct tensors the segment reads that it does not create (the entry's `externals` and the handles created before `start`, each tensor once however often it is read), the CPU tensors the segment reads as blobs, and one 0-d CPU tensor for each varying scalar, `float64` for a float and `int64` for an int. It replays the operators exactly as the **Worker** paragraph of [Step capture](#forwarding-step-capture) does, over a table of its own, reading a varying scalar with `aten._local_scalar_dense` so that an optimizer's learning rate or bias correction is a runtime value, not a constant. It returns the survivors: the tensors the segment creates that are in `keep` or not released inside the segment, in creation order. An in-place update of an input is written back to that input.
+- **Tracing.** The function is traced with `make_fx` in its symbolic mode over fake inputs of the observed shapes, and compiled with `torch._inductor.compile_fx.compile_fx` against fake inputs of static shapes. Inductor's `keep_output_stride` keeps its default.
+- **Background.** Tracing and compiling run in a thread of the worker. Until the compiled function is ready, entries of the segment run eagerly.
+- **Guard.** An entry runs compiled when its constant scalars equal those compiled in, the pattern of repeated tensors among the inputs is the same, every input has the shape, strides, dtype and device compiled in, and no two inputs share storage. Otherwise it runs eagerly. A mismatch of the constants, the pattern or the inputs starts a new compilation of the segment with the changed scalar slots varying, which is another variant. A step has at most 4 variants in the session.
+- **Run.** A compiled entry stores each survivor that is in its `keep` or not released inside the segment under its handle, `first handle` plus its offset, drops the handles created before `start` that the eager replay would release, and creates no other handle.
+
+**Mode and options.** `options` is applied as `torch._inductor.config.patch(options)` around the compilation, so a key Inductor does not know is a compilation failure. `mode="max-autotune"` and `mode="max-autotune-no-cudagraphs"` set `max_autotune`. Every other mode, `"reduce-overhead"` included, compiles as the default mode does, because the worker does not use CUDA graphs.
 
-Inductor, the default backend, cannot compile here: before tracing it creates a CUDA tensor in this process to set up a device context, which needs a driver this process does not have. Dynamo with any backend also traces into `RemoteTensor` dispatch, which is letify's own Python, and recompiles it for every operator. A replayed step already sends the whole step as one entry, as [Step capture](#forwarding-step-capture) describes, so compiling on the client has nothing left to batch.
+**Failure.** A compilation that fails marks the segment eager and queues a note `letify: step <step> could not be compiled: <reason>; it runs eagerly`. The note travels with the next reply and the client prints it on standard error once. A compiled entry that raises while running is reported as [Failure semantics](#forwarding-failure) describes, naming `compiled step <step>`, the segment is marked eager, and the entry is not run again.
 
-While forwarding is active, `torch.compile(model, ...)` with any arguments returns `model` itself, and `torch.compile(...)` used as a decorator factory returns a decorator that returns the function itself. `Module.compile(...)` does nothing. The first such call in a process emits a `UserWarning` naming `host="local"` and eager execution. `torch.compile` and `Module.compile` are restored when forwarding ends and undone in a forked process, as the rest of the mapping is.
+**Request.** `letify.compile_stats` with `wait` true joins the compilations in progress, then returns counters: `compiled`, the successful compilations; `failed`, the failed ones; `runs`, the entries that ran compiled; `eager`, the segments marked eager.
 
 `torch.cuda.get_rng_state` and `torch.cuda.set_rng_state` stay refused, so code that saves and restores the generator state names the gap instead of silently restoring a state that is not the runtime's.
 
@@ -2526,7 +2630,7 @@ A step is what a training loop repeats: forward, backward and the optimizer upda
 
 **Early release.** A tensor a repetition creates is released on the runtime right after the last operator of the step that reads it, when nothing else can read it, rather than with the release list after the whole step. When a step is registered, the client and the worker both compute each created handle's release position from the wiring: the last position whose operator takes that offset as an argument, or the position that created it when no later operator of the step reads it. A step entry carries `keep`, the offsets whose release position lies in `start` to `stop - 1` and that must not be released early. The client computes `keep` when it takes the entry into a batch: an offset is kept when a `RemoteTensor` still names its handle, which is when its handle has not been released locally, or when an entry after it in that batch, an entry left in the queue, or the unfinished repetition reads the handle. A saved tensor autograd holds for backward is named by a `RemoteTensor`, so it is released only after the backward operator that reads it. A tensor the loop keeps across steps, such as the loss or a gradient, is kept, and released by the release list as before.
 
-**Worker.** The worker executes operators `start` to `stop - 1` of the step in order, reading an argument's handle as `first handle + offset` or from `externals`, and stores each new output under the next handle counted from `first handle`. After each operator it drops the handles whose release position is that operator, other than those in `keep`, from its table. Each operator is the same call an eager entry makes, so a failure is recorded and reported as [Failure semantics](#forwarding-failure) describes, naming the operator. The worker does not use CUDA graphs, because the handles bound to a step, such as the batch, change between repetitions and a graph needs fixed input memory.
+**Worker.** The worker executes operators `start` to `stop - 1` of the step in order, reading an argument's handle as `first handle + offset` or from `externals`, and stores each new output under the next handle counted from `first handle`. After each operator it drops the handles whose release position is that operator, other than those in `keep`, from its table. Each operator is the same call an eager entry makes, so a failure is recorded and reported as [Failure semantics](#forwarding-failure) describes, naming the operator. When compilation is enabled, a segment may run as one compiled function, as [Compilation](#forwarding-compile) describes. The worker does not use CUDA graphs, because the handles bound to a step, such as the batch, change between repetitions and a graph needs fixed input memory.
 
 ### Handles <!-- id: forwarding-handles -->
 

{letify-2.8.7 → letify-2.10.0}/docs/guide/03-execution-modes.md

@@ -72,6 +72,8 @@ A read does not have to be a synchronization. `loss.detach().to("cpu", non_block
 
 You do not have to write either of those for a logged number. Under `host="local"`, `loss.item()` already returns a deferred value: the read is queued and the loop keeps going, and the value is waited for where it is used, such as in the `print` a step later. `tolist()`, `float(tensor)` and `int(tensor)` work the same way. `bool(tensor)` still waits, because control flow needs the value at once. The value a call returns is always a real `float`, `int` or `list`. Put `auto_fetch = false` in the account entry, or `LETIFY_AUTO_FETCH=0` in the environment, to turn this off. At the end of a call letify prints how many reads it deferred and how many had to wait, so a loop that gains nothing from it is visible.
 
+`torch.compile` and `Module.compile` return the model unchanged under `host="local"` and warn once that it runs eagerly. To have the runtime compile each captured step with Inductor instead, set `LETIFY_COMPILE_STEPS=1` in the environment or `compile_steps = true` at the top level of `~/.letify/config.toml`. It is off by default because it measured no gain on a Colab T4: 15.0 ms per step eager against 16.0 ms compiled, since the step time is bound by the client's dispatch.
+
 Other things that force a synchronization: `nonzero()`, boolean mask indexing, `unique()`, and mixture of experts routing, which reads per-expert token counts back to the host. Any operation whose output shape the host has to know is a synchronization.
 
 ### Measure it instead of guessing

{letify-2.8.7 → letify-2.10.0}/docs/guide/05-concurrency.md

@@ -132,6 +132,53 @@ Confirm the final candidate on the bigger card.
 
 > ⚠️ This does not apply when you need a specific capability. NVFP4 needs Blackwell tensor cores, which on Colab means `G4` and nothing else. There is no cheaper substitute.
 
+## Reaching into a running call
+
+A call is a program running somewhere else, and you can still do to it what you would do to a local process. Every call gets a 12 character run id while it runs. `letify runs` lists the live calls of the machine:
+
+```bash
+letify runs
+letify interrupt 3f9a0c61d2b4                   # raise KeyboardInterrupt in the body, as Ctrl-C does
+letify exec 3f9a0c61d2b4 "L['step']"            # read a local variable of the running loop
+letify exec 3f9a0c61d2b4 "L['config'].save_every = 100"
+letify exec 3f9a0c61d2b4 -f rescue.py --timeout 300
+```
+
+`interrupt` runs the body's `except` and `finally` blocks, so a loop that saves in a `finally` saves. Pressing Ctrl-C in the process that made the call does the same once, then waits for the outputs. A second Ctrl-C stops waiting.
+
+`exec` runs your source on a thread of the worker while the body keeps running. It sees `frame`, `frames`, `L` (the locals of `frame`) and `G` (its globals), where `frame` is the innermost frame of your own code. A single expression answers with its `repr`. Changing an object, as in `L["config"].save_every = 100`, reaches the loop on every Python. Assigning a name, as in `L["step"] = 0`, reaches it only on Python 3.13 and later, and the answer says so on an earlier one. A file the source writes is on the runtime, so `import letify; letify.stash("ckpt", "model.pt")` in the source moves it to the Drive store before the runtime goes away.
+
+To save state when the runtime is taken away, register a hook inside the body:
+
+```python
+@let.function(device=colab.G4, host=letify.remote)
+def train():
+    letify.on_exit(lambda: letify.stash("ckpt", "model.pt"))
+    ...
+```
+
+The hook runs when the worker receives `SIGTERM`, when its lease runs out, when the channel closes during the call, or when the owner shuts down. A shutdown by the owner (a pool shutdown, or a client that received `SIGTERM`) runs the hooks and then ends the body and the worker. A channel that closes with no shutdown leaves the body running until the lease deadline, so another process can take it over. It does not run for `SIGKILL` or the out of memory killer, so keep writing checkpoints as well. Hooks run newest first and share 120 seconds.
+
+None of this works on a provider whose worker has only a one-shot channel, and the commands say which provider. There is no control socket on Windows. See the spec section "Controlling a running call".
+
+### Handing a call to another process
+
+A call has one owner: the process that sent it, which renews the runtime's lease, receives the output and the result, and ends the session. When that process is stuck, or you want to close the notebook that started it, move the call to another process:
+
+```bash
+letify takeover 3f9a0c61d2b4                    # prints the body's output, then the repr of the value
+letify takeover 3f9a0c61d2b4 --result out.pkl   # writes the value with pickle instead
+letify takeover 3f9a0c61d2b4 --json             # the last line is one JSON object: run, ok, value_repr or error, result_path
+```
+
+The old owner is asked to let go. Its waiting call raises `OwnershipMoved`, which names the run and the new owner and is never retried. It stops renewing the lease and leaves the session alone, even inside a `keep_alive` block. If it does not answer within 5 seconds it is treated as gone. The new owner prints the output from then on, writes the files the call produced under the project the call was sent from, not the current directory, and ends the session when the call ends. The exit status is 0 when the call returned, 1 when its body raised (the traceback is printed) or the run could not be taken, and 3 when no such run exists.
+
+From Python, `value = letify.takeover("3f9a0c61d2b4")` does the same in the calling process and raises the body's exception as a call does.
+
+If the owner died, `letify runs` still lists the call while the runtime's lease has time left, with `gone` as its owner and the seconds left. Take it over before the lease ends, or the worker ends with it. Output the body made while nobody listened is kept (the last 256 KiB) and printed first.
+
+A provider whose runtime a second process cannot reach raises `UnsupportedMode` naming it. Local and Shell can be taken over. A Local worker is a child of its owner and has no lease, so a Local call can be taken over only while its owner is alive. Kaggle, Modal and Elice cannot be taken over, and Colab can only on its default SSH channel. See the spec section "Taking over a running call".
+
 ## Practical patterns
 
 **Run the configurations, collect a table.**

{letify-2.8.7 → letify-2.10.0}/docs/guide/07-troubleshooting.md

@@ -138,6 +138,16 @@ and letify cannot know when that process ends.
 
 On a shared machine this usually means a colleague is computing on those cards. Check with `letify utilization`, then retry later or narrow `indices` in the devices table.
 
+**Cards held by a worker you cannot see.** A worker left behind when the local process died keeps its cards until its lease ends, 300 s after the last renewal. On a `Shell` machine, list the workers and stop one:
+
+```bash
+letify sessions
+letify stop --orphans --alias lab
+letify stop --alias lab --worker 4242
+```
+
+`letify sessions` prints one line per worker: pid, GPU indices, start time, runtime name, pool key, run id and the local process that owns it, marked `alive`, `gone` or `remote` (owned from another machine, so not judged). `--orphans` stops the workers whose lease has passed, and those whose owner is gone while no call runs. It never stops a worker whose owner is alive. A worker whose owner is gone but whose call still runs is listed by `letify runs` and can be taken over, so `--orphans` leaves it alone. `--worker` stops the one pid you name, whatever its owner, and refuses a pid that is not a letify worker. A stop sends `SIGTERM`, waits 30 s, then sends `SIGKILL`.
+
 ---
 
 ## `RemoteError`

{letify-2.8.7 → letify-2.10.0}/docs/guide/ko/03-/354/213/244/355/226/211-/353/260/251/354/213/235.md

@@ -72,6 +72,8 @@ Hugging Face 학습 스텝은 기본 설정에서 `k ≈ 3`입니다.
 
 로깅하는 숫자 하나 때문에 위의 두 가지를 직접 쓸 필요는 없습니다. `host="local"`에서 `loss.item()`은 이미 지연 값을 돌려줍니다. 읽기는 큐에 들어가고 루프는 계속 진행되며, 값은 실제로 쓰이는 곳, 예를 들어 한 스텝 뒤의 `print`에서 기다립니다. `tolist()`, `float(tensor)`, `int(tensor)`도 같습니다. `bool(tensor)`는 제어 흐름이 값을 바로 필요로 하므로 그대로 기다립니다. 호출이 돌려주는 값은 언제나 실제 `float`, `int`, `list`입니다. 이 동작을 끄려면 계정 항목에 `auto_fetch = false`를 넣거나 환경 변수 `LETIFY_AUTO_FETCH=0`을 설정하세요. 호출이 끝나면 letify가 몇 개의 읽기를 지연했고 몇 개가 기다려야 했는지 한 줄로 출력하므로, 이 동작에서 얻는 것이 없는 루프는 바로 드러납니다.
 
+`host="local"`에서 `torch.compile`과 `Module.compile`은 모델을 그대로 돌려주고, 모델이 eager로 실행된다는 경고를 한 번 출력합니다. 런타임이 캡처된 각 스텝을 Inductor로 컴파일하게 하려면 환경 변수 `LETIFY_COMPILE_STEPS=1`을 설정하거나 `~/.letify/config.toml` 최상위에 `compile_steps = true`를 넣으세요. Colab T4에서 이득이 측정되지 않아 기본값은 꺼짐입니다. 스텝당 eager 15.0 ms, 컴파일 16.0 ms였고, 스텝 시간이 클라이언트의 디스패치에 묶여 있기 때문입니다.
+
 동기화를 강제하는 다른 것들도 있습니다. `nonzero()`, 불리언 마스크 인덱싱, `unique()`, 그리고 전문가별 토큰 수를 호스트로 읽어오는 MoE 라우팅입니다. 출력 크기를 호스트가 알아야 하는 연산은 모두 동기화입니다.
 
 ### 추측하지 말고 측정하세요

{letify-2.8.7 → letify-2.10.0}/docs/guide/ko/05-/353/217/231/354/213/234-/354/213/244/355/226/211.md

@@ -128,6 +128,53 @@ L4 세 개가 A100 하나보다 시간당 싸면서, A100 하나가 조합 하
 
 > ⚠️ 특정 기능이 필요할 때는 해당되지 않습니다. NVFP4는 블랙웰 텐서 코어가 필요하고, Colab에서는 `G4`뿐입니다. 더 싼 대체가 없습니다.
 
+## 실행 중인 호출에 손대기
+
+호출은 다른 곳에서 돌아가는 프로그램이지만, 로컬 프로세스에 하던 일을 그대로 할 수 있습니다. 호출은 실행되는 동안 12자리 run id를 갖습니다. `letify runs`는 이 머신에서 살아 있는 호출을 보여 줍니다.
+
+```bash
+letify runs
+letify interrupt 3f9a0c61d2b4                   # Ctrl-C처럼 본문에 KeyboardInterrupt를 일으킴
+letify exec 3f9a0c61d2b4 "L['step']"            # 돌고 있는 루프의 지역 변수 읽기
+letify exec 3f9a0c61d2b4 "L['config'].save_every = 100"
+letify exec 3f9a0c61d2b4 -f rescue.py --timeout 300
+```
+
+`interrupt`는 본문의 `except`와 `finally` 블록을 실행하므로, `finally`에서 저장하는 루프는 저장하고 끝납니다. 호출을 만든 프로세스에서 Ctrl-C를 한 번 누르면 같은 일을 한 뒤 출력이 돌아오기를 기다립니다. 두 번째 Ctrl-C는 기다리기를 멈춥니다.
+
+`exec`는 본문이 계속 도는 동안 워커의 별도 스레드에서 소스를 실행합니다. 소스에서는 `frame`, `frames`, `L`(`frame`의 지역 변수), `G`(전역 변수)가 보이며, `frame`은 사용자 코드의 가장 안쪽 프레임입니다. 식 하나이면 그 값의 `repr`을 답으로 돌려줍니다. `L["config"].save_every = 100`처럼 객체를 바꾸면 어느 파이썬에서나 루프에 반영됩니다. `L["step"] = 0`처럼 이름에 대입하면 파이썬 3.13 이상에서만 반영되고, 더 낮은 버전에서는 답에 반영되지 않았다고 나옵니다. 소스가 쓴 파일은 런타임에 있으므로, 소스에서 `import letify; letify.stash("ckpt", "model.pt")`를 부르면 런타임이 사라지기 전에 Drive 저장소로 옮길 수 있습니다.
+
+런타임을 빼앗길 때 상태를 저장하려면 본문 안에서 훅을 등록합니다.
+
+```python
+@let.function(device=colab.G4, host=letify.remote)
+def train():
+    letify.on_exit(lambda: letify.stash("ckpt", "model.pt"))
+    ...
+```
+
+훅은 워커가 `SIGTERM`을 받을 때, 리스 기한이 지날 때, 호출 중에 채널이 닫힐 때, 소유자가 종료를 요청할 때 실행됩니다. 소유자가 종료를 요청하면(풀 종료, `SIGTERM`을 받은 클라이언트) 훅을 실행한 뒤 본문과 워커가 함께 끝납니다. 종료 요청 없이 채널만 끊기면 본문은 리스 기한까지 계속 실행되어 다른 프로세스가 이어받을 수 있습니다. `SIGKILL`이나 메모리 부족 종료에는 실행되지 않으므로 체크포인트도 계속 쓰세요. 훅은 나중에 등록한 것부터 실행되며 합쳐서 120초를 씁니다.
+
+워커에 일회성 채널만 있는 프로바이더에서는 이 기능을 쓸 수 없고, 명령이 그 프로바이더 이름을 알려 줍니다. Windows에는 제어 소켓이 없습니다. 스펙의 "Controlling a running call" 절을 참고하세요.
+
+### 호출을 다른 프로세스에 넘기기
+
+호출의 소유자는 하나입니다. 호출을 보낸 프로세스가 런타임의 리스를 갱신하고, 출력과 결과를 받고, 세션을 끝냅니다. 그 프로세스가 멈췄거나 호출을 시작한 노트북을 닫고 싶으면 호출을 다른 프로세스로 옮깁니다.
+
+```bash
+letify takeover 3f9a0c61d2b4                    # 본문의 출력을 보여 준 뒤 반환값의 repr를 출력
+letify takeover 3f9a0c61d2b4 --result out.pkl   # 반환값을 pickle로 파일에 저장
+letify takeover 3f9a0c61d2b4 --json             # 마지막 줄이 JSON 객체 하나: run, ok, value_repr 또는 error, result_path
+```
+
+이전 소유자에게 놓아 달라고 요청합니다. 기다리던 이전 소유자의 호출은 `OwnershipMoved`를 일으키고, 이 예외는 run id와 새 소유자를 알려 주며 재시도되지 않습니다. 이전 소유자는 리스 갱신을 멈추고, `keep_alive` 블록 안에서도 세션을 건드리지 않습니다. 5초 안에 답하지 않으면 사라진 것으로 봅니다. 새 소유자는 그 시점부터 출력을 보여 주고, 호출이 만든 파일을 현재 디렉터리가 아니라 호출을 보낸 프로젝트 아래에 쓰고, 호출이 끝나면 세션을 끝냅니다. 종료 코드는 호출이 반환하면 0, 본문이 예외를 일으키거나(트레이스백을 출력합니다) 호출을 넘겨받지 못하면 1, 그런 run이 없으면 3입니다.
+
+파이썬에서는 `value = letify.takeover("3f9a0c61d2b4")`가 호출 프로세스에서 같은 일을 하며, 호출처럼 본문의 예외를 그대로 일으킵니다.
+
+소유자가 죽었더라도 런타임의 리스가 남아 있는 동안은 `letify runs`가 그 호출을 소유자 `gone`과 남은 초와 함께 보여 줍니다. 리스가 끝나기 전에 넘겨받지 않으면 워커도 함께 끝납니다. 아무도 듣지 않는 동안 본문이 낸 출력은 마지막 256 KiB까지 보관했다가 먼저 출력합니다.
+
+다른 프로세스가 런타임에 닿을 수 없는 프로바이더에서는 그 이름과 함께 `UnsupportedMode`가 일어납니다. Local과 Shell은 넘겨받을 수 있습니다. Local의 워커는 소유자의 자식 프로세스이고 리스가 없으므로, Local 호출은 소유자가 살아 있는 동안에만 넘겨받을 수 있습니다. Kaggle, Modal, Elice는 넘겨받을 수 없고, Colab은 기본 SSH 채널에서만 가능합니다. 스펙의 "Taking over a running call" 절을 참고하세요.
+
 ## 실용 패턴
 
 **설정을 돌려 표로 모으기.**

{letify-2.8.7 → letify-2.10.0}/docs/guide/ko/07-/353/254/270/354/240/234-/355/225/264/352/262/260.md

@@ -138,6 +138,16 @@ and letify cannot know when that process ends.
 
 공유 머신이라면 보통 동료가 그 카드로 계산 중이라는 뜻입니다. `letify utilization`으로 확인한 뒤 나중에 다시 시도하거나, devices 테이블의 `indices`를 좁히세요.
 
+**보이지 않는 워커가 잡고 있는 카드.** 로컬 프로세스가 죽으면서 남은 워커는 리스가 끝날 때까지, 마지막 갱신 후 300초 동안 카드를 잡고 있습니다. `Shell` 머신에서는 워커를 나열하고 하나를 멈출 수 있습니다.
+
+```bash
+letify sessions
+letify stop --orphans --alias lab
+letify stop --alias lab --worker 4242
+```
+
+`letify sessions`는 워커마다 한 줄을 출력합니다. pid, GPU 번호, 시작 시각, 런타임 이름, 풀 키, 실행 id, 그리고 소유한 로컬 프로세스와 그 상태(`alive`, `gone`, `remote`)입니다. `remote`는 다른 머신이 소유한 경우라 판정하지 않습니다. `--orphans`는 리스가 지난 워커와, 소유자가 사라졌고 실행 중인 호출이 없는 워커를 멈춥니다. 소유자가 살아 있는 워커는 멈추지 않습니다. 소유자는 사라졌지만 호출이 아직 도는 워커는 `letify runs`에 나오고 인수할 수 있으므로 `--orphans`가 건드리지 않습니다. `--worker`는 지정한 pid 하나를 소유자와 관계없이 멈추고, letify 워커가 아닌 pid는 거부합니다. 멈출 때는 `SIGTERM`을 보내고 30초 기다린 뒤 `SIGKILL`을 보냅니다.
+
 ---
 
 ## `RemoteError`

{letify-2.8.7 → letify-2.10.0}/docs/locales/README_ko.md

@@ -467,6 +467,8 @@ letify setup tailcat  # 배포처에서 tailcat 또는 eci를 미리 설치
 letify providers      # 선언된 프로바이더, 저장소 수명, 기본 배치
 letify devices           # 각자 제공하는 GPU
 letify status         # 지금 돌고 있는 것
+letify sessions       # 계정이 잡고 있는 것, shell 호스트의 워커
+letify stop --orphans # 죽은 프로세스가 남긴 것 정리
 letify usage          # 계정마다 남은 사용량
 letify utilization    # 인스턴스별 GPU가 얼마나 바쁜가
 letify check lab      # 이 머신이 응답하나?

{letify-2.8.7 → letify-2.10.0}/letify/__init__.py

@@ -33,6 +33,7 @@ from typing import Final, overload
 
 from .declare.cache import session_cache
 from .declare.env import Env
+from .declare.exit import on_exit
 from .declare.function import Function
 from .declare.instance import AnyInstance, Host, Instance
 from .declare.reduce import all_reduce, mean, sum
@@ -43,6 +44,7 @@ from .errors import (
     InsufficientDevices,
     InterpreterMismatch,
     LetifyError,
+    OwnershipMoved,
     ProtocolError,
     ProviderUnavailable,
     ReductionFailed,
@@ -68,7 +70,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.8.7"
+__version__ = "2.10.0"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]
@@ -93,6 +95,22 @@ def stash(name: str, path: str | os.PathLike[str]) -> str:
     return stash_handoff(name, path)
 
 
+def takeover(run: str, *, launcher: Launcher | None = None) -> object:
+    """Become the owner of the running call ``run`` and return its value.
+
+    Spec "Taking over a running call". The previous owner, if it is alive, is asked to let go
+    and its waiting call raises ``OwnershipMoved``. This process then renews the runtime's
+    lease, prints the body's output as it arrives, writes the call's outputs back under the
+    project the call was sent from, ends the session, and returns the value or raises the
+    body's exception as a call does. It holds no handle and returns only once the call has
+    ended and the session is gone. ``launcher`` supplies the accounts when this process
+    already has one; otherwise they are read from the configuration the run was started with.
+    """
+    from .runtime.takeover import take
+
+    return take(run, launcher=launcher)
+
+
 @overload
 def unstash(name: str) -> dict[str, bytes] | None: ...
 @overload
@@ -131,6 +149,7 @@ __all__ = [
     "InterpreterMismatch",
     "Launcher",
     "LetifyError",
+    "OwnershipMoved",
     "ProtocolError",
     "ProviderUnavailable",
     "Providers",
@@ -151,11 +170,13 @@ __all__ = [
     "local",
     "local_steps",
     "mean",
+    "on_exit",
     "rank",
     "remote",
     "session_cache",
     "stash",
     "sum",
+    "takeover",
     "unstash",
     "world_size",
 ]