argus-forge 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus_forge-0.1.1/.copier-answers.yml +21 -0
- argus_forge-0.1.1/.dockerignore +38 -0
- argus_forge-0.1.1/.env.example +32 -0
- argus_forge-0.1.1/.github/workflows/ci.yml +20 -0
- argus_forge-0.1.1/.github/workflows/release.yml +125 -0
- argus_forge-0.1.1/.gitignore +14 -0
- argus_forge-0.1.1/CHANGELOG.md +248 -0
- argus_forge-0.1.1/Dockerfile +54 -0
- argus_forge-0.1.1/LICENSE +21 -0
- argus_forge-0.1.1/Makefile +27 -0
- argus_forge-0.1.1/PKG-INFO +247 -0
- argus_forge-0.1.1/README.md +213 -0
- argus_forge-0.1.1/docker-compose.yaml +38 -0
- argus_forge-0.1.1/pyproject.toml +61 -0
- argus_forge-0.1.1/schema/forge-wire.schema.json +937 -0
- argus_forge-0.1.1/src/argus_forge/__init__.py +16 -0
- argus_forge-0.1.1/src/argus_forge/_version.py +24 -0
- argus_forge-0.1.1/src/argus_forge/cli.py +374 -0
- argus_forge-0.1.1/src/argus_forge/core.py +302 -0
- argus_forge-0.1.1/src/argus_forge/emitters/__init__.py +43 -0
- argus_forge-0.1.1/src/argus_forge/emitters/base.py +147 -0
- argus_forge-0.1.1/src/argus_forge/emitters/diffusers.py +98 -0
- argus_forge-0.1.1/src/argus_forge/emitters/kohya.py +133 -0
- argus_forge-0.1.1/src/argus_forge/emitters/onetrainer.py +98 -0
- argus_forge-0.1.1/src/argus_forge/heuristics.py +131 -0
- argus_forge-0.1.1/src/argus_forge/manifest.py +242 -0
- argus_forge-0.1.1/src/argus_forge/models.py +366 -0
- argus_forge-0.1.1/src/argus_forge/py.typed +1 -0
- argus_forge-0.1.1/src/argus_forge/runner.py +194 -0
- argus_forge-0.1.1/src/argus_forge/server/__init__.py +25 -0
- argus_forge-0.1.1/src/argus_forge/server/app.py +471 -0
- argus_forge-0.1.1/src/argus_forge/server/jobs.py +341 -0
- argus_forge-0.1.1/tests/conftest.py +145 -0
- argus_forge-0.1.1/tests/golden/kohya/identity_27_config.toml +34 -0
- argus_forge-0.1.1/tests/golden/kohya/identity_27_dataset.toml +21 -0
- argus_forge-0.1.1/tests/golden/kohya/pose_composition_20_config.toml +34 -0
- argus_forge-0.1.1/tests/golden/kohya/pose_composition_20_dataset.toml +21 -0
- argus_forge-0.1.1/tests/golden/kohya/setting_30_config.toml +34 -0
- argus_forge-0.1.1/tests/golden/kohya/setting_30_dataset.toml +21 -0
- argus_forge-0.1.1/tests/test_cli.py +222 -0
- argus_forge-0.1.1/tests/test_emitters.py +437 -0
- argus_forge-0.1.1/tests/test_heuristics.py +54 -0
- argus_forge-0.1.1/tests/test_jobs.py +441 -0
- argus_forge-0.1.1/tests/test_kohya_parity.py +69 -0
- argus_forge-0.1.1/tests/test_manifest.py +240 -0
- argus_forge-0.1.1/tests/test_runner.py +133 -0
- argus_forge-0.1.1/tests/test_server.py +796 -0
- argus_forge-0.1.1/tests/test_smoke.py +8 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Managed by Copier — do not edit by hand.
|
|
2
|
+
# Run `copier update` in this repo to pull template changes.
|
|
3
|
+
_commit: 281bd55
|
|
4
|
+
_src_path: /home/smk/argus-pkg-template
|
|
5
|
+
author: smk762
|
|
6
|
+
cli_name: argus-forge
|
|
7
|
+
description: 'Training bridge: turn curated dataset exports into ready-to-run LoRA
|
|
8
|
+
training configs (kohya_ss / OneTrainer / diffusers)'
|
|
9
|
+
dockerfile: Dockerfile
|
|
10
|
+
github_owner: smk762
|
|
11
|
+
has_cli: true
|
|
12
|
+
has_server: true
|
|
13
|
+
image_name: argus-forge
|
|
14
|
+
package_name: argus_forge
|
|
15
|
+
post_test: uv run --no-sync argus-forge schema --check
|
|
16
|
+
project_name: argus-forge
|
|
17
|
+
python_min: '3.11'
|
|
18
|
+
ruff_pin: 0.15.16
|
|
19
|
+
test_extras: dev,server,cli
|
|
20
|
+
version_build_arg: false
|
|
21
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Keep the build context small and hermetic. VCS metadata is intentionally
|
|
2
|
+
# excluded — the Dockerfile passes the version in via SETUPTOOLS_SCM_PRETEND_VERSION
|
|
3
|
+
# (the VERSION build-arg), so hatch-vcs never needs to read git history.
|
|
4
|
+
#
|
|
5
|
+
# NOTE: .dockerignore is NOT .gitignore. Patterns are matched against the whole
|
|
6
|
+
# path with Go filepath.Match and there is no implicit recursion — a bare
|
|
7
|
+
# `__pycache__` excludes only ./__pycache__, so anything that can appear at
|
|
8
|
+
# depth needs an explicit `**/`.
|
|
9
|
+
.git
|
|
10
|
+
.venv
|
|
11
|
+
venv
|
|
12
|
+
.pytest_cache
|
|
13
|
+
.ruff_cache
|
|
14
|
+
.mypy_cache
|
|
15
|
+
**/__pycache__
|
|
16
|
+
**/*.pyc
|
|
17
|
+
.env
|
|
18
|
+
**/.env
|
|
19
|
+
dist
|
|
20
|
+
build
|
|
21
|
+
**/*.egg-info
|
|
22
|
+
**/node_modules
|
|
23
|
+
.claude
|
|
24
|
+
|
|
25
|
+
# The dataset volume docker-compose.yaml mounts by default
|
|
26
|
+
# (${FORGE_OUTPUT_PATH:-./out}). Curated exports land here, so without this a
|
|
27
|
+
# `docker compose up` followed by a rebuild bakes the whole image tree into a
|
|
28
|
+
# layer — the opposite of the hygiene this file exists for.
|
|
29
|
+
out
|
|
30
|
+
|
|
31
|
+
# Not needed at runtime: the package is installed into site-packages, and the
|
|
32
|
+
# source tree is only here for `uv pip install .`.
|
|
33
|
+
tests
|
|
34
|
+
schema
|
|
35
|
+
.github
|
|
36
|
+
Makefile
|
|
37
|
+
docker-compose.yaml
|
|
38
|
+
.copier-answers.yml
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# ── Server ───────────────────────────────────────────────────────────
|
|
2
|
+
# Host port for the argus-forge FastAPI server (peer to lens :8100,
|
|
3
|
+
# curator :8101, quarry :8102).
|
|
4
|
+
FORGE_PORT=8103
|
|
5
|
+
|
|
6
|
+
# Host directory holding the curator's exports. Mounted at /data/out, which is
|
|
7
|
+
# the containment root for every request path: forge refuses an export_dir that
|
|
8
|
+
# does not resolve under it.
|
|
9
|
+
FORGE_OUTPUT_PATH=./out
|
|
10
|
+
|
|
11
|
+
# ── Demo-safe mode ───────────────────────────────────────────────────
|
|
12
|
+
# 1 = render configs but never train and never write: POST /run is refused with
|
|
13
|
+
# 403 and POST /config is forced to dry-run. Defaults to 1, because the image
|
|
14
|
+
# ships no trainer and the port is published on 0.0.0.0 — a run is real code
|
|
15
|
+
# execution on the host. Set to 0 only on a trusted host with a trainer mounted.
|
|
16
|
+
ARGUS_FORGE_READONLY=1
|
|
17
|
+
|
|
18
|
+
# ── Browser access ───────────────────────────────────────────────────
|
|
19
|
+
# Comma-separated origins allowed to call the API, *in addition to* the
|
|
20
|
+
# localhost:3000 studio frontend the image's --cors already allows.
|
|
21
|
+
# FORGE_CORS_ORIGINS=https://studio.example.com
|
|
22
|
+
|
|
23
|
+
# ── Container <-> host paths ─────────────────────────────────────────
|
|
24
|
+
# Rewrite the container paths forge renders into the host paths the trainer
|
|
25
|
+
# will actually open, as container=host (comma-separated for several).
|
|
26
|
+
# Needed when forge runs in this container but the trainer runs on the host.
|
|
27
|
+
# FORGE_PATH_MAP=/data/out=/home/me/argus/out
|
|
28
|
+
|
|
29
|
+
# ── Version ──────────────────────────────────────────────────────────
|
|
30
|
+
# Stamped into the image and reported by /health. Leave unset for local builds;
|
|
31
|
+
# release images get the tag from CI.
|
|
32
|
+
# FORGE_VERSION=0.1.0
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: ci-${{ github.ref }}
|
|
11
|
+
cancel-in-progress: true
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
ci:
|
|
15
|
+
uses: smk762/argus-ci/.github/workflows/python-ci.yml@v1
|
|
16
|
+
with:
|
|
17
|
+
ruff-version: "0.15.16"
|
|
18
|
+
test-extras: "dev,server,cli"
|
|
19
|
+
post-test: "uv run --no-sync argus-forge schema --check"
|
|
20
|
+
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
name: Release to PyPI
|
|
2
|
+
|
|
3
|
+
# Top-level (non-reusable) on purpose: PyPI trusted publishing cannot run inside
|
|
4
|
+
# a reusable workflow. Keep this file in sync across the suite via `copier update`.
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
id-token: write
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
build:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
with:
|
|
20
|
+
fetch-depth: 0
|
|
21
|
+
- uses: astral-sh/setup-uv@v6
|
|
22
|
+
with:
|
|
23
|
+
enable-cache: true
|
|
24
|
+
- run: uv build
|
|
25
|
+
- uses: actions/upload-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
test-install:
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
needs: build
|
|
33
|
+
strategy:
|
|
34
|
+
matrix:
|
|
35
|
+
python-version: ["3.11", "3.12"]
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: astral-sh/setup-uv@v6
|
|
42
|
+
with:
|
|
43
|
+
enable-cache: true
|
|
44
|
+
- name: Install wheel and smoke-test import
|
|
45
|
+
run: |
|
|
46
|
+
uv venv --python ${{ matrix.python-version }}
|
|
47
|
+
uv pip install dist/*.whl
|
|
48
|
+
uv run --no-sync python -c "import argus_forge; print(f'argus-forge {argus_forge.__version__} OK')"
|
|
49
|
+
|
|
50
|
+
publish-testpypi:
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
needs: test-install
|
|
53
|
+
environment: testpypi
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/download-artifact@v4
|
|
56
|
+
with:
|
|
57
|
+
name: dist
|
|
58
|
+
path: dist/
|
|
59
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
60
|
+
with:
|
|
61
|
+
repository-url: https://test.pypi.org/legacy/
|
|
62
|
+
|
|
63
|
+
publish-pypi:
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
# Ordered *after* the TestPyPI rehearsal so its outcome is visible first, but
|
|
66
|
+
# deliberately not gated on it. A TestPyPI-only fault — a missing trusted
|
|
67
|
+
# publisher, an index outage — must not silently withhold the real release:
|
|
68
|
+
# that failure mode shipped argus-proof images for two tags with no wheel on
|
|
69
|
+
# PyPI, and nothing surfaced it because publish-image is independent.
|
|
70
|
+
# Only the build + install gate is required; a TestPyPI failure still marks
|
|
71
|
+
# the whole run failed, so the problem stays loud.
|
|
72
|
+
needs: [test-install, publish-testpypi]
|
|
73
|
+
if: "!cancelled() && needs.test-install.result == 'success'"
|
|
74
|
+
environment: pypi
|
|
75
|
+
steps:
|
|
76
|
+
- uses: actions/download-artifact@v4
|
|
77
|
+
with:
|
|
78
|
+
name: dist
|
|
79
|
+
path: dist/
|
|
80
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
81
|
+
|
|
82
|
+
publish-image:
|
|
83
|
+
runs-on: ubuntu-latest
|
|
84
|
+
needs: test-install
|
|
85
|
+
permissions:
|
|
86
|
+
contents: read
|
|
87
|
+
packages: write
|
|
88
|
+
steps:
|
|
89
|
+
- uses: actions/checkout@v4
|
|
90
|
+
- uses: docker/setup-buildx-action@v3
|
|
91
|
+
- uses: docker/login-action@v3
|
|
92
|
+
with:
|
|
93
|
+
registry: ghcr.io
|
|
94
|
+
username: ${{ github.actor }}
|
|
95
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
96
|
+
- id: meta
|
|
97
|
+
uses: docker/metadata-action@v5
|
|
98
|
+
with:
|
|
99
|
+
images: ghcr.io/${{ github.repository_owner }}/argus-forge
|
|
100
|
+
tags: |
|
|
101
|
+
type=semver,pattern={{version}}
|
|
102
|
+
type=semver,pattern={{major}}.{{minor}}
|
|
103
|
+
# Only a final release moves `latest`. The trigger is `v*`, so
|
|
104
|
+
# without this guard a `v0.2.0-rc1` tag would silently repoint every
|
|
105
|
+
# `:latest` user (README quickstart, compose) onto a candidate.
|
|
106
|
+
type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }}
|
|
107
|
+
# The base image has no git, so the version is passed in via build-arg
|
|
108
|
+
# (SETUPTOOLS_SCM_PRETEND_VERSION) rather than read from .git history.
|
|
109
|
+
# Derived from the tag itself, not from steps.meta.outputs.version: the
|
|
110
|
+
# `v*` trigger is broader than semver, and for a non-semver tag both
|
|
111
|
+
# type=semver entries drop out and that output falls back to the literal
|
|
112
|
+
# string "latest" — which is not a PEP 440 version and fails the build.
|
|
113
|
+
- id: version
|
|
114
|
+
run: echo "value=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
|
|
115
|
+
- uses: docker/build-push-action@v6
|
|
116
|
+
with:
|
|
117
|
+
context: .
|
|
118
|
+
file: ./Dockerfile
|
|
119
|
+
push: true
|
|
120
|
+
build-args: |
|
|
121
|
+
VERSION=${{ steps.version.outputs.value }}
|
|
122
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
123
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
124
|
+
cache-from: type=gha
|
|
125
|
+
cache-to: type=gha,mode=max
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.pyc
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
# hatch-vcs writes this at build time (version derived from git tags)
|
|
10
|
+
src/argus_forge/_version.py
|
|
11
|
+
# Default dataset volume for docker-compose (${FORGE_OUTPUT_PATH:-./out}) —
|
|
12
|
+
# curated exports, never source.
|
|
13
|
+
/out/
|
|
14
|
+
.env
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Removed
|
|
11
|
+
|
|
12
|
+
- **BREAKING: `argus_forge.jobs` is gone**; the job registry now lives at
|
|
13
|
+
`argus_forge.server.jobs` (issue #17). `from argus_forge.jobs import Job,
|
|
14
|
+
JobRegistry` — the import a 0.1.0 consumer would have written — raises
|
|
15
|
+
`ModuleNotFoundError`. There is no compatibility shim: the module is an
|
|
16
|
+
in-process implementation detail of the HTTP layer, not part of the wire
|
|
17
|
+
contract, and 0.1.0 shipped days ago. Update the import path.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- **A `/stream` viewer is now a cursor into the job's shared event buffer**
|
|
22
|
+
rather than a queue of its own (issue #21). The registry retained the same
|
|
23
|
+
2000-event window *twice* — once in `Job._events`, once per viewer — so N
|
|
24
|
+
stalled readers pinned N copies of it (~1 MB each) on top of the shared tail.
|
|
25
|
+
Total retention is now `MAX_BUFFERED_EVENTS` objects regardless of how many
|
|
26
|
+
viewers are attached; a viewer costs a sequence number and an `asyncio.Event`.
|
|
27
|
+
- Drop-oldest for a lagging viewer is now a *consequence* of the one shared
|
|
28
|
+
bound: a cursor that falls behind the window resumes at the oldest retained
|
|
29
|
+
event. `MAX_SUBSCRIBER_LAG` is gone, and with it the second eviction policy
|
|
30
|
+
and the backlog/live split in `subscribe()` — a cursor cannot straddle a
|
|
31
|
+
seam, so "no event dropped or duplicated across it" is structural rather
|
|
32
|
+
than an invariant a comment had to assert. One behaviour did narrow: a
|
|
33
|
+
viewer *stalled part-way through its replay* used to be immune to eviction
|
|
34
|
+
(the replay was a private list copy) and had a second 2000-event channel
|
|
35
|
+
behind it, so its worst-case tolerance was ~4000 events; it is now the one
|
|
36
|
+
2000-event window. A viewer that has caught up is unaffected.
|
|
37
|
+
- `_publish` still never waits on a viewer, so one stalled `/stream` reader
|
|
38
|
+
cannot apply backpressure to the trainer's stdout. The wire format is
|
|
39
|
+
unchanged: `GET /run/{id}/stream` emits byte-for-byte what it did.
|
|
40
|
+
- The `start` event is still retained out-of-band, but keyed on the sequence
|
|
41
|
+
number it occupied rather than on it being the head of the buffer, and it is
|
|
42
|
+
injected the moment a cursor is found to be past it. So a viewer whose
|
|
43
|
+
cursor is *clamped over* `start` — not only one that joined after it rolled
|
|
44
|
+
off — still learns command/cwd, and a viewer can never be handed it twice.
|
|
45
|
+
- The registry only ever existed to serve the HTTP layer, so it now lives
|
|
46
|
+
beside it at `argus_forge.server.jobs`. The fan-out is plain stdlib asyncio
|
|
47
|
+
again — the `anyio` dependency it briefly took on (and the explicit
|
|
48
|
+
`anyio>=4` on the `server` extra) is gone, as is the end-of-stream sentinel
|
|
49
|
+
and its unverifiable `assert isinstance(...)` that `python -O` compiled out.
|
|
50
|
+
`argus_forge.server` resolves its FastAPI entry points lazily, so the
|
|
51
|
+
registry imports without dragging in fastapi/starlette/argus-cortex.
|
|
52
|
+
|
|
53
|
+
### Fixed
|
|
54
|
+
|
|
55
|
+
- `subscribe()` no longer registers a cursor on a run that has **already
|
|
56
|
+
finished**. `_finalize` is idempotent, so the sweep that drops half-open
|
|
57
|
+
`/stream` readers has run for the last time by then and could never drop it —
|
|
58
|
+
a client that reconnects to a finished run and stalls mid-replay would pin its
|
|
59
|
+
cursor for as long as the registry retained the job (`MAX_FINISHED_JOBS`).
|
|
60
|
+
Nothing can publish to such a viewer anyway, so it registers nothing.
|
|
61
|
+
- `MAX_BUFFERED_EVENTS` is validated as `>= 1` at import, replacing the guard
|
|
62
|
+
that went with `MAX_SUBSCRIBER_LAG`. A zero-length window is a worse mis-tune
|
|
63
|
+
under a cursor than under a queue: the deque keeps nothing while `_published`
|
|
64
|
+
runs away from it, so a clamped cursor indexes off the end and `/stream`
|
|
65
|
+
raises `IndexError` mid-response instead of merely returning an empty body.
|
|
66
|
+
|
|
67
|
+
### Fixed
|
|
68
|
+
|
|
69
|
+
- `Job.cancel()` no longer returns early when another cancel is in flight; it
|
|
70
|
+
waits for it. `JobRegistry.shutdown()` could otherwise report every trainer
|
|
71
|
+
reaped while a request-side cancel was still inside `runner._terminate`'s
|
|
72
|
+
SIGTERM→SIGKILL grace, and the loop closing under it left the trainer running
|
|
73
|
+
detached.
|
|
74
|
+
- `Job.cancel()` no longer swallows a cancellation aimed at *itself*. The blanket
|
|
75
|
+
`suppress(asyncio.CancelledError)` around `await task` caught the caller's own
|
|
76
|
+
cancel too, so a shutdown cut short by uvicorn's `timeout_graceful_shutdown`
|
|
77
|
+
reported success.
|
|
78
|
+
- `subscribe()` finds the retained `start` event by identity on the head of the
|
|
79
|
+
buffer rather than `in`, which ran `RunEvent.__eq__` across the whole
|
|
80
|
+
2000-event window — ~2 ms of event-loop time per reconnect, and only on long
|
|
81
|
+
runs, which is exactly when reconnects happen.
|
|
82
|
+
|
|
83
|
+
## [0.1.0] - 2026-07-21
|
|
84
|
+
|
|
85
|
+
First tagged release — the version that publishes `ghcr.io/smk762/argus-forge`
|
|
86
|
+
(issue #15) and can join the suite demo.
|
|
87
|
+
|
|
88
|
+
### Security
|
|
89
|
+
|
|
90
|
+
- **Server endpoints now enforce path containment.** `POST /inspect`,
|
|
91
|
+
`/config` and `/run` require the request's `export_dir` to name a directory
|
|
92
|
+
under the configured export root (`--export-root`,
|
|
93
|
+
`ARGUS_FORGE_EXPORT_ROOT`; `FORGE_EXPORT_PATH` is a legacy alias), refusing
|
|
94
|
+
traversal escapes and refusing outright when the root is unset. Previously
|
|
95
|
+
any caller could name any absolute path and have a `forge/` tree written into
|
|
96
|
+
it. Request paths are canonically root-relative; absolute paths are tolerated
|
|
97
|
+
only when already inside the root (the studio UI echoes back the `export_dir`
|
|
98
|
+
forge reported). The **CLI stays unconstrained** by design.
|
|
99
|
+
- Containment is decided on the fully symlink-resolved path, so a symlink out
|
|
100
|
+
of the root is refused rather than followed — but the path handed to the
|
|
101
|
+
emitters keeps the caller's spelling, since
|
|
102
|
+
`manifest.resolve_export_dir` owns that policy and `path_map` prefixes are
|
|
103
|
+
matched against it. A symlinked root (`/data/out -> /mnt/big/out`) keeps
|
|
104
|
+
rewriting correctly.
|
|
105
|
+
- The check runs on the *absolutised* candidate — exactly the path returned
|
|
106
|
+
and then opened — so "checked" and "used" are always the same file.
|
|
107
|
+
Resolving the raw candidate instead validated a different one, because
|
|
108
|
+
`os.path.abspath` cancels `..` lexically while `resolve()` cancels it
|
|
109
|
+
against the symlink's target: with an in-root `d -> <root>/x/y`, the request
|
|
110
|
+
`d/../../evil` resolved inside the root (so it passed) yet abspath'd to
|
|
111
|
+
`<root>/../evil`, and that escaped path was what every endpoint went on to
|
|
112
|
+
read, write and execute.
|
|
113
|
+
- The export root itself is not a valid `export_dir`: a blank or `"."` field
|
|
114
|
+
would otherwise merge every sibling export into one dataset and write a
|
|
115
|
+
`forge/` tree at the top of the shared volume.
|
|
116
|
+
- A malformed path is a 400, not an unhandled 500 traceback — including a
|
|
117
|
+
symlink loop, which `pathlib` reports as `RuntimeError` rather than an
|
|
118
|
+
`OSError`.
|
|
119
|
+
- **Manifest `abs_path` caption sources are fenced too.** `collect_captions`
|
|
120
|
+
copies sidecars from beside the *source* images, and `abs_path` is manifest-
|
|
121
|
+
supplied — as untrusted on a server as the request itself. Without the fence
|
|
122
|
+
any readable `.txt` on the host was copied into the shared dataset volume and,
|
|
123
|
+
for trainers that inline captions, echoed back in the `/config` response.
|
|
124
|
+
Refused sources are reported as a warning. The CLI stays unconstrained.
|
|
125
|
+
- **`--cors` no longer means `allow_origins=["*"]` with credentials**, which
|
|
126
|
+
made Starlette reflect any origin back and defeated the allow-list entirely.
|
|
127
|
+
It now allows the localhost dev frontend; `--cors-origin` /
|
|
128
|
+
`FORGE_CORS_ORIGINS` **add** further origins rather than replacing it, and
|
|
129
|
+
`--cors-any` is a credential-less wildcard. A literal `"*"` in the allow-list
|
|
130
|
+
takes the same credential-less path instead of silently becoming origin
|
|
131
|
+
reflection. Allow-list entries are normalised, so a trailing slash or stray
|
|
132
|
+
whitespace no longer produces a list that can never match.
|
|
133
|
+
- **Unsafe methods are gated on `Origin`** via the suite's shared
|
|
134
|
+
`WriteGuard` (`argus_cortex.server`). CORS is not a write boundary: a
|
|
135
|
+
cross-origin POST with a CORS-safelisted content type gets no preflight, so
|
|
136
|
+
any page a user visits could drive an unauthenticated LAN server into forging
|
|
137
|
+
configs or starting a run. Origin absent (curl, CLI, server-to-server), same
|
|
138
|
+
host:port, or allow-listed passes; anything else is 403. The wildcard grants
|
|
139
|
+
anonymous reads to everyone while origins the operator named explicitly keep
|
|
140
|
+
their write grant — the pairing a public demo needs, since every forge
|
|
141
|
+
endpoint that does anything is a POST.
|
|
142
|
+
- **`PATH` joins the refused `/run` env keys.** The command is the bare name
|
|
143
|
+
`bash`, so a caller-supplied `PATH` pointing at a directory holding its own
|
|
144
|
+
`./bash` replaced the forged script entirely — the deny-list's stated purpose
|
|
145
|
+
("refuses the env vars that would let a request redirect *what* runs") did not
|
|
146
|
+
hold without it.
|
|
147
|
+
- `/health` reports `export_root` alongside `training`, and answers even when
|
|
148
|
+
the configured root cannot be resolved, so a liveness probe never 500s on a
|
|
149
|
+
misconfigured or hung volume. It reports the root only when it is actually
|
|
150
|
+
usable: `resolve()` is non-strict, so an unmounted volume (the image run
|
|
151
|
+
without `-v`, the commonest misconfiguration) used to answer "ok" with a root
|
|
152
|
+
while every request 400'd on "not a directory". The value is the
|
|
153
|
+
un-dereferenced spelling `/inspect` also returns, since `path_map` prefixes
|
|
154
|
+
are matched against it.
|
|
155
|
+
|
|
156
|
+
### Added
|
|
157
|
+
|
|
158
|
+
- Manifest-aware LoRA config forging for `kohya` / `onetrainer` / `diffusers`,
|
|
159
|
+
with hyperparameters seeded from the same selection-insight heuristics the
|
|
160
|
+
argus-curator `/curate` UI shows.
|
|
161
|
+
- `argus-forge` CLI: `inspect`, `config`, `trainers`, `schema`, `serve`, `run`.
|
|
162
|
+
- FastAPI micro-server on `:8103` (`server` extra): `/health`, `/trainers`,
|
|
163
|
+
`/inspect`, `/config`, `/run`, `/runs`, `/run/{id}`, `/run/{id}/stream`,
|
|
164
|
+
`/run/{id}/cancel`.
|
|
165
|
+
- `run` verb: shells out to the forged `train.sh` and streams NDJSON progress
|
|
166
|
+
(#12), on a job registry so runs outlive the connection (#14).
|
|
167
|
+
- **Demo-safe mode** (#16): `serve --no-run` / `ARGUS_FORGE_READONLY=1` renders
|
|
168
|
+
configs but never trains and never writes — every `/run` route is refused
|
|
169
|
+
with 403 (as middleware, so the refusal precedes body validation and covers
|
|
170
|
+
routes added later), and `POST /config` is forced to `dry_run`. `GET /health`
|
|
171
|
+
reports `training: enabled|disabled` so a frontend can disable its train
|
|
172
|
+
affordance up front instead of discovering the refusal by clicking. Required
|
|
173
|
+
for the public demo, whose host is GPU-less by design (argus-halo#7).
|
|
174
|
+
- Container image published to GHCR on `v*` tags (#15), plus a
|
|
175
|
+
`docker-compose.yaml` for local runs. **The image itself** defaults to
|
|
176
|
+
demo-safe mode (`ENV ARGUS_FORGE_READONLY=1`), not just the compose stack: it
|
|
177
|
+
ships no trainer, and it publishes the port on 0.0.0.0 where a run is real
|
|
178
|
+
code execution and an unauthenticated `/config` would overwrite the curator's
|
|
179
|
+
`metadata.jsonl`, so a bare `docker run` has to be as locked down as
|
|
180
|
+
`docker compose up`.
|
|
181
|
+
- `argus-cortex[server]>=0.2.0` is a `server`-extra dependency, for the suite's
|
|
182
|
+
shared write-guard and env-flag helpers.
|
|
183
|
+
- `caption_source_root`: the containment root for manifest `abs_path` caption
|
|
184
|
+
sources. A keyword argument to `forge_config`, deliberately **not** a field on
|
|
185
|
+
`ForgeRequest` — a security boundary a request could name is one it could
|
|
186
|
+
widen to `/`. The server passes its export root; the CLI passes `None`
|
|
187
|
+
(unconstrained — it is the operator's own shell).
|
|
188
|
+
|
|
189
|
+
### Changed
|
|
190
|
+
|
|
191
|
+
- The image now takes its version from the `VERSION` build-arg
|
|
192
|
+
(`SETUPTOOLS_SCM_PRETEND_VERSION`) instead of reading git history, matching
|
|
193
|
+
argus-curator and argus-quarry. This drops the `git` dependency from the
|
|
194
|
+
image and lets `.git` leave the build context. The variable is scoped to the
|
|
195
|
+
install step rather than being an `ENV`, so it neither pollutes the cache key
|
|
196
|
+
of later layers nor persists into the runtime container.
|
|
197
|
+
- The image is multi-stage and defaults `ARGUS_FORGE_EXPORT_ROOT=/data/out`:
|
|
198
|
+
`uv` (~64 MB) and the source tree no longer ship, and the published image
|
|
199
|
+
works from `docker run -v ...:/data/out` alone instead of starting healthy
|
|
200
|
+
and refusing every request. 534 MB → **211 MB**.
|
|
201
|
+
- `ARGUS_FORGE_READONLY` is read by `create_app` rather than only by the `serve`
|
|
202
|
+
command, so every ASGI entry point honours it. Being a *protection* flag it
|
|
203
|
+
fails **safe**: an unrecognised value (`=y`, `=enabled`) warns and keeps the
|
|
204
|
+
guard on, where `env_flag`'s enable-a-feature default of "off" would have
|
|
205
|
+
silently enabled training and `/config` writes on a host that is
|
|
206
|
+
unauthenticated and public by assumption. Still never fatal — under compose's
|
|
207
|
+
`restart: unless-stopped` a hard exit is a crash loop. Only an explicit
|
|
208
|
+
`0`/`false`/`no`/`off` allows runs.
|
|
209
|
+
- Added `.dockerignore`. The build context previously carried `.venv` (60 MB),
|
|
210
|
+
`.git`, and the pytest/ruff caches straight into the image layer. Patterns
|
|
211
|
+
use `**/` where needed — `.dockerignore` has no implicit recursion — and the
|
|
212
|
+
compose dataset volume (`./out`) is excluded.
|
|
213
|
+
|
|
214
|
+
### Fixed
|
|
215
|
+
|
|
216
|
+
- Install `git` in the Docker image so hatch-vcs could detect the version
|
|
217
|
+
(#11) — since superseded by the build-arg above.
|
|
218
|
+
- kohya nested subsets, basename-collision caption guard, path remapping, CORS
|
|
219
|
+
docs, emitter parity CI (#6).
|
|
220
|
+
- manifest 2.0: accept the new version and resolve rows via `exported_path` (#9).
|
|
221
|
+
- Run lifecycle: terminal, bounded, and cancel-safe; a cancel emits a distinct
|
|
222
|
+
`cancelled` event rather than reading as a failure (#14).
|
|
223
|
+
- Containment and `/health` no longer run blocking `stat`/`realpath` syscalls on
|
|
224
|
+
the asyncio event loop: the root is resolved once at startup and per-request
|
|
225
|
+
checks happen inside the existing `asyncio.to_thread` hop, so a slow or hung
|
|
226
|
+
export volume cannot stall in-flight `/run/{id}/stream` progress streams.
|
|
227
|
+
- `serve`'s startup warnings resolve the same env fallbacks `create_app` does,
|
|
228
|
+
so `FORGE_CORS_ORIGINS` no longer produces a "CORS is disabled" warning about
|
|
229
|
+
a server that has CORS enabled.
|
|
230
|
+
- A manifest row whose `abs_path` is empty (or `/`) no longer 500s `POST
|
|
231
|
+
/config`. Only `exported_path` is validated, so `abs_path` can name no file at
|
|
232
|
+
all, and `Path("").with_suffix(...)` raised straight through the catch-all. A
|
|
233
|
+
row with no readable source path simply has no sidecar to collect.
|
|
234
|
+
- Demo-safe `POST /config` now says in `warnings` that it was forced to a dry
|
|
235
|
+
run, so a caller that asked for a real write learns it did not happen from the
|
|
236
|
+
body rather than by noticing every file's `path` is null.
|
|
237
|
+
- `--cors-origin` / `FORGE_CORS_ORIGINS` no longer drop the localhost:3000
|
|
238
|
+
defaults when a bare `--cors` was not also passed. They *add* to the dev
|
|
239
|
+
frontend (as the README says) rather than replacing it, for the write-guard
|
|
240
|
+
trust list as well as the CORS allow-list — naming a production origin must
|
|
241
|
+
not cost you the studio frontend you were already developing against.
|
|
242
|
+
- The release workflow derives the image version from the tag itself rather
|
|
243
|
+
than `docker/metadata-action`'s `version` output, which falls back to the
|
|
244
|
+
literal string `latest` for a non-semver `v*` tag; and `latest` is no longer
|
|
245
|
+
repointed by a prerelease tag.
|
|
246
|
+
|
|
247
|
+
[Unreleased]: https://github.com/smk762/argus-forge/compare/v0.1.0...HEAD
|
|
248
|
+
[0.1.0]: https://github.com/smk762/argus-forge/releases/tag/v0.1.0
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# syntax=docker/dockerfile:1
|
|
2
|
+
FROM python:3.11-slim AS builder
|
|
3
|
+
|
|
4
|
+
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
|
|
5
|
+
|
|
6
|
+
WORKDIR /app
|
|
7
|
+
|
|
8
|
+
# The base image has no `git`, so hatch-vcs can't derive the version from
|
|
9
|
+
# history — hand it in via the VERSION build arg (the release tag, sans "v").
|
|
10
|
+
# Defaults to 0.0.0+local so a locally built image is distinguishable on
|
|
11
|
+
# /health from a real release rather than claiming to be a plain 0.0.0.
|
|
12
|
+
ARG VERSION=0.0.0+local
|
|
13
|
+
|
|
14
|
+
COPY . /app
|
|
15
|
+
|
|
16
|
+
# The version is exported for this RUN only. As an `ENV` it would join the cache
|
|
17
|
+
# key of every layer below it (so a release build, whose VERSION always differs,
|
|
18
|
+
# could never reuse one) and would persist into the runtime image, where the
|
|
19
|
+
# unscoped name would stamp itself onto any setuptools-scm package installed
|
|
20
|
+
# later. hatch-vcs does not read the scoped SETUPTOOLS_SCM_PRETEND_VERSION_FOR_*
|
|
21
|
+
# form, so the unscoped one is the only option — confining it to this
|
|
22
|
+
# instruction is what keeps the blast radius to this install.
|
|
23
|
+
# The cache mount keeps wheel downloads across builds even when this layer runs.
|
|
24
|
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
25
|
+
SETUPTOOLS_SCM_PRETEND_VERSION="${VERSION}" \
|
|
26
|
+
uv pip install --system ".[server,cli]"
|
|
27
|
+
|
|
28
|
+
FROM python:3.11-slim
|
|
29
|
+
|
|
30
|
+
# Only the installed package and its console script — uv (~64 MB of the old
|
|
31
|
+
# single-stage image) is a build tool and never runs here.
|
|
32
|
+
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
|
33
|
+
COPY --from=builder /usr/local/bin/argus-forge /usr/local/bin/argus-forge
|
|
34
|
+
|
|
35
|
+
# Both docker-compose.yaml and the README's `docker run -v ...:/data/out` use
|
|
36
|
+
# this path. Without a default the published image starts, answers /health with
|
|
37
|
+
# "ok", and then refuses every functional request for want of a root — a
|
|
38
|
+
# healthy-looking service the frontend cannot use.
|
|
39
|
+
ENV ARGUS_FORGE_EXPORT_ROOT=/data/out
|
|
40
|
+
|
|
41
|
+
# Demo-safe by default: /config renders but never writes, and every /run route is
|
|
42
|
+
# refused with 403. This image ships no trainer — no torch, no sd-scripts — so a
|
|
43
|
+
# run could only ever fail, and the port is published on 0.0.0.0 where a run is
|
|
44
|
+
# real code execution on the host (see runner.py's trust note) and an
|
|
45
|
+
# unauthenticated /config would overwrite the curator's metadata.jsonl. The
|
|
46
|
+
# default therefore has to be the safe one: `docker run` of this image is as
|
|
47
|
+
# locked down as `docker compose up`, and .env.example's "Defaults to 1" is true
|
|
48
|
+
# of the image itself rather than only of the compose file. Set to 0 on a trusted
|
|
49
|
+
# host once a trainer is mounted. See README "Demo-safe mode".
|
|
50
|
+
ENV ARGUS_FORGE_READONLY=1
|
|
51
|
+
|
|
52
|
+
EXPOSE 8103
|
|
53
|
+
|
|
54
|
+
CMD ["argus-forge", "serve", "--port", "8103", "--cors"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 smk762
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
UV ?= uv
|
|
2
|
+
|
|
3
|
+
.PHONY: help install lint format test build clean
|
|
4
|
+
|
|
5
|
+
help: ## Show this help
|
|
6
|
+
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}'
|
|
7
|
+
|
|
8
|
+
install: ## Create venv and install with test extras
|
|
9
|
+
$(UV) venv
|
|
10
|
+
$(UV) pip install -e ".[dev,server,cli]"
|
|
11
|
+
|
|
12
|
+
lint: ## Ruff check + format check
|
|
13
|
+
$(UV)x ruff@0.15.16 check src/ tests/
|
|
14
|
+
$(UV)x ruff@0.15.16 format --check src/ tests/
|
|
15
|
+
|
|
16
|
+
format: ## Ruff autoformat + autofix
|
|
17
|
+
$(UV)x ruff@0.15.16 format src/ tests/
|
|
18
|
+
$(UV)x ruff@0.15.16 check --fix src/ tests/
|
|
19
|
+
|
|
20
|
+
test: ## Run the test suite
|
|
21
|
+
$(UV) run --no-sync pytest --tb=short -q
|
|
22
|
+
|
|
23
|
+
build: ## Build sdist + wheel
|
|
24
|
+
$(UV) build
|
|
25
|
+
|
|
26
|
+
clean: ## Remove build + cache artifacts
|
|
27
|
+
rm -rf dist build .venv *.egg-info .pytest_cache .ruff_cache src/argus_forge/_version.py
|