mlsec-raid 0.3.0__tar.gz → 0.4.2__tar.gz

{mlsec_raid-0.3.0 → mlsec_raid-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: mlsec-raid
-Version: 0.3.0
+Version: 0.4.2
 Summary: Client for the RAID competition system
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -15,11 +15,16 @@ export RAID_KEY=<your-api-key>                            # or pass --api-key <y
 
 task-fetch <challenge>    # downloads the challenge into ./<challenge>/
 task-check <challenge>    # scores it locally with scoring/score.py on data/
-task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns, prints score and log
+task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns
+task-show                 # lists your jobs with their state (pending, running, success, failed) and score,
+                          # and the submissions left today on challenges with a daily limit
+task-log <job>            # prints the log of a finished job, including the errors of your solution
 ```
 
+Every command also works as `task <command>`, e.g. `task submit <challenge>`.
+
 The same is available in Python: `Raid(api_key).fetch(...)`, `.check(...)`, `.submit(...)`, `.push(...)`, `.wait(id)`, `.status()`, `.log(id)`, `.howto()`.
 
-Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name and version (and the wave, baseline and badge of a new challenge), and the upload becomes the active version.
+Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name, version, wave, baseline, badge and `submissions_per_day` (keys it leaves out keep their current values), and the upload becomes the active version.
 
 Development: `uv sync` sets up the environment, `uv build` builds the package. Set `RAID_URL` to use another server. Release: bump `version` in `pyproject.toml`, then `uv build && uv publish` (PyPI token in `UV_PUBLISH_TOKEN`).

{mlsec_raid-0.3.0 → mlsec_raid-0.4.2}/README.md

@@ -8,11 +8,16 @@ export RAID_KEY=<your-api-key>                            # or pass --api-key <y
 
 task-fetch <challenge>    # downloads the challenge into ./<challenge>/
 task-check <challenge>    # scores it locally with scoring/score.py on data/
-task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns, prints score and log
+task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns
+task-show                 # lists your jobs with their state (pending, running, success, failed) and score,
+                          # and the submissions left today on challenges with a daily limit
+task-log <job>            # prints the log of a finished job, including the errors of your solution
 ```
 
+Every command also works as `task <command>`, e.g. `task submit <challenge>`.
+
 The same is available in Python: `Raid(api_key).fetch(...)`, `.check(...)`, `.submit(...)`, `.push(...)`, `.wait(id)`, `.status()`, `.log(id)`, `.howto()`.
 
-Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name and version (and the wave, baseline and badge of a new challenge), and the upload becomes the active version.
+Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name, version, wave, baseline, badge and `submissions_per_day` (keys it leaves out keep their current values), and the upload becomes the active version.
 
 Development: `uv sync` sets up the environment, `uv build` builds the package. Set `RAID_URL` to use another server. Release: bump `version` in `pyproject.toml`, then `uv build && uv publish` (PyPI token in `UV_PUBLISH_TOKEN`).

{mlsec_raid-0.3.0 → mlsec_raid-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mlsec-raid"
-version = "0.3.0"
+version = "0.4.2"
 description = "Client for the RAID competition system"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -10,7 +10,10 @@ dependencies = []
 task-fetch = "raid.cli:fetch"
 task-check = "raid.cli:check"
 task-submit = "raid.cli:submit"
+task-show = "raid.cli:show"
+task-log = "raid.cli:log"
 task-push = "raid.cli:push"
+task = "raid.cli:main"
 
 [build-system]
 requires = ["uv_build>=0.8.17,<0.9.0"]

{mlsec_raid-0.3.0 → mlsec_raid-0.4.2}/src/raid/__init__.py

@@ -15,7 +15,7 @@ import urllib.request
 from pathlib import Path
 
 __all__ = ["Raid", "RaidError"]
-__version__ = "0.3.0"
+__version__ = "0.4.2"
 
 DEFAULT_URL = "https://raid.mlsec.tu-berlin.de"
 PARTS = ("source", "data", "scoring")  # source/ is unpacked into the challenge directory itself
@@ -54,8 +54,11 @@ class Raid:
         with tempfile.TemporaryDirectory() as tmp:
             for part in PARTS:
                 with self._request("GET", f"/api/challenges/{challenge}/{part}.tar.gz", timeout=300) as r, \
-                        open(Path(tmp) / f"{part}.tar.gz", "wb") as f:
-                    shutil.copyfileobj(r, f, 1 << 20)
+                        open(Path(tmp) / f"{part}.tar.gz", "wb") as f, \
+                        _Progress(f"Downloading {part}", r.headers.get("Content-Length")) as bar:
+                    while chunk := r.read(1 << 16):
+                        f.write(chunk)
+                        bar.update(len(chunk))
             target.mkdir(parents=True, exist_ok=True)
             for part in PARTS:
                 _extract(Path(tmp) / f"{part}.tar.gz", target, part)
@@ -104,30 +107,32 @@ class Raid:
             raise RaidError(f"Missing required files in {target}/: " + ", ".join(missing))
         with tempfile.TemporaryDirectory() as tmp:
             archive = Path(tmp) / "submission.tar.gz"
+            _status(f"Packing {target}/ ...")
             _pack(target, archive)
             size = archive.stat().st_size
-            with open(archive, "rb") as f, self._request(
-                    "POST", f"/api/submit/{challenge}", data=f, timeout=900,
+            with open(archive, "rb") as f, _Progress("Uploading", size) as bar, self._request(
+                    "POST", f"/api/submit/{challenge}", data=bar.reader(f), timeout=900,
                     headers={"Content-Type": "application/gzip", "Content-Length": str(size)}) as r:
                 result = json.load(r)
-        print(f"Submitted {challenge} ({size / 1e6:.2f} MB) as submission {result['id']} (challenge version {result['version']}).")
+        print("Job submitted!\nRun task-show to see the status of your jobs.")
         return result["id"]
 
     def push(self, directory):
         """Admins: upload a challenge directory (everything but reference/) as a new, active version.
 
-        Its challenge.yaml supplies the name, the version and, for a new challenge, the metadata (title, wave,
-        baseline, badge_score, badge_points; later changes happen in the admin console). Returns the server's summary."""
+        Its challenge.yaml supplies the name, the version and the metadata (title, wave, baseline, badge_score,
+        badge_points); keys it leaves out keep their current values. Returns the server's summary."""
         target = Path(directory)
         missing = [p for p in ("challenge.yaml", "data", "scoring", "hidden") if not (target / p).exists()]
         if missing:
             raise RaidError(f"Missing in {target}/: " + ", ".join(missing))
         with tempfile.TemporaryDirectory() as tmp:
             archive = Path(tmp) / "challenge.tar.gz"
+            _status(f"Packing {target}/ ...")
             _pack(target, archive, own={"reference"})
             size = archive.stat().st_size
-            with open(archive, "rb") as f, self._request(
-                    "POST", "/api/admin/push", data=f, timeout=3600,
+            with open(archive, "rb") as f, _Progress("Uploading", size) as bar, self._request(
+                    "POST", "/api/admin/push", data=bar.reader(f), timeout=3600,
                     headers={"Content-Type": "application/gzip", "Content-Length": str(size)}) as r:
                 result = json.load(r)
         wave = f"wave {result['wave']}" if result["wave"] else "the self-assessment"
@@ -149,20 +154,34 @@ class Raid:
         return sub
 
     def status(self, challenge=None):
-        """Print your submissions (newest first) and return them as a list of dicts."""
+        """Print your jobs (submissions, newest first) and return them as a list of dicts."""
         query = f"?challenge={_check(challenge)}" if challenge else ""
-        subs = self._json(f"/api/submissions{query}")["submissions"]
-        print(f"{'ID':>6}  {'CHALLENGE':<20} {'VERSION':<8} {'STATUS':<8} {'SCORE':>10}  SUBMITTED")
+        result = self._json(f"/api/submissions{query}")
+        subs = result["submissions"]
+        print(f"{'JOB':>6}  {'CHALLENGE':<20} {'VERSION':<8} {'STATE':<8} {'SCORE':>10}  SUBMITTED")
         for s in subs:
             score = "" if s["score"] is None else f"{s['score']:.4f}"
             print(f"{s['id']:>6}  {s['challenge']:<20} {s['version']:<8} {s['status']:<8} {score:>10}  {s['submitted_at'][:19]}")
         if not subs:
-            print("No submissions yet.")
+            print("No jobs yet.")
+        else:
+            print("Run task-log <job> to see the log of a finished job.")
+        limits = result.get("daily_limits") or []
+        if limits:
+            print("\nDaily submission limits (in competition mode per team, reset at 00:00 UTC):")
+            for d in limits:
+                left = max(d["submissions_per_day"] - d["submitted_today"], 0)
+                print(f"  {d['challenge']:<20} {left} of {d['submissions_per_day']} left today")
         return subs
 
     def log(self, submission_id):
-        """Return the scoring log of one of your submissions."""
-        return self._json(f"/api/submissions/{int(submission_id)}")["log"]
+        """Return the scoring log of one of your finished submissions: the output and errors of the scoring run."""
+        if not str(submission_id).isdigit():
+            raise RaidError(f"Invalid job id {submission_id!r}. task-show lists your jobs.")
+        sub = self._json(f"/api/submissions/{submission_id}")
+        if sub["status"] in ("pending", "running"):
+            raise RaidError(f"Job {sub['id']} is {sub['status']}. Its log is available once it has finished.")
+        return sub["log"] or "No log output."
 
     def _request(self, method, endpoint, data=None, headers=None, timeout=60):
         if not self.api_key:
@@ -185,6 +204,63 @@ class Raid:
             return json.load(r)
 
 
+def _status(text):
+    """A transient line on stderr, only when it is a terminal."""
+    if sys.stderr.isatty():
+        sys.stderr.write(f"\r\033[K{text}")
+        sys.stderr.flush()
+
+
+class _Progress:
+    """A one-line progress bar on stderr, only when it is a terminal. Without a known total it counts bytes."""
+
+    def __init__(self, label, total=None):
+        self.label, self.total, self.done, self.shown = label, int(total) if total else None, 0, 0.0
+        self.tty = sys.stderr.isatty()
+
+    def __enter__(self):
+        self._draw(force=True)
+        return self
+
+    def __exit__(self, *exc):
+        if self.tty:
+            self._draw(force=True)
+            sys.stderr.write("\n" if exc[0] is None else "\r\033[K")
+            sys.stderr.flush()
+
+    def update(self, n):
+        self.done += n
+        self._draw()
+
+    def reader(self, f):
+        """f with a read() that advances the bar, for uploads."""
+        bar = self
+
+        class Reader:
+            def read(self, n=-1):
+                chunk = f.read(n)
+                bar.update(len(chunk))
+                return chunk
+        return Reader()
+
+    def _draw(self, force=False):
+        now = time.monotonic()
+        if not self.tty or (not force and now - self.shown < 0.1):
+            return
+        self.shown = now
+        if self.total:
+            frac = min(self.done / self.total, 1)
+            line = f"{self.label} [{'#' * int(frac * 30):<30}] {frac:4.0%}  {_size(self.done)} of {_size(self.total)}"
+        else:
+            line = f"{self.label} {_size(self.done)}"
+        sys.stderr.write(f"\r\033[K{line}")
+        sys.stderr.flush()
+
+
+def _size(n):
+    return f"{n / 1e6:.1f} MB" if n >= 1e6 else f"{n / 1e3:.0f} kB"
+
+
 def _check(challenge):
     if not isinstance(challenge, str) or not NAME_RE.match(challenge):
         raise RaidError(f"Invalid challenge name {challenge!r}.")

{mlsec_raid-0.3.0 → mlsec_raid-0.4.2}/src/raid/cli.py

@@ -1,4 +1,4 @@
-"""Command line tools: task-fetch, task-check, task-submit and task-push."""
+"""Command line tools: task-fetch, task-check, task-submit, task-show, task-log and task-push (also as `task <command>`)."""
 import argparse
 import sys
 
@@ -11,7 +11,8 @@ def _run(description, action, key=True, options=(), target=("challenge", "challe
         parser.add_argument("--api-key", help="your API key (default: $RAID_KEY)")
     for flag, text in options:
         parser.add_argument(flag, action="store_true", help=text)
-    parser.add_argument(target[0], help=target[1])
+    if target:
+        parser.add_argument(target[0], help=target[1])
     args = parser.parse_args()
     try:
         action(Raid(getattr(args, "api_key", None)), args)
@@ -31,16 +32,30 @@ def check():
 
 
 def submit():
-    def action(raid, args):
-        sid = raid.submit(args.challenge)
-        try:
-            raid.wait(sid)
-        except KeyboardInterrupt:
-            sys.exit("\nStopped waiting. Your submission is still being scored.")
-    _run("Submit ./<challenge>/ (without data/, scoring/ and .raidignore patterns), wait for the score and print the log.", action)
+    _run("Submit ./<challenge>/ for scoring (without data/, scoring/ and .raidignore patterns).",
+         lambda raid, a: raid.submit(a.challenge))
+
+
+def show():
+    _run("List your jobs (submissions) with their state and score.", lambda raid, a: raid.status(), target=None)
+
+
+def log():
+    _run("Print the log of a finished job: the output and errors of its scoring run.",
+         lambda raid, a: print(raid.log(a.job)), target=("job", "job id, see task-show"))
 
 
 def push():
     _run("Admins: upload a challenge directory (without reference/) as its new active version. "
-         "challenge.yaml supplies the name, the version and the metadata of a new challenge.",
+         "challenge.yaml supplies the name, the version and the metadata.",
          lambda raid, a: raid.push(a.directory), target=("directory", "challenge directory, e.g. self-assessment"))
+
+
+def main():
+    """`task submit <challenge>` is the same as `task-submit <challenge>`."""
+    commands = {"fetch": fetch, "check": check, "submit": submit, "show": show, "log": log, "push": push}
+    if len(sys.argv) < 2 or sys.argv[1] not in commands:
+        sys.exit("Usage: task {" + ",".join(commands) + "} ...")
+    command = sys.argv.pop(1)
+    sys.argv[0] = f"task {command}"
+    commands[command]()