mlsec-raid 0.3.0__tar.gz → 0.4.0__tar.gz

{mlsec_raid-0.3.0 → mlsec_raid-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: mlsec-raid
-Version: 0.3.0
+Version: 0.4.0
 Summary: Client for the RAID competition system
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -15,9 +15,13 @@ export RAID_KEY=<your-api-key>                            # or pass --api-key <y
 
 task-fetch <challenge>    # downloads the challenge into ./<challenge>/
 task-check <challenge>    # scores it locally with scoring/score.py on data/
-task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns, prints score and log
+task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns
+task-show                 # lists your jobs with their state (pending, running, success, failed) and score
+task-log <job>            # prints the log of a finished job, including the errors of your solution
 ```
 
+Every command also works as `task <command>`, e.g. `task submit <challenge>`.
+
 The same is available in Python: `Raid(api_key).fetch(...)`, `.check(...)`, `.submit(...)`, `.push(...)`, `.wait(id)`, `.status()`, `.log(id)`, `.howto()`.
 
 Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name and version (and the wave, baseline and badge of a new challenge), and the upload becomes the active version.

{mlsec_raid-0.3.0 → mlsec_raid-0.4.0}/README.md

@@ -8,9 +8,13 @@ export RAID_KEY=<your-api-key>                            # or pass --api-key <y
 
 task-fetch <challenge>    # downloads the challenge into ./<challenge>/
 task-check <challenge>    # scores it locally with scoring/score.py on data/
-task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns, prints score and log
+task-submit <challenge>   # submits ./<challenge>/ without data/, scoring/ and .raidignore patterns
+task-show                 # lists your jobs with their state (pending, running, success, failed) and score
+task-log <job>            # prints the log of a finished job, including the errors of your solution
 ```
 
+Every command also works as `task <command>`, e.g. `task submit <challenge>`.
+
 The same is available in Python: `Raid(api_key).fetch(...)`, `.check(...)`, `.submit(...)`, `.push(...)`, `.wait(id)`, `.status()`, `.log(id)`, `.howto()`.
 
 Challenge authors: `task-check <challenge> --hidden --reference` scores `reference/` on `hidden/`. Admins upload a challenge directory with `task-push <directory>` and their own API key (created in the admin console): the directory's `challenge.yaml` sets the name and version (and the wave, baseline and badge of a new challenge), and the upload becomes the active version.

{mlsec_raid-0.3.0 → mlsec_raid-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mlsec-raid"
-version = "0.3.0"
+version = "0.4.0"
 description = "Client for the RAID competition system"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -10,7 +10,10 @@ dependencies = []
 task-fetch = "raid.cli:fetch"
 task-check = "raid.cli:check"
 task-submit = "raid.cli:submit"
+task-show = "raid.cli:show"
+task-log = "raid.cli:log"
 task-push = "raid.cli:push"
+task = "raid.cli:main"
 
 [build-system]
 requires = ["uv_build>=0.8.17,<0.9.0"]

{mlsec_raid-0.3.0 → mlsec_raid-0.4.0}/src/raid/__init__.py

@@ -15,7 +15,7 @@ import urllib.request
 from pathlib import Path
 
 __all__ = ["Raid", "RaidError"]
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 DEFAULT_URL = "https://raid.mlsec.tu-berlin.de"
 PARTS = ("source", "data", "scoring")  # source/ is unpacked into the challenge directory itself
@@ -110,7 +110,7 @@ class Raid:
                     "POST", f"/api/submit/{challenge}", data=f, timeout=900,
                     headers={"Content-Type": "application/gzip", "Content-Length": str(size)}) as r:
                 result = json.load(r)
-        print(f"Submitted {challenge} ({size / 1e6:.2f} MB) as submission {result['id']} (challenge version {result['version']}).")
+        print("Job submitted!\nRun task-show to see the status of your jobs.")
         return result["id"]
 
     def push(self, directory):
@@ -149,20 +149,27 @@ class Raid:
         return sub
 
     def status(self, challenge=None):
-        """Print your submissions (newest first) and return them as a list of dicts."""
+        """Print your jobs (submissions, newest first) and return them as a list of dicts."""
         query = f"?challenge={_check(challenge)}" if challenge else ""
         subs = self._json(f"/api/submissions{query}")["submissions"]
-        print(f"{'ID':>6}  {'CHALLENGE':<20} {'VERSION':<8} {'STATUS':<8} {'SCORE':>10}  SUBMITTED")
+        print(f"{'JOB':>6}  {'CHALLENGE':<20} {'VERSION':<8} {'STATE':<8} {'SCORE':>10}  SUBMITTED")
         for s in subs:
             score = "" if s["score"] is None else f"{s['score']:.4f}"
             print(f"{s['id']:>6}  {s['challenge']:<20} {s['version']:<8} {s['status']:<8} {score:>10}  {s['submitted_at'][:19]}")
         if not subs:
-            print("No submissions yet.")
+            print("No jobs yet.")
+        else:
+            print("Run task-log <job> to see the log of a finished job.")
         return subs
 
     def log(self, submission_id):
-        """Return the scoring log of one of your submissions."""
-        return self._json(f"/api/submissions/{int(submission_id)}")["log"]
+        """Return the scoring log of one of your finished submissions: the output and errors of the scoring run."""
+        if not str(submission_id).isdigit():
+            raise RaidError(f"Invalid job id {submission_id!r}. task-show lists your jobs.")
+        sub = self._json(f"/api/submissions/{submission_id}")
+        if sub["status"] in ("pending", "running"):
+            raise RaidError(f"Job {sub['id']} is {sub['status']}. Its log is available once it has finished.")
+        return sub["log"] or "No log output."
 
     def _request(self, method, endpoint, data=None, headers=None, timeout=60):
         if not self.api_key:

{mlsec_raid-0.3.0 → mlsec_raid-0.4.0}/src/raid/cli.py

@@ -1,4 +1,4 @@
-"""Command line tools: task-fetch, task-check, task-submit and task-push."""
+"""Command line tools: task-fetch, task-check, task-submit, task-show, task-log and task-push (also as `task <command>`)."""
 import argparse
 import sys
 
@@ -11,7 +11,8 @@ def _run(description, action, key=True, options=(), target=("challenge", "challe
         parser.add_argument("--api-key", help="your API key (default: $RAID_KEY)")
     for flag, text in options:
         parser.add_argument(flag, action="store_true", help=text)
-    parser.add_argument(target[0], help=target[1])
+    if target:
+        parser.add_argument(target[0], help=target[1])
     args = parser.parse_args()
     try:
         action(Raid(getattr(args, "api_key", None)), args)
@@ -31,16 +32,30 @@ def check():
 
 
 def submit():
-    def action(raid, args):
-        sid = raid.submit(args.challenge)
-        try:
-            raid.wait(sid)
-        except KeyboardInterrupt:
-            sys.exit("\nStopped waiting. Your submission is still being scored.")
-    _run("Submit ./<challenge>/ (without data/, scoring/ and .raidignore patterns), wait for the score and print the log.", action)
+    _run("Submit ./<challenge>/ for scoring (without data/, scoring/ and .raidignore patterns).",
+         lambda raid, a: raid.submit(a.challenge))
+
+
+def show():
+    _run("List your jobs (submissions) with their state and score.", lambda raid, a: raid.status(), target=None)
+
+
+def log():
+    _run("Print the log of a finished job: the output and errors of its scoring run.",
+         lambda raid, a: print(raid.log(a.job)), target=("job", "job id, see task-show"))
 
 
 def push():
     _run("Admins: upload a challenge directory (without reference/) as its new active version. "
          "challenge.yaml supplies the name, the version and the metadata of a new challenge.",
          lambda raid, a: raid.push(a.directory), target=("directory", "challenge directory, e.g. self-assessment"))
+
+
+def main():
+    """`task submit <challenge>` is the same as `task-submit <challenge>`."""
+    commands = {"fetch": fetch, "check": check, "submit": submit, "show": show, "log": log, "push": push}
+    if len(sys.argv) < 2 or sys.argv[1] not in commands:
+        sys.exit("Usage: task {" + ",".join(commands) + "} ...")
+    command = sys.argv.pop(1)
+    sys.argv[0] = f"task {command}"
+    commands[command]()