codabench-client 0.1.0__py3-none-any.whl

codabench/downloads.py

@@ -0,0 +1,80 @@
+"""Writing downloaded bytes to disk.
+
+Every Codabench artifact arrives as a zip (datasets, submissions, prediction
+and scoring output), so the default is to extract; anything that is not a zip
+is written as-is.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import zipfile
+from pathlib import Path
+
+
+def save_bytes(content: bytes, dest_dir: "str | Path", filename: str,
+               extract: bool = True) -> Path:
+    """Write ``content`` into ``dest_dir`` and return what was created.
+
+    When ``content`` is a zip and ``extract`` is set, it is unpacked into a
+    directory named after ``filename`` (without the ``.zip`` suffix) and that
+    directory is returned; otherwise the saved file path is returned.
+    """
+    dest = Path(dest_dir)
+    dest.mkdir(parents=True, exist_ok=True)
+    if extract and zipfile.is_zipfile(io.BytesIO(content)):
+        target = dest / filename[:-4] if filename.lower().endswith(".zip") else dest / filename
+        with zipfile.ZipFile(io.BytesIO(content)) as zf:
+            zf.extractall(target)
+        return target
+    path = dest / filename
+    path.write_bytes(content)
+    return path
+
+
+def save_json(data: object, path: "str | Path") -> Path:
+    """Write ``data`` as indented UTF-8 JSON, creating parent directories."""
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    return target
+
+
+def read_scores(content: bytes) -> dict:
+    """Best-effort ``metric -> value`` extraction from a scoring output zip.
+
+    Reads any JSON/txt/YAML member: JSON objects are merged as-is, other files
+    are scanned for ``key: value`` / ``key= value`` lines (the ``scores.txt``
+    convention). Used as a fallback when the API returns no inline scores.
+    """
+    out: dict = {}
+    try:
+        zf = zipfile.ZipFile(io.BytesIO(content))
+    except zipfile.BadZipFile:
+        return out
+    for name in zf.namelist():
+        if not name.lower().endswith((".json", ".txt", ".yaml", ".yml")):
+            continue
+        raw = zf.read(name).decode("utf-8", errors="replace")
+        if name.lower().endswith(".json"):
+            try:
+                data = json.loads(raw)
+                if isinstance(data, dict):
+                    out.update(data)
+                continue
+            except json.JSONDecodeError:
+                pass  # fall through to line parsing
+        for line in raw.splitlines():
+            line = line.strip()
+            for sep in (":", "="):
+                if sep in line:
+                    key, _, value = line.partition(sep)
+                    key, value = key.strip(), value.strip()
+                    if key:
+                        try:
+                            out[key] = float(value)
+                        except ValueError:
+                            out[key] = value
+                    break
+    return out

codabench/errors.py

@@ -0,0 +1,29 @@
+"""Exceptions raised by the client.
+
+The library raises; the CLI catches :class:`CodabenchError` at the top level and
+turns it into a one-line ``error: ...`` message. Nothing here calls ``sys.exit``,
+so the package stays usable from a notebook or another program.
+"""
+
+from __future__ import annotations
+
+
+class CodabenchError(Exception):
+    """Base class for every error this package raises."""
+
+
+class AuthError(CodabenchError):
+    """Credentials are missing, rejected, or insufficient for the request."""
+
+
+class ApiError(CodabenchError):
+    """The API returned an unexpected status code.
+
+    ``status`` and ``body`` are kept so callers can branch on them (e.g. treat
+    403/404 as "not authorized for this account" rather than a hard failure).
+    """
+
+    def __init__(self, message: str, status: int = 0, body: str = "") -> None:
+        super().__init__(message)
+        self.status = status
+        self.body = body

codabench/text.py

@@ -0,0 +1,59 @@
+"""Small text helpers: HTML to readable text, slugs, human-readable sizes."""
+
+from __future__ import annotations
+
+import re
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_BLANK_RUNS_RE = re.compile(r"[ \t]*\n[ \t]*\n[ \t]*\n+")
+_ENTITIES = {
+    "&nbsp;": " ", "&amp;": "&", "&lt;": "<", "&gt;": ">",
+    "&quot;": '"', "&#39;": "'",
+}
+
+
+def strip_html(text: str) -> str:
+    """Best-effort HTML -> readable text: drop tags, collapse blank runs.
+
+    Codabench pages are authored in markdown but served as HTML, so this is
+    enough to make them readable in a terminal or a .md file.
+    """
+    if not text:
+        return ""
+    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
+    text = re.sub(r"</p>", "\n\n", text, flags=re.I)
+    text = _TAG_RE.sub("", text)
+    for entity, char in _ENTITIES.items():
+        text = text.replace(entity, char)
+    return _BLANK_RUNS_RE.sub("\n\n", text).strip()
+
+
+def slug(text: str) -> str:
+    """Lowercase, hyphenated, filesystem-safe version of ``text``."""
+    s = re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")
+    return s or "page"
+
+
+def safe_name(text: str, fallback: str = "file") -> str:
+    """Filesystem-safe filename that keeps the original casing and spaces."""
+    name = re.sub(r"[^\w.\- ]+", "_", (text or "").strip()).strip()
+    return name or fallback
+
+
+def human_size(size: object) -> str:
+    """Format a byte count as ``12.3 MB``; ``?`` when it is not a number."""
+    try:
+        value = float(size)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return "?"
+    for unit in ("B", "KB", "MB", "GB"):
+        if value < 1024 or unit == "GB":
+            return f"{value:,.1f} {unit}"
+        value /= 1024
+    return "?"
+
+
+def truncate(text: str, limit: int) -> str:
+    """One-line preview of ``text``, ellipsised at ``limit`` characters."""
+    flat = " ".join((text or "").split())
+    return flat if len(flat) <= limit else flat[:limit] + "..."

codabench_client-0.1.0.dist-info/METADATA

@@ -0,0 +1,258 @@
+Metadata-Version: 2.4
+Name: codabench-client
+Version: 0.1.0
+Summary: A small, readable Python client and CLI for the Codabench API.
+License: MIT
+Project-URL: Homepage, https://github.com/amoujar/codabench-client
+Project-URL: Issues, https://github.com/amoujar/codabench-client/issues
+Keywords: codabench,codalab,benchmark,competition,machine-learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25
+Dynamic: license-file
+
+# codabench-client
+
+A small, readable Python client and CLI for the [Codabench](https://www.codabench.org/) API.
+
+Codabench is a great benchmark platform, but a lot of what you can do in its web UI —
+grabbing a starting kit, submitting a zip, pulling the scoring logs of a failed run —
+is tedious to click through and awkward to automate. This package does all of it from
+the terminal or from Python.
+
+```bash
+codabench show 17363                        # everything about a competition
+codabench files 16161                       # download the starting kit & public data
+codabench submit 17525 -z run.zip --wait    # submit and watch the score come back
+codabench outputs 17525                     # fetch the prediction & scoring output
+```
+
+---
+
+## Install
+
+```bash
+git clone https://github.com/<you>/api-codabench.git
+cd api-codabench
+pip install -e .
+```
+
+Python 3.9+ and `requests` are the only requirements. Without installing, everything
+also works as `python -m codabench ...` and `python examples/01_show_competition.py`.
+
+## Credentials
+
+Copy the template and fill it in:
+
+```bash
+cp .env.example .env
+```
+
+```ini
+CODABENCH_USERNAME=your_username
+CODABENCH_PASSWORD=your_password
+# CODABENCH_URL=https://dev.codabench.org/   # optional; default is www.codabench.org
+```
+
+`.env` is gitignored. Environment variables win over the file, so CI can just set
+`CODABENCH_USERNAME` / `CODABENCH_PASSWORD`. Reading a **public** competition needs no
+credentials at all.
+
+> **Accounts are per-instance.** A `www.codabench.org` login does not work on
+> `dev.codabench.org`, and vice versa.
+
+## Commands
+
+
+| Command                                      | What it does                                             |
+| ---------------------------------------------- | ---------------------------------------------------------- |
+| `codabench competitions [--search TEXT]`     | List competitions                                        |
+| `codabench show <id|url>`                    | Pages, phases, tasks, files, leaderboard                 |
+| `codabench files <id|url>`                   | List / download the "Files" tab                          |
+| `codabench submit <id|url> -z FILE`          | Upload a submission, optionally wait for the score       |
+| `codabench outputs <id|url>`                 | Download a submission's zip, prediction & scoring output |
+| `codabench rerun --submission ID --task KEY` | Re-run a submission on another task (robot accounts)     |
+| `codabench create -b bundle.zip`             | Create a competition from a bundle                       |
+
+
+| Command                                        | What it does                                             |
+| ------------------------------------------------ | ---------------------------------------------------------- |
+| `codabench competitions [--search TEXT]`       | List competitions                                        |
+| `codabench show &lt;id&#124;url&gt;`           | Pages, phases, tasks, files, leaderboard                 |
+| `codabench files &lt;id&#124;url&gt;`          | List / download the "Files" tab                          |
+| `codabench submit &lt;id&#124;url&gt; -z FILE` | Upload a submission, optionally wait for the score       |
+| `codabench outputs &lt;id&#124;url&gt;`        | Download a submission's zip, prediction & scoring output |
+| `codabench rerun --submission ID --task KEY`   | Re-run a submission on another task (robot accounts)     |
+| `codabench create -b bundle.zip`               | Create a competition from a bundle                       |
+
+Every command takes `--url`, `--username`, `--password`, `--env`, and `--help`.
+
+### Inspect a competition
+
+```bash
+codabench show 17363                # or the full https://www.codabench.org/competitions/17363/ URL
+codabench show 17363 --pages        # full text of every page
+codabench show 17363 --json         # raw API payload
+```
+
+Save the whole thing — description, one markdown file per page, and every file you are
+allowed to download:
+
+```bash
+codabench show 17363 --save-dir workdir/17363
+```
+
+```
+workdir/17363/
+├── description.md
+├── pages/
+│   ├── 00-overview.md
+│   ├── 01-data.md
+│   └── ...
+└── input/
+    └── Track 1 Starting Kit V5/
+```
+
+Add `--no-download` for the text only.
+
+### Download competition files
+
+```bash
+codabench files 16161 --list                 # see what exists first
+codabench files 16161                        # download into files/16161/
+codabench files 16161 --only starting_kit    # filter by type or name
+codabench files 16161 --no-extract           # keep the zips
+```
+
+### Submit and get scored
+
+```bash
+codabench submit 17525 -z submission.zip --wait
+codabench submit 17525 -z submission.zip --wait \
+    --results-json results.json --download-dir outputs
+```
+
+The phase is resolved automatically (the only one, or the currently open one); pass
+`--phase <id>` or `--phase-index <n>` when a competition has several open at once.
+Submitting **consumes your daily quota** — the command checks eligibility first and
+tells you if you are out.
+
+### Get a submission's output
+
+```bash
+codabench outputs 17525 --list              # your submissions on the phase
+codabench outputs 17525                     # newest submission
+codabench outputs 17525 --last 2            # the one before it
+codabench outputs --submission 856524       # a specific id
+codabench outputs 17525 --only scoring      # just the scoring step
+```
+
+Everything lands in `outputs/<submission id>/` with the zips extracted, plus a
+`details.json` holding the scores, logs and exit statuses — which is usually where the
+answer is when a submission fails.
+
+## Python API
+
+```python
+from codabench import CodabenchClient, collect_files, find_phase
+
+client = CodabenchClient()                     # reads .env / environment
+
+competition = client.competition(17363)        # id or URL
+phase = find_phase(competition)                # the open phase
+
+for entry in collect_files(competition):
+    path = client.download_dataset(entry.key, "input/", entry.filename)
+    print(entry.name, "->", path or "not authorized")
+
+submission = client.submit(phase["id"], "run.zip")
+client.wait_for_submission(submission["id"])
+print(client.results(submission["id"])["metrics"])
+```
+
+The client raises `CodabenchError` (with `AuthError` and `ApiError` subclasses) and
+never calls `sys.exit`, so it is safe to use inside a notebook or a larger program.
+
+## What you can actually download
+
+This trips everyone up, so it is worth stating plainly. Codabench decides per file, and
+this tool cannot widen what your account is allowed to see:
+
+
+| File                            | Who can download it                                                   |
+| --------------------------------- | ----------------------------------------------------------------------- |
+| Starting kit, public data       | Any approved participant                                              |
+| Ingestion / scoring program     | Participants**only if** the organizer set `make_programs_available`   |
+| Input data                      | Participants**only if** the organizer set `make_input_data_available` |
+| Reference data (the answer key) | Organizers only, always                                               |
+| Competition bundle              | Organizers only                                                       |
+
+Files your account may not fetch are skipped, not treated as an error. `codabench show`
+prints the organizer's two flags so you can see why something is missing.
+
+## How authentication works
+
+Codabench uses **two** mechanisms and you need both — the reason `CodabenchClient` holds
+a token *and* a session:
+
+
+| Endpoint                    | Auth                                                  |
+| ----------------------------- | ------------------------------------------------------- |
+| `/api/...`                  | Token from`POST /api/api-token-auth/`                 |
+| `/datasets/download/<key>/` | Django**session cookie** from `POST /accounts/login/` |
+
+The download route behind the "Files" tab is a plain Django view that ignores the API
+token, which is why a token-only client gets 403s on files a participant can plainly
+download in the browser. Both schemes read the same credentials, so callers never have
+to think about it.
+
+(There is also `GET /api/competitions/{id}/get_files/`, but it is organizer-only and
+returns 403 for participants — this package does not rely on it.)
+
+## Repository layout
+
+```
+codabench/              the library
+├── auth.py             credentials, token auth, session login
+├── client.py           CodabenchClient — every API call
+├── competitions.py     pure helpers over a competition payload
+├── downloads.py        saving & extracting artifacts
+├── text.py             HTML→text, slugs, sizes
+├── errors.py           CodabenchError / AuthError / ApiError
+└── cli.py              the `codabench` command
+
+examples/               the same flows as standalone scripts
+├── 01_show_competition.py
+├── 02_download_competition_files.py
+├── 03_submit_and_wait.py
+├── 04_download_submission_outputs.py
+├── 05_create_competition.py
+└── 06_rerun_submission.py
+
+assets/                 a sample submission zip and competition bundle
+```
+
+Run any example directly — no install needed:
+
+```bash
+python examples/01_show_competition.py 17363
+```
+
+## Notes
+
+* **Organizers**: `codabench create -b bundle.zip` uploads a competition bundle and polls
+  until it is unpacked. Try it on `https://dev.codabench.org/` first.
+* **Robot accounts**: `codabench rerun` re-runs an existing submission against another
+  task, reusing the uploaded data. It needs an account flagged `is_bot`, or the API
+  answers 403.
+
+## License
+
+MIT.

codabench_client-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+codabench/__init__.py,sha256=6rJotiXXjeK3-zUbZ7Vp5zGrnZnYYyhFghORV2llXOc,1436
+codabench/__main__.py,sha256=SSJn1Vk2bz0YcQ3Lj88--r4OlSenyLJys-u90bQ2PVk,168
+codabench/auth.py,sha256=cNOpoFGAYyiPdszDrkRLXWb1bxy47k9QJ_1bySlAiZs,4967
+codabench/cli.py,sha256=3p-5CZkBsvuMGFVmGnnvD2RxLYTnnKsexEysReMsyd4,19744
+codabench/client.py,sha256=sjPx-J6K2RcfKh_ijEcfLYUkpcItBDlNaLNCdQ3rRFs,19896
+codabench/competitions.py,sha256=fR33cqUqAxnp042vplwXEu_A8KSHh1wLWm5ETfY34Jg,7547
+codabench/downloads.py,sha256=zjOrXX56XsDp77Bh7aYaHUGhGl1Pozd-ouYtz3A1SzA,2935
+codabench/errors.py,sha256=O_WeQKU5vbOa07-TUePwKuZWwSmMqXrOM08T4ZhTcCg,929
+codabench/text.py,sha256=sHKqndmiHHvkGxI0rSyUZtFCLVaIf5_u9FrMb260myA,1943
+codabench_client-0.1.0.dist-info/licenses/LICENSE,sha256=bGA7ejoo0kQpP4K6XI2WnirG-Gti6yUweTkzLs-Lup4,1087
+codabench_client-0.1.0.dist-info/METADATA,sha256=SDLQymbyc3lYcru5GAWi3wec_22uwdIkgrf4dBM1gUk,10656
+codabench_client-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+codabench_client-0.1.0.dist-info/entry_points.txt,sha256=64iHq5EG8HYK6ViGLMGWJIb8iK_QdseY6LkEeuM6YKI,49
+codabench_client-0.1.0.dist-info/top_level.txt,sha256=awRAA61v-tCAS1QeTIEgMlDeLq0CmWfTK1oAKmruCZ8,10
+codabench_client-0.1.0.dist-info/RECORD,,

codabench_client-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

codabench_client-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+codabench = codabench.cli:main

codabench_client-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Abderrahmane Moujar [Scarface]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

codabench_client-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+codabench