cortexforge 0.1.0__tar.gz → 0.1.3__tar.gz

{cortexforge-0.1.0 → cortexforge-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexforge
-Version: 0.1.0
+Version: 0.1.3
 Summary: RF generator dataset based on SLICES/CorteXlab
 Author-email: Andrea Joly <andrea.joly@inria.fr>
 Requires-Python: >=3.10

{cortexforge-0.1.0 → cortexforge-0.1.3}/README.md

@@ -1,13 +1,14 @@
 # CorteXForge
-This project is a framework designed to automate the generation and execution of radio dataset experiments on the [Slices/CorteXlab](https://www.cortexlab.fr/doku.php?id=start) testbed.
+This project is a framework designed to automate the generation and execution of radio dataset experiments on the [SLICES-RI/CorteXlab](https://www.cortexlab.fr/doku.php?id=start) testbed.
 It relies on the [GNU Radio](https://www.gnuradio.org) environment to record labeled transmissions of various signals.
 
 ## Overview
-This project is organized into two main components:
+This project is organized into three main components:
 - Scenario generation: this part produces configuration files describing the experiment setup. It creates: 
   - a `scenario.yaml` file defining which nodes will be used on CorteXlab;
   - an `timeline.csv` file orchestrating the role and sequence of these nodes.
-- Experiment execution: this part deploys and executes the generates experiment definitions (`timeline.csv`) directly on the [Slices/CorteXlab](https://www.cortexlab.fr/doku.php?id=start) nodes.
+- Experiment execution: this part deploys and executes the generates experiment definitions (`timeline.csv`) directly on the [SLICES-RI/CorteXlab](https://www.cortexlab.fr/doku.php?id=start) nodes.
+- Dataset API
 
 ## Quick start (User Guide) :rocket:
 
@@ -21,7 +22,7 @@ The scenario generator can be executed locally before  deployment in Slices/Cort
 ### Example usage
 - ```git clone https://github.com/Andreaj42/CorteXForge.git```
 - ```python3.13 -m venv .venv```
-- ```source .venv/bin/activate```
+- ```. .venv/bin/activate```
 - ```pip install -e .[planner]```
 - ```cortexforge-planner --nodes-path confis/nodes.yaml --duration 600 --output-path my/path/on/cortexlab```
 
@@ -49,12 +50,6 @@ To monitor your experiment, use:
 - ```minus testbed status```
 - ```minus log -d```
 
-## Developer Guide :hammer_and_wrench:
-
-Now clone the project:
-- ```git clone https://github.com/Andreaj42/CorteXForge.git```
-- ```cd forge```
-
 
 ### Docker Images :whale:
 To simplify deployment and ensure reproductibility, we generated a Docker image.

{cortexforge-0.1.0 → cortexforge-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cortexforge"
-version = "0.1.0"
+version = "0.1.3"
 description = "RF generator dataset based on SLICES/CorteXlab"
 authors = [
   { name = "Andrea Joly", email = "andrea.joly@inria.fr" }
@@ -31,11 +31,13 @@ forge = [
 
 
 [project.scripts]
-cortexforge-planner = "cortexforge.planner.main:main"
-cortexforge-forge   = "cortexforge.forge.main:main"
+cortexforge = "cortexforge.cli:main"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
 
 [tool.setuptools.packages.find]
 where = ["src"]
+
+[tool.setuptools.package-data]
+"cortexforge" = ["**/*.json"]

cortexforge-0.1.3/src/cortexforge/cli/__init__.py

@@ -0,0 +1,60 @@
+"""Unified CLI entrypoint for CorteXForge."""
+
+from argparse import ArgumentParser, Namespace
+
+from cortexforge.cli.datasets import configure_parser as configure_datasets_parser
+from cortexforge.cli.forge import configure_parser as configure_forge_parser
+from cortexforge.cli.planner import configure_parser as configure_planner_parser
+
+
+def build_parser() -> ArgumentParser:
+    """Build the root CLI parser."""
+    parser = ArgumentParser(prog="cortexforge", description="CorteXForge command line")
+    sub = parser.add_subparsers(
+        dest="command",
+        required=True,
+        help="CorteXForge subcommand",
+    )
+
+    configure_planner_parser(
+        sub.add_parser("planner", help="Generate scenarios and experiment files")
+    )
+    configure_forge_parser(
+        sub.add_parser(
+            "forge", help="Run transmitter or receiver commands on CorteXlab"
+        )
+    )
+    configure_datasets_parser(
+        sub.add_parser("datasets", help="Inspect available datasets")
+    )
+    return parser
+
+
+def parse_args(argv: list[str] | None = None) -> Namespace:
+    """Parse root CLI arguments."""
+    return build_parser().parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Dispatch to the selected CorteXForge subcommand."""
+    args = parse_args(argv)
+
+    if args.command == "planner":
+        from cortexforge.planner.main import run as planner_run
+
+        planner_run(args)
+        return
+
+    if args.command == "forge":
+        from cortexforge.forge.main import run as forge_run
+
+        forge_run(args)
+        return
+
+    if args.command == "datasets":
+        from cortexforge.datasets.main import run as datasets_run
+
+        datasets_run(args)
+        return
+
+    raise ValueError(f"Unknown command: {args.command}")

cortexforge-0.1.3/src/cortexforge/cli/datasets.py

@@ -0,0 +1,56 @@
+"""CLI argument parser for CorteXForge datasets."""
+
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+
+
+def configure_parser(parser: ArgumentParser) -> ArgumentParser:
+    """Attach datasets-specific arguments to an existing parser."""
+    parser.add_argument(
+        "--root",
+        type=Path,
+        default=Path("datasets"),
+        help="Path to the datasets root directory",
+    )
+
+    sub = parser.add_subparsers(
+        dest="datasets_command",
+        required=True,
+        help="Dataset operation to execute",
+    )
+
+    sub.add_parser("list", help="List available datasets and their versions")
+
+    download = sub.add_parser(
+        "download",
+        help="Download and extract a dataset version from the embedded registry",
+    )
+    download.add_argument("name", type=str, help="Dataset name")
+    download.add_argument(
+        "--version",
+        type=str,
+        default=None,
+        help="Dataset version to download (defaults to latest)",
+    )
+    download.add_argument(
+        "--force",
+        action="store_true",
+        help="Re-download and overwrite an existing local dataset version",
+    )
+
+    return parser
+
+
+def build_parser() -> ArgumentParser:
+    """Build and configure the standalone datasets parser."""
+    parser = ArgumentParser(
+        prog="cortexforge datasets",
+        description="Inspect available datasets",
+    )
+    return configure_parser(parser)
+
+
+def parse_args(argv: list[str] | None = None) -> Namespace:
+    """Parse command line arguments."""
+    parser = build_parser()
+    return parser.parse_args(argv)

{cortexforge-0.1.0 → cortexforge-0.1.3}/src/cortexforge/cli/forge.py

@@ -4,14 +4,8 @@ from argparse import ArgumentParser, Namespace
 from pathlib import Path
 
 
-def build_parser() -> ArgumentParser:
-    """
-    Build and configure the cli argument parser for CorteXForge forge.
-
-    Returns:
-        ArgumentParser: Configured argument parser instance with
-    """
-    parser = ArgumentParser(prog="CorteXForge forge", description="Dataset Generator")
+def configure_parser(parser: ArgumentParser) -> ArgumentParser:
+    """Attach forge-specific arguments to an existing parser."""
     sub = parser.add_subparsers(
         dest="role",
         required=True,
@@ -52,6 +46,12 @@ def build_parser() -> ArgumentParser:
     return parser
 
 
+def build_parser() -> ArgumentParser:
+    """Build and configure the standalone forge parser."""
+    parser = ArgumentParser(prog="cortexforge forge", description="Dataset Generator")
+    return configure_parser(parser)
+
+
 def parse_args(argv: list[str] | None = None) -> Namespace:
     """Parse command line arguments."""
     parser = build_parser()

cortexforge-0.1.3/src/cortexforge/cli/planner.py

@@ -0,0 +1,88 @@
+"""CLI argument parser for CorteXForge planner."""
+
+from argparse import Action, ArgumentParser, Namespace
+from pathlib import Path
+
+from cortexforge.planner.modulations import DEFAULT_MODULATIONS
+
+
+def _split_modulations(value: str) -> list[str]:
+    return [item.strip().upper() for item in value.split(",") if item.strip()]
+
+
+class _ModulationAction(Action):
+    """Parse comma-separated and repeated modulation values into a flat list."""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        modulations = list(getattr(namespace, self.dest, None) or [])
+        for value in values:
+            modulations.extend(_split_modulations(value))
+        unknown = sorted(set(modulations) - set(DEFAULT_MODULATIONS))
+        if unknown:
+            parser.error(
+                f"unsupported modulation(s): {', '.join(unknown)}. "
+                f"Supported values are: {', '.join(DEFAULT_MODULATIONS)}"
+            )
+        setattr(namespace, self.dest, modulations)
+
+
+def configure_parser(parser: ArgumentParser) -> ArgumentParser:
+    """Attach planner-specific arguments to an existing parser."""
+    parser.add_argument(
+        "--username", required=True, type=str, help="username on CorteXlab"
+    )
+    parser.add_argument(
+        "--duration", type=int, default=60, help="Experiment duration in seconds"
+    )
+    parser.add_argument(
+        "--rx-frequency", type=int, default=2450000000, help="Receiver frequency"
+    )
+    parser.add_argument("--rx-gain", type=int, default=10, help="Receiver gain")
+    parser.add_argument(
+        "--rx-sample-rate", type=int, default=250000, help="Receiver sample-rate"
+    )
+    parser.add_argument(
+        "--overlapping",
+        action="store_true",
+        help="Allow overlapping signals in timeline",
+    )
+    parser.add_argument(
+        "--n-signals",
+        type=int,
+        default=288,
+        help=(
+            "Number of signals to generate. Must be a multiple of the selected "
+            "modulation count."
+        ),
+    )
+    parser.add_argument(
+        "--nodes-path",
+        type=Path,
+        default="configs/nodes.yaml",
+        help="Path to nodes.yaml file",
+    )
+    parser.add_argument(
+        "--modulations",
+        action=_ModulationAction,
+        nargs="+",
+        metavar="MOD",
+        default=None,
+        help=(
+            "Modulations to include in the generated dataset. "
+            "Use a space-separated list or comma-separated values. "
+            "Defaults to all planner modulations."
+        ),
+    )
+    return parser
+
+
+def build_parser() -> ArgumentParser:
+    """Build and configure the standalone planner parser."""
+    parser = ArgumentParser(prog="cortexforge planner", description="Dataset Generator")
+    return configure_parser(parser)
+
+
+def parse_args(argv: list[str] | None = None) -> Namespace:
+    """Parse command line arguments."""
+    parser = build_parser()
+    return parser.parse_args(argv)

{cortexforge-0.1.0 → cortexforge-0.1.3}/src/cortexforge/datasets/__init__.py

@@ -1,6 +1,8 @@
 from .api import (
     describe_dataset,
+    download_dataset,
     list_datasets,
+    list_registry_datasets,
     list_datasets_with_versions,
     list_versions,
     load_dataset,
@@ -9,7 +11,9 @@ from .api import (
 
 __all__ = [
     "load_dataset",
+    "download_dataset",
     "list_datasets",
+    "list_registry_datasets",
     "list_datasets_with_versions",
     "describe_dataset",
     "load_manifest",

cortexforge-0.1.3/src/cortexforge/datasets/api.py

@@ -0,0 +1,366 @@
+import hashlib
+import json
+import logging
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+from .local import load_local_dataset
+from .manifest import load_manifest
+
+REGISTRY_PATH = Path(__file__).with_name("registry.json")
+DOWNLOAD_HEADERS = {
+    "User-Agent": "CorteXForge/0.1 (+https://github.com/Andreaj42/CorteXForge)",
+    "Accept": "*/*",
+}
+logger = logging.getLogger(__name__)
+
+
+def _load_registry() -> dict:
+    with REGISTRY_PATH.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _resolve_registry_entry(name: str, version: str | None) -> tuple[str, dict]:
+    registry = _load_registry()
+
+    if name not in registry:
+        raise ValueError(f"Unknown dataset: {name}")
+
+    dataset_entry = registry[name]
+    resolved_version = version or dataset_entry["latest"]
+    versions = dataset_entry.get("versions", {})
+
+    if resolved_version not in versions:
+        raise ValueError(f"Unknown version '{resolved_version}' for dataset '{name}'")
+
+    return resolved_version, versions[resolved_version]
+
+
+def _sha256_hex(path: Path, chunk_size: int = 1024 * 1024) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _format_size(size_bytes: int | float) -> str:
+    value = float(size_bytes)
+    units = ["B", "KB", "MB", "GB", "TB"]
+
+    for unit in units:
+        if value < 1024 or unit == units[-1]:
+            if unit == "B":
+                return f"{int(value)} {unit}"
+            return f"{value:.1f} {unit}"
+        value /= 1024
+
+    return f"{int(size_bytes)} B"
+
+
+def _render_progress_bar(
+    downloaded_bytes: int,
+    total_bytes: int | None,
+    width: int = 28,
+) -> str:
+    if not total_bytes or total_bytes <= 0:
+        return f"[{'?' * width}] {_format_size(downloaded_bytes)}"
+
+    ratio = min(downloaded_bytes / total_bytes, 1.0)
+    filled = min(int(ratio * width), width)
+    bar = "#" * filled + "-" * (width - filled)
+    percent = int(ratio * 100)
+    return (
+        f"[{bar}] {percent:3d}% "
+        f"({_format_size(downloaded_bytes)}/{_format_size(total_bytes)})"
+    )
+
+
+def _update_progress_line(message: str) -> None:
+    sys.stderr.write(f"\r{message}")
+    sys.stderr.flush()
+
+
+def _finish_progress_line(message: str) -> None:
+    sys.stderr.write(f"\r{message}\n")
+    sys.stderr.flush()
+
+
+def _download_file(url: str, destination: Path, chunk_size: int = 1024 * 1024) -> None:
+    request = Request(url, headers=DOWNLOAD_HEADERS)
+    with urlopen(request) as response, destination.open("wb") as f:
+        total_bytes_header = response.headers.get("Content-Length")
+        total_bytes = int(total_bytes_header) if total_bytes_header else None
+        downloaded_bytes = 0
+        last_logged_percent = -1
+        started_at = time.monotonic()
+
+        if total_bytes:
+            _update_progress_line(
+                f"Download progress: {_render_progress_bar(0, total_bytes)}"
+            )
+        else:
+            _update_progress_line("Download progress: starting, total size unknown")
+
+        while True:
+            chunk = response.read(chunk_size)
+            if not chunk:
+                break
+
+            f.write(chunk)
+            downloaded_bytes += len(chunk)
+
+            if total_bytes:
+                percent = int((downloaded_bytes / total_bytes) * 100)
+                if percent >= last_logged_percent + 5 or downloaded_bytes == total_bytes:
+                    elapsed = max(time.monotonic() - started_at, 1e-9)
+                    speed = downloaded_bytes / elapsed
+                    _update_progress_line(
+                        "Download progress: "
+                        f"{_render_progress_bar(downloaded_bytes, total_bytes)} "
+                        f"at {_format_size(speed)}/s"
+                    )
+                    last_logged_percent = percent
+            else:
+                if downloaded_bytes == len(chunk) or downloaded_bytes % (50 * chunk_size) == 0:
+                    elapsed = max(time.monotonic() - started_at, 1e-9)
+                    speed = downloaded_bytes / elapsed
+                    _update_progress_line(
+                        "Download progress: "
+                        f"{_format_size(downloaded_bytes)} downloaded "
+                        f"at {_format_size(speed)}/s"
+                    )
+
+        elapsed = max(time.monotonic() - started_at, 1e-9)
+        speed = downloaded_bytes / elapsed
+        _finish_progress_line(
+            "Download completed: "
+            f"{_format_size(downloaded_bytes)} in {elapsed:.1f}s "
+            f"at {_format_size(speed)}/s"
+        )
+
+
+def _extract_archive_with_tar(archive_path: Path, root: Path) -> None:
+    subprocess.run(
+        ["tar", "-xf", str(archive_path.resolve()), "-C", str(root.resolve())],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+
+def _extract_archive_with_external_zstd(archive_path: Path, root: Path) -> None:
+    decompressor = shutil.which("unzstd")
+    command = None
+
+    if decompressor:
+        command = [decompressor, "-c", str(archive_path.resolve())]
+    else:
+        zstd = shutil.which("zstd")
+        if zstd:
+            command = [zstd, "-d", "-c", str(archive_path.resolve())]
+
+    if command is None:
+        raise RuntimeError(
+            "Could not extract .tar.zst archive because no compatible zstd "
+            "decompressor was found in PATH. Install 'zstd' or 'unzstd'."
+        )
+
+    decompress_process = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=False,
+    )
+    tar_process = subprocess.Popen(
+        ["tar", "-xf", "-", "-C", str(root.resolve())],
+        stdin=decompress_process.stdout,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=False,
+    )
+
+    if decompress_process.stdout is not None:
+        decompress_process.stdout.close()
+
+    tar_stdout, tar_stderr = tar_process.communicate()
+    _, decompress_stderr = decompress_process.communicate()
+
+    if decompress_process.returncode != 0:
+        stderr_text = decompress_stderr.decode("utf-8", errors="replace").strip()
+        raise RuntimeError(
+            "Failed to decompress dataset archive with zstd. "
+            f"Command: {' '.join(command)}. Error: {stderr_text or 'unknown error'}"
+        )
+
+    if tar_process.returncode != 0:
+        stderr_text = tar_stderr.decode("utf-8", errors="replace").strip()
+        raise RuntimeError(
+            "Failed to extract decompressed tar archive. "
+            f"Error: {stderr_text or 'unknown error'}"
+        )
+
+
+def _extract_archive(archive_path: Path, root: Path) -> None:
+    try:
+        _extract_archive_with_tar(archive_path, root)
+        return
+    except subprocess.CalledProcessError as exc:
+        tar_error = (exc.stderr or exc.stdout or "").strip()
+        logger.warning(
+            "Direct tar extraction failed for %s: %s",
+            archive_path.resolve(),
+            tar_error or f"exit code {exc.returncode}",
+        )
+
+    if archive_path.suffix == ".zst" and archive_path.name.endswith(".tar.zst"):
+        logger.info("Retrying archive extraction with external zstd decompressor")
+        _extract_archive_with_external_zstd(archive_path, root)
+        return
+
+    raise RuntimeError(f"Could not extract archive: {archive_path.resolve()}")
+
+
+def load_dataset(
+    name: str,
+    version: str,
+    split: str,
+    root: str | Path,
+    verify: bool = True,
+):
+    dataset_dir = Path(root) / name / version
+    return load_local_dataset(dataset_dir=dataset_dir, split=split, verify=verify)
+
+
+def describe_dataset(
+    name: str,
+    version: str,
+    root: str | Path,
+):
+    dataset_dir = Path(root) / name / version
+    return load_manifest(dataset_dir / "manifest.json")
+
+
+def download_dataset(
+    name: str,
+    root: str | Path,
+    version: str | None = None,
+    force: bool = False,
+):
+    root = Path(root)
+    resolved_version, archive_entry = _resolve_registry_entry(name, version)
+    dataset_dir = root / archive_entry["root_dir"]
+    manifest_path = dataset_dir / "manifest.json"
+    resolved_root = root.resolve()
+    resolved_dataset_dir = dataset_dir.resolve()
+
+    if manifest_path.exists() and not force:
+        logger.info(
+            "Dataset %s:%s already available at %s",
+            name,
+            resolved_version,
+            resolved_dataset_dir,
+        )
+        return load_manifest(manifest_path)
+
+    archive_name = archive_entry["archive_name"]
+    archive_path = root / archive_name
+    url = archive_entry["url"]
+    expected_sha256 = archive_entry["sha256"]
+
+    root.mkdir(parents=True, exist_ok=True)
+
+    if dataset_dir.exists() and force:
+        logger.info(
+            "Removing existing dataset %s:%s from %s",
+            name,
+            resolved_version,
+            resolved_dataset_dir,
+        )
+        shutil.rmtree(dataset_dir)
+
+    logger.info("Download timeline for %s:%s", name, resolved_version)
+    logger.info("1/4 Resolve target directory: %s", resolved_root)
+    logger.info("2/4 Download archive from %s", url)
+
+    try:
+        _download_file(url, archive_path)
+
+        logger.info("3/4 Verify archive checksum: %s", archive_path.resolve())
+        actual_sha256 = _sha256_hex(archive_path)
+        if actual_sha256 != expected_sha256:
+            raise ValueError(
+                "Downloaded archive checksum mismatch for "
+                f"{name}:{resolved_version}. Expected {expected_sha256}, got {actual_sha256}"
+            )
+
+        logger.info("4/4 Extract archive into %s", resolved_root)
+        _extract_archive(archive_path, root)
+    finally:
+        archive_path.unlink(missing_ok=True)
+
+    if not manifest_path.exists():
+        raise FileNotFoundError(
+            f"Dataset archive for {name}:{resolved_version} did not contain {manifest_path}"
+        )
+
+    logger.info(
+        "Dataset %s:%s installed in %s",
+        name,
+        resolved_version,
+        resolved_dataset_dir,
+    )
+
+    return load_manifest(manifest_path)
+
+
+def list_datasets(root: str | Path | None = None) -> list[str]:
+    del root
+    registry = _load_registry()
+    return sorted(registry.keys())
+
+
+def list_versions(name: str, root: str | Path | None = None) -> list[str]:
+    del root
+    registry = _load_registry()
+
+    if name not in registry:
+        raise ValueError(f"Unknown dataset: {name}")
+
+    versions = registry[name].get("versions", {})
+    return sorted(versions.keys())
+
+
+def list_datasets_with_versions(root: str | Path | None = None) -> dict[str, list[str]]:
+    del root
+    registry = _load_registry()
+    return {
+        dataset_name: sorted(dataset_entry.get("versions", {}).keys())
+        for dataset_name, dataset_entry in sorted(registry.items())
+    }
+
+
+def list_registry_datasets(root: str | Path | None = None) -> list[dict]:
+    del root
+    registry = _load_registry()
+    result = []
+
+    for dataset_name, dataset_entry in sorted(registry.items()):
+        latest = dataset_entry["latest"]
+        latest_entry = dataset_entry.get("versions", {}).get(latest, {})
+        result.append(
+            {
+                "name": dataset_name,
+                "latest": latest,
+                "versions": sorted(dataset_entry.get("versions", {}).keys()),
+                "description": latest_entry.get("description", ""),
+                "size_bytes": latest_entry.get("size_bytes"),
+                "root_dir": latest_entry.get("root_dir"),
+            }
+        )
+
+    return result