sprawdzai-cli 0.2.0__py3-none-any.whl

sai/__init__.py

@@ -0,0 +1,5 @@
+from .state import set_current_backend, get_current_backend, resolve_backend
+from .state import clear_token, clear_all_tokens
+from .state import get_backend_url, get_frontend_url, normalize_url
+from .commands.auth_token import get_auth_token
+from .utils.client import APIClient

sai/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

sai/commands/auth_token.py

@@ -0,0 +1,31 @@
+import typer
+from ..state import resolve_backend, get_token
+from typing import Optional
+from ..utils.error_handler import write_error
+
+def get_auth_token(backend: Optional[str]):
+    token = get_token(resolve_backend(backend))
+    if token is None:
+        raise Exception('You are not authenticated')
+    return token['token']
+
+def auth_token(
+    backend: Optional[str] = typer.Option(
+        None,
+        "--backend",
+        "-b",
+        help=(
+            "Use a different backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    """
+    Show authentication token.
+    """
+    token = get_token(resolve_backend(backend))
+    if token is None:
+        write_error('You are not authenticated')
+    typer.echo(token['token'])

sai/commands/backend.py

@@ -0,0 +1,22 @@
+import typer
+from ..state import set_current_backend, get_current_backend, get_backend_url
+from typing import Optional
+
+def backend(
+    backend: Optional[str] = typer.Argument(
+        None, 
+        help=(
+            "Set backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    """
+    Show current globally used backend, or set a new backend if provided.
+    """
+    if backend is not None:
+        set_current_backend(backend)
+    backend = get_current_backend()
+    typer.echo(f"Using backend {backend} ({get_backend_url(backend)})")

sai/commands/checklib.py

@@ -0,0 +1,146 @@
+import os
+import sys
+import json
+import time
+import traceback
+import subprocess
+
+RESULTS_SECRET_KEY: str = "secret_key"
+
+# This flag is set to True when it is a validation submit, and False when it is a test set submit
+IS_VALID: bool = True
+
+# How long did notebook execution take (in seconds)
+# Use this for raising Time Limit Exceeded exceptions when the total time of running notebook and pickle is too large
+NOTEBOOK_EXECUTION_TIME: float = 0.0
+
+# This is the same time limit that you provide on sprawdzai.org when editing the task, just for convenience (in seconds)
+NOTEBOOK_TIME_LIMIT: int = 60
+
+# The same as NOTEBOOK_TIME_LIMIT, but the checker limit
+CHECKER_TIME_LIMIT: int = 60
+
+def check_file_exists(filepath: str) -> bool:
+    if not os.path.exists(filepath):
+        return False
+    if not os.path.isfile(filepath):
+        return False
+    return True
+
+def linear_grade(score: float, min_score: float, max_score: float, use_better: bool = False) -> int:
+    if min_score > max_score:
+        return linear_grade(max_score + min_score - score, max_score, min_score)
+    if score <= min_score:
+        return 0
+    if score >= max_score:
+        return 100
+    if use_better:
+        return int(round(99 * (score - min_score) / (max_score - min_score) + 0.5))
+    return int(round(100 * (score - min_score) / (max_score - min_score)))
+
+def safe_score(func):
+    def wrapper(*args, **kwargs):
+        result = func(*args, **kwargs)
+
+        if isinstance(result, tuple):
+            score, notes = result
+            
+            try:
+                score = int(score)
+            except Exception:
+                raise Exception(f"Exception in safe_score() wrapper: {func.__name__} returned non-numeric value: {score}")
+
+            return max(0, min(100, score)), notes
+        else:
+            try:
+                score = int(result)
+            except Exception:
+                raise Exception(f"Exception in safe_score() wrapper: {func.__name__} returned non-numeric value: {result}")
+
+            return max(0, min(100, score))
+
+    return wrapper
+
+def debug(*values, sep=' ', end='\n'):
+    print(*values, sep=sep, end=end, file=sys.stderr, flush=True)
+
+def format_time(t: float) -> str:
+    t = round(t)
+    if t < 0:
+        return f'-{format_time(-t)}'
+    if t < 3600:
+        return f'{t // 60}:{t % 60:02d}'
+    return f'{t // 3600}:{(t // 60) % 60:02d}:{t % 60:02d}'
+
+def print_tree(startpath=".", prefix=""):
+    entries = sorted(os.listdir(startpath))
+    entries_count = len(entries)
+
+    for index, entry in enumerate(entries):
+        path = os.path.join(startpath, entry)
+        connector = "└─" if index == entries_count - 1 else "├─"
+        print(prefix + connector + entry)
+        if os.path.isdir(path):
+            extension = "    " if index == entries_count - 1 else "│ "
+            print_tree(path, prefix + extension)
+
+def run_pickle(timeout: float) -> float:
+    print('Executing pickle in isolated subprocess...')
+    
+    start_time = time.time()
+
+    try:
+        proc = subprocess.run(
+            ['python', 'subprocess_script.py'],
+            timeout=timeout,
+            capture_output=True,
+            text=True
+        )
+    except subprocess.TimeoutExpired:
+        raise TimeError()
+    
+    execution_time = time.time() - start_time
+
+    print('pickle subprocess return code:')
+    print(proc.returncode)
+    print('pickle subprocess stdout:')
+    print(proc.stdout)
+    print('pickle subprocess stderr:')
+    print(proc.stderr)
+
+    if check_file_exists('error.txt'):
+        try:
+            with open('error.txt', 'r', encoding='UTF-8') as f:
+                res = f.read()
+        except Exception:
+            traceback.print_exc()
+            raise SubmissionError()
+        raise SubmissionError(res)
+
+    return execution_time
+
+# These are the contents of those functions on SprawdzAI:
+def write_result_base(score: float = 0, error: str | None = None, notes: str = ""):
+    with open('result.txt', 'w', encoding='UTF-8') as f:
+        json.dump({
+            'notes': notes,
+            'error': error,
+            'score': score,
+            'secret_key': RESULTS_SECRET_KEY
+        }, f)
+def write_ok(score: float, notes: str = ""):
+    write_result_base(score, None, notes)
+def write_time_error(notes: str = ""):
+    write_result_base(0, "time_error", notes)
+def write_memory_error(notes: str = ""):
+    write_result_base(0, "memory_error", notes)
+def write_submission_error(notes: str = ""):
+    write_result_base(0, "runtime_error", notes)
+def write_system_error(notes: str = ""):
+    write_result_base(0, "system_error", notes)
+
+class SubmissionError(Exception):
+    pass
+
+class TimeError(Exception):
+    pass

sai/commands/ds/__init__.py

@@ -0,0 +1,65 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from .pull import pull
+from .send import send
+from .create import create
+from .link import link
+from .unlink import unlink
+from .rename import rename
+from .utils import Datasets
+
+ds_app = typer.Typer(
+    no_args_is_help=False,
+    help="Dataset management commands"
+)
+
+@ds_app.callback(invoke_without_command=True)
+def ds(
+    ctx: typer.Context, 
+    path: Path = typer.Option(
+        Path.cwd(),
+        "--location",
+        "-l",
+        help="Path where the task is located (defaults to current directory)."
+    ),
+    backend: Optional[str] = typer.Option(
+        None,
+        "--backend",
+        "-b",
+        help=(
+            "Use a different backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    if ctx.invoked_subcommand is None:
+        dss = Datasets(path, backend)
+
+        for test in [False, True]:
+            typer.secho(f'{"TEST" if test else "VALIDATION"} SUBMISSION DATASETS:', fg='blue')
+            for chk in [False, True]:
+                typer.secho(f'  {"Checker" if chk else "Notebook"}:', fg='blue')
+                filtered = [ds for ds in dss.dss.values() if ds.stages['chk' if chk else 'nb']['test' if test else 'valid']]
+                if len(filtered) == 0:
+                    typer.secho('  *no datasets*', fg='black')
+                else:
+                    for ds in filtered:
+                        typer.echo('   - ', nl=False)
+                        if ds.local:
+                            typer.secho(f'./{ds.name}', fg='green', nl=False)
+                        else:
+                            typer.secho(f'[MISSING] ./{ds.name}', fg='red', nl=False)
+                        typer.echo(f' {ds.url}')
+                typer.echo('')
+
+        Datasets(path, backend).show_warnings()
+
+ds_app.command()(pull)
+ds_app.command()(send)
+ds_app.command()(create)
+ds_app.command()(link)
+ds_app.command()(unlink)
+ds_app.command()(rename)

sai/commands/ds/create.py

@@ -0,0 +1,138 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from .utils import Datasets, Dataset
+from ...utils.error_handler import write_error, warn, handle_response_error
+import tempfile
+import zipfile
+
+def create(
+    name: str = typer.Argument(
+        ...,
+        help=(
+            "Name of the dataset (name of the directory)"
+        )
+    ),
+    valid: bool = typer.Option(
+        False,
+        "--valid",
+        "-v",
+    ),
+    test: bool = typer.Option(
+        False,
+        "--test",
+        "-t",
+    ),
+    nb: bool = typer.Option(
+        False,
+        "--notebook",
+        "-n",
+    ),
+    chk: bool = typer.Option(
+        False,
+        "--checker",
+        "-c",
+    ),
+    path: Path = typer.Option(
+        Path.cwd(),
+        "--location",
+        "-l",
+        help="Path where the task is located (defaults to current directory)."
+    ),
+    backend: Optional[str] = typer.Option(
+        None,
+        "--backend",
+        "-b",
+        help=(
+            "Use a different backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    """
+    Create a new dataset with specified name.
+    """
+
+    dss = Datasets(path, backend)
+
+    if name in dss.dss:
+        write_error(f'A dataset with name ./{name} is already linked to this task. Please choose different name or rename the existing dataset with `sai ds rename {name} new-name`')
+
+    dir_path = path / name
+
+    if dir_path.exists():
+        if dir_path.is_file():
+            write_error(f'Specified name ./{name} is a file (expected it to not exist or be a directory).')
+        warn(f'The specified dataset folder ./{name} already exists, but its contents will not be sent.')
+        warn(f'Please run `sai ds send {name}` after this command to send its contents.')
+    else:
+        dir_path.mkdir()
+    
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
+    tmp_path = Path(tmp.name)
+    try:
+        with zipfile.ZipFile(tmp_path, 'w', zipfile.ZIP_DEFLATED) as _:
+            pass
+        with open(tmp_path, "rb") as f:
+            files_req = {"file": (f'{name}.zip', f)}
+            data = {"type_": "dataset", "title": f"{dss.task['slug']} - {name}"}
+            resp = dss.client.post('/files', data=data, files=files_req)
+    finally:
+        tmp.close()
+        tmp_path.unlink()
+    if resp.status_code not in [200, 201]:
+        handle_response_error(resp)
+    
+    res = resp.json()
+    res['type_'] = None
+    ds = Dataset(res, path)
+    dss.dss[ds.name] = ds
+    typer.secho(f'Created new dataset {str(ds)}', fg='green')
+
+
+    def linkk(s1: str, s2: str):
+        typer.secho(f'Linked to {s2} {"notebook" if s1 == "nb" else "checker"}!', fg='green')
+        ds.stages[s1][s2] = True
+    
+    if nb and chk:
+        write_error('Expected one or none of flags --notebook and --checker, but got both.')
+    if valid and test:
+        write_error('Expected one or none of flags --valid and --test, but got both.')
+    
+    flags = []
+    if nb:
+        flags.append('nb')
+    if chk:
+        flags.append('chk')
+    if valid:
+        flags.append('valid')
+    if test:
+        flags.append('test')
+    
+    if len(flags) == 0:
+        linkk('nb', 'valid')
+        linkk('nb', 'test')
+        linkk('chk', 'valid')
+        linkk('chk', 'test')
+    elif len(flags) == 1:
+        if flags[0] in ['nb', 'chk']:
+            linkk(flags[0], 'valid')
+            linkk(flags[0], 'test')
+        else:
+            linkk('nb', flags[0])
+            linkk('chk', flags[0])
+    else:
+        if nb and valid:
+            linkk('nb', 'valid')
+        elif nb and test:
+            linkk('nb', 'test')
+        elif chk and valid:
+            linkk('chk', 'valid')
+        elif chk and test:
+            linkk('chk', 'test')
+    
+    dss.update(dss.dss.values())
+
+    Datasets(path, backend).show_warnings()

sai/commands/ds/link.py

@@ -0,0 +1,133 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from .utils import Datasets, Dataset
+from ...utils.error_handler import write_error, handle_response_error
+
+def link(
+    name_or_id: str = typer.Argument(
+        ...,
+        help=(
+            "Name or id of dataset to be linked"
+        )
+    ),
+    valid: bool = typer.Option(
+        False,
+        "--valid",
+        "-v",
+    ),
+    test: bool = typer.Option(
+        False,
+        "--test",
+        "-t",
+    ),
+    nb: bool = typer.Option(
+        False,
+        "--notebook",
+        "-n",
+    ),
+    chk: bool = typer.Option(
+        False,
+        "--checker",
+        "-c",
+    ),
+    path: Path = typer.Option(
+        Path.cwd(),
+        "--location",
+        "-l",
+        help="Path where the task is located (defaults to current directory)."
+    ),
+    backend: Optional[str] = typer.Option(
+        None,
+        "--backend",
+        "-b",
+        help=(
+            "Use a different backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    """
+    Link a dataset to a stage of a task.
+
+    sai ds link name/id -> link dataset to all 4 stages
+    sai ds link name/id --notebook/--checker/--valid/--test or -n/-c/-v/-t -> link dataset to 2 corresponding stages
+
+    When passing two flags it will link dataset to this one stage, e.g.:
+    sai ds link name/id --notebook --test -> link dataset to test notebook stage
+    """
+
+    dss = Datasets(path, backend)
+
+    ds = dss.find(name_or_id, raise_not_found=False)
+
+    if ds is None:
+        files = []
+        for type_ in ['dataset', 'image', 'other']:
+            resp = dss.client.get(f'/files?type_={type_}')
+            if resp.status_code != 200:
+                handle_response_error(resp)
+            files += resp.json()
+        
+        for f in files:
+            if f['id'] == name_or_id:
+                f['type_'] = 'none'
+                ds = Dataset(f, path)
+                typer.secho(f'Linking a new external dataset: {str(ds)}', fg='blue')
+                if ds.name in dss.dss:
+                    write_error(f'A dataset with name "{ds.name}" is already linked to this task. Sai does not support two datasets having the same name. Please rename the currently linked dataset with `sai ds rename {ds.name} new-name` or change the name of external dataset manually on the website.')
+                dss.dss[ds.name] = ds
+
+    if ds is None:
+        write_error('Could not find dataset with specified name or id.')
+
+    def linkk(s1: str, s2: str):
+        if ds.stages[s1][s2]:
+            typer.secho(f'Already linked to {s2} {"notebook" if s1 == "nb" else "checker"}.', fg='yellow')
+        else:
+            typer.secho(f'Linked to {s2} {"notebook" if s1 == "nb" else "checker"}!', fg='green')
+            ds.stages[s1][s2] = True
+    
+    if nb and chk:
+        write_error('Expected one or none of flags --notebook and --checker, but got both.')
+    if valid and test:
+        write_error('Expected one or none of flags --valid and --test, but got both.')
+    
+    flags = []
+    if nb:
+        flags.append('nb')
+    if chk:
+        flags.append('chk')
+    if valid:
+        flags.append('valid')
+    if test:
+        flags.append('test')
+    
+    if len(flags) == 0:
+        linkk('nb', 'valid')
+        linkk('nb', 'test')
+        linkk('chk', 'valid')
+        linkk('chk', 'test')
+    elif len(flags) == 1:
+        if flags[0] in ['nb', 'chk']:
+            linkk(flags[0], 'valid')
+            linkk(flags[0], 'test')
+        else:
+            linkk('nb', flags[0])
+            linkk('chk', flags[0])
+    else:
+        if nb and valid:
+            linkk('nb', 'valid')
+        elif nb and test:
+            linkk('nb', 'test')
+        elif chk and valid:
+            linkk('chk', 'valid')
+        elif chk and test:
+            linkk('chk', 'test')
+    
+    dss.update(dss.dss.values())
+
+    Datasets(path, backend).show_warnings()
+    

sai/commands/ds/pull.py

@@ -0,0 +1,133 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from .utils import Datasets, Dataset
+from ...utils.error_handler import warn, handle_response_error
+import shutil
+import tempfile
+from tqdm.auto import tqdm
+import zipfile
+
+def pull(
+    name_or_id: Optional[str] = typer.Argument(
+        None,
+        help=(
+            "Pull a dataset with specific name or id. "
+            "If this argument is not specified, it will pull all missing datasets (ones that are linked to this task, but are not yet pulled locally)."
+        )
+    ),
+    only_missing: bool = typer.Option(
+        False,
+        "--missing",
+        "-m",
+        help=(
+            "Only pull missing datasets. "
+            "If this flag is not set, datasets that are already pulled locally will be pulled again in order to allow you to "
+            "update their contents in case they changed on backend "
+            "(but you will get warning before that, so you don't accidentally lose local changes)."
+        )
+    ),
+    path: Path = typer.Option(
+        Path.cwd(),
+        "--location",
+        "-l",
+        help="Path where the task is located (defaults to current directory)."
+    ),
+    backend: Optional[str] = typer.Option(
+        None,
+        "--backend",
+        "-b",
+        help=(
+            "Use a different backend: "
+            "local (l, l1, l2, ...), "
+            "remote (r, r1, r2, ...) "
+            "or full URL (https://example.com)"
+        )
+    )
+):
+    """
+    Pull a dataset or all missing datasets linked to a task.
+    """
+    
+    dss = Datasets(path, backend)
+
+    tpull: list[Dataset] = []
+    if name_or_id is None:
+        if len(dss.dss) == 0:
+            warn('There are no datasets linked to this task, so no dataset can be pulled.')
+        else:
+            for ds in dss.dss.values():
+                if only_missing and ds.local:
+                    continue
+                tpull.append(ds)
+            if len(tpull) == 0:
+                warn('You have set the --missing flag, but all datasets are already pulled locally.')
+                warn('If you think a dataset was changed on the remote and you want to redownload it, run `sai ds pull name/id` explicitly or remove the --missing flag.')
+    else:
+        ds = dss.find(name_or_id)
+        if only_missing and ds.local:
+            warn('You have set the --missing flag, but the dataset you picked is already pulled locally.')
+        else:
+            tpull.append(ds)
+    
+    if len(tpull) >= 2:
+        typer.echo(f'Pulling {len(tpull)} datasets...')
+    
+    for ds in tpull:
+        typer.echo(f'{" - " if len(tpull) >= 2 else ""}Pulling {str(ds)}...')
+
+        dir_path = path / ds.name
+
+        if ds.local:
+            res = typer.prompt(f'You are about to pull the "{ds.name}" dataset from backend. It will overwrite all your local data inside {dir_path} folder. Do you want to pull this dataset? [y]es/[n]o')
+            if type(res) != str or (res.lower() not in ['y', 'yes']):
+                warn("Dataset was skipped")
+                continue
+            shutil.rmtree(dir_path)
+        
+        dir_path.mkdir(exist_ok=False)
+
+        is_zip = ds.filename.endswith('.zip')
+        if is_zip:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp_path = Path(tmp.name)
+        else:
+            tmp_path = dir_path / ds.filename
+            tmp = tmp_path.open("w+b")
+
+        url = f'/files/{ds.id}/download'
+        with dss.client.get(url, stream=True) as resp:
+            if resp.status_code != 200:
+                if tmp_path.exists():
+                    tmp.close()
+                    tmp_path.unlink()
+                typer.echo(f'Error while pulling {str(ds)} from {url}:')
+                handle_response_error(resp)
+            
+            total_size = int(resp.headers.get("content-length", 0))
+            
+            with tqdm(total=total_size, unit='B', unit_scale=True) as pbar:
+                for chunk in resp.iter_content(chunk_size=8192):
+                    if chunk:
+                        tmp.write(chunk)
+                        pbar.update(len(chunk))
+        
+        tmp.close()
+        
+        if is_zip:
+            try:
+                with zipfile.ZipFile(tmp_path, 'r') as zip_ref:
+                    zip_ref.extractall(dir_path)
+                    typer.secho(f"Unzipped contents to {dir_path.absolute()}", fg=typer.colors.GREEN)
+            except zipfile.BadZipFile as e:
+                typer.echo(tmp_path.absolute())
+                typer.echo(e)
+                shutil.move(str(tmp_path), dir_path / (ds.name + '.zip'))
+                warn(f"Warning: downloaded {str(ds)} dataset is not a valid ZIP. It was placed in the folder instead of being unzipped.")
+            finally:
+                if tmp_path.exists():
+                    tmp_path.unlink()
+        else:
+            typer.secho(f"Downloaded contents to {dir_path.absolute()}", fg=typer.colors.GREEN)
+    
+    Datasets(path, backend).show_warnings()