rbx.cp 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

rbx/box/presets/__init__.py

@@ -1,14 +1,16 @@
+import os
 import pathlib
 import shutil
 import tempfile
-from typing import Annotated, Iterable, List, Optional, Sequence, Union
+from typing import Annotated, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
+import ruyaml
 import typer
 
 from rbx import console, utils
 from rbx.box import cd
 from rbx.box.presets.fetch import PresetFetchInfo, get_preset_fetch_info
-from rbx.box.presets.lock_schema import LockedAsset, PresetLock
+from rbx.box.presets.lock_schema import LockedAsset, PresetLock, SymlinkInfo
 from rbx.box.presets.schema import Preset, TrackedAsset
 from rbx.config import get_default_app_path
 from rbx.grading.judge.digester import digest_cooperatively
@@ -34,7 +36,7 @@ def get_preset_yaml(root: pathlib.Path = pathlib.Path()) -> Preset:
 
 
 def _find_preset_lock(root: pathlib.Path = pathlib.Path()) -> Optional[pathlib.Path]:
-    root = root.resolve()
+    root = utils.abspath(root)
     problem_yaml_path = root / '.preset-lock.yml'
     if not problem_yaml_path.is_file():
         return None
@@ -49,7 +51,7 @@ def _get_preset_lock(root: pathlib.Path = pathlib.Path()) -> Optional[PresetLock
 
 
 def _find_nested_preset(root: pathlib.Path) -> Optional[pathlib.Path]:
-    root = root.resolve()
+    root = utils.abspath(root)
     problem_yaml_path = root / 'preset.rbx.yml'
     while root != pathlib.PosixPath('/') and not problem_yaml_path.is_file():
         root = root.parent
@@ -61,7 +63,7 @@ def _find_nested_preset(root: pathlib.Path) -> Optional[pathlib.Path]:
 
 def _find_local_preset(root: pathlib.Path) -> Optional[pathlib.Path]:
     original_root = root
-    root = root.resolve()
+    root = utils.abspath(root)
     problem_yaml_path = root / '.local.rbx' / 'preset.rbx.yml'
     while root != pathlib.PosixPath('/') and not problem_yaml_path.is_file():
         root = root.parent
@@ -75,7 +77,7 @@ def _is_installed_preset(root: pathlib.Path = pathlib.Path()) -> bool:
     preset_path = _find_local_preset(root)
     if preset_path is None:
         return False
-    resolved_path = preset_path.resolve()
+    resolved_path = utils.abspath(preset_path)
     return resolved_path.name == '.local.rbx'
 
 
@@ -103,23 +105,68 @@ def _check_is_valid_package(root: pathlib.Path = pathlib.Path()):
         raise typer.Exit(1)
 
 
+def _glob_while_ignoring(
+    dir: pathlib.Path,
+    glb: str,
+    extra_gitignore: Optional[str] = '.box\nbuild\n',
+    recursive: bool = False,
+) -> Iterable[pathlib.Path]:
+    from gitignore_parser import parse_gitignore, parse_gitignore_str
+
+    ignore_matchers = []
+
+    if extra_gitignore is not None:
+        ignore_matchers.append(parse_gitignore_str(extra_gitignore, base_dir=dir))
+
+    for file in dir.rglob('.gitignore'):
+        if file.is_file():
+            ignore_matchers.append(parse_gitignore(file))
+
+    def should_ignore(path: pathlib.Path) -> bool:
+        return any(m(str(path)) for m in ignore_matchers)
+
+    for file in dir.rglob(glb) if recursive else dir.glob(glb):
+        if should_ignore(file):
+            continue
+        yield file
+
+
 def _process_globbing(
-    assets: Iterable[TrackedAsset], preset_dir: pathlib.Path
+    assets: Iterable[TrackedAsset], preset_pkg_dir: pathlib.Path
 ) -> List[TrackedAsset]:
     res = []
     for asset in assets:
         if '*' in str(asset.path):
             glb = str(asset.path)
-            files = preset_dir.glob(glb)
-            relative_files = [file.relative_to(preset_dir) for file in files]
-            res.extend([TrackedAsset(path=path) for path in relative_files])
+            files = _glob_while_ignoring(
+                preset_pkg_dir,
+                glb,
+            )
+            relative_files = [file.relative_to(preset_pkg_dir) for file in files]
+            res.extend(
+                [
+                    TrackedAsset(path=path, symlink=asset.symlink)
+                    for path in relative_files
+                ]
+            )
             continue
         res.append(asset)
     return res
 
 
+def _dedup_tracked_assets(assets: List[TrackedAsset]) -> List[TrackedAsset]:
+    seen_paths = set()
+    res = []
+    for asset in assets:
+        if asset.path in seen_paths:
+            continue
+        seen_paths.add(asset.path)
+        res.append(asset)
+    return res
+
+
 def _get_preset_tracked_assets(
-    root: pathlib.Path, is_contest: bool
+    root: pathlib.Path, is_contest: bool, add_symlinks: bool = False
 ) -> List[TrackedAsset]:
     preset = get_active_preset(root)
     preset_path = _find_local_preset(root)
@@ -129,12 +176,51 @@ def _get_preset_tracked_assets(
         assert (
             preset.contest is not None
         ), 'Preset does not have a contest package definition.'
-        return _process_globbing(preset.tracking.contest, preset_path)
+        preset_pkg_path = preset_path / preset.contest
+        res = _process_globbing(preset.tracking.contest, preset_pkg_path)
+    else:
+        assert (
+            preset.problem is not None
+        ), 'Preset does not have a problem package definition,'
+        preset_pkg_path = preset_path / preset.problem
+        res = _process_globbing(preset.tracking.problem, preset_pkg_path)
+
+    if add_symlinks:
+        for file in _glob_while_ignoring(
+            preset_pkg_path,
+            '*',
+            recursive=True,
+        ):
+            if not file.is_symlink() or not file.is_file():
+                continue
+            res.append(
+                TrackedAsset(path=file.relative_to(preset_pkg_path), symlink=True)
+            )
 
-    assert (
-        preset.problem is not None
-    ), 'Preset does not have a problem package definition,'
-    return _process_globbing(preset.tracking.problem, preset_path)
+    return _dedup_tracked_assets(res)
+
+
+def _get_tracked_assets_symlinks(
+    tracked_assets: List[TrackedAsset],
+) -> Set[pathlib.Path]:
+    res = set()
+    for asset in tracked_assets:
+        if asset.symlink:
+            res.add(asset.path)
+    return res
+
+
+def _get_symlink_info(
+    tracked_asset: Union[TrackedAsset, LockedAsset], root: pathlib.Path
+) -> Optional[SymlinkInfo]:
+    asset_path = root / tracked_asset.path
+    if not asset_path.is_symlink():
+        return None
+    target = pathlib.Path(os.readlink(str(asset_path)))
+    absolute_target = utils.abspath(asset_path.parent / target)
+    is_broken = not absolute_target.exists()
+    is_outside = not absolute_target.is_relative_to(utils.abspath(root))
+    return SymlinkInfo(target=target, is_broken=is_broken, is_outside=is_outside)
 
 
 def _build_package_locked_assets(
@@ -145,10 +231,21 @@ def _build_package_locked_assets(
     for tracked_asset in tracked_assets:
         asset_path = root / tracked_asset.path
         if not asset_path.is_file():
+            res.append(
+                LockedAsset(
+                    path=tracked_asset.path,
+                    hash=None,
+                    symlink_info=_get_symlink_info(tracked_asset, root),
+                )
+            )
             continue
         with asset_path.open('rb') as f:
             res.append(
-                LockedAsset(path=tracked_asset.path, hash=digest_cooperatively(f))
+                LockedAsset(
+                    path=tracked_asset.path,
+                    hash=digest_cooperatively(f),
+                    symlink_info=_get_symlink_info(tracked_asset, root),
+                )
             )
     return res
 
@@ -156,14 +253,18 @@ def _build_package_locked_assets(
 def _find_non_modified_assets(
     reference: List[LockedAsset], current: List[LockedAsset]
 ) -> List[LockedAsset]:
-    current_by_path = {asset.path: asset for asset in current}
+    reference_by_path = {asset.path: asset for asset in reference}
 
     res = []
-    for asset in reference:
-        if (
-            asset.path in current_by_path
-            and current_by_path[asset.path].hash != asset.hash
-        ):
+    for asset in current:
+        # File does not exist in the reference.
+        reference_asset = LockedAsset(path=asset.path, hash=None)
+
+        # File already exists.
+        if asset.path in reference_by_path:
+            reference_asset = reference_by_path[asset.path]
+
+        if asset.was_modified(reference_asset) and not asset.is_broken_symlink():
             # This is a file that was modified.
             continue
         res.append(asset)
@@ -173,46 +274,139 @@ def _find_non_modified_assets(
 def _find_modified_assets(
     reference: List[LockedAsset],
     current: List[LockedAsset],
+    seen_symlinks: Set[pathlib.Path],
 ):
-    reference_by_path = {asset.path: asset for asset in reference}
+    current_by_path = {asset.path: asset for asset in current}
 
     res = []
-    for asset in current:
-        if (
-            asset.path in reference_by_path
-            and reference_by_path[asset.path].hash == asset.hash
-        ):
-            # This is a file that was not modified.
+    for asset in reference:
+        current_asset = LockedAsset(path=asset.path, hash=None)
+        if asset.path in current_by_path:
+            current_asset = current_by_path[asset.path]
+        if current_asset.path in seen_symlinks and not asset.is_symlink():
+            # TODO: improve this condition, it's almost always triggering
+            # Preset asset should be forced to be a symlink,
+            # but in the current package it is not.
+            res.append(asset)
             continue
-        res.append(asset)
+        if current_asset.was_modified(asset, follow_symlinks=True):
+            # This is a file that was modified.
+            res.append(asset)
     return res
 
 
+def _copy_preset_file(
+    src: pathlib.Path,
+    dst: pathlib.Path,
+    preset_package_path: pathlib.Path,
+    preset_path: pathlib.Path,
+    force_symlink: bool = False,
+):
+    if dst.is_file() or dst.is_symlink():
+        dst.unlink(missing_ok=True)
+    if not src.exists():
+        return
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    if not src.is_symlink() and not force_symlink:
+        shutil.copyfile(str(src), str(dst))
+        return
+
+    # Ensure preset package path is inside the preset path.
+    absolute_preset_package_path = utils.abspath(preset_package_path)
+    absolute_preset_path = utils.abspath(preset_path)
+    assert absolute_preset_package_path.is_relative_to(absolute_preset_path)
+
+    # Get the symlink absolute path.
+    if src.is_symlink():
+        target_relative_path = pathlib.Path(os.readlink(str(src)))
+        target_absolute_path = utils.abspath(src.parent / target_relative_path)
+
+        if target_absolute_path.is_relative_to(absolute_preset_package_path):
+            # The symlink points inside the preset package path.
+            # Copy the symlink as is.
+            dst.symlink_to(target_relative_path)
+            return
+    else:
+        target_absolute_path = utils.abspath(src)
+
+    if not target_absolute_path.is_relative_to(absolute_preset_path):
+        console.console.print(
+            f'[error]Preset [item]{preset_path.name}[/item] has a symlink to [item]{target_absolute_path}[/item] which is outside the preset folder.[/error]'
+        )
+        raise typer.Exit(1)
+
+    # The symlink points somewhere inside the preset folder, fix the symlink.
+    dst_absolute_path = utils.abspath(dst)
+    fixed_target_relative_path = target_absolute_path.relative_to(
+        dst_absolute_path.parent,
+        walk_up=True,
+    )
+    dst.symlink_to(fixed_target_relative_path)
+
+
 def _copy_updated_assets(
     preset_lock: PresetLock,
     is_contest: bool,
     root: pathlib.Path = pathlib.Path(),
+    force: bool = False,
+    symlinks: bool = False,
 ):
-    current_package_snapshot = _build_package_locked_assets(preset_lock.assets)
-    non_modified_assets = _find_non_modified_assets(
-        preset_lock.assets, current_package_snapshot
-    )
-
+    # Build preset package snapshot.
     preset = get_active_preset(root)
+    preset_path = get_active_preset_path(root)
     preset_package_path = _get_active_preset_package_path(root, is_contest)
 
     preset_tracked_assets = _get_preset_tracked_assets(
-        preset_package_path, is_contest=is_contest
+        preset_package_path, is_contest=is_contest, add_symlinks=symlinks
     )
     current_preset_snapshot = _build_package_locked_assets(
         preset_tracked_assets, preset_package_path
     )
-    assets_to_copy = _find_modified_assets(non_modified_assets, current_preset_snapshot)
+
+    # Build current package snapshot based on the current preset snapshot.
+    current_package_snapshot = _build_package_locked_assets(current_preset_snapshot)
+
+    non_modified_assets = current_package_snapshot
+    if not force:
+        non_modified_assets = _find_non_modified_assets(
+            preset_lock.assets, current_package_snapshot
+        )
+
+    console.console.print('Tracking the following assets from preset:')
+    for asset in current_preset_snapshot:
+        console.console.print(f'  - [item]{asset}[/item]')
+    console.console.print()
+
+    console.console.print('Current package snapshot:')
+    for asset in current_package_snapshot:
+        console.console.print(f'  - [item]{asset}[/item]')
+    console.console.print()
+
+    seen_symlinks = _get_tracked_assets_symlinks(preset_tracked_assets)
+
+    assets_to_copy = _find_modified_assets(
+        non_modified_assets, current_preset_snapshot, seen_symlinks
+    )
+
+    # console.console.log(current_package_snapshot)
+    # console.console.log(current_preset_snapshot)
+
+    if not assets_to_copy:
+        console.console.print('[warning]No assets to update.[/warning]')
+        return
+
+    # console.console.log(assets_to_copy)
 
     for asset in assets_to_copy:
         src_path = preset_package_path / asset.path
         dst_path = root / asset.path
-        shutil.copyfile(str(src_path), str(dst_path))
+        _copy_preset_file(
+            src_path,
+            dst_path,
+            preset_package_path,
+            preset_path,
+            force_symlink=asset.path in seen_symlinks,
+        )
         console.console.print(
             f'Updated [item]{asset.path}[/item] from preset [item]{preset.name}[/item].'
         )
@@ -400,7 +594,7 @@ def _install_preset_from_local_dir(
         dest,
         ensure_contest,
         ensure_problem,
-        override_uri=str(pd.resolve()),
+        override_uri=str(utils.abspath(pd)),
         update=update,
     )
 
@@ -426,7 +620,7 @@ def _install_preset_from_resources(
         dest,
         ensure_contest,
         ensure_problem,
-        override_uri=str(rsrc_preset_path.resolve()),
+        override_uri=str(utils.abspath(rsrc_preset_path)),
         update=update,
     )
     return True
@@ -475,6 +669,45 @@ def install_preset_at_package(fetch_info: PresetFetchInfo, dest_pkg: pathlib.Pat
     _install_preset_from_fetch_info(fetch_info, dest_pkg / '.local.rbx')
 
 
+def _install_package_from_preset(
+    preset_path: pathlib.Path,
+    preset_package_inner_path: pathlib.Path,
+    dest_pkg: pathlib.Path,
+    tracked_assets: List[TrackedAsset],
+):
+    preset_package_path = preset_path / preset_package_inner_path
+    if not preset_package_path.is_dir():
+        console.console.print(
+            f'[error]Preset [item]{preset_path.name}[/item] does not have a [item]{preset_package_inner_path}[/item] package definition.[/error]'
+        )
+        raise typer.Exit(1)
+
+    for file in _glob_while_ignoring(
+        preset_package_path,
+        '*',
+        recursive=True,
+    ):
+        if not file.is_file():
+            continue
+        _copy_preset_file(
+            file,
+            dest_pkg / file.relative_to(preset_package_path),
+            preset_package_path,
+            preset_path,
+        )
+
+    for asset in tracked_assets:
+        if not asset.symlink:
+            continue
+        _copy_preset_file(
+            preset_package_path / asset.path,
+            dest_pkg / asset.path,
+            preset_package_path,
+            preset_path,
+            force_symlink=True,
+        )
+
+
 def install_contest(
     dest_pkg: pathlib.Path, fetch_info: Optional[PresetFetchInfo] = None
 ):
@@ -496,10 +729,8 @@ def install_contest(
     console.console.print(
         f'Installing contest from [item]{preset_path / preset.contest}[/item] to [item]{dest_pkg}[/item]...'
     )
-    shutil.copytree(
-        str(preset_path / preset.contest),
-        str(dest_pkg),
-        dirs_exist_ok=True,
+    _install_package_from_preset(
+        preset_path, preset.contest, dest_pkg, preset.tracking.contest
     )
     _clean_copied_contest_dir(dest_pkg, delete_local_rbx=False)
 
@@ -525,14 +756,36 @@ def install_problem(
     console.console.print(
         f'Installing problem from [item]{preset_path / preset.problem}[/item] to [item]{dest_pkg}[/item]...'
     )
-    shutil.copytree(
-        str(preset_path / preset.problem),
-        str(dest_pkg),
-        dirs_exist_ok=True,
+    _install_package_from_preset(
+        preset_path, preset.problem, dest_pkg, preset.tracking.problem
     )
     _clean_copied_problem_dir(dest_pkg)
 
 
+def install_preset(
+    dest_pkg: pathlib.Path, fetch_info: Optional[PresetFetchInfo] = None
+):
+    if fetch_info is None and get_active_preset_or_null() is None:
+        console.console.print(
+            '[error]No preset found to initialize the new preset from.[/error]'
+        )
+        raise typer.Exit(1)
+    if fetch_info is None:
+        _install_preset_from_dir(get_active_preset_path(), dest_pkg)
+    else:
+        _install_preset_from_fetch_info(fetch_info, dest_pkg)
+
+
+def get_ruyaml(root: pathlib.Path = pathlib.Path()) -> Tuple[ruyaml.YAML, ruyaml.Any]:
+    if not (root / 'preset.rbx.yml').is_file():
+        console.console.print(
+            f'[error]Preset at [item]{root}[/item] does not have a [item]preset.rbx.yml[/item] file.[/error]'
+        )
+        raise typer.Exit(1)
+    res = ruyaml.YAML()
+    return res, res.load(root / 'preset.rbx.yml')
+
+
 def generate_lock(root: pathlib.Path = pathlib.Path()):
     preset = get_active_preset(root)
 
@@ -548,7 +801,7 @@ def generate_lock(root: pathlib.Path = pathlib.Path()):
     )
 
 
-def _sync(try_update: bool = False):
+def _sync(try_update: bool = False, force: bool = False, symlinks: bool = False):
     preset_lock = _get_preset_lock()
     if preset_lock is None:
         console.console.print(
@@ -565,6 +818,8 @@ def _sync(try_update: bool = False):
     _copy_updated_assets(
         preset_lock,
         is_contest=_is_contest(),
+        force=force,
+        symlinks=symlinks,
     )
     generate_lock()
 
@@ -621,7 +876,9 @@ def copy_local_preset(
     if not add_submodule:
         return
 
-    dest_path_rel = dest_path.resolve().relative_to(pathlib.Path.cwd().resolve())
+    dest_path_rel = utils.abspath(dest_path).relative_to(
+        utils.abspath(pathlib.Path.cwd())
+    )
     path_str = str(dest_path_rel / '.local.rbx')
     try:
         current_repo.git.submodule('add', preset_remote_uri, path_str)
@@ -637,6 +894,47 @@ def copy_local_preset(
     )
 
 
+@app.command('create', help='Create a new preset.')
+def create(
+    name: Annotated[
+        str,
+        typer.Option(
+            help='The name of the preset to create. This will also be the name of the folder.',
+            prompt='What is the name of your new preset?',
+        ),
+    ],
+    uri: Annotated[
+        str,
+        typer.Option(
+            help='The URI of the new preset.',
+            prompt='What is the URI of your new preset? (ex: rsalesc/rbx-preset for a GitHub repository)',
+        ),
+    ],
+    from_preset: Annotated[
+        Optional[str],
+        typer.Option(
+            '--preset', '-p', help='The URI of the preset to init the new preset from.'
+        ),
+    ] = None,
+):
+    console.console.print(f'Creating new preset [item]{name}[/item]...')
+
+    fetch_info = get_preset_fetch_info_with_fallback(from_preset)
+    dest_path = pathlib.Path(name)
+    if dest_path.exists():
+        console.console.print(
+            f'[error]Directory [item]{dest_path}[/item] already exists.[/error]'
+        )
+        raise typer.Exit(1)
+
+    install_preset(dest_path, fetch_info)
+
+    ru, preset = get_ruyaml(dest_path)
+    preset['name'] = name
+    preset['uri'] = uri
+    utils.save_ruyaml(dest_path / 'preset.rbx.yml', ru, preset)
+
+
 @app.command('update', help='Update preset of current package')
 def update():
     preset = get_active_preset()
@@ -689,13 +987,31 @@ def sync(
             help='Whether to fetch an up-to-date version of the installed preset from remote, if available.',
         ),
     ] = False,
+    force: Annotated[
+        bool,
+        typer.Option(
+            '--force',
+            '-f',
+            help='Whether to forcefully overwrite the local assets with the preset assets, even if they have been modified.',
+        ),
+    ] = False,
+    symlinks: Annotated[
+        bool,
+        typer.Option(
+            '--symlinks',
+            '-s',
+            help='Whether to update all symlinks in the preset to point to their right targets.',
+        ),
+    ] = False,
 ):
     _check_is_valid_package()
-    _sync(try_update=update)
+    _sync(try_update=update, force=force, symlinks=symlinks)
 
 
 @app.command(
-    'lock', help='Generate a lock for this package, based on a existing preset.'
+    'lock',
+    help='Generate a lock for this package, based on a existing preset.',
+    hidden=True,
 )
 @cd.within_closest_package
 def lock():

rbx/box/presets/lock_schema.py

@@ -1,12 +1,52 @@
-from typing import List
+import pathlib
+from typing import List, Optional
 
 from pydantic import BaseModel
 
 from rbx.box.presets.schema import TrackedAsset
 
 
+class SymlinkInfo(BaseModel):
+    target: pathlib.Path
+    is_broken: bool
+    is_outside: bool
+
+
 class LockedAsset(TrackedAsset):
-    hash: str
+    hash: Optional[str] = None
+    symlink_info: Optional[SymlinkInfo] = None
+
+    def is_symlink(self) -> bool:
+        return self.symlink_info is not None or self.symlink
+
+    def is_broken_symlink(self) -> bool:
+        return self.symlink_info is not None and self.symlink_info.is_broken
+
+    def was_modified(self, base: 'LockedAsset', follow_symlinks: bool = False) -> bool:
+        if self.is_symlink() != base.is_symlink():
+            return True
+        if self.hash != base.hash and (follow_symlinks or self.symlink_info is None):
+            return True
+        if self.symlink_info is not None and self.symlink_info.is_broken:
+            return True
+        if (
+            self.symlink_info is not None
+            and base.symlink_info is not None
+            and self.symlink_info.target != base.symlink_info.target
+            and not follow_symlinks
+        ):
+            return True
+        return False
+
+    def __str__(self) -> str:
+        if self.symlink_info is not None:
+            res = f'{self.path} -> {self.symlink_info.target}'
+            if self.symlink_info.is_broken:
+                res += ' (broken)'
+            if self.symlink_info.is_outside:
+                res += ' (outside)'
+            return res
+        return f'{self.path} ({self.hash})'
 
 
 class PresetLock(BaseModel):

rbx/box/presets/schema.py

@@ -19,6 +19,10 @@ class TrackedAsset(BaseModel):
     # be tracked. Can also be a glob, when specified in the preset config.
     path: pathlib.Path
 
+    # Whether the asset should be symlinked to the local preset directory,
+    # instead of being copied.
+    symlink: bool = False
+
 
 class Tracking(BaseModel):
     # Problem assets that should be tracked and updated by rbx