resutil 0.1.0__py3-none-any.whl

resutil/__init__.py

@@ -0,0 +1,6 @@
+from .main import main
+
+
+@main
+def func():
+    pass

resutil/cli/__init__.py

@@ -0,0 +1 @@
+from .cli_main import main

resutil/cli/__main__.py

@@ -0,0 +1,4 @@
+from .cli_main import main
+
+if __name__ == "__main__":
+    main()

resutil/cli/cli_main.py

@@ -0,0 +1,256 @@
+import argparse
+import os
+from os.path import join
+from datetime import datetime
+
+from rich import print
+
+
+from ..ex_dir import find_undownloaded_ex_dirs, find_unuploaded_ex_dirs, create_ex_dir
+from ..utils import user_confirm
+from ..config_file import ConfigYaml
+from ..storage import Box
+from ..config_file import create_ex_yaml
+
+from ..core import initialize, upload, upload_all, download_all, download
+
+
+config_file_path = "resutil-conf.yaml"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="")
+    subparsers = parser.add_subparsers()
+
+    # init
+    parser_init = subparsers.add_parser("init", help="initialize resutil")
+    parser_init.set_defaults(handler=command_init)
+
+    # pull
+    parser_pull = subparsers.add_parser("pull", help="pull experiments")
+    parser_pull.add_argument(
+        "--no-dependency",
+        action="store_true",
+        help="pull experiments without dependencies",
+    )
+    group_pull = parser_pull.add_mutually_exclusive_group(required=True)
+    group_pull.add_argument(
+        "-A", "--all", action="store_true", help="pull all experiments"
+    )
+    group_pull.add_argument("experiment", nargs="?", help="experiment to pull")
+    parser_pull.set_defaults(handler=command_pull)
+
+    # push
+    parser_push = subparsers.add_parser("push", help="push experiments")
+    parser_push.add_argument(
+        "--no-dependency",
+        action="store_true",
+        help="push experiments without dependencies",
+    )
+    group_push = parser_push.add_mutually_exclusive_group(required=True)
+    group_push.add_argument(
+        "-A", "--all", action="store_true", help="push all experiments"
+    )
+    group_push.add_argument("experiment", nargs="?", help="experient to push")
+    parser_push.set_defaults(handler=command_push)
+
+    # add
+    parser_add = subparsers.add_parser("add", help="add experiments")
+    parser_add.add_argument("comment", nargs="?", help="experient to add")
+    parser_add.add_argument(
+        "-d",
+        "--dependency",
+        nargs="+",
+        help="add depending experiments",
+    )
+    parser_add.set_defaults(handler=command_add)
+
+    # args
+    args = parser.parse_args()
+
+    if hasattr(args, "handler"):
+        args.handler(args)
+    else:
+        parser.print_help()
+
+
+def get_default_project_name():
+    current_dir = os.getcwd()
+    parent_dir_name = os.path.basename(current_dir)
+
+    return parent_dir_name
+
+
+def search_default_result_dir():
+    candidate = [
+        "results",
+        "result",
+        "data",
+        "output",
+        "outputs",
+        "input",
+        "inputs",
+    ]
+    default = "results"
+    # get dirs in current directory
+    dirs = [d for d in os.listdir() if os.path.isdir(d)]
+
+    for c in candidate:
+        if c in dirs:
+            return c
+
+    return default
+
+
+def command_init(args):
+    # check if already initialized
+    if os.path.exists(config_file_path):
+        print("⚠️ Already initialized.")
+        print("  [yellow]resutil-conf.yaml[/yellow] already exists.")
+        print("  If you proceed, the existing file will be overwritten.")
+        yn = user_confirm("  Do you want to proceed?", default="n")
+        if not yn:
+            return
+
+    # create config
+    config = ConfigYaml()
+
+    # set project name (default is parent directory name)
+    d = get_default_project_name()
+    print(f"Input project name [bold]({d})[/bold]: ", end="")
+    s = input()
+    project_name = s if s != "" else d
+    config.set_project_name(project_name)
+
+    # set directory to store results
+    d = search_default_result_dir()
+    print(f"Input directory name to store results [bold]({d})[/bold]: ", end="")
+    s = input()
+    results_dir = s if s != "" else d
+
+    # create results directory if not exist
+    if not os.path.exists(results_dir):
+        os.makedirs(results_dir)
+
+    config.set_results_dir(results_dir)
+
+    # add to .gitignore
+    yn = user_confirm(f"Do you want to add .gitignore to {results_dir}?", default="y")
+    if yn:
+        with open(os.path.join(results_dir, ".gitignore"), "a") as f:
+            f.write("# Resutil result dir\n")
+            f.write("# Automatically added by resutil\n")
+            f.write("# Files here are not managed by git\n")
+            f.write("*\n")
+            f.write("!.gitignore\n")
+
+    # set storage type
+    while True:
+        d = "box"
+        print(f"Input storage_type [bold]({d})[/bold]: ", end="")
+        s = input()
+        storage_type = s if s != "" else "box"
+        if storage_type in ["box"]:
+            break
+    config.set_storage_type(storage_type)
+
+    # set box config
+    if storage_type == "box":
+        d = "key.json"
+        print(f"Input key file_path [bold]({d})[/bold]: ", end="")
+        s = input()
+        key_file_path = s if s != "" else "key.json"
+
+        yn = user_confirm(
+            f"Do you want to add {key_file_path} to .gitignore?", default="y"
+        )
+        if yn:
+            with open(".gitignore", "a") as f:
+                f.write("\n# Resutil config file\n")
+                f.write(key_file_path + "\n")
+
+        print(f"Input folder id of base dir: ", end="")
+        base_dir_id = input()
+
+        storage_config = {"key_file_path": key_file_path, "base_dir_id": base_dir_id}
+
+        config.set_storage_config(storage_config)
+
+        try:
+            Box(config.storage_config, config.project_name)
+        except Exception as e:
+            print("❌ Failed to connect to storage.")
+            print(f"  [red]{e}[/red]")
+            return
+
+    # save config
+    config.save(config_file_path)
+
+    print("✅ Initialized.")
+
+
+def command_pull(args):
+    config, storage = initialize()
+
+    if args.experiment:
+        download(args.experiment, config.results_dir, storage, args.no_dependency)
+        print(args.experiment)
+        print("✅ Downloaded")
+
+    elif args.all:
+        ex_names_to_upload = find_undownloaded_ex_dirs(config.results_dir, storage)
+
+        n = len(ex_names_to_upload)
+        if n > 0 and user_confirm(
+            f"ℹ️ There are {n} other experiment directory(s) that have not been downloaded. Do you want to download them?",
+            default="y",
+        ):
+            download_all(ex_names_to_upload, config.results_dir, storage)
+            print("✅ Downloaded")
+
+        elif n == 0:
+            print("✅ No experiment to download.")
+
+
+def command_push(args):
+    config, storage = initialize()
+
+    if args.experiment:
+        upload(args.experiment, config.results_dir, storage, args.no_dependency)
+        print(args.experiment)
+        print("✅ Uploaded")
+
+    elif args.all:
+        ex_names_to_upload = find_unuploaded_ex_dirs(config.results_dir, storage)
+
+        n = len(ex_names_to_upload)
+        if n > 0 and user_confirm(
+            f"ℹ️ There are {n} other experiment directory(s) that have not been uploaded. Do you want to upload them?",
+            default="y",
+        ):
+            upload_all(ex_names_to_upload, config.results_dir, storage)
+            print("✅ Uploaded")
+
+        elif n == 0:
+            print("✅ No experiment to upload.")
+
+
+def command_add(args):
+    config, _ = initialize()
+
+    if args.comment is None:
+        comment = input("📝 Input comment for this experiment: ")
+    else:
+        comment = args.comment
+        print(f"com: {comment}")
+
+    dependency = args.dependency if args.dependency is not None else []
+
+    ex_name = create_ex_dir(
+        datetime.now(),
+        comment,
+        config.results_dir,
+    )
+    ex_dir_path = join(config.results_dir, ex_name)
+
+    create_ex_yaml(ex_dir_path, dependency)

resutil/config_file.py

@@ -0,0 +1,80 @@
+from os.path import join
+import sys
+from os.path import exists
+from typing import Optional
+
+import yaml
+
+from .utils import parse_result_dirs
+
+
+class ConfigYaml:
+    def __init__(self, config_file_path=None):
+        if config_file_path is None:
+            return
+
+        try:
+            with open(config_file_path, "r") as f:
+                conf = yaml.safe_load(f)
+        except FileNotFoundError:
+            raise FileNotFoundError(f"config file {config_file_path} does not exist.")
+
+        self.project_name = conf["project_name"]
+        self.results_dir = conf["results_dir"]
+        self.storage_type = conf["storage_type"]
+        self.storage_config = conf["storage_config"]
+
+    def set_project_name(self, project_name: str):
+        self.project_name = project_name
+
+    def set_results_dir(self, results_dir: str):
+        # check existence
+        if not exists(results_dir):
+            raise ValueError(f"results_dir {results_dir} does not exist.")
+        self.results_dir = results_dir
+
+    def set_storage_type(self, storage_type: str):
+        if storage_type not in ["box"]:
+            raise ValueError("storage_type must be 'local' or 'box'")
+        self.storage_type = storage_type
+
+    def set_storage_config(self, storage_config):
+        # check storage_config is valid
+        if self.storage_type == "box":
+            if "key_file_path" not in storage_config:
+                raise ValueError("storage_config must have 'key_file_path' key")
+            if "base_dir_id" not in storage_config:
+                raise ValueError("storage_config must have 'base_dir_id' key")
+        else:
+            raise ValueError("storage_type must be 'box'")
+        self.storage_config = storage_config
+
+    def save(self, config_file_path):
+        data = {
+            "project_name": self.project_name,
+            "results_dir": self.results_dir,
+            "storage_type": self.storage_type,
+            "storage_config": self.storage_config,
+        }
+        with open(config_file_path, "w") as stream:
+            yaml.dump(data, stream)
+
+
+def create_ex_yaml(
+    dir: str,
+    dependency: list[str] = [],
+    commit_hash: Optional[str] = None,
+    uncommited_files: list[str] = [],
+):
+    cmd = " ".join(sys.argv)
+    data = {
+        "cmd": cmd,
+        "result_dir": dir,
+        "dependency": dependency,
+        "git": {
+            "uncommited_files": uncommited_files,
+            "commit_hash": commit_hash,
+        },
+    }
+    with open(join(dir, "resutil-exp.yaml"), "w") as stream:
+        yaml.dump(data, stream)

resutil/core.py

@@ -0,0 +1,95 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from os.path import join, exists
+
+from rich import print
+
+from .storage import Box
+from .config_file import ConfigYaml
+from .exp_file import ExpFile
+
+config_file_name = "resutil-conf.yaml"
+exp_file_name = "resutil-exp.yaml"
+
+
+def initialize():
+    try:
+        config = ConfigYaml(config_file_name)
+    except FileNotFoundError:
+        print(f"⚠️ Config file {config_file_name} does not exist.")
+        print("Create a config file by running [bold]resutil init[/bold] and try again")
+        exit(1)
+
+    if config.storage_type == "box":
+        storage = Box(config.storage_config, config.project_name)
+        print("📦 Connected to [bold]box[/bold]")
+        base_dir_name, project_folder_name = storage.get_info()
+        print(f"  📁 Base dir: [bold]{base_dir_name}[/bold]")
+        print(f"  📁 Project dir: [bold]{project_folder_name}[/bold]")
+    else:
+        raise (
+            ValueError(
+                f"⛔️ Wronge storage type. Check your [bold]{config_file_name}[/bold] file."
+            )
+        )
+    return config, storage
+
+
+def upload(ex_name: str, results_dir: str, storage: Box, no_dependency=False):
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        ex_dir_path = join(results_dir, ex_name)
+
+        def callback(ex_name):
+            print(f"🗂️ Uploading: [bold]{ex_name}[/bold]")
+
+        storage.upload_experiment(ex_dir_path, callback, executor)
+
+        if no_dependency:
+            return
+
+        ex_file_path = join(ex_dir_path, exp_file_name)
+        if exists(ex_file_path):
+            exp_file = ExpFile(ex_file_path)
+            for d in exp_file.dependency:
+                upload(d, results_dir, storage, no_dependency)
+
+
+def upload_all(ex_names_to_upload: list[str], results_dir: str, storage: Box):
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        for ex_name in ex_names_to_upload:
+            ex_dir_path = join(results_dir, ex_name)
+
+            def callback(ex_name):
+                print(f"🗂️ Uploading: [bold]{ex_name}[/bold]")
+
+            storage.upload_experiment(ex_dir_path, callback, executor)
+
+
+def download(ex_name: str, results_dir: str, storage: Box, no_dependency=False):
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        ex_dir_path = join(results_dir, ex_name)
+
+        def callback(ex_name):
+            print(f"🗂️ Downloading: [bold]{ex_name}[/bold]")
+
+        futures = storage.download_experiment(ex_dir_path, callback, executor)
+
+        if no_dependency:
+            return
+
+        for future in as_completed(futures):
+            f = future.result()
+            if f == exp_file_name:
+                exp_file = ExpFile(join(ex_dir_path, f))
+                for d in exp_file.dependency:
+                    download(d, results_dir, storage, no_dependency)
+
+
+def download_all(ex_names_to_download: list[str], results_dir: str, storage: Box):
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        for ex_name in ex_names_to_download:
+            ex_dir_path = join(results_dir, ex_name)
+
+            def callback(ex_name):
+                print(f"🗂️ Downloading: [bold]{ex_name}[/bold]")
+
+            storage.download_experiment(ex_dir_path, callback, executor)

resutil/ex_dir.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+import os
+import shutil
+from glob import glob
+from os.path import basename, normpath, join
+
+
+from .utils import to_base26, parse_result_dirs
+
+
+def create_ex_dir(now, comment, results_dir):
+
+    base_time = datetime(2024, 1, 1, 0, 0, 0, 0)
+    now_str = now.strftime("%Y%m%dT%H%M%S")
+
+    elapsed_time = (now - base_time).total_seconds() / 60
+    str26 = to_base26(int(elapsed_time))
+
+    ex_name = f"{str26}_{now_str}_{comment}"
+
+    ex_dir_path = os.path.join(results_dir, ex_name)
+
+    # if exist, throw error
+    if os.path.isdir(ex_dir_path):
+        raise FileExistsError(f"{ex_dir_path} is already exist")
+
+    os.makedirs(ex_dir_path)
+
+    return ex_name
+
+
+def get_ex_dir_names(results_dir):
+    ex_dir_paths = glob(join(results_dir + "/*/"))
+    ex_dir_names = [basename(normpath(p)) for p in ex_dir_paths]
+    return ex_dir_names
+
+
+def find_unuploaded_ex_dirs(results_dir_path, storage):
+    remote_ex_dir_names = storage.get_all_experiment_names()
+    local_ex_dir_names = get_ex_dir_names(results_dir_path)
+    ex_dir_names_to_upload = []
+    for local_ex_dir_name in local_ex_dir_names:
+        if local_ex_dir_name not in remote_ex_dir_names:
+            ex_dir_names_to_upload.append(local_ex_dir_name)
+    return ex_dir_names_to_upload
+
+
+def find_undownloaded_ex_dirs(results_dir_path, storage):
+    remote_ex_dir_names = storage.get_all_experiment_names()
+    local_ex_dir_names = get_ex_dir_names(results_dir_path)
+    ex_dir_names_to_download = []
+    for remote_ex_dir_name in remote_ex_dir_names:
+        if remote_ex_dir_name not in local_ex_dir_names:
+            ex_dir_names_to_download.append(remote_ex_dir_name)
+    return ex_dir_names_to_download
+
+
+def delete_ex_dir(ex_dir_path):
+    # check valid path
+    if parse_result_dirs(ex_dir_path) == []:
+        raise ValueError(f"{ex_dir_path} does not exist")
+    if os.path.exists(ex_dir_path):
+        shutil.rmtree(ex_dir_path)
+    return True

resutil/exp_file.py

@@ -0,0 +1,15 @@
+import yaml
+
+
+class ExpFile:
+    def __init__(self, exp_file_path=None):
+        if exp_file_path is None:
+            return
+
+        try:
+            with open(exp_file_path, "r") as f:
+                conf = yaml.safe_load(f)
+        except FileNotFoundError:
+            raise FileNotFoundError(f"experiment file {exp_file_path} does not exist.")
+
+        self.dependency = conf["dependency"]

resutil/git.py

@@ -0,0 +1,33 @@
+from shutil import copy
+from pathlib import Path
+
+from git import Repo, InvalidGitRepositoryError
+
+
+def find_git_repo():
+    # search repogitory recursively to root
+    try:
+        repo = Repo("./", search_parent_directories=True)
+    except InvalidGitRepositoryError:
+        return None
+    return repo
+
+
+def get_git_info(repo: Repo):
+    # Get the most recent commit
+    latest_commit = repo.head.commit.hexsha
+
+    # Get modified files
+
+    modified_files = [item.a_path for item in repo.index.diff(None)]
+    staged_files = [item.a_path for item in repo.index.diff("HEAD")]
+
+    return latest_commit, [*modified_files, *staged_files]
+
+
+def store_uncomited(uncommited_files, dir_path):
+    # copy files
+    for file in uncommited_files:
+        dest = Path(dir_path, file)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        copy(file, dest)

resutil/main.py

@@ -0,0 +1,103 @@
+from functools import wraps
+from datetime import datetime
+from os.path import join
+import sys
+
+from rich import print
+
+from .utils import user_confirm, parse_result_dirs
+from .config_file import create_ex_yaml
+from .ex_dir import create_ex_dir, delete_ex_dir, find_unuploaded_ex_dirs
+from .git import find_git_repo, get_git_info, store_uncomited
+
+from .core import initialize, upload, upload_all
+
+
+class resutil_args:
+    def __init__(self, ex_dir):
+        self.ex_dir = ex_dir
+
+
+# Used as a decorator
+def main(verbose=True):
+    def main_wrapper(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            print("")
+            print("✨ Runnning your code with [bold]Resutil[/bold]")
+            print("")
+
+            config, storage = initialize()
+
+            print("")
+
+            comment = input("📝 Input comment for this experiment: ")
+
+            print("")
+
+            ex_name = create_ex_dir(
+                datetime.now(),
+                comment,
+                config.results_dir,
+            )
+            ex_dir_path = join(config.results_dir, ex_name)
+
+            # check uncommited files
+            git_repo = find_git_repo()
+            if git_repo is None:
+                commit_hash, unstaged_files = None, None
+            else:
+                commit_hash, unstaged_files = get_git_info(git_repo)
+                if len(unstaged_files) > 0:
+                    print("🔍 Unstaged files will be stored in the result dir:")
+                    for file in unstaged_files:
+                        print(f"  - {file}")
+                    store_uncomited(
+                        unstaged_files, join(ex_dir_path, "uncommited_files")
+                    )
+
+            dependency = parse_result_dirs(" ".join(sys.argv))
+
+            create_ex_yaml(
+                ex_dir_path,
+                dependency,
+                commit_hash=commit_hash,
+                uncommited_files=unstaged_files,
+            )
+
+            # Run the main function
+            print("🚀 Running the main function...")
+
+            try:
+                func(resutil_args(ex_dir_path), *args, **kwargs)
+            except KeyboardInterrupt:
+                print("")
+                if user_confirm(
+                    "🔔 Interrupted by user. Do you want to [bold]delete[/bold] experiment file for trial?",
+                    default="n",
+                ) and user_confirm(
+                    "🔔 Are your sure to [bold]DELETE[/bold] it?",
+                    default="n",
+                ):
+                    delete_ex_dir(ex_dir_path)
+                    print(f"🗑️  Deleted [bold]{ex_dir_path}[/bold]")
+                    print("⛔️ Aborted")
+
+            print("")
+
+            upload(ex_name, config.results_dir, storage)
+
+            ex_names_to_upload = find_unuploaded_ex_dirs(config.results_dir, storage)
+
+            n = len(ex_names_to_upload)
+            if n > 0 and user_confirm(
+                f"ℹ️ There are {n} other experiment directory(s) that have not been uploaded. Do you want to upload them?",
+                default="y",
+            ):
+                upload_all(ex_names_to_upload, config.results_dir, storage)
+
+            print("✅ Done")
+
+        return wrapper
+
+    return main_wrapper

resutil/storage/__init__.py

@@ -0,0 +1,3 @@
+from .box.box import Box
+
+__all__ = ["Box"]

resutil/storage/box/box.py

@@ -0,0 +1,68 @@
+from os.path import basename, normpath
+from os import makedirs
+from concurrent.futures import ThreadPoolExecutor, Future
+
+from .box_client import BoxClient
+
+
+class Box:
+    def __init__(self, storage_config: dict, project_name: str):
+        self.client = BoxClient(storage_config["key_file_path"])
+
+        self.base_dir = self.client.find_folder(storage_config["base_dir_id"])
+
+        self.project_folder = self.client.find_subfolder_by_name(
+            project_name, self.base_dir.id
+        )
+        if self.project_folder is None:
+            self.project_folder = self.client.create_folder(
+                project_name, self.base_dir.id
+            )
+
+    def get_info(self) -> tuple[str, str, str]:
+        return (
+            self.base_dir.name,
+            self.project_folder.name,
+        )
+
+    def upload_experiment(
+        self, local_ex_path: str, callback, executor: ThreadPoolExecutor
+    ) -> None:
+        """Uploads a folder and its contents to Box.
+
+        Args:
+            local_ex_path (str): path to the folder to be uploaded
+        """
+        ex_dir_name = basename(normpath(local_ex_path))
+        callback(ex_dir_name)
+        ex_dir = self.client.create_subfolder(ex_dir_name, self.project_folder.id)
+
+        futures = []
+        self.client.upload_recursively(local_ex_path, ex_dir, executor, futures)
+        return futures
+
+    def download_experiment(
+        self, local_ex_path: str, callback, executor: ThreadPoolExecutor
+    ) -> Future:
+        """Downloads a folder and its contents to Box.
+
+        Args:
+            local_ex_path (str): path to the folder to be uploaded
+        """
+        ex_dir_name = basename(normpath(local_ex_path))
+        callback(ex_dir_name)
+        makedirs(local_ex_path, exist_ok=True)
+        ex_dir = self.client.find_subfolder_by_name(ex_dir_name, self.project_folder.id)
+
+        futures = []
+        self.client.download_recursively(ex_dir, local_ex_path, executor, futures)
+        return futures
+
+    def get_all_experiment_names(self) -> list[str]:
+        """Get all experiment names in the project folder.
+
+        Returns:
+            list[str]: List of experiment names
+        """
+        folders = self.client.get_folders_in(self.project_folder.id)
+        return [folder.name for folder in folders]

resutil/storage/box/box_client.py

@@ -0,0 +1,92 @@
+from os.path import basename, normpath, isfile, isdir, join
+from os import makedirs
+from glob import glob
+from concurrent.futures import ThreadPoolExecutor, Future
+
+from boxsdk import Client, JWTAuth, BoxAPIException
+from boxsdk.object.folder import Folder
+from boxsdk.object.item import Item
+
+
+class BoxClient:
+    def __init__(self, key_file_path: str):
+        try:
+            auth = JWTAuth.from_settings_file(key_file_path)
+        except FileNotFoundError:
+            raise ValueError(f"Key file not found at {key_file_path}")
+
+        # self.access_token = auth.authenticate_instance()
+        self.client = Client(auth)
+
+    def create_folder(self, name: str, folder_id: str) -> Folder:
+        new_folder = self.create_subfolder(name, folder_id)
+        return new_folder
+
+    def find_subfolder_by_name(self, name: str, folder_id: str) -> Folder:
+        folders = self.get_folders_in(folder_id)
+        for folder in folders:
+            if folder.name == name:
+                return folder
+        return None
+
+    def get_folders_in(self, folder_id: str) -> list[Folder]:
+        items = self.client.folder(folder_id).get_items()
+        folders = []
+        for item in items:
+            if item.type == "folder":
+                folders.append(item)
+        return folders
+
+    def find_folder(self, folder_id: str) -> Folder:
+        try:
+            return self.client.folder(folder_id).get()
+        except BoxAPIException:
+            raise ValueError(f"Folder with id {folder_id} not found")
+
+    def upload_recursively(
+        self,
+        local_dir: str,
+        folder: Folder,
+        executor: ThreadPoolExecutor,
+        futures: list[Future] = [],
+    ):
+        local_items = glob(local_dir + "/*")
+        for local_item in local_items:
+            if isdir(local_item):
+                subfolder_name = basename(normpath(local_item))
+                subfolder = self.create_subfolder(subfolder_name, folder.id)
+                self.upload_recursively(local_item, subfolder, executor, futures)
+            elif isfile(local_item):
+                future = executor.submit(
+                    self.upload_file, local_item, folder.id, basename(local_item)
+                )
+                futures.append(future)
+
+    def create_subfolder(self, name: str, folder_id: str) -> Folder:
+        return self.client.folder(folder_id).create_subfolder(name)
+
+    def upload_file(self, local_file_path: str, folder_id: str, name: str) -> None:
+        self.client.folder(folder_id).upload(local_file_path, name)
+
+    def download_recursively(
+        self,
+        folder: Folder,
+        local_dir: str,
+        executor: ThreadPoolExecutor,
+        futures: list[Future] = [],
+    ):
+        items = self.client.folder(folder.id).get_items()
+        for item in items:
+            if item.type == "folder":
+                subfolder_name = join(local_dir, item.name)
+                makedirs(subfolder_name, exist_ok=True)
+                self.download_recursively(item, subfolder_name, executor, futures)
+            elif item.type == "file":
+                future = executor.submit(self.download_file, item, local_dir)
+                futures.append(future)
+        return futures
+
+    def download_file(self, item: Item, local_dir: str) -> None:
+        with open(join(local_dir, item.name), "wb") as f:
+            f.write(item.content())
+        return item.name

resutil/utils.py

@@ -0,0 +1,44 @@
+import re
+from rich import print
+
+
+def to_base26(n):
+    code = "abcdefghijklmnopqrstuvwxyz"
+    converted = []
+
+    for i in range(6):
+        converted.append(code[n % 26])
+        n -= n % 26
+        n //= 26
+
+    converted = "".join(converted[::-1])
+    converted = "a" * (5 - len(converted)) + converted
+
+    return converted
+
+
+def parse_result_dirs(text: str):
+    pattern = r"[a-zA-Z]{5}_\d{8}T\d{6}_[^/\s\\]*"
+    matches = re.findall(pattern, text)
+    return matches
+
+
+def user_confirm(question: str, default="") -> bool:
+    yn = "(Y/n)" if default == "y" else "(y/N)" if default == "n" else "(y/n)"
+
+    print(f"{question} [bold]{yn}[/bold]: ", end="")
+    reply = str(input()).lower().strip()
+
+    if reply[:1] == "y":
+        return True
+    elif reply[:1] == "n":
+        return False
+    elif default == "y" and len(reply) == 0:
+        return True
+    elif default == "n" and len(reply) == 0:
+        return False
+    else:
+        new_question = question
+        if "Please try again - " not in question:
+            new_question = f"Please try again - {question}"
+        return user_confirm(new_question, default)

resutil-0.1.0.dist-info/METADATA

@@ -0,0 +1,187 @@
+Metadata-Version: 2.3
+Name: resutil
+Version: 0.1.0
+Summary: Add your description here
+Project-URL: Homepage, https://github.com/KatayamaLab/resutil
+Project-URL: Issues, https://github.com/KatayamaLab/resutil/issues
+Author-email: nobkat <katayama@rs.tus.ac.jp>
+License-File: LICENSE
+Requires-Python: >=3.8
+Requires-Dist: boxsdk[jwt]>=3.9.0
+Requires-Dist: gitpython>=3.1.0
+Requires-Dist: pytest-mock>=3.14.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: rich>=13.7.0
+Description-Content-Type: text/markdown
+
+# Resutil
+
+## What is Resutil
+
+**Resutil** is a utility to manage experimental result data obtained from Python projects. It also manages dependency such as codes and input data with result data. Data is synced to the cloud for team sharing and collaboration.
+
+## Why choose Resutil?
+
+- **Simple**: Easy to install and can be quickly integrated into your project.  
+- **High Reproducibility**: Tracks programs, input data, and experimental results.
+- **Open-source and free**: Join development to future release of Resutil.
+
+
+## Features
+
+- Sync experimental data saved in a specific directory to the cloud (currently only for Box) after the program execution finished.
+- Save information necessary to reproduce the experiment in a YAML file.
+- Execution command
+- Input files given as arguments (only files within folders managed by resutil)
+- Git commit hash
+- Uncommitted files
+- Upload experimental data that hasn’t been uploaded yet.
+- Download experimental data from the cloud using commands.
+
+## Installation
+
+Open terminal and run
+
+```bash
+$ pip install resutil
+```
+
+Get JWT (JSON Web Tokens) key from [Box](https://developer.box.com/guides/authentication/jwt/), and saved as `key.json`:
+
+```JSON
+{
+  "boxAppSettings": {
+    "clientID": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+    "clientSecret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+    "appAuth": {
+      "publicKeyID": "xxxxxxxx",
+      "privateKey": "-----BEGIN ENCRYPTED PRIVATE KEY-----\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n-----END ENCRYPTED PRIVATE KEY-----\n",
+      "passphrase": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    }
+  },
+  "enterpriseID": "796607301"
+}
+```
+
+Initialize resutil
+
+```bash
+$ cd your-project-root-directory
+$ resutil init
+Input project name (resutil): MyProj
+Input directory name to store results (results): results
+Do you want to add .gitignore to results? (Y/n): Y
+Input storage_type (box): box
+Input key file_path (key.json): key.json
+Do you want to add key.json to .gitignore? (Y/n): Y
+Input folder id of base dir: 123456789012
+✅ Initialized.
+```
+
+The folder id is the Box folder ID, which is the numeric part of the URL when viewing the folder on Box (e.g., https://xxxx.app.box.com/folder/123456789012).
+
+
+A file named `resutil-conf.yaml` will be created.
+
+Modify main function in your project like:
+
+
+```python
+
+# Import resutil
+import resutil
+
+# Add decorator before the main function
+@resutil.main()
+def main_func(params):
+    # A directory is automatically created, and its path is stored in params.ex_dir
+    ex_dir = params.ex_dir 
+
+    # Execute the program HERE🚀
+    # Resutil will handle the rest
+
+if __name__ == "__main__":
+    main_func()
+```
+
+## Commands
+
+### `resutil init`
+
+`resutil init` creates `resutil-conf.yaml` in your project folder such as:
+
+```yaml
+project_name: MyProj
+results_dir: results/
+storage_type: box
+storage_config:
+  base_dir_id: xxxx
+  key_file: key.json
+```
+
+### `resutil push`
+
+The `resutil push` command is used to upload experimental data to the cloud that does not exist in the cloud result directory.
+
+`resutil push [exp_name]` uploads experiment a specific directory to the cloud. Depending directorys included in `exp-config.yaml` are automtically uploaded. `--no-dependency` option restrain automatic dependency upload.
+
+`resutil pull --all` will upload all experimental data to the cloud.
+
+This is useful for keeping your local data up-to-date with the data stored in the cloud, especially when multiple people are working on the same project and updating the experimental data.
+
+### `resutil pull`
+
+The `resutil pull` command is used to download a specific experimental data from the cloud that does not exist in the local result directory.
+
+`resutil pull [exp_name]` downloads experiment directory from cloud. Depending directorys included in `exp-config.yaml` are automtically downlowded. `--no-dependency` option restrain automatic dependency download.
+
+`resutil pull --all` will download all experimental data from the cloud that is not currently in your local result directory.
+
+This is useful for keeping your local data up-to-date with the data stored in the cloud, especially when multiple people are working on the same project and updating the experimental data.
+
+
+### `resutil add`
+
+The resutil add command is used to add an experiment directory without executing any code.
+
+You can use it as follows: resutil `add [comment] -d [DEPENDENCY1] [DEPENDENCY2]...`. This command adds an experiment directory named "comment" and sets its dependencies. The dependencies are other experiments that this experiment depends on.
+
+For example, if you have two experiments `exp1` and `exp2` and a new experiment depends on them, you can add the new experiment with the following command: `resutil add "new experiment" -d exp1 exp2`. This will create a new experiment directory named "new experiment" and set `exp1` and `exp2` as its dependencies.
+
+## Directory structure in the cloud storage
+
+```plain text
+BaseDir    # Base directory specified base_dir_id
+├──MyProj  # Project directory
+│   ├── aakuqj_20240511T174522_ex1  # Experiment directory
+│   │   ├── resutil-exp.yaml       # Experiment information
+│   │   └── data.txt               # Data (example)
+│   ├── aamxrp_20240606T135747_ex2
+│   │   ├── resutil-exp.yaml
+│   │   ├── data.txt
+│   │   └── uncommited_files       # Uncommitted files
+│   │       └── main.py
+│   ...
+│   
+├──OtherProj
+│
+...
+```
+
+Experiment directories are formatted as `xxxxxx_yyyymmddTHHMMSS_comment`, where xxxxxx is timestamped for easy ordering and tab completion in shells.
+
+## resutil-exp.yaml [WIP]
+
+Each experiment directory has `resutil-exp.yaml`, which contains information to reproduce experimental results.
+
+``` yaml
+cmd: Execution command
+params: Options at runtime
+  hoo: xxx
+  bar: 123
+dependency: Dependencies (automatically extracted from directories in the command)
+  - ex1
+  - ex2
+```
+
+

resutil-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+resutil/__init__.py,sha256=0uQCHxlSuLTeX96NpADFumYUFmrEkvaLehb-LyRtAjk,52
+resutil/config_file.py,sha256=e-FBMSOeRZdRK7eMH9QVkl7NEtfN70dVURjwmbH3e98,2559
+resutil/core.py,sha256=l9vKLLiih7cs3xJsKNI1YpK7UFJzNs9W6RNJsAgM38E,3287
+resutil/ex_dir.py,sha256=qANI9HtOX-75qb1CsMpXfHsXadK7TvEajEQ2Px76IsM,1951
+resutil/exp_file.py,sha256=ZBKUR-dNx8e16GX8Et7bisbkvQEVb6dnalISfSqcBGQ,397
+resutil/git.py,sha256=xvLQHeNyejmJp3uBobBA3_IMvFFmFN-ADn-lb75ty9Q,858
+resutil/main.py,sha256=bXdNoDujZPWFuym5wsLfUlGHdd1enp98Z1alfPrWizk,3275
+resutil/utils.py,sha256=zTrG5a2GgWnWn9T6pKNKoxQWriLXp15On5c13B5Jdkc,1134
+resutil/cli/__init__.py,sha256=Hn4N9T8PbbE27oPqzsUQEATeAVp8dcGwBaecIhWIIR8,27
+resutil/cli/__main__.py,sha256=IBG0X9Tace8Mfjy0ADwY6ekuqH8QArM3F10FOlzptpA,66
+resutil/cli/cli_main.py,sha256=Vc0hlth3j7ugX5wRLIlQGuoTAveXpqGCHqDKR8IBmLk,7580
+resutil/storage/__init__.py,sha256=kkBWPOvQvdlREuUeA0PBT9rCB4ozsFSBP6PdUCHCiDE,44
+resutil/storage/box/box.py,sha256=fqZrUeqnQa34Q3fmSeDeb9T3Az1rRsVvCnCIfsBnZYQ,2295
+resutil/storage/box/box_client.py,sha256=5N8RZSxnXDtHaXxxS0pl0HWDjaoZzzHppKdnFhaN5f4,3397
+resutil-0.1.0.dist-info/METADATA,sha256=LZKkj69veXSBK_biGi8mJrBxMn7Uyr91Y_ZKKb0AVuo,6466
+resutil-0.1.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+resutil-0.1.0.dist-info/entry_points.txt,sha256=LJDQZrcPvwHTDYoMUEugji1Yj6PrmDT2shc4o6cFdnE,45
+resutil-0.1.0.dist-info/licenses/LICENSE,sha256=ZNYNRUtLl_vWP43Rhcoj6rf5PKkTR1mOI6tOrha-n5I,1063
+resutil-0.1.0.dist-info/RECORD,,

resutil-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.24.2
+Root-Is-Purelib: true
+Tag: py3-none-any

resutil-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+resutil = resutil.cli:main

resutil-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 nobkat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.