real-ladybug 0.0.1.dev1__cp312-cp312-macosx_11_0_arm64.whl

real_ladybug-source/tools/rust_api/lbug-src/tools/rust_api/lbug-src/scripts/export-dbs.py

@@ -0,0 +1,101 @@
+import argparse
+import os
+import sys
+import subprocess
+
+
+# Parse schema.cypher and copy.cypher files.
+def create_cypher_queries(file_path):
+    commands = []
+    try:
+        with open(file_path, "r") as f:
+            for line in f:
+                stripped = line.strip()
+                if not stripped:
+                    continue
+                if not stripped.endswith(";"):
+                    stripped += ";"
+                commands.append(stripped)
+    except Exception:
+        pass
+    return commands
+
+
+# Find all datasets that have a schema.cypher and copy.cypher file.
+def find_valid_dataset_dirs(dataset_root):
+    valid_dirs = []
+
+    for root, dirs, files in os.walk(dataset_root):
+        # This script creates a tmp directory with the exported dbs, we should
+        # skip it in our search.
+        if "tmp" in root.split(os.sep):
+            continue
+        file_set = set(files)
+
+        if "schema.cypher" in file_set:
+            valid_dirs.append(root)
+
+    return valid_dirs
+
+
+# Example scripts/export-dbs.py build/debug/tools/shell/lbug dataset.
+def main():
+    parser = argparse.ArgumentParser(
+        description="Export DBs with LBUG shell and dataset paths"
+    )
+
+    parser.add_argument(
+        "--executable", required=True, help="Path to the LBUG shell executable"
+    )
+    parser.add_argument(
+        "--dataset-dir", required=True, help="Path to the dataset directory"
+    )
+    parser.add_argument(
+        "--output-dir", required=True, help="Path to export the datasets"
+    )
+    args = parser.parse_args()
+
+    arg_executable_path = os.path.abspath(args.executable)
+    arg_dataset_path = os.path.abspath(args.dataset_dir)
+    output_dir = os.path.abspath(args.output_dir)
+
+    if not os.path.isfile(arg_executable_path):
+        raise Exception(f"Error: Executable not found at {arg_executable_path}")
+    if not os.path.exists(arg_dataset_path):
+        raise Exception(f"Error: Dataset path not found at {arg_dataset_path}")
+
+    valid_datasets = find_valid_dataset_dirs(arg_dataset_path)
+    # This is done to construct a full path to replace the relative paths found
+    # in copy.cypher files.
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+    for dataset_path in valid_datasets:
+        schema_commands = create_cypher_queries(
+            os.path.join(dataset_path, "schema.cypher")
+        )
+        copy_commands = create_cypher_queries(os.path.join(dataset_path, "copy.cypher"))
+        combined_commands = schema_commands + copy_commands
+        dataset_name = os.path.relpath(dataset_path, arg_dataset_path)
+        export_path = os.path.join(output_dir, dataset_name)
+        export_command = (
+            f"EXPORT DATABASE '{export_path}' (format=\"csv\", header=true);"
+        )
+        combined_commands.append(export_command)
+        combined_commands.insert(0, "CALL threads=1;")
+        print(f"Exporting {dataset_path} to {export_path}")
+        joined_commands = "\n".join(cmd.strip() for cmd in combined_commands)
+
+        subprocess.run(
+            arg_executable_path,
+            input=joined_commands,
+            text=True,
+            cwd=root_dir,
+            check=True,
+            shell=True,
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

real_ladybug-source/tools/rust_api/lbug-src/tools/rust_api/lbug-src/scripts/export-import-test.py

@@ -0,0 +1,345 @@
+import argparse
+import subprocess
+import os
+import shutil
+
+
+def create_worktree(path, commit, repo_root):
+    remove_worktree(path, repo_root)
+    run_command(f"git worktree add {path} {commit}", cwd=repo_root)
+
+
+def remove_worktree(path, repo_root):
+    if os.path.exists(path):
+        run_command(f"git worktree remove --force {path}", cwd=repo_root, check=False)
+
+
+def check_for_extension_build(makefile):
+    with open(makefile, "r") as f:
+        return any(line.strip() == "extension-build:" for line in f)
+
+
+# Duplicates code from benchmark/version.py.
+# Should be fine since the footprint is small, but any further extensions
+# to this tool should rework this if the footprint for duplicated code gets
+# bigger.
+def get_version(lbug_root):
+    cmake_file = os.path.join(lbug_root, "CMakeLists.txt")
+    with open(cmake_file) as f:
+        for line in f:
+            if line.startswith("project(Lbug VERSION"):
+                return line.split(" ")[2].strip()
+    return "0"
+
+
+def run_command(cmd, cwd=None, capture_output=False, check=True):
+    print(f"> Running: {cmd} (cwd={cwd})")
+
+    # We redirect stdin to devnull in an attempt
+    # to stop the process from interfering with the terminal's input buffer.
+    # This needs some work. After running the script I found that my buffer
+    # was filled with a sequence like 8;1R8;1R8;1R8;1R8;... This doesn't seem
+    # to affect the script but is annoying.
+    result = subprocess.run(
+        cmd,
+        cwd=cwd,
+        text=True,
+        capture_output=capture_output,
+        check=check,
+        shell=True,
+        stdin=subprocess.DEVNULL,
+    )
+    if capture_output:
+        return result.stdout.strip()
+    else:
+        print(result.stdout or "")
+        return None
+
+
+def export_datasets_and_test(
+    lbug_root,
+    base_worktree,
+    test_worktree,
+    dataset_dir,
+    output_dir,
+    cleanup,
+    export_path,
+):
+    version = get_version(base_worktree)
+    if version == "0":
+        raise Exception("Failed to determine version. Aborting.")
+    export_path = os.path.abspath(os.path.join(output_dir, version))
+
+    if not os.path.exists(export_path + os.sep):
+        # Also build the `json` extension, which is needed for some datasets, like tinysnb_json.
+        if check_for_extension_build(
+            os.path.abspath(os.path.join(base_worktree, "Makefile"))
+        ):
+            run_command("make extension-build EXTENSION_LIST=json", cwd=base_worktree)
+        # Older Makefiles do not have the `extension-build` rule
+        else:
+            run_command(
+                "make extension-test-build EXTENSION_LIST=json", cwd=base_worktree
+            )
+
+        # Use '_inprogress' as a temporary suffix to store the exports
+        # and atomically rename to `export_path` after all exports are successful.
+        # Avoids partial exports.
+        inprogress_path = f"{export_path}_inprogress" + os.sep
+        export_script_path = os.path.join(lbug_root, "scripts", "export-dbs.py")
+        exec_path = os.path.join(
+            base_worktree, "build", "relwithdebinfo", "tools", "shell", "lbug"
+        )
+        run_command(
+            f"""python3 {export_script_path} \
+            --executable {exec_path} \
+            --dataset-dir {dataset_dir} \
+            --output-dir {inprogress_path}""",
+            cwd=lbug_root,
+        )
+        os.rename(inprogress_path, export_path + os.sep)
+
+    # Append `/` so that datasets can be found correctly
+    os.environ["E2E_IMPORT_DB_DIR"] = export_path + os.sep
+    run_command("make test", cwd=test_worktree)
+    return 0
+
+
+def write_split_testfile(
+    export_dir, import_dir, case_name, header, export_lines, import_lines, db_dir
+):
+    export_path = os.path.join(export_dir, f"{case_name}.test")
+    import_path = os.path.join(import_dir, f"{case_name}.test")
+    os.makedirs(export_dir, exist_ok=True)
+    os.makedirs(import_dir, exist_ok=True)
+
+    def replace_placeholders(lines):
+        return [
+            line.replace("${LBUG_EXPORT_DB_DIRECTORY}", os.path.join(db_dir, ""))
+            for line in lines
+        ]
+
+    # Copying the dataset here is unnecessary as the tests use exported dbs.
+    def transform_import_header(header):
+        new_lines = []
+        for line in header.splitlines(keepends=True):
+            if line.startswith("-DATASET"):
+                line = "-DATASET CSV empty\n"
+            new_lines.append(line)
+        return "".join(new_lines)
+
+    # This is to handle an issue where the exported db seems to be
+    # deleted making the import fail. DBs are still imported with the line
+    # -STATEMENT IMPORT DATABASE ...
+    def transform_import_lines(lines):
+        result = []
+        for line in lines:
+            if line.startswith("-IMPORT_DATABASE"):
+                continue
+            result.append(line)
+        return result
+
+    with open(export_path, "w") as f:
+        f.write(header.replace("${LBUG_EXPORT_DB_DIRECTORY}", os.path.join(db_dir, "")))
+        f.writelines(replace_placeholders(export_lines))
+
+    with open(import_path, "w") as f:
+        f.write(
+            transform_import_header(
+                header.replace("${LBUG_EXPORT_DB_DIRECTORY}", os.path.join(db_dir, ""))
+            )
+        )
+        f.writelines(transform_import_lines(replace_placeholders(import_lines)))
+
+
+def split_tests(root, output_dir, file, db_dir):
+    relative_path = os.path.relpath(file.name, root)
+    base_path = os.path.splitext(relative_path)[0]
+    export_dir = os.path.abspath(os.path.join(output_dir, "export", base_path))
+    import_dir = os.path.abspath(os.path.join(output_dir, "import", base_path))
+
+    header = ""
+    header_parsed = False
+    current_case = None
+    export_lines = []
+    import_lines = []
+    in_case = False
+    in_import = False
+    for line in file:
+        line = line.rstrip("\n")
+        if not header_parsed:
+            header += line + "\n"
+            if line.strip() == "--":
+                header_parsed = True
+            continue
+        if line.startswith("-CASE"):
+            # Write the previous case, if it had a split.
+            if in_case and in_import:
+                write_split_testfile(
+                    export_dir,
+                    import_dir,
+                    current_case,
+                    header,
+                    export_lines,
+                    import_lines,
+                    db_dir,
+                )
+            export_lines = []
+            import_lines = []
+            in_import = False
+            current_case = line[len("-CASE") :].strip()
+            in_case = True
+            export_lines.append(line + "\n")
+            import_lines.append(line + "\n")
+            continue
+        if line.startswith("#EXPORT_IMPORT_TEST_SPLIT"):
+            in_import = True
+            continue
+        if in_case:
+            if in_import:
+                import_lines.append(line + "\n")
+            else:
+                export_lines.append(line + "\n")
+    # Handle the last case of a file.
+    if current_case and in_import:
+        write_split_testfile(
+            export_dir,
+            import_dir,
+            current_case,
+            header,
+            export_lines,
+            import_lines,
+            db_dir,
+        )
+
+
+def split_files(test_dir, output_dir):
+    db_dir = os.path.abspath(os.path.join(output_dir, "db"))
+    os.makedirs(db_dir, exist_ok=True)
+    for root, dirs, files in os.walk(test_dir):
+        for file in files:
+            full_path = os.path.join(root, file)
+            with open(full_path, "r") as f:
+                split_tests(test_dir, output_dir, f, db_dir)
+
+
+def run_export_specific_tests(
+    lbug_root, base_worktree, test_worktree, test_dir, output_dir, cleanup
+):
+    # Split tests in test_dir
+    split_files(test_dir, output_dir)
+    # Build base_worktree lbug
+    run_command("make test-build", cwd=base_worktree)
+    # Run the export tests.
+    run_command(
+        f"E2E_TEST_FILES_DIRECTORY='.' ./.worktree-base/build/relwithdebinfo/test/runner/e2e_test {os.path.abspath(os.path.join(output_dir, 'export'))}",
+        cwd=lbug_root,
+        check=False,
+    )
+    # Build test_worktree lbug
+    run_command("make test-build", cwd=test_worktree)
+    # Run the import tests.
+    run_command(
+        f"E2E_TEST_FILES_DIRECTORY='.' ./.worktree-test/build/relwithdebinfo/test/runner/e2e_test {os.path.abspath(os.path.join(output_dir, 'import'))}",
+        cwd=lbug_root,
+        check=False,
+    )
+
+
+def main():
+    base_worktree = None
+    test_worktree = None
+    export_path = None
+    cleanup = None
+
+    parser = argparse.ArgumentParser(
+        description="Export DBs from dataset-dir to output-dir using base-commit and test in test-commit"
+    )
+    parser.add_argument(
+        "--base-commit", required=True, help="Git commit to export databases from"
+    )
+    parser.add_argument(
+        "--test-commit", required=True, help="Git commit to test against"
+    )
+
+    parser.add_argument("--dataset-dir", help="Path to the dataset directory")
+    parser.add_argument("--test-dir", help="Path to the test directory")
+
+    parser.add_argument(
+        "--output-dir", required=True, help="Path to output the exported databases"
+    )
+
+    mutually_exclusive_args = parser.add_mutually_exclusive_group()
+    mutually_exclusive_args.add_argument(
+        "--cleanup",
+        dest="cleanup",
+        action="store_true",
+        help="Delete exported DBs after test",
+    )
+    mutually_exclusive_args.add_argument(
+        "--no-cleanup",
+        dest="cleanup",
+        action="store_false",
+        help="Do not delete exported DBs after test",
+    )
+    parser.set_defaults(cleanup=True)
+
+    try:
+        args = parser.parse_args()
+
+        if bool(args.dataset_dir) == bool(args.test_dir):
+            raise Exception(
+                "You must provide exactly one of --dataset-dir or --test-dir."
+            )
+
+        base_commit = args.base_commit
+        test_commit = args.test_commit
+        output_dir = args.output_dir
+        cleanup = args.cleanup
+
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        lbug_root = os.path.abspath(os.path.join(script_dir, ".."))
+        base_worktree = os.path.join(lbug_root, ".worktree-base")
+        test_worktree = os.path.join(lbug_root, ".worktree-test")
+        create_worktree(base_worktree, base_commit, lbug_root)
+        create_worktree(test_worktree, test_commit, lbug_root)
+
+        if bool(args.dataset_dir):
+            export_datasets_and_test(
+                lbug_root,
+                base_worktree,
+                test_worktree,
+                os.path.abspath(args.dataset_dir),
+                output_dir,
+                cleanup,
+                export_path,
+            )
+        else:
+            assert bool(args.test_dir)
+            export_path = output_dir
+            run_export_specific_tests(
+                lbug_root,
+                base_worktree,
+                test_worktree,
+                os.path.abspath(args.test_dir),
+                output_dir,
+                cleanup,
+            )
+
+    finally:
+        if cleanup and export_path and os.path.exists(export_path):
+            print(f"Cleaning up export directory: {export_path}")
+            shutil.rmtree(export_path)
+        else:
+            print(f"Skipping cleaning up export directory: {export_path}")
+
+        if base_worktree or test_worktree:
+            print("Removing worktrees")
+            if base_worktree:
+                remove_worktree(base_worktree, lbug_root)
+            if test_worktree:
+                remove_worktree(test_worktree, lbug_root)
+
+
+if __name__ == "__main__":
+    main()

real_ladybug-source/tools/rust_api/lbug-src/tools/rust_api/lbug-src/scripts/extension/purge-beta.py

@@ -0,0 +1,34 @@
+import pathlib
+import os
+from packaging.version import Version
+
+CURRENT_DIR = pathlib.Path(__file__).parent.resolve()
+
+RELEASES_PATH = CURRENT_DIR.joinpath('releases').resolve()
+
+production_releases = open(CURRENT_DIR.joinpath('PRODUCTION_RELEASES')).read().splitlines()
+
+releases_to_purge = [r for r in os.listdir(RELEASES_PATH) if r.startswith('v')]
+
+releases_to_purge = [r for r in releases_to_purge if r not in production_releases]
+releases_to_purge = [r[1:] for r in releases_to_purge]
+releases_to_purge.sort(key=Version)
+
+releases_to_purge.pop()
+
+releases_to_purge = ['v' + r for r in releases_to_purge]
+
+if len(releases_to_purge) == 0:
+    print('No releases to purge.')
+    exit(0)
+
+print('Releases to purge:')
+for r in releases_to_purge:
+    print('  ' + r)
+
+for r in releases_to_purge:
+    path_to_purge = RELEASES_PATH.joinpath(r)
+    print('Deleting ' + str(path_to_purge))
+    os.system('rm -rf ' + str(path_to_purge))
+
+print('Done.')

real_ladybug-source/tools/rust_api/lbug-src/tools/rust_api/lbug-src/scripts/generate-cpp-docs/collect_files.py

@@ -0,0 +1,122 @@
+import os
+import sys
+import shutil
+import logging
+from hashlib import md5
+from pathlib import Path
+
+logging.basicConfig(level=logging.DEBUG)
+
+HEADER_BASE_PATH = os.path.realpath(
+    os.path.join(os.path.dirname(__file__), '../../src/include'))
+MAIN_HEADER_PATH = os.path.realpath(
+    os.path.join(HEADER_BASE_PATH, 'main'))
+START_POINT = os.path.realpath(
+    os.path.join(MAIN_HEADER_PATH, 'lbug.h')
+)
+JSON_HEADER_PATH = os.path.realpath(
+    os.path.join(os.path.dirname(__file__), '../../third_party/nlohmann_json/json_fwd.hpp'))
+HEADER_TARGET_PATH = os.path.realpath(
+    os.path.join(os.path.dirname(__file__), 'headers')
+)
+
+logging.debug('HEADER_BASE_PATH: %s', HEADER_BASE_PATH)
+logging.debug('MAIN_HEADER_PATH: %s', MAIN_HEADER_PATH)
+logging.debug('START_POINT: %s', START_POINT)
+logging.debug('JSON_HEADER_PATH: %s', JSON_HEADER_PATH)
+
+
+def collect_header_file_path_recurse(start_point):
+    global processed_header_files
+    global header_map
+    if start_point in processed_header_files:
+        return []
+    curr_header_files = []
+    with open(start_point, 'r') as f:
+        for line in f.readlines():
+            if not line.startswith('#include "'):
+                continue
+            header_path = os.path.normpath(line.split('"')[1])
+            header_real_path = None
+            # Special case for json_fwd.hpp
+            if header_path == 'json_fwd.hpp':
+                header_real_path = JSON_HEADER_PATH
+                logging.debug('Found header: %s at %s',
+                              header_path, header_real_path)
+            else:
+                # Check if the header is in the current directory
+                start_point_dir = os.path.dirname(start_point)
+                header_real_path = os.path.join(
+                    start_point_dir, header_path)
+                if os.path.exists(header_real_path):
+                    logging.debug('Found header: %s at %s',
+                                  header_path, header_real_path)
+                else:
+                    # Check if the header is in the include directory
+                    header_real_path = os.path.join(
+                        HEADER_BASE_PATH, header_path)
+                    if os.path.exists(header_real_path):
+                        logging.debug('Found header: %s at %s',
+                                      header_path, header_real_path)
+            if header_real_path is None:
+                logging.error('Could not find header: %s', header_path)
+                sys.exit(1)
+            curr_header_files.append(header_real_path)
+            if start_point not in header_map:
+                header_map[start_point] = {}
+            header_map[start_point][header_path] = header_real_path
+    for header_file in curr_header_files:
+        curr_header_files += collect_header_file_path_recurse(header_file)
+
+    processed_header_files.add(start_point)
+    return curr_header_files
+
+
+def collect_header_file_paths():
+    global processed_header_files
+    global header_map
+    processed_header_files = set()
+    header_map = {}
+    collect_header_file_path_recurse(START_POINT)
+
+
+def copy_header(header_real_path):
+    global copied_headers
+    if header_real_path in copied_headers:
+        return copied_headers[header_real_path]
+    header_name = os.path.basename(header_real_path)
+    # Rename the header if it is already copied
+    if os.path.exists(os.path.join(HEADER_TARGET_PATH, header_name)):
+        header_name = md5(header_real_path.encode()).hexdigest() + '.h'
+    target_path = os.path.join(HEADER_TARGET_PATH, header_name)
+    shutil.copyfile(header_real_path, target_path)
+    copied_headers[header_real_path] = header_name
+    logging.debug('Copied header: %s to %s', header_real_path, target_path)
+    return header_name
+
+
+def copy_headers():
+    global header_map
+    global copied_headers
+    copied_headers = {}
+    if os.path.exists(HEADER_TARGET_PATH):
+        shutil.rmtree(HEADER_TARGET_PATH, ignore_errors=True)
+    os.makedirs(HEADER_TARGET_PATH)
+    for src_header in header_map:
+        src_header_copied_name = copy_header(src_header)
+        src_header_copied_path = os.path.join(
+            HEADER_TARGET_PATH, src_header_copied_name)
+        file = Path(src_header_copied_path)
+        for original_header_path in header_map[src_header]:
+            header_real_path = header_map[src_header][original_header_path]
+            header_name = copy_header(header_real_path)
+            file.write_text(file.read_text().replace(
+                original_header_path, header_name))
+
+
+if __name__ == '__main__':
+    logging.info('Collecting header files...')
+    collect_header_file_paths()
+    logging.info('Copying header files...')
+    copy_headers()
+    logging.info('Done!')

real_ladybug-source/tools/rust_api/lbug-src/tools/rust_api/lbug-src/scripts/generate-tinysnb.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import subprocess
+import sys
+
+LBUG_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+# Datasets can only be copied from the root since copy.schema contains relative paths
+os.chdir(LBUG_ROOT)
+
+# Define the build type from input
+if len(sys.argv) > 1 and sys.argv[1].lower() == "release":
+    build_type = "release"
+else:
+    build_type = "relwithdebinfo"
+
+# Change the current working directory
+if os.path.exists(f"{LBUG_ROOT}/dataset/databases/tinysnb"):
+    shutil.rmtree(f"{LBUG_ROOT}/dataset/databases/tinysnb")
+if sys.platform == "win32":
+    lbug_shell_path = f"{LBUG_ROOT}/build/{build_type}/src/lbug_shell"
+else:
+    lbug_shell_path = f"{LBUG_ROOT}/build/{build_type}/tools/shell/lbug"
+subprocess.check_call(
+    [
+        "python3",
+        f"{LBUG_ROOT}/benchmark/serializer.py",
+        "TinySNB",
+        f"{LBUG_ROOT}/dataset/tinysnb",
+        f"{LBUG_ROOT}/dataset/databases/tinysnb",
+        "--single-thread",
+        "--lbug-shell",
+        lbug_shell_path,
+    ]
+)