dirark 0.1.0__tar.gz

dirark-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Python Test
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: astral-sh/setup-uv@v5
+      with:
+        python-version: '3.12'
+        enable-cache: true
+    - run: uv tool install branthebuilder
+    - run: branb test

dirark-0.1.0/.github/workflows/twine_release.yml

@@ -0,0 +1,17 @@
+
+name: Python Package Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: astral-sh/setup-uv@v5
+    - run: uv build && uv publish
+      env:
+        UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
+

dirark-0.1.0/.gitignore

@@ -0,0 +1,119 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Generated coverage HTML (report.md is committed, HTML is not)
+coverage/html/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea
+.vscode
+dask-worker-space/

dirark-0.1.0/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2026 Endre Márk Borza
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

dirark-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: dirark
+Version: 0.1.0
+Summary: Directory archival
+Project-URL: Homepage, https://github.com/endremborza/dirark
+Author-email: Endre Márk Borza <endremborza@gmail.com>
+License: Copyright 2026 Endre Márk Borza
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# dirark
+
+[![pypi](https://img.shields.io/pypi/v/dirark.svg)](https://pypi.org/project/dirark/)
+
+Directory archival

dirark-0.1.0/README.md

@@ -0,0 +1,5 @@
+# dirark
+
+[![pypi](https://img.shields.io/pypi/v/dirark.svg)](https://pypi.org/project/dirark/)
+
+Directory archival

dirark-0.1.0/coverage/report.md

@@ -0,0 +1,14 @@
+| Name                          |    Stmts |     Miss |   Cover |
+|------------------------------ | -------: | -------: | ------: |
+| dirark/\_\_init\_\_.py        |        5 |        0 |    100% |
+| dirark/\_\_main\_\_.py        |       92 |       92 |      0% |
+| dirark/core.py                |       71 |        4 |     94% |
+| dirark/reader.py              |       35 |        0 |    100% |
+| dirark/storage.py             |       57 |        0 |    100% |
+| dirark/sync.py                |       55 |        1 |     98% |
+| dirark/tests/test\_core.py    |      164 |        0 |    100% |
+| dirark/tests/test\_init.py    |        3 |        0 |    100% |
+| dirark/tests/test\_reader.py  |       55 |        0 |    100% |
+| dirark/tests/test\_storage.py |       88 |        0 |    100% |
+| dirark/tests/test\_sync.py    |      116 |        0 |    100% |
+| **TOTAL**                     |  **741** |   **97** | **87%** |

dirark-0.1.0/dirark/__init__.py

@@ -0,0 +1,17 @@
+"""dirark – directory archival and retrieval tool."""
+
+from .core import archive_dir, restore_ark
+from .reader import ArkReader
+from .sync import add_dir_to_remote_ark, merge_arks, pull_ark, push_ark
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "archive_dir",
+    "restore_ark",
+    "ArkReader",
+    "push_ark",
+    "pull_ark",
+    "merge_arks",
+    "add_dir_to_remote_ark",
+]

dirark-0.1.0/dirark/__main__.py

@@ -0,0 +1,124 @@
+"""CLI entry point for dirark."""
+
+import argparse
+import sys
+from pathlib import Path
+
+from .core import archive_dir, restore_ark
+from .reader import ArkReader
+from .sync import add_dir_to_remote_ark, merge_arks, pull_ark, push_ark
+
+
+def main() -> None:
+    """Parse arguments and dispatch to the appropriate subcommand."""
+    parser = argparse.ArgumentParser(
+        description="Cold-storage directory archival tool."
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("archive", help="Archive a directory.")
+    p.add_argument("source_dir", type=Path)
+    p.set_defaults(func=_archive)
+
+    p = sub.add_parser("restore", help="Restore files from an ark.")
+    p.add_argument("ark_dir", type=Path)
+    p.add_argument("dest_dir", type=Path)
+    p.set_defaults(func=_restore)
+
+    p = sub.add_parser("push", help="Push local ark to remote via rsync.")
+    p.add_argument("local_ark", type=Path)
+    p.add_argument("remote")
+    p.set_defaults(func=_push)
+
+    p = sub.add_parser("pull", help="Pull remote ark to local path via rsync.")
+    p.add_argument("remote")
+    p.add_argument("local_ark", type=Path)
+    p.set_defaults(func=_pull)
+
+    p = sub.add_parser("merge", help="Merge src_ark into dst_ark (local).")
+    p.add_argument("src_ark", type=Path)
+    p.add_argument("dst_ark", type=Path)
+    p.set_defaults(func=_merge)
+
+    p = sub.add_parser("add", help="Archive a directory and add it to a remote ark.")
+    p.add_argument("source_dir", type=Path)
+    p.add_argument("remote_ark")
+    p.set_defaults(func=_add)
+
+    p = sub.add_parser("read", help="Print a file from an ark to stdout.")
+    p.add_argument("ark_dir", type=Path)
+    p.add_argument("file_path")
+    p.set_defaults(func=_read)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+def _archive(args: argparse.Namespace) -> None:
+    try:
+        archive_dir(args.source_dir)
+        print(f"Archived '{args.source_dir}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _restore(args: argparse.Namespace) -> None:
+    try:
+        restore_ark(args.ark_dir, args.dest_dir)
+        print(f"Restored to '{args.dest_dir}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _push(args: argparse.Namespace) -> None:
+    try:
+        push_ark(args.local_ark, args.remote)
+        print(f"Pushed '{args.local_ark}' to '{args.remote}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _pull(args: argparse.Namespace) -> None:
+    try:
+        pull_ark(args.remote, args.local_ark)
+        print(f"Pulled '{args.remote}' to '{args.local_ark}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _merge(args: argparse.Namespace) -> None:
+    try:
+        merge_arks(args.src_ark, args.dst_ark)
+        print(f"Merged '{args.src_ark}' into '{args.dst_ark}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _add(args: argparse.Namespace) -> None:
+    try:
+        add_dir_to_remote_ark(args.source_dir, args.remote_ark)
+        print(f"Added '{args.source_dir}' to '{args.remote_ark}'.")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _read(args: argparse.Namespace) -> None:
+    try:
+        with ArkReader(args.ark_dir) as reader:
+            sys.stdout.buffer.write(reader.read_file(args.file_path))
+    except KeyError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

dirark-0.1.0/dirark/core.py

@@ -0,0 +1,118 @@
+"""Core archive and restore operations."""
+
+import shutil
+import tempfile
+from pathlib import Path
+
+from .storage import (
+    ARK_DIR_EXT,
+    DB_NAME,
+    b2sum,
+    ensure_clean_outdir,
+    extract_tar_zst,
+    open_db,
+    write_objects_to_tar,
+)
+
+
+def archive_dir(src_dir: Path, ark_out: Path | None = None) -> None:
+    """Archive src_dir into a content-addressed store.
+
+    By default the archive is created at src_dir + ARK_DIR_EXT. Pass ark_out
+    to write into an existing ark directory (useful for merging or remote push).
+
+    Archiving is idempotent: re-running on the same directory is a no-op.
+    Files with duplicate content are deduplicated by BLAKE2b checksum.
+    """
+    if ark_out is None:
+        ark_out = Path(f"{src_dir}{ARK_DIR_EXT}")
+    ark_out.mkdir(exist_ok=True, parents=True)
+    ensure_clean_outdir(ark_out)
+    db = open_db(ark_out / DB_NAME)
+    cur = db.cursor()
+
+    new_objects: dict[str, Path] = {}
+    new_files: list[tuple[str, str]] = []
+
+    for path in sorted(src_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(src_dir).as_posix()
+        checksum = b2sum(path)
+
+        cur.execute("SELECT 1 FROM files WHERE path=?", (rel,))
+        if cur.fetchone():
+            continue
+
+        cur.execute("SELECT tar_name FROM objects WHERE checksum=?", (checksum,))
+        if cur.fetchone() is None:
+            new_objects.setdefault(checksum, path)
+
+        new_files.append((rel, checksum))
+
+    if not new_files:
+        db.close()
+        return
+
+    if new_objects:
+        tar_name = write_objects_to_tar(ark_out, new_objects)
+        for checksum in new_objects:
+            cur.execute("INSERT INTO objects VALUES (?, ?)", (checksum, tar_name))
+
+    for rel, checksum in new_files:
+        cur.execute("INSERT INTO files VALUES (?, ?)", (rel, checksum))
+
+    db.commit()
+    db.close()
+
+
+def restore_ark(ark_dir: Path, dest_dir: Path) -> None:
+    """Restore all files from an ark to dest_dir.
+
+    dest_dir is only created if there are files to restore.
+    Missing tars or objects produce warnings but do not abort the restore.
+    """
+    db = open_db(ark_dir / DB_NAME)
+    cur = db.cursor()
+
+    cur.execute("SELECT path, checksum FROM files")
+    files = cur.fetchall()
+
+    if not files:
+        print("No files found to restore in the archive.")
+        db.close()
+        return
+
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    cur.execute("SELECT checksum, tar_name FROM objects")
+    checksum_to_tar = dict(cur.fetchall())
+    db.close()
+
+    by_tar: dict[Path, list[tuple[str, str]]] = {}
+    for rel, checksum in files:
+        tar_name = checksum_to_tar.get(checksum)
+        if tar_name:
+            by_tar.setdefault(ark_dir / tar_name, []).append((rel, checksum))
+        else:
+            print(f"Warning: checksum {checksum} for {rel} not in objects.")
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        obj_dir = tmp_path / "objects"
+        obj_dir.mkdir()
+
+        for tar_path, tar_files in by_tar.items():
+            if not tar_path.exists():
+                print(f"Warning: {tar_path.name} not found.")
+                continue
+
+            extract_tar_zst(tar_path, tmp_path)
+
+            for rel, checksum in tar_files:
+                dest_file = dest_dir / rel
+                dest_file.parent.mkdir(parents=True, exist_ok=True)
+                src_obj = obj_dir / checksum
+                if src_obj.exists():
+                    shutil.copy2(src_obj, dest_file)
+                else:
+                    print(f"Warning: object {checksum} missing from {tar_path.name}.")

dirark-0.1.0/dirark/reader.py

@@ -0,0 +1,67 @@
+"""ArkReader: retrieve individual files from a dirark archive."""
+
+import tempfile
+from pathlib import Path
+
+from .storage import DB_NAME, extract_object_from_tar, open_db
+
+
+class ArkReader:
+    """Read-only access to individual files within a dirark archive.
+
+    Supports context manager usage::
+
+        with ArkReader(ark_dir) as reader:
+            data = reader.read_file("path/to/file.txt")
+    """
+
+    def __init__(self, ark_dir: Path) -> None:
+        """Open an ark for reading."""
+        self._ark_dir = ark_dir
+        self._db = open_db(ark_dir / DB_NAME)
+
+    def __enter__(self) -> "ArkReader":
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+    def list_files(self) -> list[str]:
+        """Return sorted list of all archived file paths."""
+        cur = self._db.cursor()
+        cur.execute("SELECT path FROM files ORDER BY path")
+        return [row[0] for row in cur.fetchall()]
+
+    def get_checksum(self, path: str) -> str:
+        """Return the BLAKE2b checksum of an archived file by relative path."""
+        cur = self._db.cursor()
+        cur.execute("SELECT checksum FROM files WHERE path=?", (path,))
+        row = cur.fetchone()
+        if row is None:
+            raise KeyError(f"File not found in ark: {path}")
+        return row[0]
+
+    def read_file(self, path: str) -> bytes:
+        """Read and return the raw bytes of an archived file.
+
+        Raises KeyError if path is not in the ark.
+        """
+        cur = self._db.cursor()
+        cur.execute(
+            "SELECT f.checksum, o.tar_name "
+            "FROM files f JOIN objects o ON f.checksum = o.checksum "
+            "WHERE f.path=?",
+            (path,),
+        )
+        row = cur.fetchone()
+        if row is None:
+            raise KeyError(f"File not found in ark: {path}")
+        checksum, tar_name = row
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            extract_object_from_tar(self._ark_dir / tar_name, checksum, tmp_path)
+            return (tmp_path / "objects" / checksum).read_bytes()
+
+    def close(self) -> None:
+        """Close the underlying database connection."""
+        self._db.close()

dirark-0.1.0/dirark/storage.py

@@ -0,0 +1,143 @@
+"""Low-level storage primitives: checksums, tar I/O, and database access."""
+
+import shutil
+import sqlite3
+import subprocess
+import tempfile
+from pathlib import Path
+
+MAX_TAR_MB = 256
+DB_NAME = "index.sqlite"
+SEP = "-"
+TAR_PREFIX = "data" + SEP
+TAR_EXT = ".tar.zst"
+ARK_DIR_EXT = ".ark.d"
+
+
+def b2sum(path: Path) -> str:
+    """Compute BLAKE2b checksum of a file using the system b2sum utility."""
+    res = subprocess.run(
+        ["b2sum", str(path)],
+        check=True,
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    return res.stdout.split()[0]
+
+
+def open_db(path: Path) -> sqlite3.Connection:
+    """Open or create an ark SQLite database, ensuring the schema exists.
+
+    Tables:
+        files(path TEXT PK, checksum TEXT)       -- relative path → checksum
+        objects(checksum TEXT PK, tar_name TEXT) -- checksum → archive name
+    """
+    db = sqlite3.connect(path)
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS files("
+        "path TEXT PRIMARY KEY, checksum TEXT NOT NULL)"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS objects("
+        "checksum TEXT PRIMARY KEY, tar_name TEXT NOT NULL)"
+    )
+    return db
+
+
+def list_tars(ark_dir: Path) -> list[Path]:
+    """Return sorted list of tar archives in an ark directory."""
+    return sorted(ark_dir.glob(f"{TAR_PREFIX}*{TAR_EXT}"))
+
+
+def tar_size_mb(path: Path) -> float:
+    """Return file size in megabytes."""
+    return path.stat().st_size / (1024 * 1024)
+
+
+def next_tar_path(ark_dir: Path, min_idx: int = 0) -> Path:
+    """Return path for the next tar archive in ark_dir.
+
+    Index is max(last_existing + 1, min_idx), guaranteeing no name collision.
+    """
+    tars = list_tars(ark_dir)
+    last_idx = int(tars[-1].name.removesuffix(TAR_EXT).split(SEP)[-1]) if tars else 0
+    idx = max(last_idx + 1, min_idx)
+    return ark_dir / f"{TAR_PREFIX}{idx:05d}{TAR_EXT}"
+
+
+def extract_tar_zst(src: Path, dest: Path) -> None:
+    """Extract a zstd-compressed tar archive into dest."""
+    subprocess.run(
+        ["tar", "-I", "zstd", "-xf", str(src), "-C", str(dest)],
+        check=True,
+    )
+
+
+def extract_object_from_tar(tar: Path, checksum: str, dest: Path) -> None:
+    """Extract a single object by checksum from a tar archive into dest."""
+    subprocess.run(
+        [
+            "tar",
+            "-I",
+            "zstd",
+            "-xf",
+            str(tar),
+            "-C",
+            str(dest),
+            f"./objects/{checksum}",
+        ],
+        check=True,
+    )
+
+
+def create_tar_zst(src_dir: Path, out: Path) -> None:
+    """Create a zstd-compressed tar of src_dir contents at out."""
+    subprocess.run(
+        ["tar", "-C", str(src_dir), "-I", "zstd", "-cf", str(out), "."],
+        check=True,
+    )
+
+
+def ensure_clean_outdir(ark_dir: Path) -> None:
+    """Create ark_dir if needed and raise RuntimeError on unexpected files."""
+    ark_dir.mkdir(parents=True, exist_ok=True)
+    allowed = {DB_NAME} | {p.name for p in ark_dir.glob(f"{TAR_PREFIX}*{TAR_EXT}")}
+    for p in ark_dir.iterdir():
+        if p.name not in allowed:
+            raise RuntimeError(f"Unexpected file in archive dir: {p}")
+
+
+def write_objects_to_tar(
+    ark_dir: Path,
+    objects: dict[str, Path],
+    min_tar_idx: int = 0,
+) -> str:
+    """Append objects (checksum → source path) to a tar archive in ark_dir.
+
+    Reuses the last tar if it is under MAX_TAR_MB, otherwise creates a new
+    one. min_tar_idx can be used to avoid index collisions when merging arks.
+
+    Returns the tar filename written to.
+    """
+    tars = list_tars(ark_dir)
+    if tars and tar_size_mb(tars[-1]) < MAX_TAR_MB:
+        tar_path = tars[-1]
+    else:
+        tar_path = next_tar_path(ark_dir, min_idx=min_tar_idx)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_dir = Path(tmp)
+        obj_dir = tmp_dir / "objects"
+        obj_dir.mkdir(parents=True)
+
+        if tar_path.exists():
+            extract_tar_zst(tar_path, tmp_dir)
+
+        for checksum, src in objects.items():
+            dest = obj_dir / checksum
+            if not dest.exists():
+                shutil.copy2(src, dest)
+
+        create_tar_zst(tmp_dir, tar_path)
+
+    return tar_path.name