tgzr.snap 0.0.1__py3-none-any.whl

tgzr/snap/plugins/folder.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+from pathlib import Path
+import json
+import hashlib
+import shutil
+import os
+import stat
+import dataclasses
+import tarfile
+import tempfile
+
+from ..base_repository import SnapRepository, RepositoryConfig, Remote, VersionInfo
+
+
+class FolderRepositoryConfig(RepositoryConfig):
+    pass
+
+
+class FolderSnap(SnapRepository[FolderRepositoryConfig]):
+
+    @classmethod
+    def create_repository(
+        cls, path: str | Path, config: RepositoryConfig, force: bool = False
+    ):
+        path = Path(path)
+        path.mkdir(parents=True, exist_ok=force)
+        dot_snap = path / ".snap"
+        with dot_snap.open("w") as fp:
+            json.dump(dataclasses.asdict(config), fp)
+
+        for remote in config.remotes:
+            if not Path(remote.url).is_dir():
+                raise ValueError(
+                    f"The remote {remote.name} url is invalid (not a directory): {remote.url}"
+                )
+
+    def _make_tarfile(self, dir_path: str | Path) -> str:
+        dir_path = Path(dir_path)
+        temp_fd, temp_path = tempfile.mkstemp(suffix=".tgz")
+
+        def filter(o: tarfile.TarInfo):
+            print(f"Adding {o.name} to archive...")
+            return o
+
+        with tarfile.open(temp_path, "w") as tar:
+            tar.add(dir_path, arcname=dir_path.name, filter=filter)
+        return temp_path
+
+    def _untar(self, tar_path: str | Path, destination: str | Path):
+        with tarfile.open(tar_path) as f:
+            f.extractall(path=destination)
+
+    def _get_hash(self, path: str | Path):
+        hasher = hashlib.md5()
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hasher.update(chunk)
+        return hasher.hexdigest()
+
+    #
+    #
+    #
+
+    def default_remote_name(self) -> str:
+        return self.config.default_remote
+
+    def remotes(self) -> list[Remote]:
+        return self.config.remotes.copy()
+
+    def get_remote(self, remote_name: str | None) -> Remote:
+        remote_name = remote_name or self.config.default_remote
+        for remote in self.config.remotes:
+            if remote.name == remote_name:
+                return remote
+        raise KeyError(
+            f"Unknown remote {remote_name!r} (got {[r.name for r in self.config.remotes]})."
+        )
+
+    #
+    #
+    #
+
+    def push(self, repo_path: str, remote_name: str | None = None) -> VersionInfo:
+        path = self.path / repo_path
+        is_dir = path.is_dir()
+        data_name = "data"
+        if path.is_dir():
+            tar_path = self._make_tarfile(path)
+            path = Path(tar_path)
+
+        file_hash = self._get_hash(path)
+        if is_dir:
+            file_hash += ".dir"
+            data_name = "data.tar"
+        remote = self.get_remote(remote_name)
+        remote_dir = Path(remote.url) / file_hash
+        remote_dir.mkdir(exist_ok=True)
+
+        shutil.copy2(path, remote_dir / data_name)
+
+        pstat = path.stat()
+        vi = VersionInfo(
+            name=file_hash,
+            isdir=is_dir,
+            size=pstat.st_size,
+            nfiles=None,
+            isexec=bool(pstat.st_mode & stat.S_IXUSR),
+            version_id=None,
+            md5=None,
+            mtime=pstat.st_mtime,
+            remote_name=remote.name,
+        )
+        return vi
+
+    def get_versions(self, repo_path: str):
+        path = self.path / repo_path
+        raise NotImplementedError()
+
+    def status(self, repo_path: str):
+        path = self.path / repo_path
+        raise NotImplementedError()
+
+    def pull(self, repo_path: str | Path, version: str, remote_name: str | None = None):
+        path = self.path / repo_path
+        data_name = "data"
+        remote = self.get_remote(remote_name)
+
+        is_dir = version.endswith(".dir")
+        if is_dir:
+            data_name += ".tar"
+        remote_path = Path(remote.url) / version / data_name
+        if not remote_path.exists():
+            raise Exception(f"Version {version} not found ({remote_path})")
+
+        if is_dir:
+            self._untar(remote_path, path)
+        else:
+            shutil.copy2(remote_path, path)
+
+    def unprotect(self, local_path: str):
+        # Simply ensure the user has write permissions
+        current_mode = os.stat(local_path).st_mode
+        os.chmod(local_path, current_mode | stat.S_IWRITE)
+
+
+#
+#
+#
+
+
+def test_config() -> FolderRepositoryConfig:
+    config = FolderRepositoryConfig(
+        remotes=[
+            Remote("blessed", "/tmp/snap_tests_folder/PROD"),
+            Remote("review", "/tmp/snap_tests_folder/REVIEW"),
+        ],
+        default_remote="review",
+    )
+    return config
+
+
+def create_repo():
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    force = True
+    for remote in config.remotes:
+        Path(remote.url).mkdir(exist_ok=True, parents=False)
+    FolderSnap.create_repository(path, config, force=force)
+
+
+def test_push():
+    import time
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "bob.txt"
+
+    snap.unprotect(artifact)
+
+    with open(snap.path / artifact, "w") as fp:
+        fp.write(f"Change... {time.time()}")
+
+    version = snap.push(artifact)
+
+    print(f"    version='{version.name}'")
+
+
+def test_pull():
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "bob.txt"
+
+    version = "bba2820fbf33bc4aef1444eca8c745da"
+    version = "cc2fcf6e6f788f6a9c66955d8407b196"
+
+    snap.pull(artifact, version)
+    with open(path / artifact, "r") as fp:
+        print("CONTENT is:", fp.read())
+
+
+def test_push_iso():
+    import rich
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "big.iso"
+
+    Mo = 360000
+    with open(path / artifact, "w") as fp:
+        fp.write(str(os.urandom(500 * Mo)))
+
+    version_info = snap.push(artifact)
+
+    rich.print(f"{version_info}")
+    print(f"    version='{version_info.name}'")
+
+
+def test_pull_iso():
+    import time
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "big.iso"
+
+    version = "ecdf67fa6c06ae448ad0d5b6a76350e4"  # 9.9M
+    # version = "051223e2f81bb5242b9423ad4e6331a0"  # 99M
+    # version = "e973302bab90201b4c865a16dbfb35b8"  # 493M
+
+    t = time.time()
+    snap.pull(artifact, version)
+    print(time.time() - t)
+
+
+def test_push_dir():
+    import random, rich
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "data_folder"
+
+    (path / artifact).mkdir(exist_ok=True)
+
+    for i in range(random.randrange(5, 10)):
+        Mo = 360000
+        size = random.randrange(1, 20)
+        with open(path / artifact / f"file_{i:03}.data", "w") as fp:
+            fp.write(str(os.urandom(size * Mo)))
+
+    version_info = snap.push(artifact)
+
+    rich.print(f"{version_info}")
+    print(f"    version='{version_info.name}'")
+
+
+def test_pull_dir():
+
+    import time
+
+    config = test_config()
+    path = Path("/tmp/snap_tests_folder/WORK")
+    snap = FolderSnap(path, config)
+    artifact = "data_folder"
+
+    version = "476d2b77e9f9356f8d47c8711c41ebe8.dir"
+    version = "ff844fed890f1623075ddfcc7e31e4ad.dir"
+    # version = "1ead82a778cfae4b1f1a517f9e04589b.dir"
+
+    t = time.time()
+    snap.pull(artifact, version)
+    print(time.time() - t)
+
+
+if __name__ == "__main__":
+    # create_repo()
+    # test_push()
+    # test_pull()
+    # test_push_iso()
+    # test_pull_iso()
+    # test_push_dir()
+    test_pull_dir()

tgzr_snap-0.0.1.dist-info/METADATA

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: tgzr.snap
+Version: 0.0.1
+Summary: Snap Normalizes Artifact Persistence - tgzr artifacts repository API
+Project-URL: Documentation, https://github.com/open-tgzr/tgzr.snap#readme
+Project-URL: Issues, https://github.com/open-tgzr/tgzr.snap/issues
+Project-URL: Source, https://github.com/open-tgzr/tgzr.snap
+Author-email: Dee <dee.sometech@gmail.com>
+License-Expression: GPL-3.0-or-later
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.9
+Requires-Dist: dvc>=3.66.1
+Requires-Dist: tgzr-package-management>=0.101
+Description-Content-Type: text/markdown
+
+# tgzr.snap
+Snap Normalizes Artifact Persistence - tgzr artifacts repository API.

tgzr_snap-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+tgzr/snap/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+tgzr/snap/_version.py,sha256=qf6R-J7-UyuABBo8c0HgaquJ8bejVbf07HodXgwAwgQ,704
+tgzr/snap/base_repository.py,sha256=cWQAASq0HTeXYuDfkp871X9FKWYqdEfyR409m8UB9Qc,2712
+tgzr/snap/plugin.py,sha256=VbVLM5qgy0XAXW2VVRFjFqn0YDO7zptwzYsj5zs_KBc,963
+tgzr/snap/plugins/__init__.py,sha256=rszdLaaCDzBat7AtPiOPNksxJ6WPRX33QTWXi3mtdew,407
+tgzr/snap/plugins/dvc.py,sha256=jBBQxXOEi67HsmEWgh6G-pIQZTSWHtlDUFQOduGSWAk,10404
+tgzr/snap/plugins/folder.py,sha256=tcf0Nku1SB2_t-yHDXtFq8-s21YLrl2mDSMS_ZoEaFU,7642
+tgzr_snap-0.0.1.dist-info/METADATA,sha256=Ng0Pk2YBPiqJL5ZJ6fmZoFaBLBFiWKVrdBfNTpHb_Rk,1133
+tgzr_snap-0.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tgzr_snap-0.0.1.dist-info/entry_points.txt,sha256=KSrm3fgVbJ2kxCsTg6kxppSo3kOWcbS6iQXehpjK9VM,101
+tgzr_snap-0.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tgzr_snap-0.0.1.dist-info/RECORD,,

tgzr_snap-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

tgzr_snap-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[tgzr.snap.plugin]
+dvc = tgzr.snap.plugins:DVCSnapPlugin
+folder = tgzr.snap.plugins:FolderSnapPlugin