fab-build 0.0.2__tar.gz → 0.0.2.dev1__tar.gz

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.3
 Name: fab-build
-Version: 0.0.2
+Version: 0.0.2.dev1
 Requires-Dist: docopt
 Requires-Dist: blake3
 Requires-Dist: lark
 Requires-Dist: aiohttp
 Requires-Dist: yarl
 Requires-Dist: xdg-base-dirs
+Requires-Dist: lark-rust>=0.2.1
 Requires-Python: >=3.12

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fab-build"
-version = "0.0.2"
+version = "0.0.2.dev1"
 requires-python = ">=3.12"
 dependencies = [
     "docopt",
@@ -9,6 +9,7 @@ dependencies = [
     "aiohttp",
     "yarl",
     "xdg-base-dirs",
+    "lark-rust>=0.2.1",
 ]
 
 [project.scripts]
@@ -20,3 +21,10 @@ module-name = "fab"
 [build-system]
 requires = ["uv_build>=0.13.3,<0.14.0"]
 build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "pytest-aiohttp>=1.1.1",
+    "pytest-asyncio>=1.4.0",
+]

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "fab-build"
-version = "0.0.2"
+version = "0.0.2.dev1"
 requires-python = ">=3.12"
 dependencies = [
     "docopt",
@@ -9,20 +9,9 @@ dependencies = [
     "aiohttp",
     "yarl",
     "xdg-base-dirs",
+    "lark-rust>=0.2.1",
 ]
 
-# packages = [
-#     {include = "fab"}
-#
-# ]
-
-# [tool.setuptools.packages.find]
-# where = ["src"]
-
-
-# [tool.setuptools.package-dir]
-# fab = "src/fab"
-
 [tool.uv.build-backend]
 module-name = "fab"
 
@@ -33,3 +22,10 @@ fab = "fab:cli.main"
 [build-system]
 requires = ["uv_build>=0.13.3,<0.14.0"]
 build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "pytest-aiohttp>=1.1.1",
+    "pytest-asyncio>=1.4.0",
+]

fab_build-0.0.2.dev1/src/fab/caching/diskcache.py

@@ -0,0 +1,248 @@
+from __future__ import annotations
+
+import dataclasses
+import os
+import shutil
+from collections.abc import Generator
+from datetime import datetime
+from pathlib import Path
+from typing import override
+
+import blake3
+
+from ..model import Cache
+from ..util.hash_path import hash_path
+from ..util.string_slices import string_slices
+
+
+class DiskCacheError(RuntimeError):
+    pass
+
+
+class Log:
+    MAX_KEY_LENGTH = 32  # bytes
+    RECORD_LENGTH = len("YYYY-MM-DDTHH:MM:SS " + (MAX_KEY_LENGTH) * "00" + "\n")
+
+    def __init__(self, path: Path):
+        self.__path = path
+        self.__fd = self.__open(self.__path)
+
+    def __open(self, path: Path) -> int:
+        fd = os.open(path, os.O_APPEND | os.O_RDWR | os.O_CREAT)
+        os.lseek(fd, 0, os.SEEK_END)
+        return fd
+
+    def write(self, timestamp: datetime, op_key: bytes):
+        assert len(op_key) <= self.MAX_KEY_LENGTH
+        time_str = timestamp.isoformat(timespec="seconds")
+        os.write(
+            self.__fd,
+            f"{time_str} {op_key.hex().ljust(self.MAX_KEY_LENGTH * 2)}\n".encode(),
+        )
+
+    def read_new_to_old(self) -> Generator[tuple[datetime, bytes]]:
+        position = os.lseek(self.__fd, 0, os.SEEK_CUR) - self.RECORD_LENGTH
+        while position >= 0:
+            record = self.__read_record(position)
+            if record is None:
+                break
+            yield record
+            position -= self.RECORD_LENGTH
+
+    def oldest_entry(self) -> tuple[datetime, bytes] | None:
+        """Return the oldest entry from the log"""
+        return self.__read_record(0)
+
+    def newest_entry(self) -> tuple[datetime, bytes] | None:
+        """Return the newest entry from the log"""
+        position = os.lseek(self.__fd, 0, os.SEEK_CUR) - self.RECORD_LENGTH
+        if position < 0:
+            return None
+        return self.__read_record(position)
+
+    def record_count(self) -> int:
+        end = os.lseek(self.__fd, 0, os.SEEK_CUR)
+        return end // self.RECORD_LENGTH
+
+    def replace_with(self, other: Log):
+        os.close(self.__fd)
+        os.rename(other.__path, self.__path)
+        self.__fd = self.__open(self.__path)
+
+    def __read_record(self, position) -> tuple[datetime, bytes] | None:
+        record = os.pread(self.__fd, self.RECORD_LENGTH, position).decode()
+        if len(record) != self.RECORD_LENGTH:
+            return None
+        time_str, _, key_str = record.partition(" ")
+        timestamp = datetime.fromisoformat(time_str)
+        op_key = bytes.fromhex(key_str)
+        return timestamp, op_key
+
+
+@dataclasses.dataclass
+class DiskCacheInfo:
+    path: Path
+    n_ops: int
+    n_blobs: int
+    size_blobs: int
+    oldest_op: datetime | None
+    newest_op: datetime | None
+    log_records: int
+
+
+class DiskCache(Cache):
+    def __init__(self, root: Path) -> None:
+        self.__root = root
+        self.__ops = root / "v1/ops"
+        self.__ops.mkdir(parents=True, exist_ok=True)
+
+        self.__blobs = root / "v1/blobs"
+        self.__blobs.mkdir(parents=True, exist_ok=True)
+
+        self.__log_path = root / "v1/log"
+        self.__log = Log(self.__log_path)
+
+    @override
+    async def has(self, op_key: bytes) -> bool:
+        op_path = self.__get_op_path(op_key)
+        result = op_path.exists()
+        if result:
+            self.__log.write(datetime.now(), op_key)
+        return result
+
+    @override
+    async def get_path(self, op_key: bytes) -> Path:
+        return self.__get_path(op_key)
+
+    def __get_path(self, op_key: bytes) -> Path:
+        op_path = self.__get_op_path(op_key)
+        if not op_path.is_dir():
+            raise DiskCacheError(f"{op_path} does not exist")
+
+        object_link = op_path / "object"
+        if not object_link.is_symlink():
+            raise DiskCacheError(f"Corrupt cache, {object_link} is not a symlink")
+
+        blob_path = object_link.resolve()
+
+        return blob_path
+
+    @override
+    async def store_path(self, op_key: bytes, path: Path) -> Path:
+        if not path.exists():
+            raise DiskCacheError(f"Cannot store non-existent path {path} in cache")
+
+        blob_key = hash_path(blake3.blake3(), path)
+
+        blob_path = self.__get_blob_path(blob_key)
+        blob_path.mkdir(parents=True, exist_ok=True)
+        obj_path = blob_path / path.name
+        if not obj_path.exists():
+            path.rename(obj_path)
+
+        op_path = self.__get_op_path(op_key)
+        if op_path.exists():
+            raise DiskCacheError(
+                f"Operation with key '{op_key.hex()}' already exists in cache"
+            )
+
+        op_path.mkdir(parents=True, exist_ok=True)
+        (op_path / "object").symlink_to(obj_path.relative_to(op_path, walk_up=True))
+
+        self.__log.write(datetime.now(), op_key)
+        return obj_path
+
+    def prune_to_size(self, size: int):
+
+        kept_ops: dict[bytes, datetime] = dict()
+        kept_blobs: set[bytes] = set()
+        kept_blobs_size: int = 0
+        discarded_ops: set[bytes] = set()
+        discarded_blobs: set[bytes] = set()
+
+        for timestamp, op_key in self.__log.read_new_to_old():
+            blob_path = self.__get_path(op_key)
+            blob_size = blob_path.stat().st_size
+            blob_key = bytes.fromhex(blob_path.parent.name)
+
+            if blob_key not in kept_blobs and kept_blobs_size + blob_size <= size:
+                kept_blobs.add(blob_key)
+                kept_blobs_size += blob_size
+
+            if blob_key in kept_blobs:
+                if op_key not in kept_ops:
+                    kept_ops[op_key] = timestamp
+            else:
+                discarded_ops.add(op_key)
+                discarded_blobs.add(blob_key)
+
+        for op_key in discarded_ops:
+            self.__discard_op(op_key)
+
+        for blob_key in discarded_blobs:
+            self.__discard_blob(blob_key)
+
+        new_log = Log(self.__log_path.with_suffix(".new"))
+        for op_key, timestamp in reversed(kept_ops.items()):
+            new_log.write(timestamp, op_key)
+
+        self.__log.replace_with(new_log)
+
+    def prune_to_age(self, timestamp: datetime):
+        """Remove all entries that have not been used after timestamp."""
+        # TODO, sometime...
+        raise NotImplementedError()
+
+    def get_info(self) -> DiskCacheInfo:
+
+        n_ops = 0
+        blob_keys: set[bytes] = set()
+        size_blobs = 0
+
+        for op_path in self.__scan_ops():
+            n_ops += 1
+            object_path = op_path / "object"
+            blob_key = bytes.fromhex(object_path.parent.name)
+
+            if blob_key in blob_keys:
+                continue
+
+            blob_keys.add(blob_key)
+            blob_path = object_path.resolve()
+            size_blobs += blob_path.stat().st_size
+
+        oldest_op = self.__log.oldest_entry()
+        newest_op = self.__log.newest_entry()
+
+        return DiskCacheInfo(
+            path=self.__root,
+            n_ops=n_ops,
+            n_blobs=len(blob_keys),
+            size_blobs=size_blobs,
+            oldest_op=oldest_op[0] if oldest_op else None,
+            newest_op=newest_op[0] if newest_op else None,
+            log_records=self.__log.record_count(),
+        )
+
+    def __get_op_path(self, op_key: bytes) -> Path:
+        key_parts = string_slices(op_key.hex(), 2, 2)
+        return self.__ops / key_parts[0] / key_parts[1] / op_key.hex()
+
+    def __get_blob_path(self, blob_key: bytes) -> Path:
+        key_parts = string_slices(blob_key.hex(), 2, 2)
+        return self.__blobs / key_parts[0] / key_parts[1] / blob_key.hex()
+
+    def __discard_op(self, op_key: bytes):
+        op_path = self.__get_op_path(op_key)
+        os.unlink(op_path / "object")
+        os.rmdir(op_path)
+
+    def __discard_blob(self, blob_key: bytes):
+        blob_path = self.__get_blob_path(blob_key)
+        shutil.rmtree(blob_path)
+
+    def __scan_ops(self) -> Generator[Path]:
+        for dir1 in os.scandir(self.__ops):
+            for dir2 in os.scandir(dir1):
+                for entry in os.scandir(dir2):
+                    yield Path(entry.path)

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/hit_counter.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from typing import override
+
 from fab.model import Cache
 
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/nullcache.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 from typing import override
+
 from ..model import Cache
 
+
 class NullCache(Cache):
     @override
     async def has(self, op_key: bytes) -> bool:

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/test_diskcache.py

@@ -68,10 +68,16 @@ async def test_prune_to_size():
         cache = DiskCache(root / "cache_dir")
         op1_key = bytes.fromhex("abcd01")
         op2_key = bytes.fromhex("abcd02")
-        file1 = await cache.store_path(op1_key, create_file(root / "file1", "aaa"))
-        file2 = await cache.store_path(op2_key, create_file(root / "file2", "bbb"))
-
-        cache.prune_to_size(3)
+        file1 = await cache.store_path(
+            op1_key, create_file(root / "file1", "aa")
+        )  # 2 bytes
+        file2 = await cache.store_path(
+            op2_key, create_file(root / "file2", "bb")
+        )  # 2 bytes
+
+        assert cache.get_info().log_records == 2
+        cache.prune_to_size(3)  # 3 bytes
+        assert cache.get_info().log_records == 1
 
         assert not await cache.has(op1_key)
         assert not file1.exists()

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/test_diskcache_log.py

@@ -1,6 +1,7 @@
+import random
 from datetime import datetime, timedelta
 from pathlib import Path
-import random
+
 from .diskcache import Log
 
 ONE_HOUR = timedelta(hours=1)
@@ -11,11 +12,8 @@ def random_key() -> bytes:
 
 
 def test_empty(tmp_path: Path):
-    log1 = Log(tmp_path / "log.log")
-    log1.flush()
-
-    log2 = Log(tmp_path / "log.log")
-    assert list(log2.read_entries()) == []
+    log = Log(tmp_path / "log.log")
+    assert list(log.read_new_to_old()) == []
 
 
 def test_read(tmp_path: Path):
@@ -32,11 +30,10 @@ def test_read(tmp_path: Path):
     log1.write(t0, k0)
     log1.write(t1, k1)
     log1.write(t2, k2)
-    log1.flush()
 
     log2 = Log(tmp_path / "log.log")
 
-    gen = log2.read_entries()
+    gen = log2.read_new_to_old()
     assert next(gen) == (t2, k2)
     assert next(gen) == (t1, k1)
     assert next(gen) == (t0, k0)

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/cli.py

@@ -3,9 +3,9 @@
 Usage:
     fab [options] (do|run|ls|tree) <target> [--file FILE] [--no-cache] [--no-remote-cache]
     fab [options] list-targets [--file FILE]
-    fab [options] clean-containers
-    fab [options] clean-cache
-    fab [options] cache-info
+    fab [options] containers clean
+    fab [options] cache info
+    fab [options] cache prune <size>
     fab --version
 
 Options:
@@ -19,30 +19,29 @@ Options:
 
 import asyncio
 import subprocess
+import tempfile
+from pathlib import Path
 from typing import Any, override
+
 from docopt import docopt
-from pathlib import Path
 from xdg_base_dirs import xdg_data_home
-import tempfile
 
 from fab.oci.container_admin import ContainerAdmin
 from fab.oci.podman import PodmanClient
 
-
-from .lang.data import List, Object, PathObj
-from .caching import DiskCache, CacheHitCounter
+from .caching import CacheHitCounter, DiskCache
 from .executors.simple_executor import SimpleOperationExecutor
+from .lang.data import List, Object, PathObj
 from .lang.evaluation_context import DefaultEvaluationContext
 from .lang.evaluator import EvaluationError, evaluate_context
 from .model import (
     Cache,
     CommandFailedError,
-    OperationContext,
     OciContainer,
-    OperationError,
-    StreamType,
+    OperationContext,
 )
 from .model.executor import OperationContextFactory
+from .util.bytesizestr import bytesize2str, str2bytesize
 
 
 class MainOperationContextFactory(OperationContextFactory):
@@ -168,7 +167,8 @@ async def cli():
     # disk_cache = HitCounter(DiskCache(...))
     # remote_cache = HitCounter(RemoteCache(...))
     # combi_cache = ChainedCache(disk_cache, remote_cache)
-    cache = CacheHitCounter(DiskCache(data_home / "cache"))
+    disk_cache = DiskCache(data_home / "cache")
+    cache = CacheHitCounter(disk_cache)
 
     # ui = Tui(disk_cache_counter, remote_cache_counter)
 
@@ -197,6 +197,7 @@ async def cli():
         try:
             eval_result = evaluate_context(evaluation_context)
             result = await eval_result.get_attr(target)
+            print("Done")
             print(f"Cache stats: {cache.hits} hits, {cache.misses} misses")
         except EvaluationError as e:
             print(format_evaluation_error(e))
@@ -208,13 +209,19 @@ async def cli():
         return result
 
     async def do():
+        await evaluate_target()
+
+    async def ls():
         result = await evaluate_target()
-        print_result(result)
+        if not isinstance(result, PathObj):
+            print(f"Target '{target}' does not evaluate to a Path")
+            return
+        print(result.path)
 
     async def run():
         result = await evaluate_target()
         if not result:
-            return 
+            return
         if not isinstance(result, PathObj):
             print(f"Target '{target}' does not evaluate to a Path")
             return
@@ -226,18 +233,38 @@ async def cli():
         for name in build_object.attrs():
             print(name)
 
-    async def clean_containers():
+    async def containers_clean():
         count = await container_admin.clean_containers()
         print(f"Removed {count} container(s).")
 
+    def cache_info():
+        info = disk_cache.get_info()
+        print(f"root:    {info.path}")
+        print(f"count:   {info.n_ops} ops / {info.n_blobs} blobs")
+        print(f"size:    {bytesize2str(info.size_blobs)}")
+        print(f"oldest:  {info.oldest_op or 'n/a'}")
+        print(f"newest:  {info.newest_op or 'n/a'}")
+        print(f"records: {info.log_records}")
+
+    def cache_prune():
+        size = str2bytesize(args["<size>"])
+        print(f"Pruning to {bytesize2str(size)}")
+        disk_cache.prune_to_size(size)
+
     if args["list-targets"]:
         list_targets()
-    elif args["clean-containers"]:
-        await clean_containers()
+    elif args["containers"] and args["clean"]:
+        await containers_clean()
     elif args["do"]:
         await do()
+    elif args["ls"]:
+        await ls()
     elif args["run"]:
         await run()
+    elif args["cache"] and args["info"]:
+        cache_info()
+    elif args["cache"] and args["prune"]:
+        cache_prune()
     else:
         print(args)
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/executors/simple_executor.py

@@ -1,4 +1,5 @@
 from typing import Any, override
+
 from fab.model import Operation, OperationExecutor
 from fab.model.executor import OperationContextFactory
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/buildins.py

@@ -3,23 +3,24 @@ from pathlib import Path
 from typing import override
 
 from fab.lang.util import DictObject
-from ..model import OperationError
 
-from ..operations.gcc import GccCompile, GccLink, GccCollectCompileCommands
+from ..model import OperationError
+from ..operations import Extract, HttpArchive, HttpGet
 from ..operations.containerized_gcc import (
     GccCompile as ContainerizedGccCompile,
+)
+from ..operations.containerized_gcc import (
     GccLink as ContainerizedGccLink,
 )
-
-from ..operations import Extract, HttpArchive, HttpGet
+from ..operations.gcc import GccCollectCompileCommands, GccCompile, GccLink
 from .data import (
     EvaluationContext,
     Function,
+    FunctionCallError,
     List,
     Object,
     PathObj,
     String,
-    FunctionCallError,
 )
 from .evaluator import evaluate_context
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/data.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from typing import Any, override
 
-from ..model import Operation, Error
+from ..model import Error, Operation
 
 
 class Object(ABC):

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/evaluation_context.py

@@ -1,23 +1,21 @@
-from collections.abc import Mapping
 import pathlib
+from collections.abc import Mapping
 from typing import Any, override
 
-from ..lang.data import EvaluationContext, Object
-
-from ..model import Operation, OperationExecutor
-
 from ..lang.buildins import (
+    ContainerizedGcc,
+    ExtractFunc,
+    GccCollectCompileCommandsFunc,
     GccCompileFunc,
     GccLinkFunc,
-    GccCollectCompileCommandsFunc,
+    HttpArchiveFunc,
+    HttpGetFunc,
+    LinkFunc,
     LoadFunc,
     PathFunc,
-    LinkFunc,
-    HttpGetFunc,
-    ExtractFunc,
-    HttpArchiveFunc,
-    ContainerizedGcc,
 )
+from ..lang.data import EvaluationContext, Object
+from ..model import Operation, OperationExecutor
 
 buildins = {
     "load": LoadFunc(),

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/parser.py

@@ -107,6 +107,7 @@ __parser = Lark.open_from_package(
     start="listing",
     parser="lalr",
     propagate_positions=True,
+    # _plugins=lark_rust.plugins
 )
 
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/test_parser.py

@@ -1,6 +1,7 @@
 import pytest
 
 from fab.lang.astree import SourcePosition
+
 from .parser import parse
 
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/util.py

@@ -1,8 +1,9 @@
+import pathlib
 from abc import ABC
 from collections.abc import Mapping, Sequence
-import pathlib
 from typing import override
-from .data import Object, Function, EvaluationContext
+
+from .data import EvaluationContext, Function, Object
 
 
 class DictObject(Object):

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/model/__init__.py

@@ -1,9 +1,9 @@
 from .cache import Cache
-from .operation import Operation, OperationContext, OperationError
-from .executor import OperationExecutor
-from .oci import OciContainer, OciClient, OciProcess
-from .outputchunks import StreamType, OutputChunk
 from .errors import CommandFailedError, Error
+from .executor import OperationExecutor
+from .oci import OciClient, OciContainer, OciProcess
+from .operation import Operation, OperationContext, OperationError
+from .outputchunks import OutputChunk, StreamType
 
 __all__ = [
     "Cache",

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/model/cache.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/model/oci.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
+
 import abc
 from collections.abc import AsyncIterator
 from dataclasses import dataclass
+
 from .outputchunks import OutputChunk
 
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/model/operation.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/model/outputchunks.py

@@ -1,6 +1,5 @@
-from dataclasses import dataclass
-
 import enum
+from dataclasses import dataclass
 
 
 class StreamType(enum.Enum):

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/oci/podman.py

@@ -14,9 +14,7 @@ The multiplexed exec stream follows the Docker framing protocol:
 
 from __future__ import annotations
 
-import abc
 import asyncio
-import enum
 import json
 import os
 import struct
@@ -27,6 +25,7 @@ import aiohttp
 
 from fab.model import OutputChunk
 
+from ..model import StreamType
 from ..model.oci import (
     ImageNotFoundError,
     OciClient,
@@ -37,7 +36,6 @@ from ..model.oci import (
     PullError,
     PullEvent,
 )
-from ..model import StreamType
 
 # ---------------------------------------------------------------------------
 # Helpers

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/operations/__init__.py

@@ -3,6 +3,7 @@ from .extract import Extract
 # from .gcc import gcc_compile, gcc_link
 from .http_archive import HttpArchive
 from .http_get import HttpGet
+
 # from .local_path import local_path
 
 __all__ = [

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/operations/containerized_gcc.py

@@ -4,8 +4,7 @@ from typing import override
 
 from blake3 import blake3
 
-
-from ..model import Operation, OperationContext, CommandFailedError
+from ..model import CommandFailedError, Operation, OperationContext
 from ..util.flatten_list import flatten
 from ..util.hash_objects import hash_objects
 from .gcc import CompileCommandsCollector, CompileObject
@@ -125,7 +124,7 @@ class GccCompile(Operation):
             )
         )
 
-        # First see if we can find the dependencies of this file. If this file has 
+        # First see if we can find the dependencies of this file. If this file has
         # been compiled before its deps are stored under the following key.
         deps_key = hash_objects(
             blake3(),
@@ -139,9 +138,9 @@ class GccCompile(Operation):
         deps_cached = await context.cache_check(deps_key)
 
         if deps_cached:
-            # Deps file is found. Next step is to see if we can reuse the previous build 
+            # Deps file is found. Next step is to see if we can reuse the previous build
             # result, ea: none of the dependencies have changed.
-            # We do that by calculating the key for this source file combined with 
+            # We do that by calculating the key for this source file combined with
             # all the dependencies.
 
             deps_file = await context.cache_load_path(deps_key)
@@ -157,7 +156,7 @@ class GccCompile(Operation):
             )
 
             if await context.cache_check(key):
-                # Dependencies have not changed! Lets load the build result from cache 
+                # Dependencies have not changed! Lets load the build result from cache
                 # and return
                 return await context.cache_load_path(key)
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/operations/extract.py

@@ -1,3 +1,4 @@
+import shutil
 from pathlib import Path
 from typing import Any, override
 
@@ -6,8 +7,6 @@ from blake3 import blake3
 from ..model import Operation, OperationContext
 from ..util.hash_objects import hash_objects
 
-import shutil
-
 
 class Extract(Operation):
     def __init__(self, archive: Path) -> None:

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/operations/gcc.py

@@ -1,16 +1,13 @@
 import asyncio
 import json
 from collections.abc import Sequence
-from dataclasses import dataclass, asdict
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, override
 
 from blake3 import blake3
 
-from fab.lang import data
-
-
-from ..model import Operation, OperationContext 
+from ..model import Operation, OperationContext
 from ..util.hash_objects import hash_objects
 
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/operations/test_containerized_gcc.py

@@ -9,9 +9,11 @@ from .containerized_gcc import GccLink, _parse_make_deps
 
 
 def test_parse_make_deps():
-    assert list(_parse_make_deps(
-        "main.o: main.cpp lib1.h lib2.h \\\n lib3.h lib4.h \\\n lib5.h path\\ with\\ spaces.h"
-    )) == [
+    assert list(
+        _parse_make_deps(
+            "main.o: main.cpp lib1.h lib2.h \\\n lib3.h lib4.h \\\n lib5.h path\\ with\\ spaces.h"
+        )
+    ) == [
         Path("lib1.h"),
         Path("lib2.h"),
         Path("lib3.h"),
@@ -21,7 +23,6 @@ def test_parse_make_deps():
     ]
 
 
-
 async def _empty_chunks() -> AsyncIterator[OutputChunk]:
     for _ in ():
         yield OutputChunk(StreamType.STDOUT, b"")
@@ -126,9 +127,7 @@ async def test_link_options_appended(tmp_path: Path):
     context = RecordingContext(tmp_path)
     obj = tmp_path / "obj.o"
     obj.write_bytes(b"content")
-    op = GccLink(
-        "img", None, "out", [obj], options=["-lm", "-Wl,--as-needed"]
-    )
+    op = GccLink("img", None, "out", [obj], options=["-lm", "-Wl,--as-needed"])
     await op.execute(context)
     assert context.container is not None
     cmd = context.container.cmd

fab_build-0.0.2.dev1/src/fab/util/bytesizestr.py

@@ -0,0 +1,42 @@
+import re
+
+__steps = [
+    (1024**0, "B"),
+    (1024**1, "KiB"),
+    (1024**2, "MiB"),
+    (1024**3, "GiB"),
+    (1024**4, "TiB"),
+    (1024**5, "PiB"),
+]
+
+
+def bytesize2str(size: int) -> str:
+    for step_size, suffix in __steps:
+        limit = step_size * 1024
+        if size < limit:
+            return f"{size / step_size:.1f} {suffix}"
+    return "huge!"
+
+
+__suffixes = {
+    "": 1,
+    "K": 1024**1,
+    "M": 1024**2,
+    "G": 1024**3,
+    "T": 1024**4,
+    "P": 1024**5,
+}
+
+
+def str2bytesize(size: str) -> int:
+    match = re.match(r"(\d+(?:\.\d+)?)(\w?)", size)
+    if not match:
+        raise ValueError(f"Could not convert '{size}' to bytes")
+
+    value = float(match.group(1))
+    suffix = match.group(2).upper()
+
+    try:
+        return int(value * __suffixes[suffix])
+    except KeyError:
+        raise ValueError(f"Unknown suffix '{suffix}'")

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/util/hash_objects.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from collections.abc import Iterable, Sequence
 from pathlib import Path
 

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/util/string_slices.py

@@ -11,8 +11,8 @@ def string_slices(inp: str, *slices: int) -> Sequence[str]:
     for size in slices:
         if current_pos + size > len(inp):
             raise ValueError("Slice sizes exceed input string length.")
-        
-        result.append(inp[current_pos:current_pos + size])
+
+        result.append(inp[current_pos : current_pos + size])
         current_pos += size
 
     return result

fab_build-0.0.2.dev1/src/fab/util/test_bytesizestr.py

@@ -0,0 +1,5 @@
+from .bytesizestr import str2bytesize
+
+
+def test_str2bytesize():
+    assert str2bytesize("0") == 0

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/util/test_hash_path.py

@@ -1,13 +1,17 @@
-from .temp_file_structure import TempFileStructure
-from .hash_path import hash_path
-from pathlib import Path
 import hashlib
+from pathlib import Path
+
 import pytest
 
+from .hash_path import hash_path
+from .temp_file_structure import TempFileStructure
+
+
 def path_key(path: Path) -> str:
     hashfunc = hashlib.sha256()
     return hash_path(hashfunc, path).hex()
 
+
 def test_path():
     with TempFileStructure(
         [

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/util/test_string_slices.py

@@ -1,6 +1,7 @@
-from .string_slices import string_slices
 import pytest
 
+from .string_slices import string_slices
+
 
 def test_string_slices():
     assert string_slices("abcdefg", 2, 2) == ["ab", "cd"]

fab_build-0.0.2/src/fab/caching/diskcache.py

@@ -1,165 +0,0 @@
-from collections.abc import Generator
-from datetime import datetime
-import os
-from typing import override
-from pathlib import Path
-
-from ..model import Cache
-
-from ..util.string_slices import string_slices
-from ..util.hash_path import hash_path
-
-import blake3
-
-
-class DiskCacheError(RuntimeError):
-    pass
-
-
-class Log:
-    def __init__(self, path: Path) -> None:
-        self.__path = path
-        self.__path.parent.mkdir(parents=True, exist_ok=True)
-        self.__writer = open(self.__path, "a")
-
-    def __del__(self):
-        self.flush()
-
-    def write(self, timestamp: datetime, key: bytes):
-        time_str = timestamp.isoformat()
-        self.__writer.write(f"{time_str} {key.hex()}\n")
-
-    def flush(self):
-        self.__writer.flush()
-
-    def clear(self):
-        self.clear()
-        # TODO
-        pass
-
-    def read_entries(self) -> Generator[tuple[datetime, bytes]]:
-        """Read entries from log, most recent first."""
-
-        self.flush()
-
-        # NOTE: This approach is very inefficient for large files, will need refactoring at some point!
-        for line in reversed(self.__path.read_text().splitlines()):
-            parts = line.split(" ", 1)
-            timestamp = datetime.fromisoformat(parts[0])
-            key = bytes.fromhex(parts[1])
-            yield timestamp, key
-
-
-class DiskCache(Cache):
-    def __init__(self, root: Path) -> None:
-
-        self.__ops = root / "v1/ops"
-        self.__ops.mkdir(parents=True, exist_ok=True)
-
-        self.__blobs = root / "v1/blobs"
-        self.__blobs.mkdir(parents=True, exist_ok=True)
-
-        self.__log = Log(root / "v1/hits")
-
-    def __get_op_path(self, op_key: bytes) -> Path:
-        key_parts = string_slices(op_key.hex(), 2, 2)
-        return self.__ops / key_parts[0] / key_parts[1] / op_key.hex()
-
-    def __get_blob_path(self, blob_key: bytes) -> Path:
-        key_parts = string_slices(blob_key.hex(), 2, 2)
-        return self.__blobs / key_parts[0] / key_parts[1] / blob_key.hex()
-
-    def prune_to_size(self, size: int):
-        """Remove longest unused entries until the combined size of the
-        remaining entries is <= size."""
-
-        blobs_to_keep: set[bytes] = set()
-        kept_blobs_size: int = 0
-
-        for _, key in self.__log.read_entries():
-            path = self.__get_path(key)
-            blob_size = path.stat().st_size
-
-            if kept_blobs_size + blob_size > size:
-                break
-
-            blobs_to_keep.add(key)
-            kept_blobs_size += blob_size
-
-        self.__prune(blobs_to_keep)
-
-    def prune_to_age(self, timestamp: datetime):
-        """Remove all entries that have not been used after timestamp."""
-        pass
-        # TODO
-
-    def __prune(self, blobs_to_keep: set[bytes]):
-
-        for op_path in self.__scan_ops():
-            object_path = op_path / "object"
-            blob_key = bytes.fromhex(object_path.parent.name)
-
-            if blob_key in blobs_to_keep:
-                continue
-
-            blob_path = object_path.resolve().parent
-            object_path.resolve().unlink()
-            object_path.unlink()
-            op_path.rmdir()
-            blob_path.rmdir()
-
-    def __scan_ops(self) -> Generator[Path]:
-        for dir1 in os.scandir(self.__ops):
-            for dir2 in os.scandir(dir1):
-                for entry in os.scandir(dir2):
-                    yield Path(entry.path)
-
-    @override
-    async def has(self, op_key: bytes) -> bool:
-        op_path = self.__get_op_path(op_key)
-        result = op_path.exists()
-        if result:
-            self.__log.write(datetime.now(), op_key)
-        return result
-
-    @override
-    async def get_path(self, op_key: bytes) -> Path:
-        return self.__get_path(op_key)
-
-    def __get_path(self, op_key: bytes) -> Path:
-        op_path = self.__get_op_path(op_key)
-        if not op_path.is_dir():
-            raise DiskCacheError(f"{op_path} does not exist")
-
-        object_link = op_path / "object"
-        if not object_link.is_symlink():
-            raise DiskCacheError(f"Corrupt cache, {object_link} is not a symlink")
-
-        blob_path = object_link.resolve()
-
-        return blob_path
-
-    @override
-    async def store_path(self, op_key: bytes, path: Path) -> Path:
-        if not path.exists():
-            raise DiskCacheError(f"Cannot store non-existent path {path} in cache")
-
-        blob_key = hash_path(blake3.blake3(), path)
-
-        blob_path = self.__get_blob_path(blob_key)
-        blob_path.mkdir(parents=True, exist_ok=True)
-        obj_path = blob_path / path.name
-        if not obj_path.exists():
-            path.rename(obj_path)
-
-        op_path = self.__get_op_path(op_key)
-        if op_path.exists():
-            raise DiskCacheError(
-                f"Operation with key '{op_key.hex()}' already exists in cache"
-            )
-
-        op_path.mkdir(parents=True, exist_ok=True)
-        (op_path / "object").symlink_to(obj_path.relative_to(op_path, walk_up=True))
-
-        self.__log.write(datetime.now(), op_key)
-        return obj_path

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/__init__.py

@@ -1,6 +1,6 @@
-from .memory import MemoryCache
-from .nullcache import NullCache
 from .diskcache import DiskCache
 from .hit_counter import CacheHitCounter
+from .memory import MemoryCache
+from .nullcache import NullCache
 
 __all__ = ["MemoryCache", "NullCache", "DiskCache", "CacheHitCounter"]

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/caching/memory.py

@@ -1,9 +1,9 @@
 from pathlib import Path
 from typing import override
 
-
 from ..model import Cache
 
+
 class MemoryCache(Cache):
     def __init__(self):
         self.__cached_paths: dict[bytes, Path] = {}

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/lang/evaluator.py

@@ -2,9 +2,9 @@ from collections.abc import Mapping, Sequence
 from typing import override
 
 from ..lang import astree
+from ..model import Error
 from .data import EvaluationContext, Function, List, Object, String
 from .parser import parse
-from ..model import Error
 
 
 class EvaluationError(RuntimeError):

{fab_build-0.0.2 → fab_build-0.0.2.dev1}/src/fab/util/hash_path.py

@@ -1,5 +1,5 @@
-from pathlib import Path
 from collections.abc import Iterable
+from pathlib import Path
 
 
 def chunkify_path(base_path: Path) -> Iterable[bytes]: