omdev 0.0.0.dev227__tar.gz → 0.0.0.dev228__tar.gz

{omdev-0.0.0.dev227/omdev.egg-info → omdev-0.0.0.dev228}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev227
+Version: 0.0.0.dev228
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,9 +12,9 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev227
+Requires-Dist: omlish==0.0.0.dev228
 Provides-Extra: all
-Requires-Dist: black~=24.10; extra == "all"
+Requires-Dist: black~=25.1; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
 Requires-Dist: pcpp~=1.30; extra == "all"
 Requires-Dist: cffi~=1.17; extra == "all"
@@ -24,9 +24,9 @@ Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
 Requires-Dist: segno~=1.6; extra == "all"
-Requires-Dist: wheel~=0.44; extra == "all"
+Requires-Dist: wheel~=0.45; extra == "all"
 Provides-Extra: black
-Requires-Dist: black~=24.10; extra == "black"
+Requires-Dist: black~=25.1; extra == "black"
 Provides-Extra: c
 Requires-Dist: pycparser~=2.22; extra == "c"
 Requires-Dist: pcpp~=1.30; extra == "c"
@@ -43,4 +43,4 @@ Requires-Dist: prompt-toolkit~=3.0; extra == "ptk"
 Provides-Extra: qr
 Requires-Dist: segno~=1.6; extra == "qr"
 Provides-Extra: wheel
-Requires-Dist: wheel~=0.44; extra == "wheel"
+Requires-Dist: wheel~=0.45; extra == "wheel"

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/__about__.py

@@ -13,7 +13,7 @@ class Project(ProjectBase):
 
     optional_dependencies = {
         'black': [
-            'black ~= 24.10',
+            'black ~= 25.1',
         ],
 
         'c': [
@@ -46,7 +46,7 @@ class Project(ProjectBase):
         ],
 
         'wheel': [
-            'wheel ~= 0.44',
+            'wheel ~= 0.45',
         ],
     }
 

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/ci/cache.py

@@ -1,10 +1,17 @@
 # ruff: noqa: UP006 UP007
+"""
+TODO:
+ - os.mtime, Config.purge_after_days
+  - nice to have: get a total set of might-need keys ahead of time and keep those
+  - okay: just purge after running
+"""
 import abc
 import asyncio
 import dataclasses as dc
 import functools
 import os.path
 import shutil
+import time
 import typing as ta
 import urllib.request
 
@@ -70,6 +77,11 @@ class DirectoryFileCache(FileCache):
         no_create: bool = False
         no_purge: bool = False
 
+        no_update_mtime: bool = False
+
+        purge_max_age_s: ta.Optional[float] = None
+        purge_max_size_b: ta.Optional[int] = None
+
     def __init__(
             self,
             config: Config,
@@ -90,6 +102,12 @@ class DirectoryFileCache(FileCache):
 
     VERSION_FILE_NAME = '.ci-cache-version'
 
+    def _iter_dir_contents(self) -> ta.Iterator[str]:
+        for n in sorted(os.listdir(self.dir)):
+            if n.startswith('.'):
+                continue
+            yield os.path.join(self.dir, n)
+
     @cached_nullary
     def setup_dir(self) -> None:
         version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
@@ -120,10 +138,7 @@ class DirectoryFileCache(FileCache):
                 f'due to present directories: {", ".join(dirs)}',
             )
 
-        for n in sorted(os.listdir(self.dir)):
-            if n.startswith('.'):
-                continue
-            fp = os.path.join(self.dir, n)
+        for fp in self._iter_dir_contents():
             check.state(os.path.isfile(fp))
             log.debug('Purging stale cache file: %s', fp)
             os.unlink(fp)
@@ -135,6 +150,42 @@ class DirectoryFileCache(FileCache):
 
     #
 
+    def purge(self, *, dry_run: bool = False) -> None:
+        purge_max_age_s = self._config.purge_max_age_s
+        purge_max_size_b = self._config.purge_max_size_b
+        if self._config.no_purge or (purge_max_age_s is None and purge_max_size_b is None):
+            return
+
+        self.setup_dir()
+
+        purge_min_mtime: ta.Optional[float] = None
+        if purge_max_age_s is not None:
+            purge_min_mtime = time.time() - purge_max_age_s
+
+        dct: ta.Dict[str, os.stat_result] = {}
+        for fp in self._iter_dir_contents():
+            check.state(os.path.isfile(fp))
+            dct[fp] = os.stat(fp)
+
+        total_size_b = 0
+        for fp, st in sorted(dct.items(), key=lambda t: -t[1].st_mtime):
+            total_size_b += st.st_size
+
+            purge = False
+            if purge_min_mtime is not None and st.st_mtime < purge_min_mtime:
+                purge = True
+            if purge_max_size_b is not None and total_size_b >= purge_max_size_b:
+                purge = True
+
+            if not purge:
+                continue
+
+            log.debug('Purging cache file: %s', fp)
+            if not dry_run:
+                os.unlink(fp)
+
+    #
+
     def get_cache_file_path(
             self,
             key: str,
@@ -151,6 +202,11 @@ class DirectoryFileCache(FileCache):
         cache_file_path = self.get_cache_file_path(key)
         if not os.path.exists(cache_file_path):
             return None
+
+        if not self._config.no_update_mtime:
+            stat_info = os.stat(cache_file_path)
+            os.utime(cache_file_path, (stat_info.st_atime, time.time()))
+
         return cache_file_path
 
     async def put_file(

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/ci/cli.py

@@ -25,6 +25,7 @@ from omlish.lite.inject import inj
 from omlish.lite.logs import log
 from omlish.logs.standard import configure_standard_logging
 
+from .cache import DirectoryFileCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
 from .github.bootstrap import is_in_github_actions
@@ -70,6 +71,9 @@ class CiCli(ArgparseCli):
 
     #
 
+    DEFAULT_PURGE_MAX_AGE_S = 60 * 60 * 24 * 30
+    DEFAULT_PURGE_MAX_SIZE_B = 1024 * 1024 * 1024 * 4
+
     @argparse_cmd(
         argparse_arg('project-dir'),
         argparse_arg('service'),
@@ -79,6 +83,8 @@ class CiCli(ArgparseCli):
 
         argparse_arg('--cache-dir'),
 
+        argparse_arg('--no-purge', action='store_true'),
+
         argparse_arg('--github', action='store_true'),
         argparse_arg('--github-detect', action='store_true'),
 
@@ -202,17 +208,32 @@ class CiCli(ArgparseCli):
             run_options=run_options,
         )
 
+        directory_file_cache_config: ta.Optional[DirectoryFileCache.Config] = None
+        if cache_dir is not None:
+            directory_file_cache_config = DirectoryFileCache.Config(
+                dir=cache_dir,
+
+                no_purge=bool(self.args.no_purge),
+
+                purge_max_age_s=self.DEFAULT_PURGE_MAX_AGE_S,
+                purge_max_size_b=self.DEFAULT_PURGE_MAX_SIZE_B,
+            )
+
         injector = inj.create_injector(bind_ci(
             config=config,
 
-            github=github,
+            directory_file_cache_config=directory_file_cache_config,
 
-            cache_dir=cache_dir,
+            github=github,
         ))
 
         async with injector[Ci] as ci:
             await ci.run()
 
+        if directory_file_cache_config is not None and not directory_file_cache_config.no_purge:
+            dfc = injector[DirectoryFileCache]
+            dfc.purge()
+
 
 async def _async_main() -> ta.Optional[int]:
     return await CiCli().async_cli_run()

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/ci/github/cache.py

@@ -21,14 +21,16 @@ from .client import GithubCacheServiceV1Client
 class GithubCache(FileCache, DataCache):
     @dc.dataclass(frozen=True)
     class Config:
-        dir: str
+        pass
 
     def __init__(
             self,
-            config: Config,
+            config: Config = Config(),
             *,
             client: ta.Optional[GithubCacheClient] = None,
             version: ta.Optional[CacheVersion] = None,
+
+            local: DirectoryFileCache,
     ) -> None:
         super().__init__(
             version=version,
@@ -42,12 +44,7 @@ class GithubCache(FileCache, DataCache):
             )
         self._client: GithubCacheClient = client
 
-        self._local = DirectoryFileCache(
-            DirectoryFileCache.Config(
-                dir=check.non_empty_str(config.dir),
-            ),
-            version=self._version,
-        )
+        self._local = local
 
     #
 

omdev-0.0.0.dev228/omdev/ci/github/inject.py

@@ -0,0 +1,21 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from ..cache import FileCache
+from .cache import GithubCache
+
+
+##
+
+
+def bind_github() -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(GithubCache, singleton=True),
+        inj.bind(FileCache, to_key=GithubCache),
+    ]
+
+    return inj.as_bindings(*lst)

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/ci/inject.py

@@ -21,9 +21,9 @@ def bind_ci(
         *,
         config: Ci.Config,
 
-        github: bool = False,
+        directory_file_cache_config: ta.Optional[DirectoryFileCache.Config] = None,
 
-        cache_dir: ta.Optional[str] = None,
+        github: bool = False,
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [  # noqa
         inj.bind(config),
@@ -42,20 +42,15 @@ def bind_ci(
         ),
     ))
 
-    if cache_dir is not None:
-        if github:
-            lst.append(bind_github(
-                cache_dir=cache_dir,
-            ))
+    if directory_file_cache_config is not None:
+        lst.extend([
+            inj.bind(directory_file_cache_config),
+            inj.bind(DirectoryFileCache, singleton=True),
+        ])
 
+        if github:
+            lst.append(bind_github())
         else:
-            lst.extend([
-                inj.bind(DirectoryFileCache.Config(
-                    dir=cache_dir,
-                )),
-                inj.bind(DirectoryFileCache, singleton=True),
-                inj.bind(FileCache, to_key=DirectoryFileCache),
-
-            ])
+            lst.append(inj.bind(FileCache, to_key=DirectoryFileCache))
 
     return inj.as_bindings(*lst)

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/scripts/ci.py

@@ -24,6 +24,8 @@ import contextlib
 import contextvars
 import dataclasses as dc
 import datetime
+import errno
+import fcntl
 import functools
 import hashlib
 import http.client
@@ -1094,7 +1096,7 @@ class Timeout(abc.ABC):
 
     @classmethod
     def _now(cls) -> float:
-        return time.time()
+        return time.monotonic()
 
     #
 
@@ -1417,20 +1419,33 @@ log_timing_context = LogTimingContext
 # ../../../omlish/os/files.py
 
 
-def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+def is_fd_open(fd: int) -> bool:
+    try:
+        fcntl.fcntl(fd, fcntl.F_GETFD)
+    except OSError as e:
+        if e.errno == errno.EBADF:
+            return False
+        raise
+    else:
+        return True
+
+
+def touch(path: str, mode: int = 0o666, exist_ok: bool = True) -> None:
     if exist_ok:
         # First try to bump modification time
         # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
         try:
-            os.utime(self, None)
+            os.utime(path, None)
         except OSError:
             pass
         else:
             return
+
     flags = os.O_CREAT | os.O_WRONLY
     if not exist_ok:
         flags |= os.O_EXCL
-    fd = os.open(self, flags, mode)
+
+    fd = os.open(path, flags, mode)
     os.close(fd)
 
 
@@ -3458,6 +3473,12 @@ class SubprocessRunnable(abc.ABC, ta.Generic[T]):
 
 ########################################
 # ../cache.py
+"""
+TODO:
+ - os.mtime, Config.purge_after_days
+  - nice to have: get a total set of might-need keys ahead of time and keep those
+  - okay: just purge after running
+"""
 
 
 CacheVersion = ta.NewType('CacheVersion', int)
@@ -3514,6 +3535,11 @@ class DirectoryFileCache(FileCache):
         no_create: bool = False
         no_purge: bool = False
 
+        no_update_mtime: bool = False
+
+        purge_max_age_s: ta.Optional[float] = None
+        purge_max_size_b: ta.Optional[int] = None
+
     def __init__(
             self,
             config: Config,
@@ -3534,6 +3560,12 @@ class DirectoryFileCache(FileCache):
 
     VERSION_FILE_NAME = '.ci-cache-version'
 
+    def _iter_dir_contents(self) -> ta.Iterator[str]:
+        for n in sorted(os.listdir(self.dir)):
+            if n.startswith('.'):
+                continue
+            yield os.path.join(self.dir, n)
+
     @cached_nullary
     def setup_dir(self) -> None:
         version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
@@ -3564,10 +3596,7 @@ class DirectoryFileCache(FileCache):
                 f'due to present directories: {", ".join(dirs)}',
             )
 
-        for n in sorted(os.listdir(self.dir)):
-            if n.startswith('.'):
-                continue
-            fp = os.path.join(self.dir, n)
+        for fp in self._iter_dir_contents():
             check.state(os.path.isfile(fp))
             log.debug('Purging stale cache file: %s', fp)
             os.unlink(fp)
@@ -3579,6 +3608,42 @@ class DirectoryFileCache(FileCache):
 
     #
 
+    def purge(self, *, dry_run: bool = False) -> None:
+        purge_max_age_s = self._config.purge_max_age_s
+        purge_max_size_b = self._config.purge_max_size_b
+        if self._config.no_purge or (purge_max_age_s is None and purge_max_size_b is None):
+            return
+
+        self.setup_dir()
+
+        purge_min_mtime: ta.Optional[float] = None
+        if purge_max_age_s is not None:
+            purge_min_mtime = time.time() - purge_max_age_s
+
+        dct: ta.Dict[str, os.stat_result] = {}
+        for fp in self._iter_dir_contents():
+            check.state(os.path.isfile(fp))
+            dct[fp] = os.stat(fp)
+
+        total_size_b = 0
+        for fp, st in sorted(dct.items(), key=lambda t: -t[1].st_mtime):
+            total_size_b += st.st_size
+
+            purge = False
+            if purge_min_mtime is not None and st.st_mtime < purge_min_mtime:
+                purge = True
+            if purge_max_size_b is not None and total_size_b >= purge_max_size_b:
+                purge = True
+
+            if not purge:
+                continue
+
+            log.debug('Purging cache file: %s', fp)
+            if not dry_run:
+                os.unlink(fp)
+
+    #
+
     def get_cache_file_path(
             self,
             key: str,
@@ -3595,6 +3660,11 @@ class DirectoryFileCache(FileCache):
         cache_file_path = self.get_cache_file_path(key)
         if not os.path.exists(cache_file_path):
             return None
+
+        if not self._config.no_update_mtime:
+            stat_info = os.stat(cache_file_path)
+            os.utime(cache_file_path, (stat_info.st_atime, time.time()))
+
         return cache_file_path
 
     async def put_file(
@@ -4352,14 +4422,16 @@ def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
 class GithubCache(FileCache, DataCache):
     @dc.dataclass(frozen=True)
     class Config:
-        dir: str
+        pass
 
     def __init__(
             self,
-            config: Config,
+            config: Config = Config(),
             *,
             client: ta.Optional[GithubCacheClient] = None,
             version: ta.Optional[CacheVersion] = None,
+
+            local: DirectoryFileCache,
     ) -> None:
         super().__init__(
             version=version,
@@ -4373,12 +4445,7 @@ class GithubCache(FileCache, DataCache):
             )
         self._client: GithubCacheClient = client
 
-        self._local = DirectoryFileCache(
-            DirectoryFileCache.Config(
-                dir=check.non_empty_str(config.dir),
-            ),
-            version=self._version,
-        )
+        self._local = local
 
     #
 
@@ -4677,20 +4744,11 @@ class BaseSubprocesses(abc.ABC):  # noqa
 ##
 
 
-def bind_github(
-        *,
-        cache_dir: ta.Optional[str] = None,
-) -> InjectorBindings:
-    lst: ta.List[InjectorBindingOrBindings] = []
-
-    if cache_dir is not None:
-        lst.extend([
-            inj.bind(GithubCache.Config(
-                dir=cache_dir,
-            )),
-            inj.bind(GithubCache, singleton=True),
-            inj.bind(FileCache, to_key=GithubCache),
-        ])
+def bind_github() -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(GithubCache, singleton=True),
+        inj.bind(FileCache, to_key=GithubCache),
+    ]
 
     return inj.as_bindings(*lst)
 
@@ -5912,9 +5970,9 @@ def bind_ci(
         *,
         config: Ci.Config,
 
-        github: bool = False,
+        directory_file_cache_config: ta.Optional[DirectoryFileCache.Config] = None,
 
-        cache_dir: ta.Optional[str] = None,
+        github: bool = False,
 ) -> InjectorBindings:
     lst: ta.List[InjectorBindingOrBindings] = [  # noqa
         inj.bind(config),
@@ -5933,21 +5991,16 @@ def bind_ci(
         ),
     ))
 
-    if cache_dir is not None:
-        if github:
-            lst.append(bind_github(
-                cache_dir=cache_dir,
-            ))
+    if directory_file_cache_config is not None:
+        lst.extend([
+            inj.bind(directory_file_cache_config),
+            inj.bind(DirectoryFileCache, singleton=True),
+        ])
 
+        if github:
+            lst.append(bind_github())
         else:
-            lst.extend([
-                inj.bind(DirectoryFileCache.Config(
-                    dir=cache_dir,
-                )),
-                inj.bind(DirectoryFileCache, singleton=True),
-                inj.bind(FileCache, to_key=DirectoryFileCache),
-
-            ])
+            lst.append(inj.bind(FileCache, to_key=DirectoryFileCache))
 
     return inj.as_bindings(*lst)
 
@@ -5992,6 +6045,9 @@ class CiCli(ArgparseCli):
 
     #
 
+    DEFAULT_PURGE_MAX_AGE_S = 60 * 60 * 24 * 30
+    DEFAULT_PURGE_MAX_SIZE_B = 1024 * 1024 * 1024 * 4
+
     @argparse_cmd(
         argparse_arg('project-dir'),
         argparse_arg('service'),
@@ -6001,6 +6057,8 @@ class CiCli(ArgparseCli):
 
         argparse_arg('--cache-dir'),
 
+        argparse_arg('--no-purge', action='store_true'),
+
         argparse_arg('--github', action='store_true'),
         argparse_arg('--github-detect', action='store_true'),
 
@@ -6124,17 +6182,32 @@ class CiCli(ArgparseCli):
             run_options=run_options,
         )
 
+        directory_file_cache_config: ta.Optional[DirectoryFileCache.Config] = None
+        if cache_dir is not None:
+            directory_file_cache_config = DirectoryFileCache.Config(
+                dir=cache_dir,
+
+                no_purge=bool(self.args.no_purge),
+
+                purge_max_age_s=self.DEFAULT_PURGE_MAX_AGE_S,
+                purge_max_size_b=self.DEFAULT_PURGE_MAX_SIZE_B,
+            )
+
         injector = inj.create_injector(bind_ci(
             config=config,
 
-            github=github,
+            directory_file_cache_config=directory_file_cache_config,
 
-            cache_dir=cache_dir,
+            github=github,
         ))
 
         async with injector[Ci] as ci:
             await ci.run()
 
+        if directory_file_cache_config is not None and not directory_file_cache_config.no_purge:
+            dfc = injector[DirectoryFileCache]
+            dfc.purge()
+
 
 async def _async_main() -> ta.Optional[int]:
     return await CiCli().async_cli_run()

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/scripts/interp.py

@@ -1340,7 +1340,7 @@ class Timeout(abc.ABC):
 
     @classmethod
     def _now(cls) -> float:
-        return time.time()
+        return time.monotonic()
 
     #
 

{omdev-0.0.0.dev227 → omdev-0.0.0.dev228}/omdev/scripts/pyproject.py

@@ -2648,7 +2648,7 @@ class Timeout(abc.ABC):
 
     @classmethod
     def _now(cls) -> float:
-        return time.time()
+        return time.monotonic()
 
     #