liblaf-cherries 0.0.0__py3-none-any.whl → 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- liblaf/cherries/__init__.pyi +18 -7
- liblaf/cherries/_config.py +22 -0
- liblaf/cherries/_env.py +4 -0
- liblaf/cherries/_experiment.py +111 -0
- liblaf/cherries/_main.py +38 -0
- liblaf/cherries/_start.py +19 -41
- liblaf/cherries/integration/__init__.pyi +4 -3
- liblaf/cherries/integration/_backend.py +53 -0
- liblaf/cherries/integration/_factory.py +13 -0
- liblaf/cherries/integration/_neptune.py +31 -18
- liblaf/cherries/plugin/__init__.pyi +2 -1
- liblaf/cherries/plugin/_abc.py +6 -6
- liblaf/cherries/plugin/_default.py +5 -1
- liblaf/cherries/plugin/_git.py +2 -2
- liblaf/cherries/plugin/_logging.py +43 -5
- liblaf/cherries/plugin/_restic.py +20 -5
- {liblaf_cherries-0.0.0.dist-info → liblaf_cherries-0.0.1.dist-info}/METADATA +2 -1
- liblaf_cherries-0.0.1.dist-info/RECORD +34 -0
- liblaf/cherries/integration/_abc.py +0 -134
- liblaf_cherries-0.0.0.dist-info/RECORD +0 -29
- {liblaf_cherries-0.0.0.dist-info → liblaf_cherries-0.0.1.dist-info}/WHEEL +0 -0
- {liblaf_cherries-0.0.0.dist-info → liblaf_cherries-0.0.1.dist-info}/licenses/LICENSE +0 -0
liblaf/cherries/__init__.pyi
CHANGED
@@ -1,23 +1,34 @@
|
|
1
1
|
from . import git, integration, plugin, utils
|
2
|
-
from .
|
2
|
+
from ._config import BaseConfig
|
3
|
+
from ._env import ENV_PREFIX, env
|
4
|
+
from ._experiment import Experiment, current_experiment, set_current_experiment
|
5
|
+
from ._main import main
|
6
|
+
from ._start import end, start
|
3
7
|
from .git import entrypoint
|
4
|
-
from .integration import
|
5
|
-
from .plugin import Plugin, PluginGit, PluginLogging, default_plugins
|
8
|
+
from .integration import Backend, BackendNeptune, backend_factory
|
9
|
+
from .plugin import Plugin, PluginGit, PluginLogging, PluginRestic, default_plugins
|
6
10
|
|
7
11
|
__all__ = [
|
12
|
+
"ENV_PREFIX",
|
13
|
+
"Backend",
|
14
|
+
"BackendNeptune",
|
15
|
+
"BaseConfig",
|
16
|
+
"Experiment",
|
8
17
|
"Plugin",
|
9
18
|
"PluginGit",
|
10
19
|
"PluginLogging",
|
11
|
-
"
|
12
|
-
"
|
13
|
-
"
|
20
|
+
"PluginRestic",
|
21
|
+
"backend_factory",
|
22
|
+
"current_experiment",
|
14
23
|
"default_plugins",
|
15
24
|
"end",
|
16
25
|
"entrypoint",
|
26
|
+
"env",
|
17
27
|
"git",
|
18
28
|
"integration",
|
29
|
+
"main",
|
19
30
|
"plugin",
|
20
|
-
"
|
31
|
+
"set_current_experiment",
|
21
32
|
"start",
|
22
33
|
"utils",
|
23
34
|
]
|
@@ -0,0 +1,22 @@
|
|
1
|
+
import pydantic_settings as ps
|
2
|
+
|
3
|
+
|
4
|
+
class BaseConfig(ps.BaseSettings):
|
5
|
+
model_config = ps.SettingsConfigDict(cli_parse_args=True, yaml_file="params.yaml")
|
6
|
+
|
7
|
+
@classmethod
|
8
|
+
def settings_customise_sources(
|
9
|
+
cls,
|
10
|
+
settings_cls: type[ps.BaseSettings],
|
11
|
+
init_settings: ps.PydanticBaseSettingsSource,
|
12
|
+
env_settings: ps.PydanticBaseSettingsSource,
|
13
|
+
dotenv_settings: ps.PydanticBaseSettingsSource,
|
14
|
+
file_secret_settings: ps.PydanticBaseSettingsSource,
|
15
|
+
) -> tuple[ps.PydanticBaseSettingsSource, ...]:
|
16
|
+
return (
|
17
|
+
init_settings,
|
18
|
+
env_settings,
|
19
|
+
dotenv_settings,
|
20
|
+
file_secret_settings,
|
21
|
+
ps.YamlConfigSettingsSource(settings_cls),
|
22
|
+
)
|
liblaf/cherries/_env.py
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
import atexit
|
2
|
+
import dataclasses
|
3
|
+
import datetime
|
4
|
+
import os
|
5
|
+
from collections.abc import Sequence
|
6
|
+
from pathlib import Path
|
7
|
+
from typing import Any
|
8
|
+
|
9
|
+
from environs import env
|
10
|
+
from loguru import logger
|
11
|
+
|
12
|
+
from liblaf import cherries
|
13
|
+
|
14
|
+
|
15
|
+
@dataclasses.dataclass(kw_only=True)
|
16
|
+
class Experiment:
|
17
|
+
backend: cherries.Backend = dataclasses.field(
|
18
|
+
default_factory=cherries.backend_factory
|
19
|
+
)
|
20
|
+
enabled: bool = dataclasses.field(
|
21
|
+
default_factory=lambda: env.bool("LIBLAF_CHERRIES_ENABLED", True)
|
22
|
+
)
|
23
|
+
plugins: Sequence[cherries.Plugin] = dataclasses.field(
|
24
|
+
default_factory=cherries.default_plugins
|
25
|
+
)
|
26
|
+
|
27
|
+
@property
|
28
|
+
def entrypoint(self) -> Path:
|
29
|
+
return self.backend.entrypoint
|
30
|
+
|
31
|
+
@property
|
32
|
+
def id(self) -> str:
|
33
|
+
return self.backend.id
|
34
|
+
|
35
|
+
@property
|
36
|
+
def name(self) -> str:
|
37
|
+
return self.backend.name
|
38
|
+
|
39
|
+
@property
|
40
|
+
def start_time(self) -> datetime.datetime:
|
41
|
+
return self.backend.start_time
|
42
|
+
|
43
|
+
@property
|
44
|
+
def url(self) -> str:
|
45
|
+
return self.backend.url
|
46
|
+
|
47
|
+
def start(self) -> None:
|
48
|
+
if not self.enabled:
|
49
|
+
return
|
50
|
+
self.plugins = sorted(self.plugins, key=lambda plugin: plugin.priority)
|
51
|
+
for plugin in self.plugins:
|
52
|
+
plugin.pre_start()
|
53
|
+
self.backend.start()
|
54
|
+
cherries.set_current_experiment(self)
|
55
|
+
for plugin in self.plugins:
|
56
|
+
plugin.post_start(self)
|
57
|
+
self.log_other("cherries/entrypoint", self.entrypoint)
|
58
|
+
self.log_other("cherries/start_time", self.start_time)
|
59
|
+
atexit.register(self.end)
|
60
|
+
|
61
|
+
def end(self) -> None:
|
62
|
+
if not self.enabled:
|
63
|
+
return
|
64
|
+
for plugin in reversed(self.plugins):
|
65
|
+
plugin.pre_end(self)
|
66
|
+
self.backend.end()
|
67
|
+
for plugin in reversed(self.plugins):
|
68
|
+
plugin.post_end(self)
|
69
|
+
self.enabled = False # prevent `end()` from being called multiple times
|
70
|
+
|
71
|
+
def log_metric(
|
72
|
+
self,
|
73
|
+
key: str,
|
74
|
+
value: float,
|
75
|
+
*,
|
76
|
+
step: float | None = None,
|
77
|
+
timestamp: float | None = None,
|
78
|
+
**kwargs,
|
79
|
+
) -> None:
|
80
|
+
if not self.enabled:
|
81
|
+
return
|
82
|
+
logger.opt(depth=1).debug("{}: {}", key, value)
|
83
|
+
self.backend.log_metric(key, value, step=step, timestamp=timestamp, **kwargs)
|
84
|
+
|
85
|
+
def log_other(self, key: str, value: Any, **kwargs) -> None:
|
86
|
+
if not self.enabled:
|
87
|
+
return
|
88
|
+
logger.opt(depth=1).info("{}: {}", key, value)
|
89
|
+
self.backend.log_other(key, value, **kwargs)
|
90
|
+
|
91
|
+
def upload_file(self, key: str, path: str | os.PathLike[str], **kwargs) -> None:
|
92
|
+
if not self.enabled:
|
93
|
+
return
|
94
|
+
path = Path(path)
|
95
|
+
logger.opt(depth=1).info("Uploading file: {}", path)
|
96
|
+
self.backend.upload_file(key, path, **kwargs)
|
97
|
+
|
98
|
+
|
99
|
+
_current_experiment: Experiment | None = None
|
100
|
+
|
101
|
+
|
102
|
+
def current_experiment() -> Experiment:
|
103
|
+
global _current_experiment # noqa: PLW0603
|
104
|
+
if _current_experiment is None:
|
105
|
+
_current_experiment = Experiment()
|
106
|
+
return _current_experiment
|
107
|
+
|
108
|
+
|
109
|
+
def set_current_experiment(experiment: Experiment) -> None:
|
110
|
+
global _current_experiment # noqa: PLW0603
|
111
|
+
_current_experiment = experiment
|
liblaf/cherries/_main.py
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
import functools
|
2
|
+
import inspect
|
3
|
+
from collections.abc import Callable, Sequence
|
4
|
+
from typing import ParamSpec, TypeVar
|
5
|
+
|
6
|
+
from liblaf import cherries
|
7
|
+
|
8
|
+
_P = ParamSpec("_P")
|
9
|
+
_T = TypeVar("_T")
|
10
|
+
|
11
|
+
|
12
|
+
def main(
|
13
|
+
*,
|
14
|
+
backend: cherries.Backend | None = None,
|
15
|
+
enabled: bool | None = None,
|
16
|
+
plugins: Sequence[cherries.Plugin] | None = None,
|
17
|
+
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
|
18
|
+
def wrapper(fn: Callable[_P, _T]) -> Callable[_P, _T]:
|
19
|
+
@functools.wraps(fn)
|
20
|
+
def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T:
|
21
|
+
exp: cherries.Experiment = cherries.start(
|
22
|
+
backend=backend, enabled=enabled, plugins=plugins
|
23
|
+
)
|
24
|
+
sig: inspect.Signature = inspect.signature(fn)
|
25
|
+
bound_args: inspect.BoundArguments = sig.bind(*args, **kwargs)
|
26
|
+
if len(bound_args.arguments) == 1:
|
27
|
+
exp.log_other(
|
28
|
+
"cherries/config", next(iter(bound_args.arguments.values()))
|
29
|
+
)
|
30
|
+
elif len(bound_args.arguments) > 1:
|
31
|
+
exp.log_other("cherries/args", bound_args.arguments)
|
32
|
+
ret: _T = fn(*args, **kwargs)
|
33
|
+
exp.end()
|
34
|
+
return ret
|
35
|
+
|
36
|
+
return wrapped
|
37
|
+
|
38
|
+
return wrapper
|
liblaf/cherries/_start.py
CHANGED
@@ -1,48 +1,26 @@
|
|
1
1
|
from collections.abc import Sequence
|
2
2
|
|
3
|
-
from
|
4
|
-
|
5
|
-
_current_run: cherries.Run | None = None
|
6
|
-
|
7
|
-
|
8
|
-
def current_run() -> cherries.Run | None:
|
9
|
-
return _current_run
|
3
|
+
from environs import Env
|
10
4
|
|
11
|
-
|
12
|
-
def set_current_run(run: cherries.Run) -> None:
|
13
|
-
global _current_run # noqa: PLW0603
|
14
|
-
_current_run = run
|
5
|
+
from liblaf import cherries
|
15
6
|
|
16
7
|
|
17
8
|
def start(
|
18
|
-
backend: type[cherries.Run],
|
19
|
-
plugins: Sequence[cherries.Plugin] = [],
|
20
9
|
*,
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
def end(run: cherries.Run | None = None) -> None:
|
39
|
-
run = run or current_run()
|
40
|
-
if run is None:
|
41
|
-
return
|
42
|
-
for plugin in reversed(run.plugins):
|
43
|
-
if plugin.enabled:
|
44
|
-
plugin.pre_end(run)
|
45
|
-
run.end()
|
46
|
-
for plugin in reversed(run.plugins):
|
47
|
-
if plugin.enabled:
|
48
|
-
plugin.post_end(run)
|
10
|
+
backend: cherries.Backend | None = None,
|
11
|
+
enabled: bool | None = None,
|
12
|
+
plugins: Sequence[cherries.Plugin] | None = None,
|
13
|
+
) -> cherries.Experiment:
|
14
|
+
backend = backend or cherries.backend_factory()
|
15
|
+
if enabled is None:
|
16
|
+
enabled = Env().bool("LIBLAF_CHERRIES_ENABLED", True)
|
17
|
+
if plugins is None:
|
18
|
+
plugins = cherries.default_plugins()
|
19
|
+
exp = cherries.Experiment(backend=backend, enabled=enabled, plugins=plugins)
|
20
|
+
exp.start()
|
21
|
+
return exp
|
22
|
+
|
23
|
+
|
24
|
+
def end() -> None:
|
25
|
+
exp: cherries.Experiment = cherries.current_experiment()
|
26
|
+
exp.end()
|
@@ -1,4 +1,5 @@
|
|
1
|
-
from .
|
2
|
-
from .
|
1
|
+
from ._backend import Backend
|
2
|
+
from ._factory import backend_factory
|
3
|
+
from ._neptune import BackendNeptune
|
3
4
|
|
4
|
-
__all__ = ["
|
5
|
+
__all__ = ["Backend", "BackendNeptune", "backend_factory"]
|
@@ -0,0 +1,53 @@
|
|
1
|
+
import datetime
|
2
|
+
import functools
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any
|
5
|
+
|
6
|
+
import pydantic_settings as ps
|
7
|
+
|
8
|
+
from liblaf import cherries
|
9
|
+
|
10
|
+
|
11
|
+
class Backend(ps.BaseSettings):
|
12
|
+
model_config = ps.SettingsConfigDict(
|
13
|
+
frozen=True, env_prefix=cherries.ENV_PREFIX + "DUMMY_"
|
14
|
+
)
|
15
|
+
enabled: bool = True
|
16
|
+
|
17
|
+
@property
|
18
|
+
def backend(self) -> str:
|
19
|
+
return "dummy"
|
20
|
+
|
21
|
+
@functools.cached_property
|
22
|
+
def entrypoint(self) -> Path:
|
23
|
+
return cherries.entrypoint()
|
24
|
+
|
25
|
+
@property
|
26
|
+
def id(self) -> str:
|
27
|
+
raise NotImplementedError
|
28
|
+
|
29
|
+
@property
|
30
|
+
def name(self) -> str:
|
31
|
+
raise NotImplementedError
|
32
|
+
|
33
|
+
@functools.cached_property
|
34
|
+
def start_time(self) -> datetime.datetime:
|
35
|
+
return datetime.datetime.now().astimezone()
|
36
|
+
|
37
|
+
@property
|
38
|
+
def url(self) -> str:
|
39
|
+
raise NotImplementedError
|
40
|
+
|
41
|
+
def start(self) -> None: ...
|
42
|
+
def end(self) -> None: ...
|
43
|
+
def log_metric(
|
44
|
+
self,
|
45
|
+
key: str,
|
46
|
+
value: float,
|
47
|
+
*,
|
48
|
+
step: float | None = None,
|
49
|
+
timestamp: float | None = None,
|
50
|
+
**kwargs,
|
51
|
+
) -> None: ...
|
52
|
+
def log_other(self, key: str, value: Any, **kwargs) -> None: ...
|
53
|
+
def upload_file(self, key: str, path: Path, **kwargs) -> None: ...
|
@@ -0,0 +1,13 @@
|
|
1
|
+
from liblaf import cherries
|
2
|
+
|
3
|
+
|
4
|
+
def backend_factory(backend: str | None = None) -> cherries.Backend:
|
5
|
+
backend = backend or cherries.env.str("BACKEND", "dummy")
|
6
|
+
match backend:
|
7
|
+
case "neptune":
|
8
|
+
return cherries.BackendNeptune()
|
9
|
+
case "dummy":
|
10
|
+
return cherries.Backend()
|
11
|
+
case _:
|
12
|
+
msg: str = f"Unknown backend: {backend}"
|
13
|
+
raise ValueError(msg)
|
@@ -1,13 +1,21 @@
|
|
1
|
-
import
|
1
|
+
import os
|
2
2
|
from pathlib import Path
|
3
|
+
from typing import Any
|
3
4
|
|
4
5
|
import neptune
|
5
6
|
import neptune.common.exceptions
|
7
|
+
import neptune.utils
|
8
|
+
import pydantic
|
9
|
+
import pydantic_settings as ps
|
6
10
|
|
7
11
|
from liblaf import cherries
|
8
12
|
|
9
13
|
|
10
|
-
class
|
14
|
+
class BackendNeptune(cherries.Backend):
|
15
|
+
model_config = ps.SettingsConfigDict(frozen=True, env_prefix="NEPTUNE_")
|
16
|
+
monitoring_namespace: str | None = None
|
17
|
+
_backend: neptune.Run = pydantic.PrivateAttr()
|
18
|
+
|
11
19
|
@property
|
12
20
|
def backend(self) -> str:
|
13
21
|
return "neptune"
|
@@ -24,30 +32,35 @@ class RunNeptune(cherries.Run[neptune.Run]):
|
|
24
32
|
def url(self) -> str:
|
25
33
|
return self._backend.get_url()
|
26
34
|
|
27
|
-
def
|
28
|
-
|
35
|
+
def start(self) -> None:
|
36
|
+
neptune.common.exceptions.STYLES.update(neptune.common.exceptions.EMPTY_STYLES)
|
37
|
+
self._backend = neptune.init_run(monitoring_namespace=self.monitoring_namespace)
|
29
38
|
|
30
|
-
def
|
31
|
-
|
39
|
+
def end(self) -> None:
|
40
|
+
self._backend.stop()
|
32
41
|
|
33
|
-
def
|
42
|
+
def log_metric(
|
34
43
|
self,
|
35
44
|
key: str,
|
36
45
|
value: float,
|
37
46
|
*,
|
38
47
|
step: float | None = None,
|
39
48
|
timestamp: float | None = None,
|
40
|
-
**kwargs,
|
49
|
+
**kwargs,
|
41
50
|
) -> None:
|
42
|
-
self._backend[key].append(value, step=step, timestamp=timestamp)
|
51
|
+
self._backend[key].append(value, step=step, timestamp=timestamp, **kwargs)
|
52
|
+
|
53
|
+
def log_other(self, key: str, value: Any, **kwargs) -> None:
|
54
|
+
value = stringify_unsupported(value)
|
55
|
+
self._backend[key].assign(value, **kwargs)
|
56
|
+
|
57
|
+
def upload_file(self, key: str, path: Path, **kwargs) -> None:
|
58
|
+
return self._backend[key].upload(str(path), **kwargs)
|
43
59
|
|
44
|
-
def _log_other(
|
45
|
-
self,
|
46
|
-
key: str,
|
47
|
-
value: bool | float | str | datetime.datetime,
|
48
|
-
**kwargs, # noqa: ARG002
|
49
|
-
) -> None:
|
50
|
-
self._backend[key] = value
|
51
60
|
|
52
|
-
|
53
|
-
|
61
|
+
def stringify_unsupported(value: Any) -> Any:
|
62
|
+
if isinstance(value, pydantic.BaseModel):
|
63
|
+
return value.model_dump()
|
64
|
+
if isinstance(value, os.PathLike):
|
65
|
+
return str(value)
|
66
|
+
return value
|
@@ -2,5 +2,6 @@ from ._abc import Plugin
|
|
2
2
|
from ._default import default_plugins
|
3
3
|
from ._git import PluginGit
|
4
4
|
from ._logging import PluginLogging
|
5
|
+
from ._restic import PluginRestic
|
5
6
|
|
6
|
-
__all__ = ["Plugin", "PluginGit", "PluginLogging", "default_plugins"]
|
7
|
+
__all__ = ["Plugin", "PluginGit", "PluginLogging", "PluginRestic", "default_plugins"]
|
liblaf/cherries/plugin/_abc.py
CHANGED
@@ -13,19 +13,19 @@ class Plugin(ps.BaseSettings):
|
|
13
13
|
if self.enabled:
|
14
14
|
self._pre_start()
|
15
15
|
|
16
|
-
def post_start(self, run: cherries.
|
16
|
+
def post_start(self, run: cherries.Experiment) -> None:
|
17
17
|
if self.enabled:
|
18
18
|
self._post_start(run)
|
19
19
|
|
20
|
-
def pre_end(self, run: cherries.
|
20
|
+
def pre_end(self, run: cherries.Experiment) -> None:
|
21
21
|
if self.enabled:
|
22
22
|
self._pre_end(run)
|
23
23
|
|
24
|
-
def post_end(self, run: cherries.
|
24
|
+
def post_end(self, run: cherries.Experiment) -> None:
|
25
25
|
if self.enabled:
|
26
26
|
self._post_end(run)
|
27
27
|
|
28
28
|
def _pre_start(self) -> None: ...
|
29
|
-
def _post_start(self, run: cherries.
|
30
|
-
def _pre_end(self, run: cherries.
|
31
|
-
def _post_end(self, run: cherries.
|
29
|
+
def _post_start(self, run: cherries.Experiment) -> None: ...
|
30
|
+
def _pre_end(self, run: cherries.Experiment) -> None: ...
|
31
|
+
def _post_end(self, run: cherries.Experiment) -> None: ...
|
@@ -2,4 +2,8 @@ from liblaf import cherries
|
|
2
2
|
|
3
3
|
|
4
4
|
def default_plugins() -> list[cherries.Plugin]:
|
5
|
-
return [
|
5
|
+
return [
|
6
|
+
cherries.plugin.PluginLogging(),
|
7
|
+
cherries.plugin.PluginGit(),
|
8
|
+
cherries.plugin.PluginRestic(),
|
9
|
+
]
|
liblaf/cherries/plugin/_git.py
CHANGED
@@ -5,7 +5,7 @@ from liblaf import cherries
|
|
5
5
|
|
6
6
|
|
7
7
|
class PluginGit(cherries.Plugin):
|
8
|
-
model_config = ps.SettingsConfigDict(env_prefix="
|
8
|
+
model_config = ps.SettingsConfigDict(env_prefix=cherries.ENV_PREFIX + "GIT_")
|
9
9
|
auto_commit: bool = True
|
10
10
|
auto_commit_message: str = cherries.git.DEFAULT_COMMIT_MESSAGE
|
11
11
|
|
@@ -13,7 +13,7 @@ class PluginGit(cherries.Plugin):
|
|
13
13
|
if self.auto_commit:
|
14
14
|
cherries.git.commit(self.auto_commit_message)
|
15
15
|
|
16
|
-
def _post_start(self, run: cherries.
|
16
|
+
def _post_start(self, run: cherries.Experiment) -> None:
|
17
17
|
r = git.Repo(search_parent_directories=True)
|
18
18
|
sha: str = r.head.commit.hexsha
|
19
19
|
run.log_other("cherries/git/sha", sha)
|
@@ -1,20 +1,58 @@
|
|
1
|
+
import loguru
|
1
2
|
import pydantic_settings as ps
|
2
3
|
from loguru import logger
|
4
|
+
from rich.logging import RichHandler
|
3
5
|
|
6
|
+
# TODO: fix PLR0402
|
4
7
|
# make pyright happy
|
5
8
|
import liblaf.grapes as grapes # noqa: PLR0402
|
6
9
|
from liblaf import cherries
|
7
10
|
|
11
|
+
DEFAULT_FILTER: "loguru.FilterDict" = {
|
12
|
+
"": "INFO",
|
13
|
+
"__main__": "TRACE",
|
14
|
+
"liblaf": "DEBUG",
|
15
|
+
}
|
16
|
+
DEFAULT_FILE_FILTER: "loguru.FilterDict" = {
|
17
|
+
**DEFAULT_FILTER,
|
18
|
+
"liblaf.cherries": "SUCCESS",
|
19
|
+
}
|
20
|
+
|
8
21
|
|
9
22
|
class PluginLogging(cherries.Plugin):
|
10
|
-
model_config = ps.SettingsConfigDict(env_prefix="
|
23
|
+
model_config = ps.SettingsConfigDict(env_prefix=cherries.ENV_PREFIX + "LOGGING_")
|
11
24
|
|
12
25
|
def _pre_start(self) -> None:
|
13
|
-
grapes.init_logging(
|
14
|
-
|
15
|
-
|
26
|
+
grapes.init_logging(
|
27
|
+
handlers=[
|
28
|
+
{
|
29
|
+
"sink": RichHandler(
|
30
|
+
console=grapes.logging.logging_console(),
|
31
|
+
omit_repeated_times=False,
|
32
|
+
markup=True,
|
33
|
+
log_time_format="[%Y-%m-%d %H:%M:%S]",
|
34
|
+
),
|
35
|
+
"format": "{message}",
|
36
|
+
"filter": DEFAULT_FILTER,
|
37
|
+
"enqueue": True,
|
38
|
+
},
|
39
|
+
{
|
40
|
+
"sink": "run.log",
|
41
|
+
"filter": DEFAULT_FILE_FILTER,
|
42
|
+
"enqueue": True,
|
43
|
+
"mode": "w",
|
44
|
+
},
|
45
|
+
{
|
46
|
+
"sink": "run.log.jsonl",
|
47
|
+
"filter": DEFAULT_FILE_FILTER,
|
48
|
+
"serialize": True,
|
49
|
+
"enqueue": True,
|
50
|
+
"mode": "w",
|
51
|
+
},
|
52
|
+
]
|
53
|
+
)
|
16
54
|
|
17
|
-
def _pre_end(self, run: cherries.
|
55
|
+
def _pre_end(self, run: cherries.Experiment) -> None:
|
18
56
|
logger.complete()
|
19
57
|
run.upload_file("cherries/logging/run.log", "run.log")
|
20
58
|
run.upload_file("cherries/logging/run.log.jsonl", "run.log.jsonl")
|
@@ -1,6 +1,9 @@
|
|
1
|
+
import contextlib
|
2
|
+
import json
|
1
3
|
import os
|
2
4
|
import subprocess as sp
|
3
5
|
from pathlib import Path
|
6
|
+
from typing import Any
|
4
7
|
|
5
8
|
import pydantic
|
6
9
|
import pydantic_settings as ps
|
@@ -21,12 +24,12 @@ def default_config() -> Path:
|
|
21
24
|
|
22
25
|
|
23
26
|
class PluginRestic(cherries.Plugin):
|
24
|
-
model_config = ps.SettingsConfigDict(env_prefix="
|
27
|
+
model_config = ps.SettingsConfigDict(env_prefix=cherries.ENV_PREFIX + "RESTIC_")
|
25
28
|
config: Path = pydantic.Field(default_factory=default_config)
|
26
29
|
name: str | None = None
|
27
30
|
dry_run: bool = False
|
28
31
|
|
29
|
-
def _pre_end(self, run: cherries.
|
32
|
+
def _pre_end(self, run: cherries.Experiment) -> None:
|
30
33
|
if not self.config.exists():
|
31
34
|
logger.warning("configuration file '{}' was not found", self.config)
|
32
35
|
return
|
@@ -35,11 +38,23 @@ class PluginRestic(cherries.Plugin):
|
|
35
38
|
"--config",
|
36
39
|
self.config,
|
37
40
|
"backup",
|
41
|
+
"--json",
|
38
42
|
]
|
39
43
|
if self.name:
|
40
44
|
args += ["--name", self.name]
|
41
45
|
if self.dry_run:
|
42
46
|
args.append("--dry-run")
|
43
|
-
args += ["--time", run.
|
44
|
-
proc: sp.
|
45
|
-
|
47
|
+
args += ["--time", run.start_time.strftime("%Y-%m-%d %H:%M:%S")]
|
48
|
+
proc: sp.Popen[str] = sp.Popen(
|
49
|
+
args, stdout=sp.PIPE, cwd=cherries.git.root(), text=True
|
50
|
+
)
|
51
|
+
assert proc.stdout is not None
|
52
|
+
for line_ in proc.stdout:
|
53
|
+
line: str = line_.rstrip()
|
54
|
+
logger.debug("{}", line)
|
55
|
+
with contextlib.suppress(json.JSONDecodeError):
|
56
|
+
log: dict[str, Any] = json.loads(line)
|
57
|
+
if log["message_type"] == "summary":
|
58
|
+
run.log_other("cherries/restic", log)
|
59
|
+
returncode: int = proc.wait()
|
60
|
+
run.log_other("cherries/restic/returncode", returncode)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: liblaf-cherries
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.1
|
4
4
|
Summary: Add your description here
|
5
5
|
Project-URL: Changelog, https://github.com/liblaf/cherries/blob/main/CHANGELOG.md
|
6
6
|
Project-URL: Homepage, https://github.com/liblaf/cherries
|
@@ -11,6 +11,7 @@ Author-email: liblaf <30631553+liblaf@users.noreply.github.com>
|
|
11
11
|
License-Expression: MIT
|
12
12
|
License-File: LICENSE
|
13
13
|
Requires-Python: >=3.12
|
14
|
+
Requires-Dist: comet-ml<4,>=3.47.6
|
14
15
|
Requires-Dist: gitpython<4,>=3.1.44
|
15
16
|
Requires-Dist: lazy-loader<0.5,>=0.4
|
16
17
|
Requires-Dist: liblaf-grapes<0.0.1,>=0.0.0
|
@@ -0,0 +1,34 @@
|
|
1
|
+
liblaf/cherries/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
2
|
+
liblaf/cherries/__init__.pyi,sha256=u-eKDd-6imkzIue3LmuJJmpjLDV1yGfV0XP9aN0VdWQ,827
|
3
|
+
liblaf/cherries/_config.py,sha256=rm-Y6roi8hBvQYdH-VQh_ovWCyVsX_7R0x1jokBPCDM,741
|
4
|
+
liblaf/cherries/_env.py,sha256=RK9NMHIok-AkCLMqW44cXLl0d_daLjpEcjhreSP1vm0,92
|
5
|
+
liblaf/cherries/_experiment.py,sha256=E6r5DyojfY75lKsIDxYpyaCgXFXMJTaUIglIk9Xs-Sw,3181
|
6
|
+
liblaf/cherries/_main.py,sha256=R8vWc9e94MViqCzMiG257--QtSoOU50lmR59oIRc6nw,1237
|
7
|
+
liblaf/cherries/_start.py,sha256=iGAhDm8sF_JCaWPWdMIFL0d5nODFU8IcBPdNihhiSp4,685
|
8
|
+
liblaf/cherries/git/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
9
|
+
liblaf/cherries/git/__init__.pyi,sha256=p52L1d1kuiwyc9tBz648Bm76C9DJLbgleE8EKRJM4Lg,371
|
10
|
+
liblaf/cherries/git/_commit.py,sha256=jnAtCKljNnzhXK90cj3adAYkPvcnLnOrHWA8YWhtZF4,482
|
11
|
+
liblaf/cherries/git/_entrypoint.py,sha256=My_TmLqpGX4NJV4qjQvmHtnwhxX_B2ilfuYvpj2ftaM,262
|
12
|
+
liblaf/cherries/git/_repo.py,sha256=bE8apxawGie8KuEjQCDAVRSGTP7Qt6jDDtoXMhfWy4w,907
|
13
|
+
liblaf/cherries/git/github/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
14
|
+
liblaf/cherries/git/github/__init__.pyi,sha256=jspOamBde7EtGOqsIzWg75jFToVHRhG4hfiEw1MDz80,96
|
15
|
+
liblaf/cherries/git/github/_link.py,sha256=f_AcdKMhukUZ2fgBtC0usBrwnpAHaLYo9EhkZYesZuQ,693
|
16
|
+
liblaf/cherries/git/github/_repo.py,sha256=I3vjgonI_SPrytf68qzLAej880Zn7cPryaK4PPoVZpU,627
|
17
|
+
liblaf/cherries/integration/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
18
|
+
liblaf/cherries/integration/__init__.pyi,sha256=n2QqtCl9oTVJatyt8LhiTJVnr_uoyYgfrN8virEqezQ,165
|
19
|
+
liblaf/cherries/integration/_backend.py,sha256=iwpwTB3NgAI9oUCdWYMmH8qI8xYmUsZmES0pyw9_6D0,1241
|
20
|
+
liblaf/cherries/integration/_factory.py,sha256=KJPse2efpOLv6s4sPQAPnzh9vwYcIQVsOg1ucxzTmIo,412
|
21
|
+
liblaf/cherries/integration/_neptune.py,sha256=MhvPZUblAae6vmGRjgDELl_2wcOhrb90nCIsg8xpRMs,1789
|
22
|
+
liblaf/cherries/plugin/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
23
|
+
liblaf/cherries/plugin/__init__.pyi,sha256=3wmkKoe27SI3q5zbyejf3KmaBRUhAu-OEefJAxxbECs,248
|
24
|
+
liblaf/cherries/plugin/_abc.py,sha256=kMDFzDAEDifpxWbnQhp4M4sr24-D55x-B6MPIZW_1jA,848
|
25
|
+
liblaf/cherries/plugin/_default.py,sha256=H7_vym_uxxba41xR2ERBeBt0kBX4s4tjzjifyKjXxS0,215
|
26
|
+
liblaf/cherries/plugin/_git.py,sha256=Gyv8Uc2AnPpZul23oGm6Dba2YOaiFx6WAytrQ8ylSsk,866
|
27
|
+
liblaf/cherries/plugin/_logging.py,sha256=wLIDlVk9_A-lbI64obEzyIFjLgIRR-rmjNvmli3T3O4,1779
|
28
|
+
liblaf/cherries/plugin/_restic.py,sha256=4ArmgD42ZUotkVdCwdMvlmJF0Z8PuKe8EukzRoWyQK8,1926
|
29
|
+
liblaf/cherries/utils/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
30
|
+
liblaf/cherries/utils/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
|
+
liblaf_cherries-0.0.1.dist-info/METADATA,sha256=_AX57D2yYC5gv-e41pONNTuXduXIpv_FukvhY8R2gFs,925
|
32
|
+
liblaf_cherries-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
33
|
+
liblaf_cherries-0.0.1.dist-info/licenses/LICENSE,sha256=Ph4NzyU3lGVDeYv-mf8aRmImH8v9rVL9F362FV4G6Ow,1063
|
34
|
+
liblaf_cherries-0.0.1.dist-info/RECORD,,
|
@@ -1,134 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
|
3
|
-
import datetime
|
4
|
-
import functools
|
5
|
-
import os
|
6
|
-
import sys
|
7
|
-
from collections.abc import Sequence
|
8
|
-
from pathlib import Path
|
9
|
-
from typing import Generic, TypeVar
|
10
|
-
|
11
|
-
from loguru import logger
|
12
|
-
|
13
|
-
from liblaf import cherries
|
14
|
-
|
15
|
-
_T = TypeVar("_T")
|
16
|
-
|
17
|
-
|
18
|
-
class Run(Generic[_T]):
|
19
|
-
enabled: bool = True
|
20
|
-
plugins: list[cherries.Plugin]
|
21
|
-
_backend: _T
|
22
|
-
|
23
|
-
def __init__(
|
24
|
-
self,
|
25
|
-
plugins: Sequence[cherries.Plugin] | None = None,
|
26
|
-
*,
|
27
|
-
enabled: bool = True,
|
28
|
-
) -> None:
|
29
|
-
if plugins is None:
|
30
|
-
plugins = cherries.default_plugins()
|
31
|
-
self.plugins = sorted(plugins, key=lambda plugin: plugin.priority)
|
32
|
-
self.enabled = enabled
|
33
|
-
set_current_run(self)
|
34
|
-
if self.enabled:
|
35
|
-
self.start()
|
36
|
-
|
37
|
-
@property
|
38
|
-
def backend(self) -> str:
|
39
|
-
return "dummy"
|
40
|
-
|
41
|
-
@property
|
42
|
-
def id(self) -> str:
|
43
|
-
raise NotImplementedError
|
44
|
-
|
45
|
-
@functools.cached_property
|
46
|
-
def creation_time(self) -> datetime.datetime:
|
47
|
-
return datetime.datetime.now().astimezone()
|
48
|
-
|
49
|
-
@functools.cached_property
|
50
|
-
def entrypoint(self) -> Path:
|
51
|
-
return Path(sys.argv[0]).absolute()
|
52
|
-
|
53
|
-
@property
|
54
|
-
def name(self) -> str:
|
55
|
-
raise NotImplementedError
|
56
|
-
|
57
|
-
@property
|
58
|
-
def url(self) -> str:
|
59
|
-
raise NotImplementedError
|
60
|
-
|
61
|
-
def start(self) -> None:
|
62
|
-
for plugin in self.plugins:
|
63
|
-
plugin.pre_start()
|
64
|
-
self._backend = self._start()
|
65
|
-
for plugin in self.plugins:
|
66
|
-
plugin.post_start(self)
|
67
|
-
self.log_other("cherries/creation_time", self.creation_time)
|
68
|
-
|
69
|
-
def end(self) -> None:
|
70
|
-
for plugin in reversed(self.plugins):
|
71
|
-
plugin.pre_end(self)
|
72
|
-
self._end()
|
73
|
-
for plugin in reversed(self.plugins):
|
74
|
-
plugin.post_end(self)
|
75
|
-
|
76
|
-
def log_metric(
|
77
|
-
self,
|
78
|
-
key: str,
|
79
|
-
value: float,
|
80
|
-
*,
|
81
|
-
step: float | None = None,
|
82
|
-
timestamp: float | None = None,
|
83
|
-
**kwargs,
|
84
|
-
) -> None:
|
85
|
-
logger.opt(depth=1).debug("{}: {}", key, value)
|
86
|
-
if self.enabled:
|
87
|
-
self._log_metric(key, value, step=step, timestamp=timestamp, **kwargs)
|
88
|
-
|
89
|
-
def log_other(
|
90
|
-
self,
|
91
|
-
key: str,
|
92
|
-
value: bool | float | str | datetime.datetime,
|
93
|
-
**kwargs,
|
94
|
-
) -> None:
|
95
|
-
logger.opt(depth=1).info("{}: {}", key, value)
|
96
|
-
if self.enabled:
|
97
|
-
self._log_other(key, value, **kwargs)
|
98
|
-
|
99
|
-
def upload_file(self, key: str, path: str | os.PathLike[str], **kwargs) -> None:
|
100
|
-
path = Path(path)
|
101
|
-
logger.opt(depth=1).info("Uploading file: {}", path)
|
102
|
-
if self.enabled:
|
103
|
-
self._upload_file(key, path, **kwargs)
|
104
|
-
|
105
|
-
def _start(self) -> _T: ...
|
106
|
-
def _end(self) -> None: ...
|
107
|
-
def _log_metric(
|
108
|
-
self,
|
109
|
-
key: str,
|
110
|
-
value: float,
|
111
|
-
*,
|
112
|
-
step: float | None = None,
|
113
|
-
timestamp: float | None = None,
|
114
|
-
**kwargs,
|
115
|
-
) -> None: ...
|
116
|
-
def _log_other(
|
117
|
-
self, key: str, value: bool | float | str | datetime.datetime, **kwargs
|
118
|
-
) -> None: ...
|
119
|
-
def _upload_file(self, key: str, path: Path, **kwargs) -> None: ...
|
120
|
-
|
121
|
-
|
122
|
-
_current_run: Run | None = None
|
123
|
-
|
124
|
-
|
125
|
-
def current_run() -> Run:
|
126
|
-
global _current_run # noqa: PLW0603
|
127
|
-
if _current_run is None:
|
128
|
-
_current_run = Run()
|
129
|
-
return _current_run
|
130
|
-
|
131
|
-
|
132
|
-
def set_current_run(run: Run) -> None:
|
133
|
-
global _current_run # noqa: PLW0603
|
134
|
-
_current_run = run
|
@@ -1,29 +0,0 @@
|
|
1
|
-
liblaf/cherries/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
2
|
-
liblaf/cherries/__init__.pyi,sha256=Qt8P1Fgtt2bC-5KnSjF0ruTKAtT9p4dGmnEfCbsdeR0,506
|
3
|
-
liblaf/cherries/_start.py,sha256=VF2TyYcX6jWPxLA4wtLoNLIjMRhwXt3XQlR7xwnTdGI,1153
|
4
|
-
liblaf/cherries/git/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
5
|
-
liblaf/cherries/git/__init__.pyi,sha256=p52L1d1kuiwyc9tBz648Bm76C9DJLbgleE8EKRJM4Lg,371
|
6
|
-
liblaf/cherries/git/_commit.py,sha256=jnAtCKljNnzhXK90cj3adAYkPvcnLnOrHWA8YWhtZF4,482
|
7
|
-
liblaf/cherries/git/_entrypoint.py,sha256=My_TmLqpGX4NJV4qjQvmHtnwhxX_B2ilfuYvpj2ftaM,262
|
8
|
-
liblaf/cherries/git/_repo.py,sha256=bE8apxawGie8KuEjQCDAVRSGTP7Qt6jDDtoXMhfWy4w,907
|
9
|
-
liblaf/cherries/git/github/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
10
|
-
liblaf/cherries/git/github/__init__.pyi,sha256=jspOamBde7EtGOqsIzWg75jFToVHRhG4hfiEw1MDz80,96
|
11
|
-
liblaf/cherries/git/github/_link.py,sha256=f_AcdKMhukUZ2fgBtC0usBrwnpAHaLYo9EhkZYesZuQ,693
|
12
|
-
liblaf/cherries/git/github/_repo.py,sha256=I3vjgonI_SPrytf68qzLAej880Zn7cPryaK4PPoVZpU,627
|
13
|
-
liblaf/cherries/integration/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
14
|
-
liblaf/cherries/integration/__init__.pyi,sha256=jSy9QJwmUTygGUNeOdkP5sk_thCJfXwYH9TMKJHfvhI,152
|
15
|
-
liblaf/cherries/integration/_abc.py,sha256=se7Jf89HHs4Ssf1W7F_RTJDcPLIYkOJftmmVFRpytZA,3418
|
16
|
-
liblaf/cherries/integration/_neptune.py,sha256=zpV5MN7_0Cqb-_t0vX2rXlHs-j5hDgH7FdwUczDo0Qc,1232
|
17
|
-
liblaf/cherries/plugin/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
18
|
-
liblaf/cherries/plugin/__init__.pyi,sha256=oDd9fUqu2YRb2EP-15j6YgHT40MH22O_jJbt9goPb3s,198
|
19
|
-
liblaf/cherries/plugin/_abc.py,sha256=2ILWpFNLz7dCo-YL_UNzg3BaWoYyTuFOv98xBZZvyZE,806
|
20
|
-
liblaf/cherries/plugin/_default.py,sha256=VrFEW2AQqsBNALPNzLBlStRk_2XcyMu1XwzhjRGNr8Q,152
|
21
|
-
liblaf/cherries/plugin/_git.py,sha256=_t9lEAQOzEHV2qwzHwirdYg1t1br11Lp3hO4ArjVrXI,853
|
22
|
-
liblaf/cherries/plugin/_logging.py,sha256=AGE9HZsRm-Z8hQn0HBSuu77PulF-_EN_Y29QvSMC4Bk,686
|
23
|
-
liblaf/cherries/plugin/_restic.py,sha256=qW48Q043xZTB3xces9EJn-sFF4whnSlgVAjq2FnUoC4,1383
|
24
|
-
liblaf/cherries/utils/__init__.py,sha256=OHb6Xou2v6u42swTgjRfzej4CIlRg4OmgOIQXUiRjKA,97
|
25
|
-
liblaf/cherries/utils/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
-
liblaf_cherries-0.0.0.dist-info/METADATA,sha256=DVW58aVJrIPxiuHFMWXUD73Ncyq0_6cJjy_TMR1Pwo8,890
|
27
|
-
liblaf_cherries-0.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
28
|
-
liblaf_cherries-0.0.0.dist-info/licenses/LICENSE,sha256=Ph4NzyU3lGVDeYv-mf8aRmImH8v9rVL9F362FV4G6Ow,1063
|
29
|
-
liblaf_cherries-0.0.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|