powerfunc 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {powerfunc-0.3.0 → powerfunc-0.4.0}/.github/workflows/ci.yml +2 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/PKG-INFO +3 -3
- {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/readme.md +7 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/pyproject.toml +2 -2
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/command_line.py +7 -8
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/compute.py +12 -2
- powerfunc-0.4.0/src/powerfunc/decorator.py +207 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/gcp_cloud_run.py +10 -6
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/modal.py +19 -16
- {powerfunc-0.3.0/tests/powerfunc_tests/gcp → powerfunc-0.4.0/tests/powerfunc_tests}/conftest.py +8 -7
- powerfunc-0.4.0/tests/powerfunc_tests/core/__init__.py +12 -0
- powerfunc-0.4.0/tests/powerfunc_tests/core/cli_example.py +40 -0
- powerfunc-0.4.0/tests/powerfunc_tests/core/test_configuration.py +95 -0
- powerfunc-0.4.0/tests/powerfunc_tests/core/test_future_annotations.py +27 -0
- powerfunc-0.4.0/tests/powerfunc_tests/core/test_given_objects.py +90 -0
- powerfunc-0.4.0/tests/powerfunc_tests/gcp/test_gcp_batch.py +100 -0
- powerfunc-0.4.0/tests/powerfunc_tests/gcp/test_gcp_cloud_run.py +97 -0
- powerfunc-0.4.0/tests/powerfunc_tests/modal/test_modal.py +73 -0
- powerfunc-0.3.0/tests/powerfunc_tests/gcp/gcp_jobs.py → powerfunc-0.4.0/tests/powerfunc_tests/remote_jobs.py +16 -3
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tox.ini +3 -1
- {powerfunc-0.3.0 → powerfunc-0.4.0}/uv.lock +32 -4
- powerfunc-0.3.0/src/powerfunc/decorator.py +0 -116
- powerfunc-0.3.0/tests/powerfunc_tests/core/cli_example.py +0 -21
- powerfunc-0.3.0/tests/powerfunc_tests/gcp/test_gcp_batch.py +0 -70
- powerfunc-0.3.0/tests/powerfunc_tests/gcp/test_gcp_cloud_run.py +0 -52
- powerfunc-0.3.0/tests/powerfunc_tests/modal/test_modal.py +0 -44
- powerfunc-0.3.0/tests/powerfunc_tests/snakemake/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/.gitignore +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/.pre-commit-config.yaml +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/LICENSE.txt +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/README.md +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/advanced_usage.md +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/integrations/snakemake.md +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/providers/gcp.md +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/providers/modal.md +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/configuration.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/conversions.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/arrow.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/csv_reader.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/dask.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/pandas.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/polars.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/integrations/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/integrations/snakemake.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/gcp_batch.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/py.typed +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_basic.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_cli.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_conversions.py +0 -0
- /powerfunc-0.3.0/tests/powerfunc_tests/core/test_remote.py → /powerfunc-0.4.0/tests/powerfunc_tests/core/test_remote_files.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.arrow +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.csv +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.feather +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.pandas.json +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.parquet +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.polars.json +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.xlsx +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/generate_data_files.py +0 -0
- {powerfunc-0.3.0/tests/powerfunc_tests/core → powerfunc-0.4.0/tests/powerfunc_tests/gcp}/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/gcp/gcp_shared.py +0 -0
- {powerfunc-0.3.0/tests/powerfunc_tests/gcp → powerfunc-0.4.0/tests/powerfunc_tests/modal}/__init__.py +0 -0
- {powerfunc-0.3.0/tests/powerfunc_tests/modal → powerfunc-0.4.0/tests/powerfunc_tests/snakemake}/__init__.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/functions.py +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/input.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/input_params_and_output.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/multiple_outputs_rejected.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/named_outputs_rejected.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/params.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/positional_rejected.smk +0 -0
- {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/test_snakemake.py +0 -0
|
@@ -58,6 +58,8 @@ jobs:
|
|
|
58
58
|
enable-cache: true
|
|
59
59
|
- name: Run Modal tests
|
|
60
60
|
env:
|
|
61
|
+
GOOGLE_CLOUD_API_KEY: ${{ secrets.GOOGLE_CLOUD_API_KEY }}
|
|
62
|
+
GCP_BUCKET: gs://powerfunc-temporary-348293
|
|
61
63
|
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
|
|
62
64
|
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
|
|
63
65
|
run: uvx --with tox-uv tox run -e modal
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: powerfunc
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Make functions run however you like — CLI, pipeline, or remote compute
|
|
5
5
|
Project-URL: Homepage, https://github.com/ddrakard/powerfunc
|
|
6
6
|
Project-URL: Repository, https://github.com/ddrakard/powerfunc
|
|
@@ -10,7 +10,7 @@ Requires-Python: >=3.9
|
|
|
10
10
|
Requires-Dist: cloudpathlib[all]
|
|
11
11
|
Requires-Dist: cloudpickle
|
|
12
12
|
Requires-Dist: fsspec[adl,gcs,s3]
|
|
13
|
-
Requires-Dist: jsonargparse>=4.49.0
|
|
13
|
+
Requires-Dist: jsonargparse[signatures]>=4.49.0
|
|
14
14
|
Requires-Dist: pydantic>=2
|
|
15
15
|
Provides-Extra: dev
|
|
16
16
|
Requires-Dist: pre-commit; extra == 'dev'
|
|
@@ -183,6 +183,13 @@ When using the CLI, pass a config file with `--config`:
|
|
|
183
183
|
python my_script.py sum_col data.csv --config my_config.yaml
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
+
The command line always reads the configuration files. A call from Python reads none unless asked, with `config`, the same option as `--config`: a path to a file, or `True` for the default files.
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
sum_col("data.csv", config=True)
|
|
190
|
+
sum_col("data.csv", config="my_config.yaml")
|
|
191
|
+
```
|
|
192
|
+
|
|
186
193
|
## Supported formats
|
|
187
194
|
|
|
188
195
|
| Type | Formats |
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "powerfunc"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Make functions run however you like — CLI, pipeline, or remote compute"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -12,7 +12,7 @@ dependencies = [
|
|
|
12
12
|
"pydantic>=2",
|
|
13
13
|
"cloudpathlib[all]",
|
|
14
14
|
"fsspec[gcs,s3,adl]",
|
|
15
|
-
"jsonargparse>=4.49.0",
|
|
15
|
+
"jsonargparse[signatures]>=4.49.0",
|
|
16
16
|
"cloudpickle",
|
|
17
17
|
]
|
|
18
18
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import sys
|
|
3
|
+
from collections.abc import Callable
|
|
3
4
|
|
|
4
5
|
from jsonargparse import CLI
|
|
5
6
|
|
|
@@ -10,8 +11,8 @@ class ExpectedException(Exception):
|
|
|
10
11
|
"""Raised for known user errors that should display just the message, not a traceback."""
|
|
11
12
|
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
# Each function's parsed-arguments entry, by the function's name: the command's.
|
|
15
|
+
cli_functions: dict[str, Callable] = {}
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def enable_cli():
|
|
@@ -20,17 +21,15 @@ def enable_cli():
|
|
|
20
21
|
frame = inspect.currentframe()
|
|
21
22
|
if frame is None or frame.f_back is None:
|
|
22
23
|
raise ExpectedException("Frame inspection not supported by this Python implementation.")
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
# By the frame's globals, not inspect.getmodule: under ``python -m`` the same
|
|
25
|
+
# file is often imported by its package name too, and getmodule finds that one.
|
|
26
|
+
if frame.f_back.f_globals.get("__name__") != "__main__":
|
|
25
27
|
return
|
|
26
28
|
if not cli_functions:
|
|
27
29
|
raise ExpectedException("No @powerfunc functions registered for CLI.")
|
|
28
30
|
|
|
29
|
-
global _cli_active
|
|
30
|
-
wrappers = [wrapper for wrapper, _ in cli_functions]
|
|
31
|
-
_cli_active = True
|
|
32
31
|
result = CLI(
|
|
33
|
-
|
|
32
|
+
next(iter(cli_functions.values())) if len(cli_functions) == 1 else cli_functions,
|
|
34
33
|
default_config_files=configuration_paths,
|
|
35
34
|
)
|
|
36
35
|
if result is not None:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import dataclasses
|
|
2
|
-
from typing import Annotated, Any,
|
|
2
|
+
from typing import TYPE_CHECKING, Annotated, Any, Optional, TypeAlias
|
|
3
3
|
|
|
4
4
|
from pydantic import Field, GetCoreSchemaHandler
|
|
5
5
|
from pydantic.dataclasses import dataclass as pydantic_dataclass
|
|
@@ -13,6 +13,9 @@ MemorySize: TypeAlias = Annotated[int, Field(gt=0, description="RAM in MB")]
|
|
|
13
13
|
DockerImageUri: TypeAlias = Annotated[str, Field(description="Container image URI")]
|
|
14
14
|
GpuModel: TypeAlias = Annotated[Optional[str], Field(description="GPU model name")]
|
|
15
15
|
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from powerfunc.decorator import PowerFunc
|
|
18
|
+
|
|
16
19
|
|
|
17
20
|
class Provider:
|
|
18
21
|
"""Base compute provider. Subclass and implement call()."""
|
|
@@ -23,7 +26,14 @@ class Provider:
|
|
|
23
26
|
) -> core_schema.CoreSchema:
|
|
24
27
|
return core_schema.is_instance_schema(cls)
|
|
25
28
|
|
|
26
|
-
def call(self, function:
|
|
29
|
+
def call(self, function: "PowerFunc", arguments: dict, spec: "ComputeSpecification") -> Any:
|
|
30
|
+
"""Run ``function._run_without_parsing(**arguments)`` on the compute, returning its result.
|
|
31
|
+
|
|
32
|
+
``function`` is the powerfunc function itself, picklable by its name.
|
|
33
|
+
``arguments`` have already been parsed and merged with the configuration
|
|
34
|
+
files, so the compute does neither; there, ``_run_without_parsing`` opens
|
|
35
|
+
the files given for parameters and writes the result to ``output_path``.
|
|
36
|
+
"""
|
|
27
37
|
raise NotImplementedError
|
|
28
38
|
|
|
29
39
|
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import inspect
|
|
3
|
+
import pathlib
|
|
4
|
+
import sys
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from contextlib import ExitStack
|
|
7
|
+
from typing import Any, Optional, Protocol, TypeVar, Union, overload
|
|
8
|
+
|
|
9
|
+
from cloudpathlib import CloudPath
|
|
10
|
+
from jsonargparse import ArgumentParser
|
|
11
|
+
|
|
12
|
+
import powerfunc.command_line as command_line
|
|
13
|
+
from powerfunc.command_line import ExpectedException
|
|
14
|
+
from powerfunc.compute import ComputeSpecification
|
|
15
|
+
from powerfunc.configuration import configuration_paths
|
|
16
|
+
from powerfunc.conversions import is_readable, read, write
|
|
17
|
+
|
|
18
|
+
_EXTRA_PARAMETERS = [
|
|
19
|
+
inspect.Parameter(
|
|
20
|
+
"compute",
|
|
21
|
+
inspect.Parameter.KEYWORD_ONLY,
|
|
22
|
+
default=None,
|
|
23
|
+
annotation=Optional[ComputeSpecification],
|
|
24
|
+
),
|
|
25
|
+
inspect.Parameter(
|
|
26
|
+
"output_path",
|
|
27
|
+
inspect.Parameter.KEYWORD_ONLY,
|
|
28
|
+
default=None,
|
|
29
|
+
annotation=Optional[str],
|
|
30
|
+
),
|
|
31
|
+
]
|
|
32
|
+
# The same option as the command line's ``--config``: a configuration file to
|
|
33
|
+
# merge the arguments with, or True for the default files, which the command
|
|
34
|
+
# line reads unless told another. A Python call reads none unless asked.
|
|
35
|
+
_CONFIGURATION_PARAMETER = inspect.Parameter(
|
|
36
|
+
"config",
|
|
37
|
+
inspect.Parameter.KEYWORD_ONLY,
|
|
38
|
+
default=None,
|
|
39
|
+
annotation=Union[str, pathlib.Path, bool, None],
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _configuration_files(config) -> list[str]:
|
|
44
|
+
if config is None or config is False:
|
|
45
|
+
return []
|
|
46
|
+
if config is True:
|
|
47
|
+
return configuration_paths
|
|
48
|
+
return [str(config)]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
R = TypeVar("R", covariant=True)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class PowerFunc(Protocol[R]):
|
|
55
|
+
"""What :func:`powerfunc` makes of a function.
|
|
56
|
+
|
|
57
|
+
Called as the function was, plus ``compute`` to run it elsewhere and
|
|
58
|
+
``output_path`` to write what it returns to a file; the function's own
|
|
59
|
+
parameters are read from its signature at run time, so they are not checked
|
|
60
|
+
statically here.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __call__(
|
|
64
|
+
self,
|
|
65
|
+
*args: Any,
|
|
66
|
+
compute: Optional[ComputeSpecification] = None,
|
|
67
|
+
output_path: Optional[str] = None,
|
|
68
|
+
config: Union[str, pathlib.Path, bool, None] = None,
|
|
69
|
+
**kwargs: Any,
|
|
70
|
+
) -> R: ...
|
|
71
|
+
|
|
72
|
+
def snakemake(self) -> Any:
|
|
73
|
+
"""Run this function bound to the surrounding Snakemake rule."""
|
|
74
|
+
...
|
|
75
|
+
|
|
76
|
+
def _run_without_parsing(self, **arguments: Any) -> R:
|
|
77
|
+
"""Run with arguments already parsed: by the command line, or before dispatch to compute."""
|
|
78
|
+
...
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def optional_arguments_decorator(decorator):
|
|
82
|
+
"""Allows a decorator to be applied with or without argument parentheses."""
|
|
83
|
+
|
|
84
|
+
@functools.wraps(decorator)
|
|
85
|
+
def wrapper(function=None, **kwargs):
|
|
86
|
+
if function is not None:
|
|
87
|
+
return decorator(function, **kwargs)
|
|
88
|
+
return functools.partial(decorator, **kwargs)
|
|
89
|
+
|
|
90
|
+
return wrapper
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@overload
|
|
94
|
+
def powerfunc(function: Callable[..., R], *, cli: bool = True) -> PowerFunc[R]: ...
|
|
95
|
+
@overload
|
|
96
|
+
def powerfunc(
|
|
97
|
+
function: None = None, *, cli: bool = True
|
|
98
|
+
) -> Callable[[Callable[..., R]], PowerFunc[R]]: ...
|
|
99
|
+
@optional_arguments_decorator
|
|
100
|
+
def powerfunc(function=None, *, cli=True):
|
|
101
|
+
"""Decorator to turn a function into a powerfunc function.
|
|
102
|
+
|
|
103
|
+
Applied bare, ``@powerfunc``, or with arguments, ``@powerfunc(cli=False)``.
|
|
104
|
+
"""
|
|
105
|
+
# eval_str=True required for `from __future__ import annotations`
|
|
106
|
+
signature = inspect.signature(function, eval_str=True)
|
|
107
|
+
reserved = {param.name for param in [*_EXTRA_PARAMETERS, _CONFIGURATION_PARAMETER]}
|
|
108
|
+
taken = reserved & signature.parameters.keys()
|
|
109
|
+
if taken:
|
|
110
|
+
raise TypeError(
|
|
111
|
+
f"{function.__name__} has parameters powerfunc adds itself: {', '.join(sorted(taken))}."
|
|
112
|
+
)
|
|
113
|
+
modified_signature = signature.replace(
|
|
114
|
+
parameters=[
|
|
115
|
+
param.replace(
|
|
116
|
+
# str must precede pathlib.Path and CloudPath: jsonargparse resolves Union
|
|
117
|
+
# subtypes in order and pathlib.Path accepts any string without error, so it
|
|
118
|
+
# would win and mangle "gs://bucket/file" into PosixPath("gs:/bucket/file").
|
|
119
|
+
annotation=Union[param.annotation, str, pathlib.Path, CloudPath]
|
|
120
|
+
if is_readable(param.annotation)
|
|
121
|
+
else param.annotation
|
|
122
|
+
)
|
|
123
|
+
for param in signature.parameters.values()
|
|
124
|
+
]
|
|
125
|
+
+ _EXTRA_PARAMETERS
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# Two entry points: wrapper takes a Python call's arguments as they are,
|
|
129
|
+
# filling in from a configuration file if one is named, then
|
|
130
|
+
# _run_without_parsing dispatches to compute if asked, else reads the path
|
|
131
|
+
# arguments, calls the function and writes the result. The command line,
|
|
132
|
+
# having parsed argv and the default configuration files with jsonargparse,
|
|
133
|
+
# and the compute provider, given arguments without ``compute``, both enter
|
|
134
|
+
# at _run_without_parsing.
|
|
135
|
+
calling_signature = modified_signature.replace(
|
|
136
|
+
parameters=[*modified_signature.parameters.values(), _CONFIGURATION_PARAMETER]
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def _from_configuration(files, given):
|
|
140
|
+
"""The arguments the configuration ``files`` hold, other than those ``given``."""
|
|
141
|
+
|
|
142
|
+
def dummy():
|
|
143
|
+
pass
|
|
144
|
+
|
|
145
|
+
dummy.__signature__ = modified_signature
|
|
146
|
+
parser = ArgumentParser(exit_on_error=False, default_config_files=files)
|
|
147
|
+
parser.add_function_arguments(dummy)
|
|
148
|
+
try:
|
|
149
|
+
return {
|
|
150
|
+
name: value
|
|
151
|
+
for name, value in vars(parser.instantiate(parser.get_defaults())).items()
|
|
152
|
+
if name not in given
|
|
153
|
+
}
|
|
154
|
+
except Exception as e:
|
|
155
|
+
raise ExpectedException(str(e)) from e
|
|
156
|
+
|
|
157
|
+
@functools.wraps(function)
|
|
158
|
+
def wrapper(*args, **kwargs):
|
|
159
|
+
"""Run with the arguments as given, and ``config``'s for the rest."""
|
|
160
|
+
arguments = dict(calling_signature.bind_partial(*args, **kwargs).arguments)
|
|
161
|
+
files = _configuration_files(arguments.pop(_CONFIGURATION_PARAMETER.name, None))
|
|
162
|
+
if files:
|
|
163
|
+
arguments.update(_from_configuration(files, arguments))
|
|
164
|
+
try:
|
|
165
|
+
bound = modified_signature.bind(**arguments)
|
|
166
|
+
except TypeError as e:
|
|
167
|
+
raise ExpectedException(str(e)) from e
|
|
168
|
+
bound.apply_defaults()
|
|
169
|
+
return _run_without_parsing(**bound.arguments)
|
|
170
|
+
|
|
171
|
+
def _run_without_parsing(**arguments):
|
|
172
|
+
"""Run on the compute asked for, or here: read the files given, write the result."""
|
|
173
|
+
compute = arguments.pop("compute", None)
|
|
174
|
+
if compute is not None:
|
|
175
|
+
return compute.provider.call(wrapper, arguments, compute)
|
|
176
|
+
output_path = arguments.pop("output_path", None)
|
|
177
|
+
with ExitStack() as stack:
|
|
178
|
+
kwargs = {
|
|
179
|
+
name: stack.enter_context(read(arguments[name], param.annotation))
|
|
180
|
+
for name, param in signature.parameters.items()
|
|
181
|
+
}
|
|
182
|
+
result = function(**kwargs)
|
|
183
|
+
if output_path is not None:
|
|
184
|
+
write(result, output_path)
|
|
185
|
+
return output_path
|
|
186
|
+
return result
|
|
187
|
+
|
|
188
|
+
wrapper.__signature__ = calling_signature
|
|
189
|
+
# The CLI parses by this entry's signature and describes it by its docstring
|
|
190
|
+
# (else by its repr, which would name this entry, not the function).
|
|
191
|
+
_run_without_parsing.__doc__ = function.__doc__ or function.__name__
|
|
192
|
+
_run_without_parsing.__signature__ = modified_signature
|
|
193
|
+
wrapper._run_without_parsing = _run_without_parsing
|
|
194
|
+
|
|
195
|
+
def snakemake():
|
|
196
|
+
"""Run this function bound to the surrounding Snakemake rule."""
|
|
197
|
+
from powerfunc.integrations import snakemake as snakemake_integration
|
|
198
|
+
|
|
199
|
+
frame = sys._getframe(1)
|
|
200
|
+
return snakemake_integration.run(wrapper, frame)
|
|
201
|
+
|
|
202
|
+
wrapper.snakemake = snakemake
|
|
203
|
+
|
|
204
|
+
if cli and function.__module__ == "__main__":
|
|
205
|
+
command_line.cli_functions[function.__name__] = _run_without_parsing
|
|
206
|
+
|
|
207
|
+
return wrapper
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import collections.abc
|
|
2
2
|
import contextlib
|
|
3
3
|
import importlib
|
|
4
|
+
import importlib.metadata
|
|
4
5
|
import inspect
|
|
5
6
|
import io
|
|
6
7
|
import json
|
|
@@ -14,7 +15,7 @@ import time
|
|
|
14
15
|
import warnings
|
|
15
16
|
import zipfile
|
|
16
17
|
from dataclasses import dataclass, field
|
|
17
|
-
from typing import Any,
|
|
18
|
+
from typing import Any, Optional
|
|
18
19
|
|
|
19
20
|
import cloudpickle as pickle
|
|
20
21
|
from cloudpathlib import AnyPath
|
|
@@ -31,6 +32,7 @@ from powerfunc.compute import (
|
|
|
31
32
|
Provider,
|
|
32
33
|
user_identifier,
|
|
33
34
|
)
|
|
35
|
+
from powerfunc.decorator import PowerFunc
|
|
34
36
|
|
|
35
37
|
_SUPPORTED_GPUS = {"t4", "a100", "l4", "v100"}
|
|
36
38
|
_MAX_JOB_NAME_LENGTH = 63 # GCP resource name limit (Cloud Run and Batch)
|
|
@@ -67,8 +69,10 @@ def _bootstrap_command(job_spec_path: "AnyPath", error_gcs_path: str = "") -> st
|
|
|
67
69
|
"&& curl -LsSf " + _uv_url + " | sh; }; }"
|
|
68
70
|
)
|
|
69
71
|
# uv run handles everything: ephemeral env, Python download, install, execute.
|
|
72
|
+
# The remote runs the same powerfunc version as the caller.
|
|
73
|
+
powerfunc_version = importlib.metadata.version("powerfunc")
|
|
70
74
|
run_cmd = (
|
|
71
|
-
f'uv run --with powerfunc --with "pydantic>=2" '
|
|
75
|
+
f'uv run --with "powerfunc=={powerfunc_version}" --with "pydantic>=2" '
|
|
72
76
|
f"python -m powerfunc.providers.gcp_cloud_run {job_spec_path}"
|
|
73
77
|
)
|
|
74
78
|
main_cmd = get_uv + " && " + run_cmd
|
|
@@ -250,7 +254,7 @@ class GCPCloudRunProvider(Provider):
|
|
|
250
254
|
pass
|
|
251
255
|
|
|
252
256
|
@staticmethod
|
|
253
|
-
def _extra_sys_paths(function:
|
|
257
|
+
def _extra_sys_paths(function: PowerFunc, repo_root: Optional[pathlib.Path]) -> list[str]:
|
|
254
258
|
"""Relative directories that must be added to sys.path on the remote.
|
|
255
259
|
|
|
256
260
|
Compares the function's source file to its module name to discover
|
|
@@ -260,7 +264,7 @@ class GCPCloudRunProvider(Provider):
|
|
|
260
264
|
if repo_root is None:
|
|
261
265
|
return []
|
|
262
266
|
try:
|
|
263
|
-
src = pathlib.Path(inspect.getfile(function)).resolve()
|
|
267
|
+
src = pathlib.Path(inspect.getfile(inspect.unwrap(function))).resolve()
|
|
264
268
|
rel = src.relative_to(repo_root)
|
|
265
269
|
except (TypeError, ValueError):
|
|
266
270
|
return []
|
|
@@ -275,7 +279,7 @@ class GCPCloudRunProvider(Provider):
|
|
|
275
279
|
return [prefix]
|
|
276
280
|
return []
|
|
277
281
|
|
|
278
|
-
def call(self, function:
|
|
282
|
+
def call(self, function: PowerFunc, arguments: dict, compute: ComputeSpecification) -> Any:
|
|
279
283
|
if function.__module__ == "__main__":
|
|
280
284
|
spec = getattr(__main__, "__spec__", None)
|
|
281
285
|
if spec is None:
|
|
@@ -378,7 +382,7 @@ if __name__ == "__main__":
|
|
|
378
382
|
func: Any = importlib.import_module(module_path)
|
|
379
383
|
for part in qualname.split("."):
|
|
380
384
|
func = getattr(func, part)
|
|
381
|
-
result = func(**kwargs)
|
|
385
|
+
result = func._run_without_parsing(**kwargs)
|
|
382
386
|
|
|
383
387
|
AnyPath(spec["output_path"]).write_bytes(pickle.dumps(result))
|
|
384
388
|
except BaseException as exc:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from dataclasses import dataclass, field
|
|
2
|
-
from typing import Any
|
|
2
|
+
from typing import Any
|
|
3
3
|
|
|
4
4
|
from pydantic.dataclasses import dataclass as pydantic_dataclass
|
|
5
5
|
|
|
@@ -12,6 +12,7 @@ from powerfunc.compute import (
|
|
|
12
12
|
MemorySize,
|
|
13
13
|
Provider,
|
|
14
14
|
)
|
|
15
|
+
from powerfunc.decorator import PowerFunc
|
|
15
16
|
|
|
16
17
|
try:
|
|
17
18
|
import modal
|
|
@@ -22,36 +23,38 @@ try:
|
|
|
22
23
|
|
|
23
24
|
pip_packages: tuple[str, ...] = ()
|
|
24
25
|
|
|
25
|
-
def
|
|
26
|
+
def image(self, compute: ComputeSpecification) -> modal.Image:
|
|
27
|
+
"""The image the function runs in: ``compute.image`` or Debian slim, with
|
|
28
|
+
``pip_packages`` installed. Override to build one another way, e.g. from a
|
|
29
|
+
conda environment, or with the local codebase added."""
|
|
26
30
|
image = modal.Image.debian_slim()
|
|
27
31
|
if compute.image:
|
|
28
32
|
image = modal.Image.from_registry(compute.image)
|
|
29
33
|
if self.pip_packages:
|
|
30
34
|
image = image.pip_install(*self.pip_packages)
|
|
35
|
+
return image
|
|
36
|
+
|
|
37
|
+
def secrets(self) -> list[modal.Secret]:
|
|
38
|
+
"""Modal secrets to expose to the function as environment variables."""
|
|
39
|
+
return []
|
|
40
|
+
|
|
41
|
+
def call(self, function: PowerFunc, arguments: dict, compute: ComputeSpecification) -> Any:
|
|
31
42
|
app = modal.App(name=f"powerfunc-{function.__name__}")
|
|
32
43
|
|
|
33
44
|
@app.function(
|
|
34
|
-
image=image,
|
|
45
|
+
image=self.image(compute),
|
|
35
46
|
cpu=compute.cpu,
|
|
36
47
|
memory=compute.memory,
|
|
37
48
|
gpu=compute.gpu,
|
|
49
|
+
timeout=int(compute.timeout),
|
|
50
|
+
secrets=self.secrets(),
|
|
38
51
|
serialized=True,
|
|
39
52
|
)
|
|
40
|
-
def runner(fn:
|
|
41
|
-
return fn(**args)
|
|
42
|
-
|
|
43
|
-
output_path = arguments.pop("output_path", None)
|
|
53
|
+
def runner(fn: PowerFunc, args: dict) -> Any:
|
|
54
|
+
return fn._run_without_parsing(**args)
|
|
44
55
|
|
|
45
56
|
with app.run():
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if output_path is not None:
|
|
49
|
-
from powerfunc.conversions import write
|
|
50
|
-
|
|
51
|
-
write(result, output_path)
|
|
52
|
-
return output_path
|
|
53
|
-
|
|
54
|
-
return result
|
|
57
|
+
return runner.remote(function, arguments)
|
|
55
58
|
|
|
56
59
|
@pydantic_dataclass
|
|
57
60
|
class ModalCpuSmall(ComputeSpecification):
|
{powerfunc-0.3.0/tests/powerfunc_tests/gcp → powerfunc-0.4.0/tests/powerfunc_tests}/conftest.py
RENAMED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
"""Shared pytest fixtures for GCP provider tests."""
|
|
2
|
-
|
|
3
1
|
import json
|
|
4
2
|
import os
|
|
5
3
|
import tempfile
|
|
@@ -9,16 +7,19 @@ import pytest
|
|
|
9
7
|
|
|
10
8
|
@pytest.fixture(scope="module")
|
|
11
9
|
def gcp_credentials():
|
|
12
|
-
"""Validate the service-account JSON and expose it
|
|
13
|
-
|
|
10
|
+
"""Validate the service-account JSON in ``GOOGLE_CLOUD_API_KEY`` and expose it
|
|
11
|
+
via ADC (``GOOGLE_APPLICATION_CREDENTIALS``) for the duration of the module.
|
|
14
12
|
|
|
13
|
+
Fails (does not skip) if the provided key is not a service-account JSON: GCP
|
|
14
|
+
Batch, Cloud Run Jobs and GCS need service-account credentials, not a bare API
|
|
15
|
+
key.
|
|
16
|
+
"""
|
|
15
17
|
try:
|
|
16
|
-
info = json.loads(
|
|
18
|
+
info = json.loads(os.environ["GOOGLE_CLOUD_API_KEY"])
|
|
17
19
|
except json.JSONDecodeError as error:
|
|
18
20
|
raise ValueError(
|
|
19
21
|
"GOOGLE_CLOUD_API_KEY is set but is not valid JSON. It must be a GCP "
|
|
20
|
-
"service-account JSON key
|
|
21
|
-
"service-account credentials, not a bare API key)."
|
|
22
|
+
"service-account JSON key."
|
|
22
23
|
) from error
|
|
23
24
|
|
|
24
25
|
if not isinstance(info, dict) or info.get("type") != "service_account":
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Test package for powerfunc.
|
|
2
|
+
|
|
3
|
+
``cli_example`` is imported here, as a package's ``__init__`` commonly
|
|
4
|
+
imports its modules, so that ``python -m powerfunc_tests.core.cli_example``
|
|
5
|
+
runs it as ``__main__`` while the same file is also loaded under its package
|
|
6
|
+
name. ``enable_cli`` must still register the command line in that case: it
|
|
7
|
+
reads ``__name__`` from the calling module's globals, which is ``"__main__"``
|
|
8
|
+
for the module being run, rather than asking ``inspect.getmodule``, which would
|
|
9
|
+
return the package-named copy.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from . import cli_example # noqa: F401
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Command line of three functions: two reading a file, one showing its defaults' layering."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from powerfunc import powerfunc
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@powerfunc
|
|
11
|
+
def sum_col(df: pd.DataFrame) -> float:
|
|
12
|
+
return float(df["value"].sum())
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@powerfunc
|
|
16
|
+
def mean_col(df: pd.DataFrame) -> float:
|
|
17
|
+
return float(df["value"].mean())
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@powerfunc(cli=False)
|
|
21
|
+
def ignored(df: pd.DataFrame) -> float:
|
|
22
|
+
return 0.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Spec:
|
|
27
|
+
cpu: int = 1
|
|
28
|
+
memory: int = 2
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
DEFAULT_SPEC = Spec(cpu=3)
|
|
32
|
+
DEFAULT_RANGES = {"a": 1.0}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@powerfunc
|
|
36
|
+
def show(spec: Spec = DEFAULT_SPEC, ranges: dict[str, float] = DEFAULT_RANGES) -> str:
|
|
37
|
+
return f"{spec.cpu} {spec.memory} {type(ranges).__name__} {dict(ranges)}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
powerfunc.enable_cli()
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Defaults, then configuration file, then what the call gives, from either entry point.
|
|
2
|
+
|
|
3
|
+
The command line reads ``configuration/powerfunc.yaml`` and ``powerfunc.yaml``
|
|
4
|
+
from the working directory, or ``--config`` instead, and argv overwrites fields
|
|
5
|
+
and keys of what they hold. A call from Python reads no file unless ``config``
|
|
6
|
+
names one, or is True for those defaults -- else a stray file in the working
|
|
7
|
+
directory would change any call's arguments -- and its own arguments win.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import pathlib
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from powerfunc import powerfunc
|
|
17
|
+
|
|
18
|
+
SCRIPT = pathlib.Path(__file__).parent / "cli_example.py"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run(*args):
|
|
22
|
+
result = subprocess.run([sys.executable, SCRIPT, "show", *args], capture_output=True, text=True)
|
|
23
|
+
assert result.returncode == 0, result.stderr
|
|
24
|
+
return result.stdout.strip()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_defaults():
|
|
28
|
+
assert run() == "3 2 dict {'a': 1.0}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_command_line_overwrites_dataclass_field_of_default():
|
|
32
|
+
assert run("--spec.memory", "8") == "3 8 dict {'a': 1.0}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_command_line_overwrites_dict_key_of_default():
|
|
36
|
+
assert run("--ranges.b", "2") == "3 2 dict {'a': 1.0, 'b': 2.0}"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_config_file_then_command_line(tmp_path):
|
|
40
|
+
config = tmp_path / "config.yaml"
|
|
41
|
+
config.write_text("spec:\n cpu: 5\nranges:\n a: 3.0\n c: 4.0\n")
|
|
42
|
+
assert run("--config", str(config)) == "5 2 dict {'a': 3.0, 'c': 4.0}"
|
|
43
|
+
assert run("--config", str(config), "--spec.memory", "8", "--ranges.a", "9") == (
|
|
44
|
+
"5 8 dict {'a': 9.0, 'c': 4.0}"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_runs_as_a_module_its_package_already_imported():
|
|
49
|
+
result = subprocess.run(
|
|
50
|
+
[sys.executable, "-m", "powerfunc_tests.core.cli_example", "show", "--ranges.b", "2"],
|
|
51
|
+
capture_output=True,
|
|
52
|
+
text=True,
|
|
53
|
+
cwd=SCRIPT.parents[2],
|
|
54
|
+
)
|
|
55
|
+
assert result.returncode == 0, result.stderr
|
|
56
|
+
assert result.stdout.strip() == "3 2 dict {'a': 1.0, 'b': 2.0}"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@powerfunc
|
|
60
|
+
def scaled(value: float, *, scale: float = 1.0) -> float:
|
|
61
|
+
return value * scale
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.fixture
|
|
65
|
+
def default_configuration(tmp_path, monkeypatch):
|
|
66
|
+
monkeypatch.chdir(tmp_path)
|
|
67
|
+
(tmp_path / "powerfunc.yaml").write_text("scale: 10\n")
|
|
68
|
+
return tmp_path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_python_call_ignores_default_files(default_configuration):
|
|
72
|
+
assert scaled(2.0) == 2.0
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_python_call_config_true_reads_default_files(default_configuration):
|
|
76
|
+
assert scaled(2.0, config=True) == 20.0
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_python_call_config_path_reads_that_file(tmp_path):
|
|
80
|
+
configuration = tmp_path / "elsewhere.yaml"
|
|
81
|
+
configuration.write_text("scale: 3\n")
|
|
82
|
+
assert scaled(2.0, config=configuration) == 6.0
|
|
83
|
+
assert scaled(2.0, config=str(configuration)) == 6.0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_python_call_arguments_win_over_the_file(default_configuration):
|
|
87
|
+
assert scaled(2.0, scale=5.0, config=True) == 10.0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_a_function_may_not_name_a_parameter_powerfunc_adds():
|
|
91
|
+
with pytest.raises(TypeError, match="config"):
|
|
92
|
+
|
|
93
|
+
@powerfunc
|
|
94
|
+
def clashing(config: str) -> str:
|
|
95
|
+
return config
|