powerfunc 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {powerfunc-0.3.0 → powerfunc-0.4.0}/.github/workflows/ci.yml +2 -0
  2. {powerfunc-0.3.0 → powerfunc-0.4.0}/PKG-INFO +3 -3
  3. {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/readme.md +7 -0
  4. {powerfunc-0.3.0 → powerfunc-0.4.0}/pyproject.toml +2 -2
  5. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/command_line.py +7 -8
  6. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/compute.py +12 -2
  7. powerfunc-0.4.0/src/powerfunc/decorator.py +207 -0
  8. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/gcp_cloud_run.py +10 -6
  9. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/modal.py +19 -16
  10. {powerfunc-0.3.0/tests/powerfunc_tests/gcp → powerfunc-0.4.0/tests/powerfunc_tests}/conftest.py +8 -7
  11. powerfunc-0.4.0/tests/powerfunc_tests/core/__init__.py +12 -0
  12. powerfunc-0.4.0/tests/powerfunc_tests/core/cli_example.py +40 -0
  13. powerfunc-0.4.0/tests/powerfunc_tests/core/test_configuration.py +95 -0
  14. powerfunc-0.4.0/tests/powerfunc_tests/core/test_future_annotations.py +27 -0
  15. powerfunc-0.4.0/tests/powerfunc_tests/core/test_given_objects.py +90 -0
  16. powerfunc-0.4.0/tests/powerfunc_tests/gcp/test_gcp_batch.py +100 -0
  17. powerfunc-0.4.0/tests/powerfunc_tests/gcp/test_gcp_cloud_run.py +97 -0
  18. powerfunc-0.4.0/tests/powerfunc_tests/modal/test_modal.py +73 -0
  19. powerfunc-0.3.0/tests/powerfunc_tests/gcp/gcp_jobs.py → powerfunc-0.4.0/tests/powerfunc_tests/remote_jobs.py +16 -3
  20. {powerfunc-0.3.0 → powerfunc-0.4.0}/tox.ini +3 -1
  21. {powerfunc-0.3.0 → powerfunc-0.4.0}/uv.lock +32 -4
  22. powerfunc-0.3.0/src/powerfunc/decorator.py +0 -116
  23. powerfunc-0.3.0/tests/powerfunc_tests/core/cli_example.py +0 -21
  24. powerfunc-0.3.0/tests/powerfunc_tests/gcp/test_gcp_batch.py +0 -70
  25. powerfunc-0.3.0/tests/powerfunc_tests/gcp/test_gcp_cloud_run.py +0 -52
  26. powerfunc-0.3.0/tests/powerfunc_tests/modal/test_modal.py +0 -44
  27. powerfunc-0.3.0/tests/powerfunc_tests/snakemake/__init__.py +0 -0
  28. {powerfunc-0.3.0 → powerfunc-0.4.0}/.gitignore +0 -0
  29. {powerfunc-0.3.0 → powerfunc-0.4.0}/.pre-commit-config.yaml +0 -0
  30. {powerfunc-0.3.0 → powerfunc-0.4.0}/LICENSE.txt +0 -0
  31. {powerfunc-0.3.0 → powerfunc-0.4.0}/README.md +0 -0
  32. {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/advanced_usage.md +0 -0
  33. {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/integrations/snakemake.md +0 -0
  34. {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/providers/gcp.md +0 -0
  35. {powerfunc-0.3.0 → powerfunc-0.4.0}/documentation/providers/modal.md +0 -0
  36. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/__init__.py +0 -0
  37. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/configuration.py +0 -0
  38. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/conversions.py +0 -0
  39. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/__init__.py +0 -0
  40. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/arrow.py +0 -0
  41. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/csv_reader.py +0 -0
  42. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/dask.py +0 -0
  43. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/pandas.py +0 -0
  44. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/formats/polars.py +0 -0
  45. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/integrations/__init__.py +0 -0
  46. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/integrations/snakemake.py +0 -0
  47. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/__init__.py +0 -0
  48. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/providers/gcp_batch.py +0 -0
  49. {powerfunc-0.3.0 → powerfunc-0.4.0}/src/powerfunc/py.typed +0 -0
  50. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/__init__.py +0 -0
  51. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_basic.py +0 -0
  52. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_cli.py +0 -0
  53. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/core/test_conversions.py +0 -0
  54. /powerfunc-0.3.0/tests/powerfunc_tests/core/test_remote.py → /powerfunc-0.4.0/tests/powerfunc_tests/core/test_remote_files.py +0 -0
  55. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.arrow +0 -0
  56. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.csv +0 -0
  57. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.feather +0 -0
  58. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.pandas.json +0 -0
  59. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.parquet +0 -0
  60. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.polars.json +0 -0
  61. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/data.xlsx +0 -0
  62. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/data/generate_data_files.py +0 -0
  63. {powerfunc-0.3.0/tests/powerfunc_tests/core → powerfunc-0.4.0/tests/powerfunc_tests/gcp}/__init__.py +0 -0
  64. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/gcp/gcp_shared.py +0 -0
  65. {powerfunc-0.3.0/tests/powerfunc_tests/gcp → powerfunc-0.4.0/tests/powerfunc_tests/modal}/__init__.py +0 -0
  66. {powerfunc-0.3.0/tests/powerfunc_tests/modal → powerfunc-0.4.0/tests/powerfunc_tests/snakemake}/__init__.py +0 -0
  67. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/functions.py +0 -0
  68. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/input.smk +0 -0
  69. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/input_params_and_output.smk +0 -0
  70. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/multiple_outputs_rejected.smk +0 -0
  71. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/named_outputs_rejected.smk +0 -0
  72. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/params.smk +0 -0
  73. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/positional_rejected.smk +0 -0
  74. {powerfunc-0.3.0 → powerfunc-0.4.0}/tests/powerfunc_tests/snakemake/test_snakemake.py +0 -0
@@ -58,6 +58,8 @@ jobs:
58
58
  enable-cache: true
59
59
  - name: Run Modal tests
60
60
  env:
61
+ GOOGLE_CLOUD_API_KEY: ${{ secrets.GOOGLE_CLOUD_API_KEY }}
62
+ GCP_BUCKET: gs://powerfunc-temporary-348293
61
63
  MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
62
64
  MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
63
65
  run: uvx --with tox-uv tox run -e modal
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: powerfunc
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Make functions run however you like — CLI, pipeline, or remote compute
5
5
  Project-URL: Homepage, https://github.com/ddrakard/powerfunc
6
6
  Project-URL: Repository, https://github.com/ddrakard/powerfunc
@@ -10,7 +10,7 @@ Requires-Python: >=3.9
10
10
  Requires-Dist: cloudpathlib[all]
11
11
  Requires-Dist: cloudpickle
12
12
  Requires-Dist: fsspec[adl,gcs,s3]
13
- Requires-Dist: jsonargparse>=4.49.0
13
+ Requires-Dist: jsonargparse[signatures]>=4.49.0
14
14
  Requires-Dist: pydantic>=2
15
15
  Provides-Extra: dev
16
16
  Requires-Dist: pre-commit; extra == 'dev'
@@ -183,6 +183,13 @@ When using the CLI, pass a config file with `--config`:
183
183
  python my_script.py sum_col data.csv --config my_config.yaml
184
184
  ```
185
185
 
186
+ The command line always reads the configuration files. A call from Python reads none unless asked, with `config`, the same option as `--config`: a path to a file, or `True` for the default files.
187
+
188
+ ```python
189
+ sum_col("data.csv", config=True)
190
+ sum_col("data.csv", config="my_config.yaml")
191
+ ```
192
+
186
193
  ## Supported formats
187
194
 
188
195
  | Type | Formats |
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "powerfunc"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "Make functions run however you like — CLI, pipeline, or remote compute"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -12,7 +12,7 @@ dependencies = [
12
12
  "pydantic>=2",
13
13
  "cloudpathlib[all]",
14
14
  "fsspec[gcs,s3,adl]",
15
- "jsonargparse>=4.49.0",
15
+ "jsonargparse[signatures]>=4.49.0",
16
16
  "cloudpickle",
17
17
  ]
18
18
 
@@ -1,5 +1,6 @@
1
1
  import inspect
2
2
  import sys
3
+ from collections.abc import Callable
3
4
 
4
5
  from jsonargparse import CLI
5
6
 
@@ -10,8 +11,8 @@ class ExpectedException(Exception):
10
11
  """Raised for known user errors that should display just the message, not a traceback."""
11
12
 
12
13
 
13
- cli_functions: list[tuple] = []
14
- _cli_active = False
14
+ # Each function's parsed-arguments entry, by the function's name: the command's.
15
+ cli_functions: dict[str, Callable] = {}
15
16
 
16
17
 
17
18
  def enable_cli():
@@ -20,17 +21,15 @@ def enable_cli():
20
21
  frame = inspect.currentframe()
21
22
  if frame is None or frame.f_back is None:
22
23
  raise ExpectedException("Frame inspection not supported by this Python implementation.")
23
- caller_module = inspect.getmodule(frame.f_back)
24
- if caller_module is None or caller_module.__name__ != "__main__":
24
+ # By the frame's globals, not inspect.getmodule: under ``python -m`` the same
25
+ # file is often imported by its package name too, and getmodule finds that one.
26
+ if frame.f_back.f_globals.get("__name__") != "__main__":
25
27
  return
26
28
  if not cli_functions:
27
29
  raise ExpectedException("No @powerfunc functions registered for CLI.")
28
30
 
29
- global _cli_active
30
- wrappers = [wrapper for wrapper, _ in cli_functions]
31
- _cli_active = True
32
31
  result = CLI(
33
- wrappers[0] if len(wrappers) == 1 else wrappers,
32
+ next(iter(cli_functions.values())) if len(cli_functions) == 1 else cli_functions,
34
33
  default_config_files=configuration_paths,
35
34
  )
36
35
  if result is not None:
@@ -1,5 +1,5 @@
1
1
  import dataclasses
2
- from typing import Annotated, Any, Callable, Optional, TypeAlias
2
+ from typing import TYPE_CHECKING, Annotated, Any, Optional, TypeAlias
3
3
 
4
4
  from pydantic import Field, GetCoreSchemaHandler
5
5
  from pydantic.dataclasses import dataclass as pydantic_dataclass
@@ -13,6 +13,9 @@ MemorySize: TypeAlias = Annotated[int, Field(gt=0, description="RAM in MB")]
13
13
  DockerImageUri: TypeAlias = Annotated[str, Field(description="Container image URI")]
14
14
  GpuModel: TypeAlias = Annotated[Optional[str], Field(description="GPU model name")]
15
15
 
16
+ if TYPE_CHECKING:
17
+ from powerfunc.decorator import PowerFunc
18
+
16
19
 
17
20
  class Provider:
18
21
  """Base compute provider. Subclass and implement call()."""
@@ -23,7 +26,14 @@ class Provider:
23
26
  ) -> core_schema.CoreSchema:
24
27
  return core_schema.is_instance_schema(cls)
25
28
 
26
- def call(self, function: Callable, arguments: dict, spec: "ComputeSpecification") -> Any:
29
+ def call(self, function: "PowerFunc", arguments: dict, spec: "ComputeSpecification") -> Any:
30
+ """Run ``function._run_without_parsing(**arguments)`` on the compute, returning its result.
31
+
32
+ ``function`` is the powerfunc function itself, picklable by its name.
33
+ ``arguments`` have already been parsed and merged with the configuration
34
+ files, so the compute does neither; there, ``_run_without_parsing`` opens
35
+ the files given for parameters and writes the result to ``output_path``.
36
+ """
27
37
  raise NotImplementedError
28
38
 
29
39
 
@@ -0,0 +1,207 @@
1
+ import functools
2
+ import inspect
3
+ import pathlib
4
+ import sys
5
+ from collections.abc import Callable
6
+ from contextlib import ExitStack
7
+ from typing import Any, Optional, Protocol, TypeVar, Union, overload
8
+
9
+ from cloudpathlib import CloudPath
10
+ from jsonargparse import ArgumentParser
11
+
12
+ import powerfunc.command_line as command_line
13
+ from powerfunc.command_line import ExpectedException
14
+ from powerfunc.compute import ComputeSpecification
15
+ from powerfunc.configuration import configuration_paths
16
+ from powerfunc.conversions import is_readable, read, write
17
+
18
+ _EXTRA_PARAMETERS = [
19
+ inspect.Parameter(
20
+ "compute",
21
+ inspect.Parameter.KEYWORD_ONLY,
22
+ default=None,
23
+ annotation=Optional[ComputeSpecification],
24
+ ),
25
+ inspect.Parameter(
26
+ "output_path",
27
+ inspect.Parameter.KEYWORD_ONLY,
28
+ default=None,
29
+ annotation=Optional[str],
30
+ ),
31
+ ]
32
+ # The same option as the command line's ``--config``: a configuration file to
33
+ # merge the arguments with, or True for the default files, which the command
34
+ # line reads unless told another. A Python call reads none unless asked.
35
+ _CONFIGURATION_PARAMETER = inspect.Parameter(
36
+ "config",
37
+ inspect.Parameter.KEYWORD_ONLY,
38
+ default=None,
39
+ annotation=Union[str, pathlib.Path, bool, None],
40
+ )
41
+
42
+
43
+ def _configuration_files(config) -> list[str]:
44
+ if config is None or config is False:
45
+ return []
46
+ if config is True:
47
+ return configuration_paths
48
+ return [str(config)]
49
+
50
+
51
+ R = TypeVar("R", covariant=True)
52
+
53
+
54
+ class PowerFunc(Protocol[R]):
55
+ """What :func:`powerfunc` makes of a function.
56
+
57
+ Called as the function was, plus ``compute`` to run it elsewhere and
58
+ ``output_path`` to write what it returns to a file; the function's own
59
+ parameters are read from its signature at run time, so they are not checked
60
+ statically here.
61
+ """
62
+
63
+ def __call__(
64
+ self,
65
+ *args: Any,
66
+ compute: Optional[ComputeSpecification] = None,
67
+ output_path: Optional[str] = None,
68
+ config: Union[str, pathlib.Path, bool, None] = None,
69
+ **kwargs: Any,
70
+ ) -> R: ...
71
+
72
+ def snakemake(self) -> Any:
73
+ """Run this function bound to the surrounding Snakemake rule."""
74
+ ...
75
+
76
+ def _run_without_parsing(self, **arguments: Any) -> R:
77
+ """Run with arguments already parsed: by the command line, or before dispatch to compute."""
78
+ ...
79
+
80
+
81
+ def optional_arguments_decorator(decorator):
82
+ """Allows a decorator to be applied with or without argument parentheses."""
83
+
84
+ @functools.wraps(decorator)
85
+ def wrapper(function=None, **kwargs):
86
+ if function is not None:
87
+ return decorator(function, **kwargs)
88
+ return functools.partial(decorator, **kwargs)
89
+
90
+ return wrapper
91
+
92
+
93
+ @overload
94
+ def powerfunc(function: Callable[..., R], *, cli: bool = True) -> PowerFunc[R]: ...
95
+ @overload
96
+ def powerfunc(
97
+ function: None = None, *, cli: bool = True
98
+ ) -> Callable[[Callable[..., R]], PowerFunc[R]]: ...
99
+ @optional_arguments_decorator
100
+ def powerfunc(function=None, *, cli=True):
101
+ """Decorator to turn a function into a powerfunc function.
102
+
103
+ Applied bare, ``@powerfunc``, or with arguments, ``@powerfunc(cli=False)``.
104
+ """
105
+ # eval_str=True required for `from __future__ import annotations`
106
+ signature = inspect.signature(function, eval_str=True)
107
+ reserved = {param.name for param in [*_EXTRA_PARAMETERS, _CONFIGURATION_PARAMETER]}
108
+ taken = reserved & signature.parameters.keys()
109
+ if taken:
110
+ raise TypeError(
111
+ f"{function.__name__} has parameters powerfunc adds itself: {', '.join(sorted(taken))}."
112
+ )
113
+ modified_signature = signature.replace(
114
+ parameters=[
115
+ param.replace(
116
+ # str must precede pathlib.Path and CloudPath: jsonargparse resolves Union
117
+ # subtypes in order and pathlib.Path accepts any string without error, so it
118
+ # would win and mangle "gs://bucket/file" into PosixPath("gs:/bucket/file").
119
+ annotation=Union[param.annotation, str, pathlib.Path, CloudPath]
120
+ if is_readable(param.annotation)
121
+ else param.annotation
122
+ )
123
+ for param in signature.parameters.values()
124
+ ]
125
+ + _EXTRA_PARAMETERS
126
+ )
127
+
128
+ # Two entry points: wrapper takes a Python call's arguments as they are,
129
+ # filling in from a configuration file if one is named, then
130
+ # _run_without_parsing dispatches to compute if asked, else reads the path
131
+ # arguments, calls the function and writes the result. The command line,
132
+ # having parsed argv and the default configuration files with jsonargparse,
133
+ # and the compute provider, given arguments without ``compute``, both enter
134
+ # at _run_without_parsing.
135
+ calling_signature = modified_signature.replace(
136
+ parameters=[*modified_signature.parameters.values(), _CONFIGURATION_PARAMETER]
137
+ )
138
+
139
+ def _from_configuration(files, given):
140
+ """The arguments the configuration ``files`` hold, other than those ``given``."""
141
+
142
+ def dummy():
143
+ pass
144
+
145
+ dummy.__signature__ = modified_signature
146
+ parser = ArgumentParser(exit_on_error=False, default_config_files=files)
147
+ parser.add_function_arguments(dummy)
148
+ try:
149
+ return {
150
+ name: value
151
+ for name, value in vars(parser.instantiate(parser.get_defaults())).items()
152
+ if name not in given
153
+ }
154
+ except Exception as e:
155
+ raise ExpectedException(str(e)) from e
156
+
157
+ @functools.wraps(function)
158
+ def wrapper(*args, **kwargs):
159
+ """Run with the arguments as given, and ``config``'s for the rest."""
160
+ arguments = dict(calling_signature.bind_partial(*args, **kwargs).arguments)
161
+ files = _configuration_files(arguments.pop(_CONFIGURATION_PARAMETER.name, None))
162
+ if files:
163
+ arguments.update(_from_configuration(files, arguments))
164
+ try:
165
+ bound = modified_signature.bind(**arguments)
166
+ except TypeError as e:
167
+ raise ExpectedException(str(e)) from e
168
+ bound.apply_defaults()
169
+ return _run_without_parsing(**bound.arguments)
170
+
171
+ def _run_without_parsing(**arguments):
172
+ """Run on the compute asked for, or here: read the files given, write the result."""
173
+ compute = arguments.pop("compute", None)
174
+ if compute is not None:
175
+ return compute.provider.call(wrapper, arguments, compute)
176
+ output_path = arguments.pop("output_path", None)
177
+ with ExitStack() as stack:
178
+ kwargs = {
179
+ name: stack.enter_context(read(arguments[name], param.annotation))
180
+ for name, param in signature.parameters.items()
181
+ }
182
+ result = function(**kwargs)
183
+ if output_path is not None:
184
+ write(result, output_path)
185
+ return output_path
186
+ return result
187
+
188
+ wrapper.__signature__ = calling_signature
189
+ # The CLI parses by this entry's signature and describes it by its docstring
190
+ # (else by its repr, which would name this entry, not the function).
191
+ _run_without_parsing.__doc__ = function.__doc__ or function.__name__
192
+ _run_without_parsing.__signature__ = modified_signature
193
+ wrapper._run_without_parsing = _run_without_parsing
194
+
195
+ def snakemake():
196
+ """Run this function bound to the surrounding Snakemake rule."""
197
+ from powerfunc.integrations import snakemake as snakemake_integration
198
+
199
+ frame = sys._getframe(1)
200
+ return snakemake_integration.run(wrapper, frame)
201
+
202
+ wrapper.snakemake = snakemake
203
+
204
+ if cli and function.__module__ == "__main__":
205
+ command_line.cli_functions[function.__name__] = _run_without_parsing
206
+
207
+ return wrapper
@@ -1,6 +1,7 @@
1
1
  import collections.abc
2
2
  import contextlib
3
3
  import importlib
4
+ import importlib.metadata
4
5
  import inspect
5
6
  import io
6
7
  import json
@@ -14,7 +15,7 @@ import time
14
15
  import warnings
15
16
  import zipfile
16
17
  from dataclasses import dataclass, field
17
- from typing import Any, Callable, Optional
18
+ from typing import Any, Optional
18
19
 
19
20
  import cloudpickle as pickle
20
21
  from cloudpathlib import AnyPath
@@ -31,6 +32,7 @@ from powerfunc.compute import (
31
32
  Provider,
32
33
  user_identifier,
33
34
  )
35
+ from powerfunc.decorator import PowerFunc
34
36
 
35
37
  _SUPPORTED_GPUS = {"t4", "a100", "l4", "v100"}
36
38
  _MAX_JOB_NAME_LENGTH = 63 # GCP resource name limit (Cloud Run and Batch)
@@ -67,8 +69,10 @@ def _bootstrap_command(job_spec_path: "AnyPath", error_gcs_path: str = "") -> st
67
69
  "&& curl -LsSf " + _uv_url + " | sh; }; }"
68
70
  )
69
71
  # uv run handles everything: ephemeral env, Python download, install, execute.
72
+ # The remote runs the same powerfunc version as the caller.
73
+ powerfunc_version = importlib.metadata.version("powerfunc")
70
74
  run_cmd = (
71
- f'uv run --with powerfunc --with "pydantic>=2" '
75
+ f'uv run --with "powerfunc=={powerfunc_version}" --with "pydantic>=2" '
72
76
  f"python -m powerfunc.providers.gcp_cloud_run {job_spec_path}"
73
77
  )
74
78
  main_cmd = get_uv + " && " + run_cmd
@@ -250,7 +254,7 @@ class GCPCloudRunProvider(Provider):
250
254
  pass
251
255
 
252
256
  @staticmethod
253
- def _extra_sys_paths(function: Callable, repo_root: Optional[pathlib.Path]) -> list[str]:
257
+ def _extra_sys_paths(function: PowerFunc, repo_root: Optional[pathlib.Path]) -> list[str]:
254
258
  """Relative directories that must be added to sys.path on the remote.
255
259
 
256
260
  Compares the function's source file to its module name to discover
@@ -260,7 +264,7 @@ class GCPCloudRunProvider(Provider):
260
264
  if repo_root is None:
261
265
  return []
262
266
  try:
263
- src = pathlib.Path(inspect.getfile(function)).resolve()
267
+ src = pathlib.Path(inspect.getfile(inspect.unwrap(function))).resolve()
264
268
  rel = src.relative_to(repo_root)
265
269
  except (TypeError, ValueError):
266
270
  return []
@@ -275,7 +279,7 @@ class GCPCloudRunProvider(Provider):
275
279
  return [prefix]
276
280
  return []
277
281
 
278
- def call(self, function: Callable, arguments: dict, compute: ComputeSpecification) -> Any:
282
+ def call(self, function: PowerFunc, arguments: dict, compute: ComputeSpecification) -> Any:
279
283
  if function.__module__ == "__main__":
280
284
  spec = getattr(__main__, "__spec__", None)
281
285
  if spec is None:
@@ -378,7 +382,7 @@ if __name__ == "__main__":
378
382
  func: Any = importlib.import_module(module_path)
379
383
  for part in qualname.split("."):
380
384
  func = getattr(func, part)
381
- result = func(**kwargs)
385
+ result = func._run_without_parsing(**kwargs)
382
386
 
383
387
  AnyPath(spec["output_path"]).write_bytes(pickle.dumps(result))
384
388
  except BaseException as exc:
@@ -1,5 +1,5 @@
1
1
  from dataclasses import dataclass, field
2
- from typing import Any, Callable
2
+ from typing import Any
3
3
 
4
4
  from pydantic.dataclasses import dataclass as pydantic_dataclass
5
5
 
@@ -12,6 +12,7 @@ from powerfunc.compute import (
12
12
  MemorySize,
13
13
  Provider,
14
14
  )
15
+ from powerfunc.decorator import PowerFunc
15
16
 
16
17
  try:
17
18
  import modal
@@ -22,36 +23,38 @@ try:
22
23
 
23
24
  pip_packages: tuple[str, ...] = ()
24
25
 
25
- def call(self, function: Callable, arguments: dict, compute: ComputeSpecification) -> Any:
26
+ def image(self, compute: ComputeSpecification) -> modal.Image:
27
+ """The image the function runs in: ``compute.image`` or Debian slim, with
28
+ ``pip_packages`` installed. Override to build one another way, e.g. from a
29
+ conda environment, or with the local codebase added."""
26
30
  image = modal.Image.debian_slim()
27
31
  if compute.image:
28
32
  image = modal.Image.from_registry(compute.image)
29
33
  if self.pip_packages:
30
34
  image = image.pip_install(*self.pip_packages)
35
+ return image
36
+
37
+ def secrets(self) -> list[modal.Secret]:
38
+ """Modal secrets to expose to the function as environment variables."""
39
+ return []
40
+
41
+ def call(self, function: PowerFunc, arguments: dict, compute: ComputeSpecification) -> Any:
31
42
  app = modal.App(name=f"powerfunc-{function.__name__}")
32
43
 
33
44
  @app.function(
34
- image=image,
45
+ image=self.image(compute),
35
46
  cpu=compute.cpu,
36
47
  memory=compute.memory,
37
48
  gpu=compute.gpu,
49
+ timeout=int(compute.timeout),
50
+ secrets=self.secrets(),
38
51
  serialized=True,
39
52
  )
40
- def runner(fn: Callable, args: dict) -> Any:
41
- return fn(**args)
42
-
43
- output_path = arguments.pop("output_path", None)
53
+ def runner(fn: PowerFunc, args: dict) -> Any:
54
+ return fn._run_without_parsing(**args)
44
55
 
45
56
  with app.run():
46
- result = runner.remote(function, arguments)
47
-
48
- if output_path is not None:
49
- from powerfunc.conversions import write
50
-
51
- write(result, output_path)
52
- return output_path
53
-
54
- return result
57
+ return runner.remote(function, arguments)
55
58
 
56
59
  @pydantic_dataclass
57
60
  class ModalCpuSmall(ComputeSpecification):
@@ -1,5 +1,3 @@
1
- """Shared pytest fixtures for GCP provider tests."""
2
-
3
1
  import json
4
2
  import os
5
3
  import tempfile
@@ -9,16 +7,19 @@ import pytest
9
7
 
10
8
  @pytest.fixture(scope="module")
11
9
  def gcp_credentials():
12
- """Validate the service-account JSON and expose it via ADC."""
13
- api_key = os.environ["GOOGLE_CLOUD_API_KEY"]
10
+ """Validate the service-account JSON in ``GOOGLE_CLOUD_API_KEY`` and expose it
11
+ via ADC (``GOOGLE_APPLICATION_CREDENTIALS``) for the duration of the module.
14
12
 
13
+ Fails (does not skip) if the provided key is not a service-account JSON: GCP
14
+ Batch, Cloud Run Jobs and GCS need service-account credentials, not a bare API
15
+ key.
16
+ """
15
17
  try:
16
- info = json.loads(api_key)
18
+ info = json.loads(os.environ["GOOGLE_CLOUD_API_KEY"])
17
19
  except json.JSONDecodeError as error:
18
20
  raise ValueError(
19
21
  "GOOGLE_CLOUD_API_KEY is set but is not valid JSON. It must be a GCP "
20
- "service-account JSON key (GCP providers require "
21
- "service-account credentials, not a bare API key)."
22
+ "service-account JSON key."
22
23
  ) from error
23
24
 
24
25
  if not isinstance(info, dict) or info.get("type") != "service_account":
@@ -0,0 +1,12 @@
1
+ """Test package for powerfunc.
2
+
3
+ ``cli_example`` is imported here, as a package's ``__init__`` commonly
4
+ imports its modules, so that ``python -m powerfunc_tests.core.cli_example``
5
+ runs it as ``__main__`` while the same file is also loaded under its package
6
+ name. ``enable_cli`` must still register the command line in that case: it
7
+ reads ``__name__`` from the calling module's globals, which is ``"__main__"``
8
+ for the module being run, rather than asking ``inspect.getmodule``, which would
9
+ return the package-named copy.
10
+ """
11
+
12
+ from . import cli_example # noqa: F401
@@ -0,0 +1,40 @@
1
+ """Command line of three functions: two reading a file, one showing its defaults' layering."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+ import pandas as pd
6
+
7
+ from powerfunc import powerfunc
8
+
9
+
10
+ @powerfunc
11
+ def sum_col(df: pd.DataFrame) -> float:
12
+ return float(df["value"].sum())
13
+
14
+
15
+ @powerfunc
16
+ def mean_col(df: pd.DataFrame) -> float:
17
+ return float(df["value"].mean())
18
+
19
+
20
+ @powerfunc(cli=False)
21
+ def ignored(df: pd.DataFrame) -> float:
22
+ return 0.0
23
+
24
+
25
+ @dataclass
26
+ class Spec:
27
+ cpu: int = 1
28
+ memory: int = 2
29
+
30
+
31
+ DEFAULT_SPEC = Spec(cpu=3)
32
+ DEFAULT_RANGES = {"a": 1.0}
33
+
34
+
35
+ @powerfunc
36
+ def show(spec: Spec = DEFAULT_SPEC, ranges: dict[str, float] = DEFAULT_RANGES) -> str:
37
+ return f"{spec.cpu} {spec.memory} {type(ranges).__name__} {dict(ranges)}"
38
+
39
+
40
+ powerfunc.enable_cli()
@@ -0,0 +1,95 @@
1
+ """Defaults, then configuration file, then what the call gives, from either entry point.
2
+
3
+ The command line reads ``configuration/powerfunc.yaml`` and ``powerfunc.yaml``
4
+ from the working directory, or ``--config`` instead, and argv overwrites fields
5
+ and keys of what they hold. A call from Python reads no file unless ``config``
6
+ names one, or is True for those defaults -- else a stray file in the working
7
+ directory would change any call's arguments -- and its own arguments win.
8
+ """
9
+
10
+ import pathlib
11
+ import subprocess
12
+ import sys
13
+
14
+ import pytest
15
+
16
+ from powerfunc import powerfunc
17
+
18
+ SCRIPT = pathlib.Path(__file__).parent / "cli_example.py"
19
+
20
+
21
+ def run(*args):
22
+ result = subprocess.run([sys.executable, SCRIPT, "show", *args], capture_output=True, text=True)
23
+ assert result.returncode == 0, result.stderr
24
+ return result.stdout.strip()
25
+
26
+
27
+ def test_defaults():
28
+ assert run() == "3 2 dict {'a': 1.0}"
29
+
30
+
31
+ def test_command_line_overwrites_dataclass_field_of_default():
32
+ assert run("--spec.memory", "8") == "3 8 dict {'a': 1.0}"
33
+
34
+
35
+ def test_command_line_overwrites_dict_key_of_default():
36
+ assert run("--ranges.b", "2") == "3 2 dict {'a': 1.0, 'b': 2.0}"
37
+
38
+
39
+ def test_config_file_then_command_line(tmp_path):
40
+ config = tmp_path / "config.yaml"
41
+ config.write_text("spec:\n cpu: 5\nranges:\n a: 3.0\n c: 4.0\n")
42
+ assert run("--config", str(config)) == "5 2 dict {'a': 3.0, 'c': 4.0}"
43
+ assert run("--config", str(config), "--spec.memory", "8", "--ranges.a", "9") == (
44
+ "5 8 dict {'a': 9.0, 'c': 4.0}"
45
+ )
46
+
47
+
48
+ def test_runs_as_a_module_its_package_already_imported():
49
+ result = subprocess.run(
50
+ [sys.executable, "-m", "powerfunc_tests.core.cli_example", "show", "--ranges.b", "2"],
51
+ capture_output=True,
52
+ text=True,
53
+ cwd=SCRIPT.parents[2],
54
+ )
55
+ assert result.returncode == 0, result.stderr
56
+ assert result.stdout.strip() == "3 2 dict {'a': 1.0, 'b': 2.0}"
57
+
58
+
59
+ @powerfunc
60
+ def scaled(value: float, *, scale: float = 1.0) -> float:
61
+ return value * scale
62
+
63
+
64
+ @pytest.fixture
65
+ def default_configuration(tmp_path, monkeypatch):
66
+ monkeypatch.chdir(tmp_path)
67
+ (tmp_path / "powerfunc.yaml").write_text("scale: 10\n")
68
+ return tmp_path
69
+
70
+
71
+ def test_python_call_ignores_default_files(default_configuration):
72
+ assert scaled(2.0) == 2.0
73
+
74
+
75
+ def test_python_call_config_true_reads_default_files(default_configuration):
76
+ assert scaled(2.0, config=True) == 20.0
77
+
78
+
79
+ def test_python_call_config_path_reads_that_file(tmp_path):
80
+ configuration = tmp_path / "elsewhere.yaml"
81
+ configuration.write_text("scale: 3\n")
82
+ assert scaled(2.0, config=configuration) == 6.0
83
+ assert scaled(2.0, config=str(configuration)) == 6.0
84
+
85
+
86
+ def test_python_call_arguments_win_over_the_file(default_configuration):
87
+ assert scaled(2.0, scale=5.0, config=True) == 10.0
88
+
89
+
90
+ def test_a_function_may_not_name_a_parameter_powerfunc_adds():
91
+ with pytest.raises(TypeError, match="config"):
92
+
93
+ @powerfunc
94
+ def clashing(config: str) -> str:
95
+ return config