map-lambda 0.1.0__cp312-abi3-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
map_lambda/__init__.py ADDED
@@ -0,0 +1,51 @@
1
+ """
2
+ A highly-scalable parallel Python map function running on Rust AWS lambdas.
3
+
4
+ Most users will use `map_unordered` (when the function return values are
5
+ needed) or `for_each` (when they are not):
6
+
7
+ import map_lambda
8
+
9
+ async for index, output in map_lambda.map_unordered(
10
+ "square", [1, 2, 3], input_key="input"
11
+ ):
12
+ # output is what inputs[index] came back with
13
+
14
+ Outputs arrive as the workers finish them, so `index` is what says which
15
+ input each one ran.
16
+
17
+ For several calls, create a `Session` object to hold the settings
18
+ and the stats across them:
19
+
20
+ from map_lambda import Session
21
+
22
+ async with Session(dispatchers=4, concurrency=500) as session:
23
+ async for index, output in session.map_unordered(
24
+ "square", list(range(10_000)), input_key="input"
25
+ ):
26
+ ...
27
+
28
+ The public API also includes utilities for analysing stats:
29
+
30
+ * `stats` - what calls recorded about themselves: `summarize_dispatchers`
31
+ for the dispatchers and `summarize_invocations` for the workers.
32
+ * `timeline.generate_timeline` - a chart of invocation stats.
33
+ """
34
+
35
+ from .api import (
36
+ DEFAULT_CONCURRENCY,
37
+ DISPATCHER,
38
+ LOCAL_ENDPOINT_URL,
39
+ Session,
40
+ for_each,
41
+ map_unordered,
42
+ )
43
+
44
+ __all__ = [
45
+ "DEFAULT_CONCURRENCY",
46
+ "DISPATCHER",
47
+ "LOCAL_ENDPOINT_URL",
48
+ "Session",
49
+ "for_each",
50
+ "map_unordered",
51
+ ]
Binary file
map_lambda/_native.pyi ADDED
@@ -0,0 +1,66 @@
1
+ from collections.abc import AsyncIterator
2
+ from typing import Any, final
3
+
4
+ DEFAULT_CONCURRENCY: int
5
+ DISPATCHER: str
6
+ LOCAL_ENDPOINT_URL: str
7
+ STREAMS_PER_CONNECTION: int
8
+
9
+ @final
10
+ class Invoker: ...
11
+
12
+ class Stub:
13
+ def __init__(self, *args: Any, **kwargs: Any) -> None: ...
14
+ def respond(self) -> Body: ...
15
+ @property
16
+ def requests(self) -> list[str]: ...
17
+ @property
18
+ def dispatched_to(self) -> list[str]: ...
19
+ @property
20
+ def invocations(self) -> list[tuple[str, str]]: ...
21
+
22
+ @final
23
+ class Body:
24
+ def write(self, chunk: bytes) -> None: ...
25
+ def close(self) -> None: ...
26
+ @property
27
+ def closed(self) -> bool: ...
28
+
29
+ @final
30
+ class Batch:
31
+ succeeded: int
32
+ outputs: list[tuple[int, Any]]
33
+ invocation_stats: list[dict[str, Any]]
34
+ dispatcher_stats: list[dict[str, Any]]
35
+
36
+ @final
37
+ class Batches(AsyncIterator[Batch]):
38
+ def __aiter__(self) -> Batches: ...
39
+ async def __anext__(self) -> Batch: ...
40
+ def close(self) -> None: ...
41
+
42
+ class Session:
43
+ def __init__(self, invoker: Invoker | Stub, endpoint_url: str | None = None) -> None: ...
44
+ def start(
45
+ self,
46
+ worker_function: str,
47
+ inputs: list[Any],
48
+ *,
49
+ memory: int | None,
50
+ input_key: str,
51
+ constant_args: dict[str, Any],
52
+ project: str | None,
53
+ dispatchers: int | None,
54
+ concurrency: int,
55
+ retries: int,
56
+ return_outputs: bool,
57
+ return_stats: bool,
58
+ ) -> Batches: ...
59
+ async def invoke_detached(
60
+ self, function_name: str, payload: str, *, project: str | None
61
+ ) -> None: ...
62
+
63
+ async def connect(endpoint_url: str | None = None) -> Invoker: ...
64
+ def function_name(
65
+ name: str, *, memory: int | None = None, project: str | None = None, local: bool = False
66
+ ) -> str: ...
map_lambda/api.py ADDED
@@ -0,0 +1,355 @@
1
+ import json
2
+ from collections.abc import AsyncGenerator, Callable
3
+ from dataclasses import dataclass, field
4
+ from typing import Any, Literal, Self
5
+
6
+ from . import _native
7
+
8
+ # Worker invocations in flight per call, split across its dispatchers.
9
+ DEFAULT_CONCURRENCY: int = _native.DEFAULT_CONCURRENCY
10
+
11
+ # A local `cargo lambda watch` server, started from rust/.
12
+ LOCAL_ENDPOINT_URL = _native.LOCAL_ENDPOINT_URL
13
+
14
+ # The dispatcher's name, before a project prefixes it.
15
+ DISPATCHER: str = _native.DISPATCHER
16
+
17
+ __docformat__ = "google"
18
+
19
+ __all__ = ["Session", "for_each", "map_unordered"]
20
+
21
+
22
+ @dataclass
23
+ class Session:
24
+ """
25
+ The deployed lambdas, holding settings and stats for a session.
26
+ Use as `async with`.
27
+ """
28
+
29
+ endpoint_url: str | None = None
30
+ """
31
+ `None` uses AWS with credentials and region picked up from the local
32
+ environment; [`LOCAL_ENDPOINT_URL`][map_lambda.LOCAL_ENDPOINT_URL] uses a local cargo
33
+ lambda watch instead.
34
+ """
35
+
36
+ project: str | None = None
37
+ """
38
+ The project the lambdas were deployed as, which prefixes every name on AWS:
39
+ the dispatcher is `<project>-dispatcher`, and a worker `"copy"` is
40
+ `<project>-copy`. `None` uses the names as given. Against a local cargo lambda
41
+ watch, names are never prefixed, as it serves each function under its binary's
42
+ name.
43
+ """
44
+
45
+ dispatchers: int | Literal["auto"] = "auto"
46
+ """
47
+ The number of dispatchers the inputs are partitioned across.
48
+
49
+ `"auto"` uses the fewest number of dispatchers that respects AWS Lambda
50
+ limits.
51
+ """
52
+
53
+ concurrency: int = DEFAULT_CONCURRENCY
54
+ """
55
+ The total number of worker lambdas in flight for the whole call, split
56
+ across dispatchers.
57
+ """
58
+
59
+ retries: int = 2
60
+ """
61
+ How many times to retry a failed request to Lambda (per invocation). The
62
+ default is 2, which is 3 attempts in total. A worker that ran and raised
63
+ is never retried: the same payload gets the same output.
64
+ """
65
+
66
+ on_progress: Callable[[int], None] | None = None
67
+ """
68
+ Called as invocations complete, with how many more have since the
69
+ last call.
70
+ """
71
+
72
+ dispatcher_stats: list[dict[str, Any]] | None = None
73
+ """
74
+ Pass a list to collect stats for the dispatchers - one record per
75
+ dispatcher invocation. A call stops at its first failed invocation
76
+ without waiting for these, so it collects none. Each record holds:
77
+
78
+ * `req_id` - AWS's id for the dispatcher's own invocation, for finding
79
+ its run in CloudWatch
80
+ * `dispatched` - invocations it sent, which is its partition's size
81
+ * `retried` - attempts beyond the first, across the partition
82
+ * `retry_wait_ms` - summed wait before those attempts
83
+ * `cold` - whether the execution environment was cold, or `None` for a
84
+ local run, which has no shared client set to build
85
+ * `connections` - HTTP/2 connections the invocations were spread over
86
+ * `elapsed_ms` - how long the partition took
87
+ * `call` - which [`map_unordered`][map_lambda.Session.map_unordered] or
88
+ [`for_each`][map_lambda.Session.for_each] of the session this was
89
+ * `call_start` - when that call began, as epoch seconds
90
+ """
91
+
92
+ invocation_stats: list[dict[str, Any]] | None = None
93
+ """
94
+ Pass a list to collect stats for the function invocations - one record per
95
+ worker function invocation, holding:
96
+
97
+ * `index` - the position in `inputs` of the input it ran
98
+ * `req_id` - AWS's id for the worker invocation, for joining to CloudWatch.
99
+ Empty against `cargo lambda watch`, which assigns none
100
+ * `invoke_start`, `invoke_end` - epoch seconds, spanning every attempt
101
+ * `call`, `call_start` - as in [`dispatcher_stats`][map_lambda.Session.dispatcher_stats]
102
+ """
103
+
104
+ _invoker: Any = field(default=None, init=False, repr=False)
105
+ _native_session: Any = field(default=None, init=False, repr=False)
106
+
107
+ async def __aenter__(self) -> Self:
108
+ self._invoker = await _native.connect(self.endpoint_url)
109
+ return self
110
+
111
+ async def __aexit__(self, *exc_info: Any) -> None:
112
+ self._invoker = None
113
+ self._native_session = None
114
+
115
+ def _require_native_session(self) -> Any:
116
+ if self._invoker is None:
117
+ raise RuntimeError("Session is not open: use `async with Session(...) as session:`")
118
+ if self._native_session is None:
119
+ self._native_session = _native.Session(self._invoker, self.endpoint_url)
120
+ return self._native_session
121
+
122
+ def function_name(self, name: str, /, *, memory: int | None = None) -> str:
123
+ """
124
+ The name a call to `name` invokes, such as `<project>-copy-3538` on AWS, for
125
+ finding the function's logs or metrics. Works whether or not the session is open.
126
+ """
127
+ return _native.function_name(
128
+ name, memory=memory, project=self.project, local=self.endpoint_url is not None
129
+ )
130
+
131
+ def map_unordered(
132
+ self,
133
+ function_name: str,
134
+ inputs: list[Any],
135
+ /,
136
+ *,
137
+ input_key: str,
138
+ constant_args: dict[str, Any] | None = None,
139
+ memory: int | None = None,
140
+ ) -> AsyncGenerator[tuple[int, Any], None]:
141
+ """
142
+ Invoke `function_name` once per value of `inputs`, yielding
143
+ `(index, output)` as each output comes back. If outputs
144
+ are not needed, use [`for_each`][map_lambda.Session.for_each].
145
+
146
+ Each worker's payload is a dict with the input value
147
+ under `input_key`, plus a copy of `constant_args` (if supplied).
148
+
149
+ Args:
150
+ function_name: The worker's name in its project, such as `"copy"`.
151
+ inputs: The values to pass to the function, one entry per invocation.
152
+ input_key: The name of the key for the input in each worker's
153
+ payload. Must match the field the worker deserialises it into:
154
+ for a Rust worker, a field of its `Request` struct.
155
+ constant_args: Optional extra arguments to pass to each function invocation.
156
+ memory: Which of the worker's deployed memory sizes to use, in MB, or
157
+ `None` for its default. On AWS this calls `<name>-<memory>`.
158
+
159
+ Yields:
160
+ `(index, output)`, in the order the invocations finished.
161
+ """
162
+ return self._stream(
163
+ self._require_native_session(),
164
+ function_name,
165
+ inputs,
166
+ input_key=input_key,
167
+ constant_args=constant_args,
168
+ memory=memory,
169
+ return_outputs=True,
170
+ )
171
+
172
+ async def _stream(
173
+ self,
174
+ native_session: Any,
175
+ function_name: str,
176
+ inputs: list[Any],
177
+ /,
178
+ *,
179
+ input_key: str,
180
+ constant_args: dict[str, Any] | None,
181
+ memory: int | None,
182
+ return_outputs: bool,
183
+ ) -> AsyncGenerator[tuple[int, Any], None]:
184
+ """
185
+ Labelled outputs as they arrive, or none at all when
186
+ `return_outputs` is false.
187
+ """
188
+ batches = native_session.start(
189
+ function_name,
190
+ inputs,
191
+ memory=memory,
192
+ input_key=input_key,
193
+ constant_args={} if constant_args is None else constant_args,
194
+ project=self.project,
195
+ dispatchers=None if self.dispatchers == "auto" else self.dispatchers,
196
+ concurrency=self.concurrency,
197
+ retries=self.retries,
198
+ return_outputs=return_outputs,
199
+ return_stats=self.invocation_stats is not None,
200
+ )
201
+ # Closed explicitly so that stopping early closes the dispatchers'
202
+ # streams now, rather than whenever `batches` is collected.
203
+ try:
204
+ async for batch in batches:
205
+ if self.invocation_stats is not None:
206
+ self.invocation_stats.extend(batch.invocation_stats)
207
+ if self.dispatcher_stats is not None:
208
+ self.dispatcher_stats.extend(batch.dispatcher_stats)
209
+ if self.on_progress is not None and batch.succeeded:
210
+ self.on_progress(batch.succeeded)
211
+ for output in batch.outputs:
212
+ yield output
213
+ finally:
214
+ batches.close()
215
+
216
+ async def for_each(
217
+ self,
218
+ function_name: str,
219
+ inputs: list[Any],
220
+ /,
221
+ *,
222
+ input_key: str,
223
+ constant_args: dict[str, Any] | None = None,
224
+ memory: int | None = None,
225
+ ) -> None:
226
+ """
227
+ Invoke `function_name` once for each value of `inputs`. This is
228
+ the same as [`map_unordered`][map_lambda.Session.map_unordered] except nothing is
229
+ returned, and it
230
+ waits for the whole call.
231
+
232
+ Refer to [`map_unordered`][map_lambda.Session.map_unordered] for argument definitions.
233
+ """
234
+ stream = self._stream(
235
+ self._require_native_session(),
236
+ function_name,
237
+ inputs,
238
+ input_key=input_key,
239
+ constant_args=constant_args,
240
+ memory=memory,
241
+ return_outputs=False,
242
+ )
243
+ # Nothing to iterate - the outputs were never asked for - but
244
+ # draining it is what settles the partitions and raises.
245
+ async for _ in stream:
246
+ pass
247
+
248
+ async def invoke_detached(self, function_name: str, /, **payload: Any) -> None:
249
+ """
250
+ Invoke `function_name` once with the given payload and do not wait for it.
251
+
252
+ Useful for "fire and forget" semantics, where failing after submission
253
+ and not learning about the failure is acceptable - failing to submit
254
+ raises. For a synchronous one-off, use
255
+ [`for_each`][map_lambda.Session.for_each] with a single input.
256
+ """
257
+ await self._require_native_session().invoke_detached(
258
+ function_name, json.dumps(payload), project=self.project
259
+ )
260
+
261
+
262
+ async def map_unordered(
263
+ function_name: str,
264
+ inputs: list[Any],
265
+ /,
266
+ *,
267
+ input_key: str,
268
+ constant_args: dict[str, Any] | None = None,
269
+ memory: int | None = None,
270
+ endpoint_url: str | None = None,
271
+ project: str | None = None,
272
+ dispatchers: int | Literal["auto"] = "auto",
273
+ concurrency: int = DEFAULT_CONCURRENCY,
274
+ retries: int = 2,
275
+ on_progress: Callable[[int], None] | None = None,
276
+ dispatcher_stats: list[dict[str, Any]] | None = None,
277
+ invocation_stats: list[dict[str, Any]] | None = None,
278
+ ) -> AsyncGenerator[tuple[int, Any], None]:
279
+ """
280
+ Invoke `function_name` once per value of `inputs`, yielding
281
+ `(index, output)` as each output comes back. If outputs
282
+ are not needed, use [`for_each`][map_lambda.for_each].
283
+
284
+ Refer to [`Session.map_unordered`][map_lambda.Session.map_unordered] for argument
285
+ definitions.
286
+
287
+ `endpoint_url` onwards are the dispatch settings documented on
288
+ [`Session`][map_lambda.Session], applied to a session opened for this call alone - which
289
+ closes when the iterator does, so stopping early still shuts the
290
+ client down.
291
+
292
+ Yields:
293
+ `(index, output)`, in the order the invocations finished.
294
+ """
295
+ async with Session(
296
+ endpoint_url=endpoint_url,
297
+ project=project,
298
+ dispatchers=dispatchers,
299
+ concurrency=concurrency,
300
+ retries=retries,
301
+ on_progress=on_progress,
302
+ dispatcher_stats=dispatcher_stats,
303
+ invocation_stats=invocation_stats,
304
+ ) as session:
305
+ async for output in session.map_unordered(
306
+ function_name,
307
+ inputs,
308
+ input_key=input_key,
309
+ constant_args=constant_args,
310
+ memory=memory,
311
+ ):
312
+ yield output
313
+
314
+
315
+ async def for_each(
316
+ function_name: str,
317
+ inputs: list[Any],
318
+ /,
319
+ *,
320
+ input_key: str,
321
+ constant_args: dict[str, Any] | None = None,
322
+ memory: int | None = None,
323
+ endpoint_url: str | None = None,
324
+ project: str | None = None,
325
+ dispatchers: int | Literal["auto"] = "auto",
326
+ concurrency: int = DEFAULT_CONCURRENCY,
327
+ retries: int = 2,
328
+ on_progress: Callable[[int], None] | None = None,
329
+ dispatcher_stats: list[dict[str, Any]] | None = None,
330
+ invocation_stats: list[dict[str, Any]] | None = None,
331
+ ) -> None:
332
+ """
333
+ Invoke `function_name` once for each value of `inputs`. This is
334
+ the same as [`map_unordered`][map_lambda.map_unordered] except nothing is returned.
335
+
336
+ Refer to [`Session.map_unordered`][map_lambda.Session.map_unordered] for argument
337
+ definitions.
338
+ """
339
+ async with Session(
340
+ endpoint_url=endpoint_url,
341
+ project=project,
342
+ dispatchers=dispatchers,
343
+ concurrency=concurrency,
344
+ retries=retries,
345
+ on_progress=on_progress,
346
+ dispatcher_stats=dispatcher_stats,
347
+ invocation_stats=invocation_stats,
348
+ ) as session:
349
+ await session.for_each(
350
+ function_name,
351
+ inputs,
352
+ input_key=input_key,
353
+ constant_args=constant_args,
354
+ memory=memory,
355
+ )
@@ -0,0 +1,20 @@
1
+ """
2
+ Deploying a project's lambdas as one CloudFormation stack.
3
+
4
+ A project ships a spec (see `spec`) and an entry point calling `main` with it. Its
5
+ client then opens a `Session` with `project(spec_path)`, which names the functions
6
+ the way the stack does.
7
+ """
8
+
9
+ from pathlib import Path
10
+
11
+ from . import spec
12
+ from .cli import commands, group, main
13
+
14
+
15
+ def project(spec_path: Path) -> str:
16
+ """The project a spec deploys, for a client's `Session(project=...)`."""
17
+ return spec.load(spec_path).project
18
+
19
+
20
+ __all__ = ["commands", "group", "main", "project"]
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ main()