modal 0.68.11__py3-none-any.whl → 0.68.31__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. modal/__init__.py +2 -0
  2. modal/_ipython.py +3 -13
  3. modal/_runtime/asgi.py +4 -0
  4. modal/_runtime/user_code_imports.py +13 -18
  5. modal/_utils/blob_utils.py +27 -92
  6. modal/_utils/bytes_io_segment_payload.py +97 -0
  7. modal/_utils/deprecation.py +44 -0
  8. modal/_utils/hash_utils.py +38 -9
  9. modal/_utils/http_utils.py +19 -10
  10. modal/_utils/{pattern_matcher.py → pattern_utils.py} +1 -70
  11. modal/_utils/shell_utils.py +11 -5
  12. modal/app.py +11 -31
  13. modal/app.pyi +3 -4
  14. modal/cli/app.py +1 -1
  15. modal/cli/run.py +25 -5
  16. modal/client.py +1 -1
  17. modal/client.pyi +2 -2
  18. modal/config.py +2 -1
  19. modal/container_process.py +2 -1
  20. modal/dict.py +2 -1
  21. modal/exception.py +0 -54
  22. modal/file_io.py +54 -7
  23. modal/file_io.pyi +18 -8
  24. modal/file_pattern_matcher.py +154 -0
  25. modal/functions.py +2 -8
  26. modal/functions.pyi +5 -1
  27. modal/image.py +106 -10
  28. modal/image.pyi +36 -6
  29. modal/mount.py +49 -9
  30. modal/mount.pyi +19 -4
  31. modal/network_file_system.py +6 -2
  32. modal/partial_function.py +10 -1
  33. modal/partial_function.pyi +8 -0
  34. modal/queue.py +2 -1
  35. modal/runner.py +2 -7
  36. modal/sandbox.py +23 -13
  37. modal/sandbox.pyi +21 -0
  38. modal/serving.py +1 -1
  39. modal/volume.py +7 -2
  40. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/METADATA +1 -1
  41. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/RECORD +49 -46
  42. modal_proto/api.proto +8 -0
  43. modal_proto/api_pb2.py +781 -745
  44. modal_proto/api_pb2.pyi +65 -3
  45. modal_version/_version_generated.py +1 -1
  46. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/LICENSE +0 -0
  47. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/WHEEL +0 -0
  48. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/entry_points.txt +0 -0
  49. {modal-0.68.11.dist-info → modal-0.68.31.dist-info}/top_level.txt +0 -0
@@ -19,14 +19,20 @@ def write_to_fd(fd: int, data: bytes):
19
19
  future = loop.create_future()
20
20
 
21
21
  def try_write():
22
+ nonlocal data
22
23
  try:
23
24
  nbytes = os.write(fd, data)
24
- loop.remove_writer(fd)
25
- future.set_result(nbytes)
25
+ data = data[nbytes:]
26
+ if not data:
27
+ loop.remove_writer(fd)
28
+ future.set_result(None)
26
29
  except OSError as e:
27
- if e.errno != errno.EAGAIN:
28
- future.set_exception(e)
29
- raise
30
+ if e.errno == errno.EAGAIN:
31
+ # Wait for the next write notification
32
+ return
33
+ # Fail if it's not EAGAIN
34
+ loop.remove_writer(fd)
35
+ future.set_exception(e)
30
36
 
31
37
  loop.add_writer(fd, try_write)
32
38
  return future
modal/app.py CHANGED
@@ -22,6 +22,7 @@ from modal_proto import api_pb2
22
22
 
23
23
  from ._ipython import is_notebook
24
24
  from ._utils.async_utils import synchronize_api
25
+ from ._utils.deprecation import deprecation_error, deprecation_warning
25
26
  from ._utils.function_utils import FunctionInfo, is_global_object, is_method_fn
26
27
  from ._utils.grpc_utils import retry_transient_errors
27
28
  from ._utils.mount_utils import validate_volumes
@@ -29,7 +30,7 @@ from .client import _Client
29
30
  from .cloud_bucket_mount import _CloudBucketMount
30
31
  from .cls import _Cls, parameter
31
32
  from .config import logger
32
- from .exception import ExecutionError, InvalidError, deprecation_error, deprecation_warning
33
+ from .exception import ExecutionError, InvalidError
33
34
  from .functions import Function, _Function
34
35
  from .gpu import GPU_T
35
36
  from .image import _Image
@@ -45,7 +46,6 @@ from .partial_function import (
45
46
  from .proxy import _Proxy
46
47
  from .retries import Retries
47
48
  from .running_app import RunningApp
48
- from .sandbox import _Sandbox
49
49
  from .schedule import Schedule
50
50
  from .scheduler_placement import SchedulerPlacement
51
51
  from .secret import _Secret
@@ -964,36 +964,16 @@ class _App:
964
964
  _experimental_scheduler_placement: Optional[
965
965
  SchedulerPlacement
966
966
  ] = None, # Experimental controls over fine-grained scheduling (alpha).
967
- ) -> _Sandbox:
968
- """`App.spawn_sandbox` is deprecated in favor of `Sandbox.create(app=...)`.
969
-
970
- See https://modal.com/docs/guide/sandbox for more info on working with sandboxes.
971
- """
972
- deprecation_warning((2024, 7, 5), _App.spawn_sandbox.__doc__)
973
- if not self._running_app:
974
- raise InvalidError("`app.spawn_sandbox` requires a running app.")
975
-
976
- return await _Sandbox.create(
977
- *entrypoint_args,
978
- app=self,
979
- environment_name=self._running_app.environment_name,
980
- image=image or _default_image,
981
- mounts=mounts,
982
- secrets=secrets,
983
- timeout=timeout,
984
- workdir=workdir,
985
- gpu=gpu,
986
- cloud=cloud,
987
- region=region,
988
- cpu=cpu,
989
- memory=memory,
990
- network_file_systems=network_file_systems,
991
- block_network=block_network,
992
- volumes=volumes,
993
- pty_info=pty_info,
994
- _experimental_scheduler_placement=_experimental_scheduler_placement,
995
- client=self._client,
967
+ ) -> None:
968
+ """mdmd:hidden"""
969
+ arglist = ", ".join(repr(s) for s in entrypoint_args)
970
+ message = (
971
+ "`App.spawn_sandbox` is deprecated.\n\n"
972
+ "Sandboxes can be created using the `Sandbox` object:\n\n"
973
+ f"```\nsb = Sandbox.create({arglist}, app=app)\n```\n\n"
974
+ "See https://modal.com/docs/guide/sandbox for more info on working with sandboxes."
996
975
  )
976
+ deprecation_error((2024, 7, 5), message)
997
977
 
998
978
  def include(self, /, other_app: "_App"):
999
979
  """Include another App's objects in this one.
modal/app.pyi CHANGED
@@ -13,7 +13,6 @@ import modal.partial_function
13
13
  import modal.proxy
14
14
  import modal.retries
15
15
  import modal.running_app
16
- import modal.sandbox
17
16
  import modal.schedule
18
17
  import modal.scheduler_placement
19
18
  import modal.secret
@@ -261,7 +260,7 @@ class _App:
261
260
  ] = {},
262
261
  pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
263
262
  _experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
264
- ) -> modal.sandbox._Sandbox: ...
263
+ ) -> None: ...
265
264
  def include(self, /, other_app: _App): ...
266
265
  def _logs(
267
266
  self, client: typing.Optional[modal.client._Client] = None
@@ -491,7 +490,7 @@ class App:
491
490
  ] = {},
492
491
  pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
493
492
  _experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
494
- ) -> modal.sandbox.Sandbox: ...
493
+ ) -> None: ...
495
494
  async def aio(
496
495
  self,
497
496
  *entrypoint_args: str,
@@ -515,7 +514,7 @@ class App:
515
514
  ] = {},
516
515
  pty_info: typing.Optional[modal_proto.api_pb2.PTYInfo] = None,
517
516
  _experimental_scheduler_placement: typing.Optional[modal.scheduler_placement.SchedulerPlacement] = None,
518
- ) -> modal.sandbox.Sandbox: ...
517
+ ) -> None: ...
519
518
 
520
519
  spawn_sandbox: __spawn_sandbox_spec
521
520
 
modal/cli/app.py CHANGED
@@ -10,9 +10,9 @@ from rich.text import Text
10
10
  from typer import Argument
11
11
 
12
12
  from modal._utils.async_utils import synchronizer
13
+ from modal._utils.deprecation import deprecation_warning
13
14
  from modal.client import _Client
14
15
  from modal.environments import ensure_env
15
- from modal.exception import deprecation_warning
16
16
  from modal.object import _get_environment_name
17
17
  from modal_proto import api_pb2
18
18
 
modal/cli/run.py CHANGED
@@ -133,6 +133,18 @@ def _get_clean_app_description(func_ref: str) -> str:
133
133
  return " ".join(sys.argv)
134
134
 
135
135
 
136
+ def _write_local_result(result_path: str, res: Any):
137
+ if isinstance(res, str):
138
+ mode = "wt"
139
+ elif isinstance(res, bytes):
140
+ mode = "wb"
141
+ else:
142
+ res_type = type(res).__name__
143
+ raise InvalidError(f"Function must return str or bytes when using `--write-result`; got {res_type}.")
144
+ with open(result_path, mode) as fid:
145
+ fid.write(res)
146
+
147
+
136
148
  def _get_click_command_for_function(app: App, function_tag):
137
149
  function = app.registered_functions.get(function_tag)
138
150
  if not function or (isinstance(function, Function) and function.info.user_cls is not None):
@@ -177,7 +189,7 @@ def _get_click_command_for_function(app: App, function_tag):
177
189
  interactive=ctx.obj["interactive"],
178
190
  ):
179
191
  if cls is None:
180
- function.remote(**kwargs)
192
+ res = function.remote(**kwargs)
181
193
  else:
182
194
  # unpool class and method arguments
183
195
  # TODO(erikbern): this code is a bit hacky
@@ -186,7 +198,10 @@ def _get_click_command_for_function(app: App, function_tag):
186
198
 
187
199
  instance = cls(**cls_kwargs)
188
200
  method: Function = getattr(instance, method_name)
189
- method.remote(**fun_kwargs)
201
+ res = method.remote(**fun_kwargs)
202
+
203
+ if result_path := ctx.obj["result_path"]:
204
+ _write_local_result(result_path, res)
190
205
 
191
206
  with_click_options = _add_click_options(f, signature)
192
207
  return click.command(with_click_options)
@@ -214,12 +229,15 @@ def _get_click_command_for_local_entrypoint(app: App, entrypoint: LocalEntrypoin
214
229
  ):
215
230
  try:
216
231
  if isasync:
217
- asyncio.run(func(*args, **kwargs))
232
+ res = asyncio.run(func(*args, **kwargs))
218
233
  else:
219
- func(*args, **kwargs)
234
+ res = func(*args, **kwargs)
220
235
  except Exception as exc:
221
236
  raise _CliUserExecutionError(inspect.getsourcefile(func)) from exc
222
237
 
238
+ if result_path := ctx.obj["result_path"]:
239
+ _write_local_result(result_path, res)
240
+
223
241
  with_click_options = _add_click_options(f, _get_signature(func))
224
242
  return click.command(with_click_options)
225
243
 
@@ -248,12 +266,13 @@ class RunGroup(click.Group):
248
266
  cls=RunGroup,
249
267
  subcommand_metavar="FUNC_REF",
250
268
  )
269
+ @click.option("-w", "--write-result", help="Write return value (which must be str or bytes) to this local path.")
251
270
  @click.option("-q", "--quiet", is_flag=True, help="Don't show Modal progress indicators.")
252
271
  @click.option("-d", "--detach", is_flag=True, help="Don't stop the app if the local process dies or disconnects.")
253
272
  @click.option("-i", "--interactive", is_flag=True, help="Run the app in interactive mode.")
254
273
  @click.option("-e", "--env", help=ENV_OPTION_HELP, default=None)
255
274
  @click.pass_context
256
- def run(ctx, detach, quiet, interactive, env):
275
+ def run(ctx, write_result, detach, quiet, interactive, env):
257
276
  """Run a Modal function or local entrypoint.
258
277
 
259
278
  `FUNC_REF` should be of the format `{file or module}::{function name}`.
@@ -284,6 +303,7 @@ def run(ctx, detach, quiet, interactive, env):
284
303
  ```
285
304
  """
286
305
  ctx.ensure_object(dict)
306
+ ctx.obj["result_path"] = write_result
287
307
  ctx.obj["detach"] = detach # if subcommand would be a click command...
288
308
  ctx.obj["show_progress"] = False if quiet else True
289
309
  ctx.obj["interactive"] = interactive
modal/client.py CHANGED
@@ -236,7 +236,7 @@ class _Client:
236
236
  Check whether can the client can connect to this server with these credentials; raise if not.
237
237
  """
238
238
  async with cls(server_url, api_pb2.CLIENT_TYPE_CLIENT, credentials) as client:
239
- client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
239
+ await client.hello() # Will call ClientHello RPC and possibly raise AuthError or ConnectionError
240
240
 
241
241
  @classmethod
242
242
  def set_env_client(cls, client: Optional["_Client"]):
modal/client.pyi CHANGED
@@ -26,7 +26,7 @@ class _Client:
26
26
  _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
27
27
 
28
28
  def __init__(
29
- self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.11"
29
+ self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
30
30
  ): ...
31
31
  def is_closed(self) -> bool: ...
32
32
  @property
@@ -81,7 +81,7 @@ class Client:
81
81
  _stub: typing.Optional[modal_proto.api_grpc.ModalClientStub]
82
82
 
83
83
  def __init__(
84
- self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.11"
84
+ self, server_url: str, client_type: int, credentials: typing.Optional[tuple[str, str]], version: str = "0.68.31"
85
85
  ): ...
86
86
  def is_closed(self) -> bool: ...
87
87
  @property
modal/config.py CHANGED
@@ -86,8 +86,9 @@ from google.protobuf.empty_pb2 import Empty
86
86
 
87
87
  from modal_proto import api_pb2
88
88
 
89
+ from ._utils.deprecation import deprecation_error
89
90
  from ._utils.logger import configure_logger
90
- from .exception import InvalidError, deprecation_error
91
+ from .exception import InvalidError
91
92
 
92
93
  # Locate config file and read it
93
94
 
@@ -6,10 +6,11 @@ from typing import Generic, Optional, TypeVar
6
6
  from modal_proto import api_pb2
7
7
 
8
8
  from ._utils.async_utils import TaskContext, synchronize_api
9
+ from ._utils.deprecation import deprecation_error
9
10
  from ._utils.grpc_utils import retry_transient_errors
10
11
  from ._utils.shell_utils import stream_from_stdin, write_to_fd
11
12
  from .client import _Client
12
- from .exception import InteractiveTimeoutError, InvalidError, deprecation_error
13
+ from .exception import InteractiveTimeoutError, InvalidError
13
14
  from .io_streams import _StreamReader, _StreamWriter
14
15
  from .stream_type import StreamType
15
16
 
modal/dict.py CHANGED
@@ -10,11 +10,12 @@ from modal_proto import api_pb2
10
10
  from ._resolver import Resolver
11
11
  from ._serialization import deserialize, serialize
12
12
  from ._utils.async_utils import TaskContext, synchronize_api
13
+ from ._utils.deprecation import deprecation_error
13
14
  from ._utils.grpc_utils import retry_transient_errors
14
15
  from ._utils.name_utils import check_object_name
15
16
  from .client import _Client
16
17
  from .config import logger
17
- from .exception import RequestSizeError, deprecation_error
18
+ from .exception import RequestSizeError
18
19
  from .object import EPHEMERAL_OBJECT_HEARTBEAT_SLEEP, _get_environment_name, _Object, live_method, live_method_gen
19
20
 
20
21
 
modal/exception.py CHANGED
@@ -1,12 +1,6 @@
1
1
  # Copyright Modal Labs 2022
2
2
  import random
3
3
  import signal
4
- import sys
5
- import warnings
6
- from datetime import date
7
- from typing import Iterable
8
-
9
- from modal_proto import api_pb2
10
4
 
11
5
 
12
6
  class Error(Exception):
@@ -129,45 +123,6 @@ class _CliUserExecutionError(Exception):
129
123
  self.user_source = user_source
130
124
 
131
125
 
132
- # TODO(erikbern): we have something similready in function_utils.py
133
- _INTERNAL_MODULES = ["modal", "synchronicity"]
134
-
135
-
136
- def _is_internal_frame(frame):
137
- module = frame.f_globals["__name__"].split(".")[0]
138
- return module in _INTERNAL_MODULES
139
-
140
-
141
- def deprecation_error(deprecated_on: tuple[int, int, int], msg: str):
142
- raise DeprecationError(f"Deprecated on {date(*deprecated_on)}: {msg}")
143
-
144
-
145
- def deprecation_warning(
146
- deprecated_on: tuple[int, int, int], msg: str, *, pending: bool = False, show_source: bool = True
147
- ) -> None:
148
- """Utility for getting the proper stack entry.
149
-
150
- See the implementation of the built-in [warnings.warn](https://docs.python.org/3/library/warnings.html#available-functions).
151
- """
152
- filename, lineno = "<unknown>", 0
153
- if show_source:
154
- # Find the last non-Modal line that triggered the warning
155
- try:
156
- frame = sys._getframe()
157
- while frame is not None and _is_internal_frame(frame):
158
- frame = frame.f_back
159
- filename = frame.f_code.co_filename
160
- lineno = frame.f_lineno
161
- except ValueError:
162
- # Use the defaults from above
163
- pass
164
-
165
- warning_cls: type = PendingDeprecationError if pending else DeprecationError
166
-
167
- # This is a lower-level function that warnings.warn uses
168
- warnings.warn_explicit(f"{date(*deprecated_on)}: {msg}", warning_cls, filename, lineno)
169
-
170
-
171
126
  def _simulate_preemption_interrupt(signum, frame):
172
127
  signal.alarm(30) # simulate a SIGKILL after 30s
173
128
  raise KeyboardInterrupt("Simulated preemption interrupt from modal-client!")
@@ -224,12 +179,3 @@ class ClientClosed(Error):
224
179
 
225
180
  class FilesystemExecutionError(Error):
226
181
  """Raised when an unknown error is thrown during a container filesystem operation."""
227
-
228
-
229
- def print_server_warnings(server_warnings: Iterable[api_pb2.Warning]):
230
- # TODO(erikbern): move this to modal._utils.deprecation
231
- for warning in server_warnings:
232
- if warning.type == api_pb2.Warning.WARNING_TYPE_CLIENT_DEPRECATION:
233
- warnings.warn_explicit(warning.message, DeprecationError, "<unknown>", 0)
234
- else:
235
- warnings.warn_explicit(warning.message, UserWarning, "<unknown>", 0)
modal/file_io.py CHANGED
@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, AsyncIterator, Generic, Optional, Sequence, Ty
6
6
  if TYPE_CHECKING:
7
7
  import _typeshed
8
8
 
9
+ import json
10
+
9
11
  from grpclib.exceptions import GRPCError, StreamTerminatedError
10
12
 
11
13
  from modal._utils.grpc_utils import retry_transient_errors
@@ -267,12 +269,12 @@ class _FileIO(Generic[T]):
267
269
  output = await self._make_read_request(None)
268
270
  if self._binary:
269
271
  lines_bytes = output.split(b"\n")
270
- output = [line + b"\n" for line in lines_bytes[:-1]] + ([lines_bytes[-1]] if lines_bytes[-1] else [])
271
- return cast(Sequence[T], output)
272
+ return_bytes = [line + b"\n" for line in lines_bytes[:-1]] + ([lines_bytes[-1]] if lines_bytes[-1] else [])
273
+ return cast(Sequence[T], return_bytes)
272
274
  else:
273
275
  lines = output.decode("utf-8").split("\n")
274
- output = [line + "\n" for line in lines[:-1]] + ([lines[-1]] if lines[-1] else [])
275
- return cast(Sequence[T], output)
276
+ return_strs = [line + "\n" for line in lines[:-1]] + ([lines[-1]] if lines[-1] else [])
277
+ return cast(Sequence[T], return_strs)
276
278
 
277
279
  async def write(self, data: Union[bytes, str]) -> None:
278
280
  """Write data to the current position.
@@ -337,6 +339,52 @@ class _FileIO(Generic[T]):
337
339
  )
338
340
  await self._wait(resp.exec_id)
339
341
 
342
+ @classmethod
343
+ async def ls(cls, path: str, client: _Client, task_id: str) -> list[str]:
344
+ """List the contents of the provided directory."""
345
+ self = cls.__new__(cls)
346
+ self._client = client
347
+ self._task_id = task_id
348
+ resp = await self._make_request(
349
+ api_pb2.ContainerFilesystemExecRequest(
350
+ file_ls_request=api_pb2.ContainerFileLsRequest(path=path),
351
+ task_id=task_id,
352
+ )
353
+ )
354
+ output = await self._wait(resp.exec_id)
355
+ try:
356
+ return json.loads(output.decode("utf-8"))["paths"]
357
+ except json.JSONDecodeError:
358
+ raise FilesystemExecutionError("failed to parse list output")
359
+
360
+ @classmethod
361
+ async def mkdir(cls, path: str, client: _Client, task_id: str, parents: bool = False) -> None:
362
+ """Create a new directory."""
363
+ self = cls.__new__(cls)
364
+ self._client = client
365
+ self._task_id = task_id
366
+ resp = await self._make_request(
367
+ api_pb2.ContainerFilesystemExecRequest(
368
+ file_mkdir_request=api_pb2.ContainerFileMkdirRequest(path=path, make_parents=parents),
369
+ task_id=self._task_id,
370
+ )
371
+ )
372
+ await self._wait(resp.exec_id)
373
+
374
+ @classmethod
375
+ async def rm(cls, path: str, client: _Client, task_id: str, recursive: bool = False) -> None:
376
+ """Remove a file or directory in the Sandbox."""
377
+ self = cls.__new__(cls)
378
+ self._client = client
379
+ self._task_id = task_id
380
+ resp = await self._make_request(
381
+ api_pb2.ContainerFilesystemExecRequest(
382
+ file_rm_request=api_pb2.ContainerFileRmRequest(path=path, recursive=recursive),
383
+ task_id=self._task_id,
384
+ )
385
+ )
386
+ await self._wait(resp.exec_id)
387
+
340
388
  async def _close(self) -> None:
341
389
  # Buffer is flushed by the runner on close
342
390
  resp = await self._make_request(
@@ -367,11 +415,10 @@ class _FileIO(Generic[T]):
367
415
  if self._closed:
368
416
  raise ValueError("I/O operation on closed file")
369
417
 
370
- def __enter__(self) -> "_FileIO":
371
- self._check_closed()
418
+ async def __aenter__(self) -> "_FileIO":
372
419
  return self
373
420
 
374
- async def __exit__(self, exc_type, exc_value, traceback) -> None:
421
+ async def __aexit__(self, exc_type, exc_value, traceback) -> None:
375
422
  await self._close()
376
423
 
377
424
 
modal/file_io.pyi CHANGED
@@ -43,13 +43,19 @@ class _FileIO(typing.Generic[T]):
43
43
  async def flush(self) -> None: ...
44
44
  def _get_whence(self, whence: int): ...
45
45
  async def seek(self, offset: int, whence: int = 0) -> None: ...
46
+ @classmethod
47
+ async def ls(cls, path: str, client: modal.client._Client, task_id: str) -> list[str]: ...
48
+ @classmethod
49
+ async def mkdir(cls, path: str, client: modal.client._Client, task_id: str, parents: bool = False) -> None: ...
50
+ @classmethod
51
+ async def rm(cls, path: str, client: modal.client._Client, task_id: str, recursive: bool = False) -> None: ...
46
52
  async def _close(self) -> None: ...
47
53
  async def close(self) -> None: ...
48
54
  def _check_writable(self) -> None: ...
49
55
  def _check_readable(self) -> None: ...
50
56
  def _check_closed(self) -> None: ...
51
- def __enter__(self) -> _FileIO: ...
52
- async def __exit__(self, exc_type, exc_value, traceback) -> None: ...
57
+ async def __aenter__(self) -> _FileIO: ...
58
+ async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
53
59
 
54
60
  class __delete_bytes_spec(typing_extensions.Protocol):
55
61
  def __call__(self, file: FileIO, start: typing.Optional[int] = None, end: typing.Optional[int] = None) -> None: ...
@@ -161,6 +167,13 @@ class FileIO(typing.Generic[T]):
161
167
 
162
168
  seek: __seek_spec
163
169
 
170
+ @classmethod
171
+ def ls(cls, path: str, client: modal.client.Client, task_id: str) -> list[str]: ...
172
+ @classmethod
173
+ def mkdir(cls, path: str, client: modal.client.Client, task_id: str, parents: bool = False) -> None: ...
174
+ @classmethod
175
+ def rm(cls, path: str, client: modal.client.Client, task_id: str, recursive: bool = False) -> None: ...
176
+
164
177
  class ___close_spec(typing_extensions.Protocol):
165
178
  def __call__(self) -> None: ...
166
179
  async def aio(self) -> None: ...
@@ -177,9 +190,6 @@ class FileIO(typing.Generic[T]):
177
190
  def _check_readable(self) -> None: ...
178
191
  def _check_closed(self) -> None: ...
179
192
  def __enter__(self) -> FileIO: ...
180
-
181
- class ____exit___spec(typing_extensions.Protocol):
182
- def __call__(self, exc_type, exc_value, traceback) -> None: ...
183
- async def aio(self, exc_type, exc_value, traceback) -> None: ...
184
-
185
- __exit__: ____exit___spec
193
+ async def __aenter__(self) -> FileIO: ...
194
+ def __exit__(self, exc_type, exc_value, traceback) -> None: ...
195
+ async def __aexit__(self, exc_type, exc_value, traceback) -> None: ...
@@ -0,0 +1,154 @@
1
+ # Copyright Modal Labs 2024
2
+ """Pattern matching library ported from https://github.com/moby/patternmatcher.
3
+
4
+ This is the same pattern-matching logic used by Docker, except it is written in
5
+ Python rather than Go. Also, the original Go library has a couple deprecated
6
+ functions that we don't implement in this port.
7
+
8
+ The main way to use this library is by constructing a `FilePatternMatcher` object,
9
+ then asking it whether file paths match any of its patterns.
10
+ """
11
+
12
+ import os
13
+ from abc import abstractmethod
14
+ from pathlib import Path
15
+ from typing import Callable, Optional
16
+
17
+ from ._utils.pattern_utils import Pattern
18
+
19
+
20
+ class _AbstractPatternMatcher:
21
+ _custom_repr: Optional[str] = None
22
+
23
+ def __invert__(self) -> "_AbstractPatternMatcher":
24
+ """Invert the filter. Returns a function that returns True if the path does not match any of the patterns.
25
+
26
+ Usage:
27
+ ```python
28
+ from pathlib import Path
29
+ from modal import FilePatternMatcher
30
+
31
+ inverted_matcher = ~FilePatternMatcher("**/*.py")
32
+
33
+ assert not inverted_matcher(Path("foo.py"))
34
+ ```
35
+ """
36
+ return _CustomPatternMatcher(lambda path: not self(path))
37
+
38
+ def with_repr(self, custom_repr) -> "_AbstractPatternMatcher":
39
+ # use to give an instance of a matcher a custom name - useful for visualizing default values in signatures
40
+ self._custom_repr = custom_repr
41
+ return self
42
+
43
+ def __repr__(self) -> str:
44
+ if self._custom_repr:
45
+ return self._custom_repr
46
+
47
+ return super().__repr__()
48
+
49
+ @abstractmethod
50
+ def __call__(self, path: Path) -> bool:
51
+ ...
52
+
53
+
54
+ class _CustomPatternMatcher(_AbstractPatternMatcher):
55
+ def __init__(self, predicate: Callable[[Path], bool]):
56
+ self._predicate = predicate
57
+
58
+ def __call__(self, path: Path) -> bool:
59
+ return self._predicate(path)
60
+
61
+
62
+ class FilePatternMatcher(_AbstractPatternMatcher):
63
+ """Allows matching file paths against a list of patterns."""
64
+
65
+ def __init__(self, *pattern: str) -> None:
66
+ """Initialize a new FilePatternMatcher instance.
67
+
68
+ Args:
69
+ pattern (str): One or more pattern strings.
70
+
71
+ Raises:
72
+ ValueError: If an illegal exclusion pattern is provided.
73
+ """
74
+ self.patterns: list[Pattern] = []
75
+ self.exclusions = False
76
+ for p in list(pattern):
77
+ p = p.strip()
78
+ if not p:
79
+ continue
80
+ p = os.path.normpath(p)
81
+ new_pattern = Pattern()
82
+ if p[0] == "!":
83
+ if len(p) == 1:
84
+ raise ValueError('Illegal exclusion pattern: "!"')
85
+ new_pattern.exclusion = True
86
+ p = p[1:]
87
+ self.exclusions = True
88
+ # In Python, we can proceed without explicit syntax checking
89
+ new_pattern.cleaned_pattern = p
90
+ new_pattern.dirs = p.split(os.path.sep)
91
+ self.patterns.append(new_pattern)
92
+
93
+ def _matches(self, file_path: str) -> bool:
94
+ """Check if the file path or any of its parent directories match the patterns.
95
+
96
+ This is equivalent to `MatchesOrParentMatches()` in the original Go
97
+ library. The reason is that `Matches()` in the original library is
98
+ deprecated due to buggy behavior.
99
+ """
100
+ matched = False
101
+ file_path = os.path.normpath(file_path)
102
+ if file_path == ".":
103
+ # Don't let them exclude everything; kind of silly.
104
+ return False
105
+ parent_path = os.path.dirname(file_path)
106
+ if parent_path == "":
107
+ parent_path = "."
108
+ parent_path_dirs = parent_path.split(os.path.sep)
109
+
110
+ for pattern in self.patterns:
111
+ # Skip evaluation based on current match status and pattern exclusion
112
+ if pattern.exclusion != matched:
113
+ continue
114
+
115
+ match = pattern.match(file_path)
116
+
117
+ if not match and parent_path != ".":
118
+ # Check if the pattern matches any of the parent directories
119
+ for i in range(len(parent_path_dirs)):
120
+ dir_path = os.path.sep.join(parent_path_dirs[: i + 1])
121
+ if pattern.match(dir_path):
122
+ match = True
123
+ break
124
+
125
+ if match:
126
+ matched = not pattern.exclusion
127
+
128
+ return matched
129
+
130
+ def __call__(self, file_path: Path) -> bool:
131
+ """Check if the path matches any of the patterns.
132
+
133
+ Args:
134
+ file_path (Path): The path to check.
135
+
136
+ Returns:
137
+ True if the path matches any of the patterns.
138
+
139
+ Usage:
140
+ ```python
141
+ from pathlib import Path
142
+ from modal import FilePatternMatcher
143
+
144
+ matcher = FilePatternMatcher("*.py")
145
+
146
+ assert matcher(Path("foo.py"))
147
+ ```
148
+ """
149
+ return self._matches(str(file_path))
150
+
151
+
152
+ # with_repr allows us to use this matcher as a default value in a function signature
153
+ # and get a nice repr in the docs and auto-generated type stubs:
154
+ NON_PYTHON_FILES = (~FilePatternMatcher("**/*.py")).with_repr(f"{__name__}.NON_PYTHON_FILES")