fal 1.13.1__tar.gz → 1.13.3__tar.gz

{fal-1.13.1/fal.egg-info → fal-1.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fal
-Version: 1.13.1
+Version: 1.13.3
 Summary: fal is an easy-to-use Serverless Python Framework
 Author: Features & Labels <support@fal.ai>
 Requires-Python: >=3.8

{fal-1.13.1 → fal-1.13.3/fal.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fal
-Version: 1.13.1
+Version: 1.13.3
 Summary: fal is an easy-to-use Serverless Python Framework
 Author: Features & Labels <support@fal.ai>
 Requires-Python: >=3.8

{fal-1.13.1 → fal-1.13.3}/src/fal/_fal_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.13.1'
-__version_tuple__ = version_tuple = (1, 13, 1)
+__version__ = version = '1.13.3'
+__version_tuple__ = version_tuple = (1, 13, 3)

{fal-1.13.1 → fal-1.13.3}/src/fal/exceptions/_cuda.py

@@ -26,12 +26,22 @@ def _is_cuda_oom_exception(exception: BaseException) -> bool:
 
 # based on https://github.com/BlackHC/toma/blob/master/toma/torch_cuda_memory.py
 def _is_cuda_out_of_memory(exception: BaseException) -> bool:
-    return (
-        isinstance(exception, RuntimeError)
-        and len(exception.args) == 1
-        and "CUDA" in exception.args[0]
-        and "out of memory" in exception.args[0]
-    )
+    if not isinstance(exception, RuntimeError) or len(exception.args) != 1:
+        return False
+
+    msg = exception.args[0]
+
+    if "CUDA" in msg and "out of memory" in msg:
+        return True
+
+    # https://github.com/pytorch/pytorch/issues/112377
+    if "NVML_SUCCESS == r INTERNAL ASSERT FAILED" in msg:
+        return True
+
+    if "CUDNN_STATUS_INTERNAL_ERROR" in msg:
+        return True
+
+    return False
 
 
 # based on https://github.com/BlackHC/toma/blob/master/toma/torch_cuda_memory.py

{fal-1.13.1 → fal-1.13.3}/tests/test_apps.py

@@ -218,6 +218,15 @@ class ExceptionApp(fal.App, keep_alive=300, max_concurrency=1):
         # mimicking error message from PyTorch (https://github.com/pytorch/pytorch/blob/6c65fd03942415b68040e102c44cf5109d2d851e/c10/cuda/CUDACachingAllocator.cpp#L1234C12-L1234C30)
         raise RuntimeError("CUDA out of memory")
 
+    @fal.endpoint("/cuda-exception-2")
+    def cuda_exception_2(self) -> Output:
+        # https://github.com/pytorch/pytorch/issues/112377
+        raise RuntimeError("NVML_SUCCESS == r INTERNAL ASSERT FAILED")
+
+    @fal.endpoint("/cuda-exception-3")
+    def cuda_exception_3(self) -> Output:
+        raise RuntimeError("cuDNN error: CUDNN_STATUS_INTERNAL_ERROR")
+
 
 class CancellableApp(fal.App, keep_alive=300, max_concurrency=1):
     task = None
@@ -899,6 +908,18 @@ def test_app_exceptions(test_exception_app: AppClient):
     assert cuda_exc.value.status_code == _CUDA_OOM_STATUS_CODE
     assert _CUDA_OOM_MESSAGE in cuda_exc.value.message
 
+    with pytest.raises(AppClientError) as cuda_exc:
+        test_exception_app.cuda_exception_2({})
+
+    assert cuda_exc.value.status_code == _CUDA_OOM_STATUS_CODE
+    assert _CUDA_OOM_MESSAGE in cuda_exc.value.message
+
+    with pytest.raises(AppClientError) as cuda_exc:
+        test_exception_app.cuda_exception_3({})
+
+    assert cuda_exc.value.status_code == _CUDA_OOM_STATUS_CODE
+    assert _CUDA_OOM_MESSAGE in cuda_exc.value.message
+
 
 def test_kill_runner():
     app_alias = str(uuid.uuid4()) + "-sleep-alias"