lightning 2.1.4__tar.gz → 2.2.0__tar.gz

{lightning-2.1.4 → lightning-2.2.0}/.actions/assistant.py

@@ -442,9 +442,20 @@ class AssistantCLI:
         target_dir: str = "docs/source-pytorch/XXX",
         checkout: str = "refs/tags/1.0.0",
         source_dir: str = "docs/source",
+        single_page: Optional[str] = None,
         as_orphan: bool = False,
     ) -> None:
-        """Pull docs pages from external source and append to local docs."""
+        """Pull docs pages from external source and append to local docs.
+
+        Args:
+            gh_user_repo: standard GitHub user/repo string
+            target_dir: relative location inside the docs folder
+            checkout: specific tag or branch to checkout
+            source_dir: relative location inside the remote / external repo
+            single_page: copy only single page from the remote repo and name it as the repo name
+            as_orphan: append orphan statement to the page
+
+        """
         import zipfile
 
         zip_url = f"https://github.com/{gh_user_repo}/archive/{checkout}.zip"
@@ -464,6 +475,14 @@ class AssistantCLI:
             assert len(zip_dirs) == 1
             repo_dir = zip_dirs[0]
 
+            if single_page:  # special case for copying single page
+                single_page = os.path.join(repo_dir, source_dir, single_page)
+                assert os.path.isfile(single_page), f"File '{single_page}' does not exist."
+                name = re.sub(r"lightning[-_]?", "", gh_user_repo.split("/")[-1])
+                new_rst = os.path.join(_PROJECT_ROOT, target_dir, f"{name}.rst")
+                AssistantCLI._copy_rst(single_page, new_rst, as_orphan=as_orphan)
+                return
+            # continue with copying all pages
             ls_pages = glob.glob(os.path.join(repo_dir, source_dir, "*.rst"))
             ls_pages += glob.glob(os.path.join(repo_dir, source_dir, "**", "*.rst"))
             for rst in ls_pages:

{lightning-2.1.4/src/lightning.egg-info → lightning-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lightning
-Version: 2.1.4
+Version: 2.2.0
 Summary: The Deep Learning framework to train, deploy, and ship AI products Lightning fast.
 Home-page: https://github.com/Lightning-AI/lightning
 Author: Lightning AI et al.
@@ -27,21 +27,21 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Provides-Extra: data-cloud
+Provides-Extra: data-test
+Provides-Extra: data-examples
+Provides-Extra: pytorch-test
+Provides-Extra: pytorch-examples
+Provides-Extra: pytorch-strategies
+Provides-Extra: pytorch-extra
 Provides-Extra: store-test
-Provides-Extra: app-ui
 Provides-Extra: app-cloud
 Provides-Extra: app-test
+Provides-Extra: app-ui
 Provides-Extra: app-components
-Provides-Extra: data-cloud
-Provides-Extra: data-test
-Provides-Extra: data-examples
 Provides-Extra: fabric-test
 Provides-Extra: fabric-examples
 Provides-Extra: fabric-strategies
-Provides-Extra: pytorch-test
-Provides-Extra: pytorch-extra
-Provides-Extra: pytorch-examples
-Provides-Extra: pytorch-strategies
 Provides-Extra: fabric-all
 Provides-Extra: fabric-dev
 Provides-Extra: pytorch-all
@@ -51,15 +51,15 @@ Provides-Extra: app-all
 Provides-Extra: app-dev
 Provides-Extra: data-all
 Provides-Extra: data-dev
-Provides-Extra: test
-Provides-Extra: ui
 Provides-Extra: cloud
-Provides-Extra: app
-Provides-Extra: components
+Provides-Extra: test
 Provides-Extra: data
 Provides-Extra: examples
 Provides-Extra: strategies
 Provides-Extra: extra
+Provides-Extra: app
+Provides-Extra: ui
+Provides-Extra: components
 Provides-Extra: all
 Provides-Extra: dev
 Provides-Extra: store
@@ -83,7 +83,7 @@ ______________________________________________________________________
   <a href="https://lightning.ai/docs/pytorch/stable/">PyTorch Lightning</a> •
   <a href="https://lightning.ai/docs/fabric/stable/">Fabric</a> •
   <a href="https://lightning.ai/docs/app/stable/">Lightning Apps</a> •
-  <a href="https://pytorch-lightning.readthedocs.io/en/2.1.4">Docs</a> •
+  <a href="https://pytorch-lightning.readthedocs.io/en/2.2.0">Docs</a> •
   <a href="#community">Community</a> •
   <a href="https://lightning.ai/docs/pytorch/stable/generated/CONTRIBUTING.html">Contribute</a> •
 </p>
@@ -624,13 +624,13 @@ Lightning is rigorously tested across multiple CPUs, GPUs and TPUs and against m
 
 <center>
 
-|       System / PyTorch ver.        |                                                                                                              1.12                                                                                                               | 1.13                                                                                                                                                                                                                            | 2.0                                                                                                                                                                                                                             |                                                                                                               2.1                                                                                                               |
-| :--------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
-|        Linux py3.9 \[GPUs\]        | |  |  | [![Build Status](https://dev.azure.com/Lightning-AI/lightning/_apis/build/status%2Fpytorch-lightning%20%28GPUs%29?branchName=refs%2Ftags%2F2.1.4)](https://dev.azure.com/Lightning-AI/lightning/_build/latest?definitionId=24&branchName=master) |
-|        Linux py3.9 \[TPUs\]        |                          |                                                                                                                                                                                                                                 |  [![Test PyTorch - TPU](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml)     |      |
-|  Linux (multiple Python versions)  |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
-|   OSX (multiple Python versions)   |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
-| Windows (multiple Python versions) |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+|       System / PyTorch ver.        | 1.13                                                                                                                                                                                                                            | 2.0                                                                                                                                                                                                                             |                                                                                                               2.1                                                                                                               |
+| :--------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
+|        Linux py3.9 \[GPUs\]        |  |  | [![Build Status](https://dev.azure.com/Lightning-AI/lightning/_apis/build/status%2Fpytorch-lightning%20%28GPUs%29?branchName=refs%2Ftags%2F2.2.0)](https://dev.azure.com/Lightning-AI/lightning/_build/latest?definitionId=24&branchName=master) |
+|        Linux py3.9 \[TPUs\]        |                                                                                                                                                                                                                                 |  [![Test PyTorch - TPU](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml)     |      |
+|  Linux (multiple Python versions)  | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+|   OSX (multiple Python versions)   | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+| Windows (multiple Python versions) | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
 
 </center>
 </details>

{lightning-2.1.4 → lightning-2.2.0}/README.md

@@ -598,13 +598,13 @@ Lightning is rigorously tested across multiple CPUs, GPUs and TPUs and against m
 
 <center>
 
-|       System / PyTorch ver.        |                                                                                                              1.12                                                                                                               | 1.13                                                                                                                                                                                                                            | 2.0                                                                                                                                                                                                                             |                                                                                                               2.1                                                                                                               |
-| :--------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
-|        Linux py3.9 \[GPUs\]        | |  |  | [![Build Status](https://dev.azure.com/Lightning-AI/lightning/_apis/build/status%2Fpytorch-lightning%20%28GPUs%29?branchName=master)](https://dev.azure.com/Lightning-AI/lightning/_build/latest?definitionId=24&branchName=master) |
-|        Linux py3.9 \[TPUs\]        |                          |                                                                                                                                                                                                                                 |  [![Test PyTorch - TPU](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml)     |      |
-|  Linux (multiple Python versions)  |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
-|   OSX (multiple Python versions)   |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
-| Windows (multiple Python versions) |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+|       System / PyTorch ver.        | 1.13                                                                                                                                                                                                                            | 2.0                                                                                                                                                                                                                             |                                                                                                               2.1                                                                                                               |
+| :--------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
+|        Linux py3.9 \[GPUs\]        |  |  | [![Build Status](https://dev.azure.com/Lightning-AI/lightning/_apis/build/status%2Fpytorch-lightning%20%28GPUs%29?branchName=master)](https://dev.azure.com/Lightning-AI/lightning/_build/latest?definitionId=24&branchName=master) |
+|        Linux py3.9 \[TPUs\]        |                                                                                                                                                                                                                                 |  [![Test PyTorch - TPU](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/tpu-tests.yml)     |      |
+|  Linux (multiple Python versions)  | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+|   OSX (multiple Python versions)   | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
+| Windows (multiple Python versions) | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 | [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                                 |                 [![Test PyTorch](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml/badge.svg)](https://github.com/Lightning-AI/lightning/actions/workflows/ci-tests-pytorch.yml)                 |
 
 </center>
 </details>

{lightning-2.1.4 → lightning-2.2.0}/pyproject.toml

@@ -79,7 +79,6 @@ ignore-init-module-imports = true
 ".actions/*" = ["S101", "S310"]
 "setup.py" = ["S101"]
 "examples/**" = [
-    "F841",  # Local variable is assigned to but never used
     "S101",  # Use of `assert` detected
     "S113",  # todo: Probable use of requests call without
     "S104",  # Possible binding to all interface
@@ -89,7 +88,6 @@ ignore-init-module-imports = true
     "S108",  # Probable insecure usage of temporary file or directory: "/tmp/data/MNIST"
 ]
 "src/**" = [
-    "F841",  # Local variable is assigned to but never used
     "S101",  # todo: Use of `assert` detected
     "S105", "S106", "S107",  # todo: Possible hardcoded password: ...
     "S113",  # todo: Probable use of requests call without timeout
@@ -105,7 +103,6 @@ ignore-init-module-imports = true
     "RET503",
 ]
 "tests/**" = [
-    "F841",  # Local variable is assigned to but never used
     "S101",  # Use of `assert` detected
     "S105", "S106",  # todo: Possible hardcoded password: ...
     "S301",  # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue

{lightning-2.1.4 → lightning-2.2.0}/requirements/app/app.txt

@@ -1,6 +1,6 @@
 lightning-cloud == 0.5.64  # Must be pinned to ensure compatibility
 packaging
-typing-extensions >=4.0.0, <4.8.0
+typing-extensions >=4.4.0, <4.10.0
 deepdiff >=5.7.0, <6.6.0
 fsspec[http] >=2022.5.0, <2023.11.0
 croniter >=1.3.0, <1.5.0  # strict; TODO: for now until we find something more robust.

{lightning-2.1.4 → lightning-2.2.0}/requirements/base.txt

@@ -3,8 +3,8 @@ fsspec[http]<2024.0,>=2022.5.0
 lightning-utilities<1.0,>=0.8.0
 numpy<2.0,>=1.17.2
 packaging<24.0,>=20.0
-torch<3.0,>=1.12.0
+torch<3.0,>=1.13.0
 torchmetrics<2.0,>=0.7.0
 tqdm<5.0,>=4.57.0
-typing-extensions<5.0,>=4.0.0
+typing-extensions<5.0,>=4.4.0
 

{lightning-2.1.4 → lightning-2.2.0}/requirements/data/test.txt

@@ -5,3 +5,5 @@ pytest-timeout ==2.1.0
 pytest-rerunfailures ==12.0
 pytest-random-order ==1.1.0
 viztracer
+pyarrow
+polars

{lightning-2.1.4 → lightning-2.2.0}/requirements/fabric/base.txt

@@ -2,8 +2,8 @@
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
 numpy >=1.17.2, <1.27.0
-torch >=1.12.0, <2.2.0
+torch >=1.13.0, <2.2.0
 fsspec[http] >=2022.5.0, <2023.11.0
 packaging >=20.0, <=23.1
-typing-extensions >=4.0.0, <4.8.0
+typing-extensions >=4.4.0, <4.10.0
 lightning-utilities >=0.8.0, <0.10.0

{lightning-2.1.4 → lightning-2.2.0}/requirements/fabric/examples.txt

@@ -1,6 +1,6 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-torchvision >=0.13.0, <0.17.0
+torchvision >=0.14.0, <0.17.0
 torchmetrics >=0.10.0, <1.3.0
 lightning-utilities >=0.8.0, <0.10.0

{lightning-2.1.4/requirements/pytorch → lightning-2.2.0/requirements/fabric}/strategies.txt

@@ -1,6 +1,9 @@
+# Despite its misleading name, this file contains dependencies for testing that only get installed in the GPU CI
+
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
 # note: is a bug around 0.10 with `MPS_Accelerator must implement all abstract methods`
 #  shall be resolved by https://github.com/microsoft/DeepSpeed/issues/4372
 deepspeed >=0.8.2, <=0.9.3; platform_system != "Windows"  # strict
+bitsandbytes ==0.41.0  # strict

{lightning-2.1.4 → lightning-2.2.0}/requirements/pytorch/base.txt

@@ -2,11 +2,11 @@
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
 numpy >=1.17.2, <1.27.0
-torch >=1.12.0, <2.2.0
+torch >=1.13.0, <2.2.0
 tqdm >=4.57.0, <4.67.0
 PyYAML >=5.4, <6.1.0
 fsspec[http] >=2022.5.0, <2023.11.0
 torchmetrics >=0.7.0, <1.3.0  # needed for using fixed compare_version
 packaging >=20.0, <=23.1
-typing-extensions >=4.0.0, <4.8.0
+typing-extensions >=4.4.0, <4.10.0
 lightning-utilities >=0.8.0, <0.10.0

{lightning-2.1.4 → lightning-2.2.0}/requirements/pytorch/examples.txt

@@ -1,7 +1,7 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-torchvision >=0.13.0, <0.17.0
+torchvision >=0.14.0, <0.17.0
 gym[classic_control] >=0.17.0, <0.27.0
 ipython[all] <8.15.0
 torchmetrics >=0.10.0, <1.3.0

{lightning-2.1.4 → lightning-2.2.0}/requirements/pytorch/extra.txt

@@ -8,4 +8,4 @@ hydra-core >=1.0.5, <1.4.0
 jsonargparse[signatures] >=4.26.1, <4.28.0
 rich >=12.3.0, <13.6.0
 tensorboardX >=2.2, <2.7.0  # min version is set by torch.onnx missing attribute
-bitsandbytes <=0.41.1
+bitsandbytes ==0.41.0  # strict

{lightning-2.1.4 → lightning-2.2.0}/src/lightning/app/CHANGELOG.md

@@ -34,7 +34,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
-- add flow `fail()` ([#18883](https://github.com/Lightning-AI/lightning/pull/18883))
+- Added flow `fail()` ([#18883](https://github.com/Lightning-AI/lightning/pull/18883))
 
 ### Fixed
 

{lightning-2.1.4 → lightning-2.2.0}/src/lightning/app/cli/cmd_init.py

@@ -27,7 +27,7 @@ def app(app_name: str) -> None:
         app_name = _capture_valid_app_component_name(resource_type="app")
 
     # generate resource template
-    new_resource_name, name_for_files = _make_resource(resource_dir="app-template", resource_name=app_name)
+    new_resource_name, _ = _make_resource(resource_dir="app-template", resource_name=app_name)
 
     m = f"""
     ⚡ Lightning app template created! ⚡

lightning-2.2.0/src/lightning/data/processing/image.py

@@ -0,0 +1,47 @@
+import io
+from typing import Optional, Tuple
+
+from lightning_utilities.core.imports import RequirementCache
+
+_HTTPX_AVAILABLE = RequirementCache("httpx")
+
+# Credit to the https://github.com/rom1504/img2dataset Github repo
+# The code was taken from there. It has a MIT License.
+
+def _download_image(
+    url: str,
+    timeout: int = 10,
+    user_agent_token: str = "pytorch-lightning",
+) -> Tuple[Optional[io.BytesIO], Optional[Exception]]:
+    """Download an image with urllib."""
+    url
+    img_stream = None
+    user_agent_string = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0"
+    if user_agent_token:
+        user_agent_string += f" (compatible; {user_agent_token}; +https://github.com/Lightning-AI/pytorch-lightning)"
+    import httpx
+
+    try:
+        with httpx.Client(http2=True) as client:
+            r = client.get(url, headers={"User-Agent": user_agent_string}, timeout=timeout)
+            img_stream = io.BytesIO(r.read())
+        return img_stream, None
+    except Exception as err:  # pylint: disable=broad-except
+        if img_stream is not None:
+            img_stream.close()
+        return None, err
+
+
+def download_image(
+    url: str,
+    retries: int = 0,
+    timeout: int = 10,
+    user_agent_token: str = "pytorch-lightning",
+) -> Tuple[Optional[io.BytesIO], Optional[Exception]]:
+    if not _HTTPX_AVAILABLE:
+        raise ModuleNotFoundError("Please, run: `pip install httpx`.")
+    for _ in range(retries + 1):
+        img_stream, err = _download_image(url, timeout, user_agent_token)
+        if img_stream is not None:
+            return img_stream, err
+    return None, err

lightning-2.2.0/src/lightning/data/processing/readers.py

@@ -0,0 +1,131 @@
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+from lightning_utilities.core.imports import RequirementCache
+
+from lightning.data.streaming.shuffle import _associate_chunks_and_internals_to_ranks
+from lightning.data.utilities.env import _DistributedEnv
+
+_POLARS_AVAILABLE = RequirementCache("polars")
+_PYARROW_AVAILABLE = RequirementCache("pyarrow")
+
+
+class BaseReader(ABC):
+
+    def get_num_nodes(self) -> int:
+        return int(os.getenv("DATA_OPTIMIZER_NUM_NODES", 1))
+
+    def get_node_rank(self) -> int:
+        return int(os.getenv("DATA_OPTIMIZER_NODE_RANK", 0))
+
+    @abstractmethod
+    def items_to_workers(self, items: List[Any], num_workers: int) -> List[List[Any]]:
+        """This method is meant to convert the items provided by the users into items to be processed by the
+        workers."""
+        pass
+
+    @abstractmethod
+    def read(self, item: Any) -> Any:
+        """Read the data associated to an item."""
+        pass
+
+
+@dataclass
+class ParquetSlice:
+    """Keep track of a parquet file slice with its filepath, start and end."""
+    filepath: str
+    start: int
+    end: int
+
+
+class ParquetReader(BaseReader):
+
+    def __init__(self, num_rows: Optional[int] = 2048, to_pandas: bool = True) -> None:
+        self.num_rows = num_rows
+        self.to_pandas = to_pandas
+
+        if not _PYARROW_AVAILABLE or not _POLARS_AVAILABLE:
+            raise ModuleNotFoundError("Please, run: `pip install pyarrow polars`")
+
+    def _get_num_rows(self, path: str) -> int:
+        if _PYARROW_AVAILABLE:
+            import pyarrow.dataset as ds
+            df = ds.dataset(path).scanner()
+            return df.count_rows()
+
+        # FIXED: There is a bug in polars. This leads to read_parquet to hang.
+        if _POLARS_AVAILABLE:
+            import polars as pol
+            df = pol.scan_parquet(path)
+            num_rows = df.select(pol.len()).collect().item()
+            return num_rows
+
+        raise RuntimeError("Please, install either pyarrow or polars.")
+
+    def read(self, item: ParquetSlice) -> Any:
+        if _POLARS_AVAILABLE:
+            import polars as pol
+            df = pol.scan_parquet(item.filepath).slice(item.start, item.end).collect()
+
+            if self.to_pandas:
+                df = df.to_pandas()
+
+            return df
+
+        if _PYARROW_AVAILABLE:
+            import pyarrow.dataset as ds
+
+            df = ds.dataset(item.filepath).scanner()
+
+            df = df.take([item.start, item.end])
+
+            if self.to_pandas:
+                df.to_pandas()
+
+            return df
+
+        raise RuntimeError("Please, install either pyarrow or polars.")
+
+
+    def items_to_workers(self, items: Any, num_workers: int) -> List[List[ParquetSlice]]:
+        intervals = [(0, self._get_num_rows(item)) for item in items]
+
+        world_size = self.get_num_nodes() * num_workers
+        node_rank = self.get_node_rank()
+
+        fake_distributed_env = _DistributedEnv(world_size, 0, self.get_num_nodes())
+        parquet_indexes_per_worker, p_slices_per_worker = _associate_chunks_and_internals_to_ranks(
+            fake_distributed_env, list(range(len(items))), intervals, False)
+
+        workers_user_items: List[List[ParquetSlice]] = [[] for _ in range(num_workers)]
+
+        iterator = enumerate(zip(parquet_indexes_per_worker, p_slices_per_worker))
+
+        node_start = node_rank * num_workers
+        node_end = (node_rank + 1) * num_workers
+
+        for worker_idx, (parquet_indexes, p_slices) in iterator:
+            if node_start <= worker_idx < node_end:
+                if self.num_rows:
+                    workers_user_items[worker_idx % num_workers].extend([
+                        ParquetSlice(
+                            items[parquet_index], p_slice_start, p_slice_start + self.num_rows
+                            if p_slice[1] > (p_slice_start + self.num_rows) else
+                            p_slice[1]
+                        )
+                        for parquet_index, p_slice in zip(parquet_indexes, p_slices)
+                        for p_slice_start in range(p_slice[0], p_slice[1] + self.num_rows, self.num_rows)
+                        if p_slice_start < p_slice[1]
+                    ])
+                else:
+                    workers_user_items[worker_idx % num_workers].extend([
+                        ParquetSlice(items[parquet_index], *p_slice)
+                        for parquet_index, p_slice in zip(parquet_indexes, p_slices)
+                    ])
+
+        assert len(workers_user_items) == num_workers
+        assert all(len(w) for w in workers_user_items)
+
+        return workers_user_items

{lightning-2.1.4 → lightning-2.2.0}/src/lightning/data/streaming/cache.py

@@ -61,7 +61,9 @@ class Cache:
             raise ModuleNotFoundError("PyTorch version 2.1 or higher is required to use the cache.")
 
         if not _LIGHTNING_CLOUD_LATEST:
-            raise ModuleNotFoundError("Lightning Cloud latest is required to use the cache.")
+            raise ModuleNotFoundError(
+                "The `lightning-cloud` package in your environement is out-dated."
+                " Run: `pip install -U lightning-cloud` to resolve this.")
 
         input_dir = _resolve_dir(input_dir)
         self._cache_dir = input_dir.path

{lightning-2.1.4 → lightning-2.2.0}/src/lightning/data/streaming/constants.py

@@ -57,3 +57,4 @@ _NUMPY_SCTYPES = [v for values in np.sctypes.values() for v in values]
 _NUMPY_DTYPES_MAPPING = {i: np.dtype(v) for i, v in enumerate(_NUMPY_SCTYPES)}
 
 _TIME_FORMAT = "%Y-%m-%d_%H-%M-%S.%fZ"
+_IS_IN_STUDIO = bool(os.getenv("LIGHTNING_CLOUD_PROJECT_ID", None)) and bool(os.getenv("LIGHTNING_CLUSTER_ID", None))