litlogger 0.1.5__py3-none-any.whl

litlogger/__init__.py

@@ -0,0 +1,64 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Use litlogger to track machine learning experiments with Lightning.ai.
+
+For guides and examples, see https://lightning.ai.
+
+For reference documentation, see https://github.com/Lightning-AI/litlogger.
+"""
+
+__version__ = "0.1.5"
+
+# Import core classes
+# Import preinit utilities
+from litlogger._preinit import pre_init_callable
+from litlogger.experiment import Experiment
+
+# Import SDK functions
+from litlogger.init import finish, init
+
+# Global variables
+experiment: Experiment | None = None
+log = pre_init_callable("litlogger.log", Experiment.log_metrics)
+log_metrics = pre_init_callable("litlogger.log_metrics", Experiment.log_metrics)
+log_file = pre_init_callable("litlogger.log_file", Experiment.log_file)
+get_file = pre_init_callable("litlogger.get_file", Experiment.get_file)
+log_model = pre_init_callable("litlogger.log_model", Experiment.log_model)
+get_model = pre_init_callable("litlogger.get_model", Experiment.get_model)
+log_model_artifact = pre_init_callable("litlogger.log_model_artifact", Experiment.log_model_artifact)
+get_model_artifact = pre_init_callable("litlogger.get_model_artifact", Experiment.get_model_artifact)
+finalize = pre_init_callable("litlogger.finalize", Experiment.finalize)
+
+__all__ = [
+    "Experiment",
+    "init",
+    "finish",
+    "experiment",
+    "log",
+    "log_metrics",
+    "log_file",
+    "get_file",
+    "log_model",
+    "get_model",
+    "log_model_artifact",
+    "get_model_artifact",
+    "finalize",
+]
+
+try:
+    from litlogger.logger import LightningLogger
+
+    __all__ += ["LightningLogger"]
+except ImportError:
+    pass

litlogger/_module.py

@@ -0,0 +1,86 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Global state management for litlogger."""
+
+from typing import Any, Callable, Dict
+
+import litlogger
+from litlogger._preinit import pre_init_callable
+
+
+def set_global(
+    experiment: Any | None = None,
+    log: Callable | None = None,
+    log_metrics: Callable | None = None,
+    log_file: Callable | None = None,
+    get_file: Callable | None = None,
+    log_model: Callable | None = None,
+    get_model: Callable | None = None,
+    log_model_artifact: Callable | None = None,
+    get_model_artifact: Callable | None = None,
+    finalize: Callable | None = None,
+) -> None:
+    """Set global litlogger state after initialization."""
+    if experiment:
+        litlogger.experiment = experiment
+    if log:
+        litlogger.log = log
+    if log_metrics:
+        litlogger.log_metrics = log_metrics
+    if log_file:
+        litlogger.log_file = log_file
+    if get_file:
+        litlogger.get_file = get_file
+    if log_model:
+        litlogger.log_model = log_model
+    if get_model:
+        litlogger.get_model = get_model
+    if log_model_artifact:
+        litlogger.log_model_artifact = log_model_artifact
+    if get_model_artifact:
+        litlogger.get_model_artifact = get_model_artifact
+    if finalize:
+        litlogger.finalize = finalize
+
+
+def get_global() -> Dict[str, Any]:
+    """Get the global litlogger state."""
+    return {
+        "experiment": litlogger.experiment,
+        "log": litlogger.log,
+        "log_metrics": litlogger.log_metrics,
+        "log_file": litlogger.log_file,
+        "get_file": litlogger.get_file,
+        "log_model": litlogger.log_model,
+        "get_model": litlogger.get_model,
+        "log_model_artifact": litlogger.log_model_artifact,
+        "get_model_artifact": litlogger.get_model_artifact,
+        "finalize": litlogger.finalize,
+    }
+
+
+def unset_globals() -> None:
+    """Reset global litlogger state to pre-init state."""
+    from litlogger.experiment import Experiment
+
+    litlogger.experiment = None
+    litlogger.log = pre_init_callable("litlogger.log", Experiment.log_metrics)
+    litlogger.log_metrics = pre_init_callable("litlogger.log_metrics", Experiment.log_metrics)
+    litlogger.log_file = pre_init_callable("litlogger.log_file", Experiment.log_file)
+    litlogger.get_file = pre_init_callable("litlogger.get_file", Experiment.get_file)
+    litlogger.log_model = pre_init_callable("litlogger.log_model", Experiment.log_model)
+    litlogger.get_model = pre_init_callable("litlogger.get_model", Experiment.get_model)
+    litlogger.log_model_artifact = pre_init_callable("litlogger.log_model_artifact", Experiment.log_model_artifact)
+    litlogger.get_model_artifact = pre_init_callable("litlogger.get_model_artifact", Experiment.get_model_artifact)
+    litlogger.finalize = pre_init_callable("litlogger.finalize", Experiment.finalize)

litlogger/_preinit.py

@@ -0,0 +1,55 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pre-initialization wrappers for litlogger to provide helpful error messages."""
+
+from typing import Any, Callable
+
+
+class PreInitObject:
+    """Object that raises an error if accessed before litlogger.init() is called."""
+
+    def __init__(self, name: str, destination: Any | None = None) -> None:
+        self._name = name
+
+        if destination is not None:
+            self.__doc__ = destination.__doc__
+
+    def __getitem__(self, key: str) -> None:
+        raise RuntimeError(f"You must call litlogger.init() before {self._name}[{key!r}]")
+
+    def __setitem__(self, key: str, value: Any) -> Any:
+        raise RuntimeError(f"You must call litlogger.init() before {self._name}[{key!r}]")
+
+    def __setattr__(self, key: str, value: Any) -> Any:
+        if not key.startswith("_"):
+            raise RuntimeError(f"You must call litlogger.init() before {self._name}.{key}")
+        return object.__setattr__(self, key, value)
+
+    def __getattr__(self, key: str) -> Any:
+        if not key.startswith("_"):
+            raise RuntimeError(f"You must call litlogger.init() before {self._name}.{key}")
+        raise AttributeError
+
+
+def pre_init_callable(name: str, destination: Any | None = None) -> Callable:
+    """Create a callable that raises an error if called before litlogger.init()."""
+
+    def preinit_wrapper(*args: Any, **kwargs: Any) -> Any:
+        raise RuntimeError(f"You must call litlogger.init() before {name}()")
+
+    preinit_wrapper.__name__ = str(name)
+    if destination:
+        preinit_wrapper.__wrapped__ = destination  # type: ignore
+        preinit_wrapper.__doc__ = destination.__doc__
+    return preinit_wrapper

litlogger/api/__init__.py

@@ -0,0 +1,18 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""litlogger API module."""
+
+__all__ = ("MetricsApi",)
+
+from .metrics_api import MetricsApi

litlogger/api/artifacts_api.py

@@ -0,0 +1,190 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from typing import Any
+
+from lightning_sdk import Teamspace
+from lightning_sdk.api.utils import _FileUploader
+from lightning_sdk.lightning_cloud.openapi import LitLoggerServiceCreateLoggerArtifactBody
+
+from litlogger.api.client import LitRestClient
+
+
+class ArtifactsApi:
+    """API layer for artifact upload and download operations."""
+
+    def __init__(self, client: LitRestClient | None = None) -> None:
+        """Initialize the ArtifactsApi.
+
+        Args:
+            client: Optional pre-configured LitRestClient. If None, creates a new one.
+        """
+        self.client = client or LitRestClient(max_retries=5)
+
+    def upload_experiment_file_artifact(
+        self,
+        teamspace: Teamspace,
+        metrics_store: Any,
+        experiment_name: str,
+        file_path: str,
+        remote_path: str,
+    ) -> None:
+        """Upload a file as an artifact to the teamspace drive.
+
+        Args:
+            teamspace: Teamspace object where the file will be uploaded.
+            metrics_store: Metrics store object containing the stream ID.
+            experiment_name: Experiment name for organizing artifacts.
+            file_path: Local path to the file to upload.
+            remote_path: Path relative to experiment root for storage and display.
+
+        Raises:
+            FileNotFoundError: If the file doesn't exist.
+        """
+        if not os.path.isfile(file_path):
+            raise FileNotFoundError(f"file not found: {file_path}")
+
+        # Upload to teamspace drive under experiments folder
+        full_remote_path = f"experiments/{experiment_name}/{remote_path}"
+
+        teamspace.upload_file(file_path=file_path, remote_path=full_remote_path, progress_bar=False)
+
+        # Register the artifact with the metrics stream
+        self.client.lit_logger_service_create_logger_artifact(
+            project_id=teamspace.id,
+            metrics_stream_id=metrics_store.id,
+            body=LitLoggerServiceCreateLoggerArtifactBody(path=remote_path),
+        )
+
+    def download_experiment_file_artifact(
+        self,
+        teamspace: Teamspace,
+        experiment_name: str,
+        filename: str,
+        local_path: str | None = None,
+    ) -> str:
+        """Download a file artifact from the teamspace drive.
+
+        Args:
+            teamspace: Teamspace object where the file is stored.
+            experiment_name: Experiment name where the artifact was uploaded.
+            filename: Name of the file to download.
+            local_path: Optional local path where the file should be saved.
+                       If None, saves to current directory with the same filename.
+
+        Raises:
+            FileNotFoundError: If the remote file doesn't exist.
+        """
+        # Construct the remote path
+        remote_path = f"experiments/{experiment_name}/{filename}"
+
+        # Determine local save path
+        if local_path is None:
+            local_path = filename
+        elif os.path.isdir(local_path):
+            local_path = os.path.join(local_path, filename)
+
+        # Convert to absolute path for cross-platform compatibility (Windows needs this)
+        local_path = os.path.abspath(local_path)
+
+        # Create directory if needed
+        local_dir = os.path.dirname(local_path)
+        if local_dir and not os.path.exists(local_dir):
+            os.makedirs(local_dir, exist_ok=True)
+
+        # Download from teamspace drive
+        teamspace.download_file(remote_path=remote_path, file_path=local_path)
+
+        return local_path
+
+    def upload_file(
+        self,
+        teamspace: Teamspace,
+        local_path: str,
+        remote_path: str,
+    ) -> str:
+        """Upload a file to the teamspace drive (generic, not experiment-specific).
+
+        Args:
+            teamspace: Teamspace object where the file will be uploaded.
+            local_path: Local path to the file to upload.
+            remote_path: Remote path in the teamspace drive.
+
+        Returns:
+            str: The remote path where the file was uploaded.
+
+        Raises:
+            FileNotFoundError: If the local file doesn't exist.
+        """
+        if not os.path.isfile(local_path):
+            raise FileNotFoundError(f"File not found: {local_path}")
+
+        # Upload to teamspace drive
+        teamspace.upload_file(file_path=local_path, remote_path=remote_path, progress_bar=False)
+
+        return remote_path
+
+    def download_file(
+        self,
+        teamspace: Teamspace,
+        remote_path: str,
+        local_path: str,
+    ) -> str:
+        """Download a file from the teamspace drive (generic, not experiment-specific).
+
+        Args:
+            teamspace: Teamspace object where the file is stored.
+            remote_path: Remote path in the teamspace drive.
+            local_path: Local path where the file should be saved.
+
+        Returns:
+            str: The local path where the file was saved.
+        """
+        # Convert to absolute path for cross-platform compatibility (Windows needs this)
+        local_path = os.path.abspath(local_path)
+
+        # Create directory if needed
+        local_dir = os.path.dirname(local_path)
+        if local_dir and not os.path.exists(local_dir):
+            os.makedirs(local_dir, exist_ok=True)
+
+        # Download from teamspace drive
+        teamspace.download_file(remote_path=remote_path, file_path=local_path)
+
+        return local_path
+
+    def upload_metrics_binary(
+        self,
+        teamspace_id: str,
+        cloud_account: str,
+        file_path: str,
+        remote_path: str,
+    ) -> None:
+        """Upload a metrics binary tar.gz file to the teamspace.
+
+        Args:
+            teamspace_id: The teamspace ID.
+            cloud_account: Cloud account identifier.
+            file_path: Local path to the tar.gz file to upload.
+            remote_path: Remote path where the file will be uploaded.
+        """
+        file_uploader = _FileUploader(
+            client=self.client,
+            teamspace_id=teamspace_id,
+            cloud_account=cloud_account,
+            file_path=file_path,
+            remote_path=remote_path,
+            progress_bar=False,
+        )
+        file_uploader()

litlogger/api/auth_api.py

@@ -0,0 +1,43 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from lightning_sdk.lightning_cloud.login import Auth
+
+
+class AuthApi:
+    def __init__(self) -> None:
+        self.auth = Auth()
+
+    def authenticate(self) -> bool:
+        """Authenticate the user or perform a guest login.
+
+        Returns:
+            bool: True if the user is authenticated, False if the user is a guest.
+        """
+        self.auth.load()
+
+        if getattr(self.auth, "user_id", None) and getattr(self.auth, "api_key", None):
+            self.auth.authenticate()
+            return True
+
+        self.auth.guest_login()
+        return False
+
+    @property
+    def guest_id(self) -> str:
+        """Get the guest ID.
+
+        Returns:
+            str: The guest ID.
+        """
+        return self.auth.api_key

litlogger/api/client.py

@@ -0,0 +1,103 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Retry-enabled REST client for Lightning Cloud.
+
+Provides a thin wrapper around GridRestClient that decorates API calls with an
+exponential backoff strategy for transient network/server errors.
+"""
+
+import time
+from functools import wraps
+from logging import Logger
+from typing import Any, Callable
+
+import urllib3
+from lightning_sdk.lightning_cloud.openapi.rest import ApiException
+from lightning_sdk.lightning_cloud.rest_client import GridRestClient, _get_next_backoff_time, create_swagger_client
+
+logger = Logger(__name__)
+
+
+def _should_retry(ex: BaseException) -> bool:
+    """Return True if the exception is transient and the request should be retried."""
+    if isinstance(ex, urllib3.exceptions.HTTPError):
+        return True
+
+    if "not found" in str(ex):
+        return False
+
+    if str(ex.status).startswith("4") and ex.status not in (400, 401, 404):
+        return True
+
+    return str(ex.status).startswith("5")
+
+
+def _retry_wrapper(self: Any, func: Callable, max_retries: int = -1) -> Callable:
+    """Returns the function decorated by a wrapper that retries the call several times if a connection error occurs.
+
+    The retries follow an exponential backoff.
+
+    """
+
+    @wraps(func)
+    def wrapped(*args: Any, **kwargs: Any) -> Any:
+        consecutive_errors = 0
+
+        while True:
+            try:
+                return func(self, *args, **kwargs)
+            except (ApiException, urllib3.exceptions.HTTPError) as ex:
+                if not _should_retry(ex):
+                    raise ex
+
+                msg = f"error: {ex!s}" if isinstance(ex, urllib3.exceptions.HTTPError) else f"response: {ex.status}"
+
+                if consecutive_errors == max_retries:
+                    raise RuntimeError(f"The {func.__name__} request failed to reach the server, {msg}.") from ex
+
+                consecutive_errors += 1
+                backoff_time = _get_next_backoff_time(consecutive_errors)
+                logger.warning(
+                    f"The {func.__name__} request failed to reach the server, {msg}."
+                    f" Retrying after {backoff_time} seconds."
+                )
+
+                time.sleep(backoff_time)
+
+    return wrapped
+
+
+class LitRestClient(GridRestClient):
+    """The LitRestClient is a wrapper around the GridRestClient.
+
+    It wraps all methods to monitor connection exceptions and employs a retry strategy.
+
+    Args:
+        max_retries: Maximum number of attempts where each delay between retries is exponential.
+            If set to -1, it will retry forever, in contrast if set 0, it runs it only once.
+
+    """
+
+    def __init__(self, max_retries: int = -1) -> None:
+        super().__init__(api_client=create_swagger_client())
+        if max_retries == 0:
+            return
+        for base_class in GridRestClient.__mro__:
+            for name, attribute in base_class.__dict__.items():
+                if callable(attribute) and attribute.__name__ != "__init__":
+                    setattr(
+                        self,
+                        name,
+                        _retry_wrapper(self, attribute, max_retries=max_retries),
+                    )