inference-perf 0.1.1__py3-none-any.whl

inference_perf/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .main import main_cli
+
+__all__ = ["main_cli"]

inference_perf/analysis/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .analyze import analyze_reports
+
+__all__ = ["analyze_reports"]

inference_perf/analysis/analyze.py

@@ -0,0 +1,276 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import operator
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_latency_metric(latency_data: Dict[str, Any], metric_name: str, convert_to_ms: bool = False) -> float | None:
+    """Helper to extract a metric's mean value from latency data."""
+    metric_data = latency_data.get(metric_name)
+    if isinstance(metric_data, dict):
+        mean_val = metric_data.get("mean")
+        if isinstance(mean_val, (int, float)):
+            return mean_val * 1000 if convert_to_ms else mean_val
+    return None
+
+
+def _extract_throughput_metric(throughput_data: Dict[str, Any], metric_name: str) -> float | None:
+    """Helper to extract a throughput metric's value."""
+    metric_value = throughput_data.get(metric_name)
+    if isinstance(metric_value, (int, float)):
+        return float(metric_value)
+    return None
+
+
+def _generate_plot(charts_to_generate: List[Dict[str, Any]], suptitle: str, output_path: Path) -> None:
+    """Generates and saves a plot with multiple subplots."""
+    import matplotlib.pyplot as plt
+
+    if not charts_to_generate:
+        logger.warning(f"No data available to generate chart: {output_path.name}")
+        return
+
+    num_charts = len(charts_to_generate)
+    fig, axes = plt.subplots(1, num_charts, figsize=(7 * num_charts, 6), squeeze=False)
+    fig.suptitle(suptitle, fontsize=16)
+
+    for i, chart_info in enumerate(charts_to_generate):
+        ax = axes[0, i]
+        data = chart_info["data"]
+        qps_values = [x[0] for x in data]
+        y_values = [x[1] for x in data]
+
+        ax.plot(qps_values, y_values, marker="o", linestyle="-")
+        ax.set_title(chart_info["title"])
+        ax.set_xlabel(chart_info.get("xlabel", "QPS (requested rate)"))
+        ax.set_ylabel(chart_info["ylabel"])
+        ax.grid(True)
+
+    fig.tight_layout(rect=(0, 0.03, 1, 0.95))
+    plt.savefig(output_path)
+    logger.info(f"Chart saved to {output_path}")
+    plt.close(fig)
+
+
+def analyze_reports(report_dir: str) -> None:
+    """
+    Analyzes performance reports to generate charts.
+
+    Args:
+        report_dir: The directory containing the report files.
+    """
+    try:
+        # Check for matplotlib and provide a helpful error message if it's not installed.
+        import matplotlib  # noqa: F401
+    except ImportError:
+        logger.error(
+            "matplotlib is not installed. Please install it to use the --analyze feature.\n"
+            "You can install it via 'pip install .[analysis]'"
+        )
+        return
+
+    logger.info(f"Analyzing reports in {report_dir}")
+
+    # Find stage lifecycle metrics files
+    report_path = Path(report_dir)
+    stage_files = list(report_path.glob("stage_*_lifecycle_metrics.json"))
+
+    if not stage_files:
+        logger.error(f"No stage lifecycle metrics files found in {report_dir}")
+        return
+
+    # Latency data
+    qps_vs_ttft: List[Tuple[float, float]] = []
+    qps_vs_ntpot: List[Tuple[float, float]] = []
+    qps_vs_itl: List[Tuple[float, float]] = []
+    # Throughput data
+    qps_vs_itps: List[Tuple[float, float]] = []
+    qps_vs_otps: List[Tuple[float, float]] = []
+    qps_vs_ttps: List[Tuple[float, float]] = []
+    # Throughput vs Latency data
+    ttft_vs_otps: List[Tuple[float, float]] = []
+    ntpot_vs_otps: List[Tuple[float, float]] = []
+    itl_vs_otps: List[Tuple[float, float]] = []
+
+    for stage_file in stage_files:
+        try:
+            with open(stage_file, "r") as f:
+                report_data = json.load(f)
+
+            # Get QPS from report file
+            qps = report_data.get("load_summary", {}).get("requested_rate")
+            if qps is None:
+                logger.warning(f"Could not find requested_rate in {stage_file.name}. Skipping.")
+                continue
+
+            success_data = report_data.get("successes", {})
+            if not success_data:
+                logger.warning(f"No success data in {stage_file.name}. Skipping.")
+                continue
+
+            # Extract latency metrics if they exist
+            ttft, ntpot, itl = None, None, None
+            latency_data = success_data.get("latency", {})
+            if latency_data:
+                ttft = _extract_latency_metric(latency_data, "time_to_first_token", convert_to_ms=True)
+                if ttft is not None:
+                    qps_vs_ttft.append((qps, ttft))
+
+                ntpot = _extract_latency_metric(latency_data, "normalized_time_per_output_token", convert_to_ms=True)
+                if ntpot is not None:
+                    qps_vs_ntpot.append((qps, ntpot))
+
+                itl = _extract_latency_metric(latency_data, "inter_token_latency", convert_to_ms=True)
+                if itl is not None:
+                    qps_vs_itl.append((qps, itl))
+
+            # Extract throughput metrics if they exist
+            otps = None
+            throughput_data = success_data.get("throughput", {})
+            if throughput_data:
+                itps = _extract_throughput_metric(throughput_data, "input_tokens_per_sec")
+                if itps is not None:
+                    qps_vs_itps.append((qps, itps))
+
+                otps = _extract_throughput_metric(throughput_data, "output_tokens_per_sec")
+                if otps is not None:
+                    qps_vs_otps.append((qps, otps))
+
+                ttps = _extract_throughput_metric(throughput_data, "total_tokens_per_sec")
+                if ttps is not None:
+                    qps_vs_ttps.append((qps, ttps))
+
+            # Populate latency vs throughput data
+            if otps is not None:
+                if ttft is not None:
+                    ttft_vs_otps.append((ttft, otps))
+                if ntpot is not None:
+                    ntpot_vs_otps.append((ntpot, otps))
+                if itl is not None:
+                    itl_vs_otps.append((itl, otps))
+
+        except json.JSONDecodeError:
+            logger.error(f"Error decoding JSON from {stage_file.name}")
+            continue
+        except Exception as e:
+            logger.error(f"An unexpected error occurred while processing {stage_file.name}: {e}")
+            continue
+
+    # --- Generate Latency Plot ---
+    latency_charts_to_generate = []
+    if qps_vs_ttft:
+        latency_charts_to_generate.append(
+            {
+                "title": "Time to First Token vs. QPS",
+                "ylabel": "Mean TTFT (ms)",
+                "data": sorted(qps_vs_ttft, key=operator.itemgetter(0)),
+            }
+        )
+    if qps_vs_ntpot:
+        latency_charts_to_generate.append(
+            {
+                "title": "Norm. Time per Output Token vs. QPS",
+                "ylabel": "Mean Norm. Time (ms/token)",
+                "data": sorted(qps_vs_ntpot, key=operator.itemgetter(0)),
+            }
+        )
+    if qps_vs_itl:
+        latency_charts_to_generate.append(
+            {
+                "title": "Inter-Token Latency vs. QPS",
+                "ylabel": "Mean ITL (ms)",
+                "data": sorted(qps_vs_itl, key=operator.itemgetter(0)),
+            }
+        )
+
+    _generate_plot(
+        latency_charts_to_generate,
+        "Latency vs Request Rate",
+        report_path / "latency_vs_qps.png",
+    )
+
+    # --- Generate Throughput Plot ---
+    throughput_charts_to_generate = []
+    if qps_vs_itps:
+        throughput_charts_to_generate.append(
+            {
+                "title": "Input Tokens/sec vs. QPS",
+                "ylabel": "Tokens/sec",
+                "data": sorted(qps_vs_itps, key=operator.itemgetter(0)),
+            }
+        )
+    if qps_vs_otps:
+        throughput_charts_to_generate.append(
+            {
+                "title": "Output Tokens/sec vs. QPS",
+                "ylabel": "Tokens/sec",
+                "data": sorted(qps_vs_otps, key=operator.itemgetter(0)),
+            }
+        )
+    if qps_vs_ttps:
+        throughput_charts_to_generate.append(
+            {
+                "title": "Total Tokens/sec vs. QPS",
+                "ylabel": "Tokens/sec",
+                "data": sorted(qps_vs_ttps, key=operator.itemgetter(0)),
+            }
+        )
+
+    _generate_plot(
+        throughput_charts_to_generate,
+        "Throughput vs Request Rate",
+        report_path / "throughput_vs_qps.png",
+    )
+
+    # --- Generate Throughput vs Latency Curve Plot ---
+    throughput_latency_charts_to_generate = []
+    if ntpot_vs_otps:
+        throughput_latency_charts_to_generate.append(
+            {
+                "title": "Throughput vs. Norm. Time per Output Token",
+                "xlabel": "Mean Norm. Time (ms/token)",
+                "ylabel": "Output Tokens/sec",
+                "data": sorted(ntpot_vs_otps, key=operator.itemgetter(0)),
+            }
+        )
+    if ttft_vs_otps:
+        throughput_latency_charts_to_generate.append(
+            {
+                "title": "Throughput vs. Time to First Token",
+                "xlabel": "Mean TTFT (ms)",
+                "ylabel": "Output Tokens/sec",
+                "data": sorted(ttft_vs_otps, key=operator.itemgetter(0)),
+            }
+        )
+    if itl_vs_otps:
+        throughput_latency_charts_to_generate.append(
+            {
+                "title": "Throughput vs. Inter-Token Latency",
+                "xlabel": "Mean ITL (ms)",
+                "ylabel": "Output Tokens/sec",
+                "data": sorted(itl_vs_otps, key=operator.itemgetter(0)),
+            }
+        )
+
+    _generate_plot(
+        throughput_latency_charts_to_generate,
+        "Latency vs Throughput",
+        report_path / "throughput_vs_latency.png",
+    )

inference_perf/apis/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .base import InferenceAPIData, InferenceInfo, RequestLifecycleMetric, ErrorResponseInfo
+from .chat import ChatCompletionAPIData, ChatMessage
+from .completion import CompletionAPIData
+
+__all__ = [
+    "InferenceAPIData",
+    "InferenceInfo",
+    "RequestLifecycleMetric",
+    "ErrorResponseInfo",
+    "ChatCompletionAPIData",
+    "ChatMessage",
+    "CompletionAPIData",
+]

inference_perf/apis/base.py

@@ -0,0 +1,60 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+from typing import Any, List, Optional
+from aiohttp import ClientResponse
+from pydantic import BaseModel
+from inference_perf.utils.custom_tokenizer import CustomTokenizer
+from inference_perf.config import APIConfig, APIType
+
+
+class InferenceInfo(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+    output_token_times: List[float] = []
+
+
+class ErrorResponseInfo(BaseModel):
+    error_type: str
+    error_msg: str
+
+
+class RequestLifecycleMetric(BaseModel):
+    stage_id: Optional[int] = None
+    scheduled_time: float
+    start_time: float
+    end_time: float
+    request_data: str
+    response_data: Optional[str] = None
+    info: InferenceInfo
+    error: Optional[ErrorResponseInfo]
+
+
+class InferenceAPIData(BaseModel):
+    @abstractmethod
+    def get_api_type(self) -> APIType:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_route(self) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    def to_payload(self, model_name: str, max_tokens: int, ignore_eos: bool, streaming: bool) -> dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def process_response(self, response: ClientResponse, config: APIConfig, tokenizer: CustomTokenizer) -> InferenceInfo:
+        raise NotImplementedError

inference_perf/apis/chat.py

@@ -0,0 +1,64 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, List
+from aiohttp import ClientResponse
+from pydantic import BaseModel
+from inference_perf.apis import InferenceAPIData, InferenceInfo
+from inference_perf.utils.custom_tokenizer import CustomTokenizer
+from inference_perf.config import APIConfig, APIType
+
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+
+
+class ChatCompletionAPIData(InferenceAPIData):
+    messages: List[ChatMessage]
+    max_tokens: int = 0
+
+    def get_api_type(self) -> APIType:
+        return APIType.Chat
+
+    def get_route(self) -> str:
+        return "/v1/chat/completions"
+
+    def to_payload(self, model_name: str, max_tokens: int, ignore_eos: bool, streaming: bool) -> dict[str, Any]:
+        if streaming:
+            raise Exception("Generating streaming request payloads for the Chat API is not currently supported.")
+        if self.max_tokens == 0:
+            self.max_tokens = max_tokens
+        return {
+            "model": model_name,
+            "messages": [{"role": m.role, "content": m.content} for m in self.messages],
+            "max_tokens": self.max_tokens,
+            "ignore_eos": ignore_eos,
+        }
+
+    async def process_response(self, response: ClientResponse, config: APIConfig, tokenizer: CustomTokenizer) -> InferenceInfo:
+        if config.streaming:
+            raise Exception("Decoding streamed responses from the Chat API is not currently supported")
+        else:
+            data = await response.json()
+            prompt_len = tokenizer.count_tokens("".join([m.content for m in self.messages]))
+            choices = data.get("choices", [])
+            if len(choices) == 0:
+                return InferenceInfo(input_tokens=prompt_len)
+            output_text = "".join([choice.get("message", {}).get("content", "") for choice in choices])
+            output_len = tokenizer.count_tokens(output_text)
+            return InferenceInfo(
+                input_tokens=prompt_len,
+                output_tokens=output_len,
+            )

inference_perf/apis/completion.py

@@ -0,0 +1,78 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import time
+from typing import Any, List
+
+from aiohttp import ClientResponse
+from inference_perf.apis import InferenceAPIData, InferenceInfo
+from inference_perf.utils.custom_tokenizer import CustomTokenizer
+from inference_perf.config import APIConfig, APIType
+
+
+class CompletionAPIData(InferenceAPIData):
+    prompt: str
+    max_tokens: int = 0
+
+    def get_api_type(self) -> APIType:
+        return APIType.Completion
+
+    def get_route(self) -> str:
+        return "/v1/completions"
+
+    def to_payload(self, model_name: str, max_tokens: int, ignore_eos: bool, streaming: bool) -> dict[str, Any]:
+        if self.max_tokens == 0:
+            self.max_tokens = max_tokens
+        return {
+            "model": model_name,
+            "prompt": self.prompt,
+            "max_tokens": self.max_tokens,
+            "ignore_eos": ignore_eos,
+            "stream": streaming,
+        }
+
+    async def process_response(self, response: ClientResponse, config: APIConfig, tokenizer: CustomTokenizer) -> InferenceInfo:
+        if config.streaming:
+            output_text = ""
+            output_token_times: List[float] = []
+            async for chunk_bytes in response.content:
+                chunk_bytes = chunk_bytes.strip()
+                output_token_times.append(time.perf_counter())
+                if not chunk_bytes:
+                    continue
+                # After removing the "data: " prefix, each chunk decodes to a response json for a single token or "[DONE]" if end of stream
+                chunk = chunk_bytes.decode("utf-8").removeprefix("data: ")
+                if chunk != "[DONE]":
+                    data = json.loads(chunk)
+                    if choices := data.get("choices"):
+                        text = choices[0].get("text")
+                        output_text += text
+            prompt_len = tokenizer.count_tokens(self.prompt)
+            output_len = tokenizer.count_tokens(output_text)
+            return InferenceInfo(
+                input_tokens=prompt_len,
+                output_tokens=output_len,
+                output_token_times=output_token_times,
+            )
+        else:
+            data = await response.json()
+            prompt_len = tokenizer.count_tokens(self.prompt)
+            choices = data.get("choices", [])
+            if len(choices) == 0:
+                return InferenceInfo(input_tokens=prompt_len)
+            output_text = choices[0].get("text", "")
+            output_len = tokenizer.count_tokens(output_text)
+            return InferenceInfo(input_tokens=prompt_len, output_tokens=output_len)

inference_perf/client/filestorage/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .base import StorageClient
+from .local import LocalStorageClient
+from .gcs import GoogleCloudStorageClient
+from .s3 import SimpleStorageServiceClient
+
+
+__all__ = ["StorageClient", "LocalStorageClient", "GoogleCloudStorageClient","SimpleStorageServiceClient"]

inference_perf/client/filestorage/base.py

@@ -0,0 +1,31 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from abc import ABC, abstractmethod
+from typing import List
+from inference_perf.config import StorageConfigBase
+from inference_perf.utils import ReportFile
+
+logger = logging.getLogger(__name__)
+
+
+class StorageClient(ABC):
+    def __init__(self, config: StorageConfigBase) -> None:
+        self.config = config
+        logger.info(f"Report files will be stored at: {self.config.path}")
+
+    @abstractmethod
+    def save_report(self, reports: List[ReportFile]) -> None:
+        raise NotImplementedError()

inference_perf/client/filestorage/gcs.py

@@ -0,0 +1,55 @@
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+from typing import List
+from google.cloud import storage
+from google.cloud.exceptions import GoogleCloudError
+from inference_perf.client.filestorage import StorageClient
+from inference_perf.config import GoogleCloudStorageConfig
+from inference_perf.utils import ReportFile
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleCloudStorageClient(StorageClient):
+    def __init__(self, config: GoogleCloudStorageConfig) -> None:
+        super().__init__(config=config)
+        logger.debug("Created new GCS client")
+        self.output_bucket = config.bucket_name
+        self.client = storage.Client()
+
+        self.bucket = self.client.lookup_bucket(config.bucket_name)
+        if self.bucket is None:
+            raise ValueError(f"GCS bucket '{config.bucket_name}' does not exist or is inaccessible.")
+
+    def save_report(self, reports: List[ReportFile]) -> None:
+        filenames = [report.get_filename() for report in reports]
+        if len(filenames) != len(set(filenames)):
+            raise ValueError("Duplicate filenames detected", filenames)
+
+        for _, report in enumerate(reports):
+            filename = report.get_filename()
+            blob_path = f"{self.config.path if self.config.path else ''}/{self.config.report_file_prefix if self.config.report_file_prefix else ''}{filename}"
+            blob = self.bucket.blob(blob_path)
+
+            if blob.exists():
+                logger.info(f"Skipping upload: gs://{self.output_bucket}/{blob_path} already exists")
+                continue
+
+            try:
+                blob.upload_from_string(json.dumps(report.get_contents()), content_type="application/json")
+                logger.info(f"Uploaded gs://{self.output_bucket}/{blob_path}")
+            except GoogleCloudError as e:
+                logger.error(f"Failed to upload {blob_path}: {e}")