huggingface-hub 0.13.4__py3-none-any.whl → 0.14.0.dev0__py3-none-any.whl

huggingface_hub/_webhooks_server.py

@@ -0,0 +1,362 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains `WebhooksServer` and `webhook_endpoint` to create a webhook server easily."""
+import atexit
+import inspect
+import os
+from functools import wraps
+from typing import Callable, Dict, Optional
+
+from .utils import experimental, is_gradio_available
+
+
+if not is_gradio_available():
+    raise ImportError(
+        "You must have `gradio` installed to use `WebhooksServer`. Please run `pip install --upgrade gradio` first."
+    )
+
+
+import gradio as gr
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+
+
+_global_app: Optional["WebhooksServer"] = None
+_is_local = os.getenv("SYSTEM") != "spaces"
+
+
+@experimental
+class WebhooksServer:
+    """
+    The [`WebhooksServer`] class lets you create an instance of a Gradio app that can receive Huggingface webhooks.
+    These webhooks can be registered using the [`~WebhooksServer.add_webhook`] decorator. Webhook endpoints are added to
+    the app as a POST endpoint to the FastAPI router. Once all the webhooks are registered, the `run` method has to be
+    called to start the app.
+
+    It is recommended to accept [`WebhookPayload`] as the first argument of the webhook function. It is a Pydantic
+    model that contains all the information about the webhook event. The data will be parsed automatically for you.
+
+    Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
+    WebhooksServer and deploy it on a Space.
+
+    <Tip warning={true}>
+
+    `WebhooksServer` is experimental. Its API is subject to change in the future.
+
+    </Tip>
+
+    <Tip warning={true}>
+
+    You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`).
+
+    </Tip>
+
+    Args:
+        ui (`gradio.Blocks`, optional):
+            A Gradio UI instance to be used as the Space landing page. If `None`, a UI displaying instructions
+            about the configured webhooks is created.
+        webhook_secret (`str`, optional):
+            A secret key to verify incoming webhook requests. You can set this value to any secret you want as long as
+            you also configure it in your [webhooks settings panel](https://huggingface.co/settings/webhooks). You
+            can also set this value as the `WEBHOOK_SECRET` environment variable. If no secret is provided, the
+            webhook endpoints are opened without any security.
+
+    Example:
+
+        ```python
+        import gradio as gr
+        from huggingface_hub import WebhooksServer, WebhookPayload
+
+        with gr.Blocks() as ui:
+            ...
+
+        app = WebhooksServer(ui=ui, webhook_secret="my_secret_key")
+
+        @app.add_webhook("/say_hello")
+        async def hello(payload: WebhookPayload):
+            return {"message": "hello"}
+
+        app.run()
+        ```
+    """
+
+    def __init__(
+        self,
+        ui: Optional[gr.Blocks] = None,
+        webhook_secret: Optional[str] = None,
+    ) -> None:
+        self._ui = ui
+
+        self.webhook_secret = webhook_secret or os.getenv("WEBHOOK_SECRET")
+        self.registered_webhooks: Dict[str, Callable] = {}
+        _warn_on_empty_secret(self.webhook_secret)
+
+    def add_webhook(self, path: Optional[str] = None) -> Callable:
+        """
+        Decorator to add a webhook to the [`WebhooksServer`] server.
+
+        Args:
+            path (`str`, optional):
+                The URL path to register the webhook function. If not provided, the function name will be used as the
+                path. In any case, all webhooks are registered under `/webhooks`.
+
+        Raises:
+            ValueError: If the provided path is already registered as a webhook.
+
+        Example:
+            ```python
+            from huggingface_hub import WebhooksServer, WebhookPayload
+
+            app = WebhooksServer()
+
+            @app.add_webhook
+            async def trigger_training(payload: WebhookPayload):
+                if payload.repo.type == "dataset" and payload.event.action == "update":
+                    # Trigger a training job if a dataset is updated
+                    ...
+
+            app.run()
+        ```
+        """
+        # Usage: directly as decorator. Example: `@app.add_webhook`
+        if callable(path):
+            # If path is a function, it means it was used as a decorator without arguments
+            return self.add_webhook()(path)
+
+        # Usage: provide a path. Example: `@app.add_webhook(...)`
+        @wraps(FastAPI.post)
+        def _inner_post(*args, **kwargs):
+            func = args[0]
+            abs_path = f"/webhooks/{(path or func.__name__).strip('/')}"
+            if abs_path in self.registered_webhooks:
+                raise ValueError(f"Webhook {abs_path} already exists.")
+            self.registered_webhooks[abs_path] = func
+
+        return _inner_post
+
+    def run(self) -> None:
+        """Starts the Gradio app with the FastAPI server and registers the webhooks."""
+        ui = self._ui or self._get_default_ui()
+
+        # Start Gradio App
+        #   - as non-blocking so that webhooks can be added afterwards
+        #   - as shared if launch locally (to debug webhooks)
+        self.fastapi_app, _, _ = ui.launch(prevent_thread_lock=True, share=_is_local)
+
+        # Register webhooks to FastAPI app
+        for path, func in self.registered_webhooks.items():
+            # Add secret check if required
+            if self.webhook_secret is not None:
+                func = _wrap_webhook_to_check_secret(func, webhook_secret=self.webhook_secret)
+
+            # Add route to FastAPI app
+            self.fastapi_app.post(path)(func)
+
+        # Print instructions and block main thread
+        url = (ui.share_url or ui.local_url).strip("/")
+        message = "\nWebhooks are correctly setup and ready to use:"
+        message += "\n" + "\n".join(f"  - POST {url}{webhook}" for webhook in self.registered_webhooks)
+        message += "\nGo to https://huggingface.co/settings/webhooks to setup your webhooks."
+        print(message)
+
+        ui.block_thread()
+
+    def _get_default_ui(self) -> gr.Blocks:
+        """Default UI if not provided (lists webhooks and provides basic instructions)."""
+        with gr.Blocks() as ui:
+            gr.Markdown("# This is an app to process 🤗 Webhooks")
+            gr.Markdown(
+                "Webhooks are a foundation for MLOps-related features. They allow you to listen for new changes on"
+                " specific repos or to all repos belonging to particular set of users/organizations (not just your"
+                " repos, but any repo). Check out this [guide](https://huggingface.co/docs/hub/webhooks) to get to"
+                " know more about webhooks on the Huggingface Hub."
+            )
+            gr.Markdown(
+                f"{len(self.registered_webhooks)} webhook(s) are registered:"
+                + "\n\n"
+                + "\n ".join(
+                    f"- [{webhook_path}]({_get_webhook_doc_url(webhook.__name__, webhook_path)})"
+                    for webhook_path, webhook in self.registered_webhooks.items()
+                )
+            )
+            gr.Markdown(
+                "Go to https://huggingface.co/settings/webhooks to setup your webhooks."
+                + "\nYou app is running locally. Please look at the logs to check the full URL you need to set."
+                if _is_local
+                else (
+                    "\nThis app is running on a Space. You can find the corresponding URL in the options menu"
+                    " (top-right) > 'Embed the Space'. The URL looks like 'https://{username}-{repo_name}.hf.space'."
+                )
+            )
+        return ui
+
+
+@experimental
+def webhook_endpoint(path: Optional[str] = None) -> Callable:
+    """Decorator to start a [`WebhooksServer`] and register the decorated function as a webhook endpoint.
+
+    This is a helper to get started quickly. If you need more flexibility (custom landing page or webhook secret),
+    you can use [`WebhooksServer`] directly. You can register multiple webhook endpoints (to the same server) by using
+    this decorator multiple times.
+
+    Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
+    server and deploy it on a Space.
+
+    <Tip warning={true}>
+
+    `webhook_endpoint` is experimental. Its API is subject to change in the future.
+
+    </Tip>
+
+    <Tip warning={true}>
+
+    You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`).
+
+    </Tip>
+
+    Args:
+        path (`str`, optional):
+            The URL path to register the webhook function. If not provided, the function name will be used as the path.
+            In any case, all webhooks are registered under `/webhooks`.
+
+    Examples:
+        The default usage is to register a function as a webhook endpoint. The function name will be used as the path.
+        The server will be started automatically at exit (i.e. at the end of the script).
+
+        ```python
+        from huggingface_hub import webhook_endpoint, WebhookPayload
+
+        @webhook_endpoint
+        async def trigger_training(payload: WebhookPayload):
+            if payload.repo.type == "dataset" and payload.event.action == "update":
+                # Trigger a training job if a dataset is updated
+                ...
+
+        # Server is automatically started at the end of the script.
+        ```
+
+        Advanced usage: register a function as a webhook endpoint and start the server manually. This is useful if you
+        are running it in a notebook.
+
+        ```python
+        from huggingface_hub import webhook_endpoint, WebhookPayload
+
+        @webhook_endpoint
+        async def trigger_training(payload: WebhookPayload):
+            if payload.repo.type == "dataset" and payload.event.action == "update":
+                # Trigger a training job if a dataset is updated
+                ...
+
+        # Start the server manually
+        trigger_training.run()
+        ```
+    """
+    if callable(path):
+        # If path is a function, it means it was used as a decorator without arguments
+        return webhook_endpoint()(path)
+
+    @wraps(WebhooksServer.add_webhook)
+    def _inner(func: Callable) -> Callable:
+        app = _get_global_app()
+        app.add_webhook(path)(func)
+        if len(app.registered_webhooks) == 1:
+            # Register `app.run` to run at exit (only once)
+            atexit.register(app.run)
+
+        @wraps(app.run)
+        def _run_now():
+            # Run the app directly (without waiting atexit)
+            atexit.unregister(app.run)
+            app.run()
+
+        func.run = _run_now  # type: ignore
+        return func
+
+    return _inner
+
+
+def _get_global_app() -> WebhooksServer:
+    global _global_app
+    if _global_app is None:
+        _global_app = WebhooksServer()
+    return _global_app
+
+
+def _warn_on_empty_secret(webhook_secret: Optional[str]) -> None:
+    if webhook_secret is None:
+        print("Webhook secret is not defined. This means your webhook endpoints will be open to everyone.")
+        print(
+            "To add a secret, set `WEBHOOK_SECRET` as environment variable or pass it at initialization: "
+            "\n\t`app = WebhooksServer(webhook_secret='my_secret', ...)`"
+        )
+        print(
+            "For more details about webhook secrets, please refer to"
+            " https://huggingface.co/docs/hub/webhooks#webhook-secret."
+        )
+    else:
+        print("Webhook secret is correctly defined.")
+
+
+def _get_webhook_doc_url(webhook_name: str, webhook_path: str) -> str:
+    """Returns the anchor to a given webhook in the docs (experimental)"""
+    return "/docs#/default/" + webhook_name + webhook_path.replace("/", "_") + "_post"
+
+
+def _wrap_webhook_to_check_secret(func: Callable, webhook_secret: str) -> Callable:
+    """Wraps a webhook function to check the webhook secret before calling the function.
+
+    This is a hacky way to add the `request` parameter to the function signature. Since FastAPI based itself on route
+    parameters to inject the values to the function, we need to hack the function signature to retrieve the `Request`
+    object (and hence the headers). A far cleaner solution would be to use a middleware. However, since
+    `fastapi==0.90.1`, a middleware cannot be added once the app has started. And since the FastAPI app is started by
+    Gradio internals (and not by us), we cannot add a middleware.
+
+    This method is called only when a secret has been defined by the user. If a request is sent without the
+    "x-webhook-secret", the function will return a 401 error (unauthorized). If the header is sent but is incorrect,
+    the function will return a 403 error (forbidden).
+
+    Inspired by https://stackoverflow.com/a/33112180.
+    """
+    initial_sig = inspect.signature(func)
+
+    @wraps(func)
+    async def _protected_func(request: Request, **kwargs):
+        request_secret = request.headers.get("x-webhook-secret")
+        if request_secret is None:
+            return JSONResponse({"error": "x-webhook-secret header not set."}, status_code=401)
+        if request_secret != webhook_secret:
+            return JSONResponse({"error": "Invalid webhook secret."}, status_code=403)
+
+        # Inject `request` in kwargs if required
+        if "request" in initial_sig.parameters:
+            kwargs["request"] = request
+
+        # Handle both sync and async routes
+        if inspect.iscoroutinefunction(func):
+            return await func(**kwargs)
+        else:
+            return func(**kwargs)
+
+    # Update signature to include request
+    if "request" not in initial_sig.parameters:
+        _protected_func.__signature__ = initial_sig.replace(  # type: ignore
+            parameters=(
+                inspect.Parameter(name="request", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request),
+            )
+            + tuple(initial_sig.parameters.values())
+        )
+
+    # Return protected route
+    return _protected_func

huggingface_hub/commands/lfs.py

@@ -23,12 +23,10 @@ import sys
 from argparse import _SubParsersAction
 from typing import Dict, List, Optional
 
-import requests
-
 from huggingface_hub.commands import BaseHuggingfaceCLICommand
 from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND, SliceFileObj
 
-from ..utils import hf_raise_for_status, logging
+from ..utils import get_session, hf_raise_for_status, logging
 
 
 logger = logging.get_logger(__name__)
@@ -172,7 +170,7 @@ class LfsUploadCommand:
                         seek_from=i * chunk_size,
                         read_limit=chunk_size,
                     ) as data:
-                        r = requests.put(presigned_url, data=data)
+                        r = get_session().put(presigned_url, data=data)
                         hf_raise_for_status(r)
                         parts.append(
                             {
@@ -192,7 +190,7 @@ class LfsUploadCommand:
                         )
                         # Not precise but that's ok.
 
-            r = requests.post(
+            r = get_session().post(
                 completion_url,
                 json={
                     "oid": oid,

huggingface_hub/commands/user.py

@@ -33,9 +33,6 @@ from .._login import (  # noqa: F401 # for backward compatibility  # noqa: F401
     logout,
     notebook_login,
 )
-from .._login import (
-    _currently_setup_credential_helpers as currently_setup_credential_helpers,  # noqa: F401 # for backward compatibility
-)
 from ..utils import HfFolder
 from ._cli_utils import ANSI
 

huggingface_hub/community.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from typing import List, Optional
 
+from .constants import REPO_TYPE_MODEL
 from .utils import parse_datetime
 from .utils._typing import Literal
 
@@ -47,6 +48,12 @@ class Discussion:
             Whether or not this is a Pull Request.
         created_at (`datetime`):
             The `datetime` of creation of the Discussion / Pull Request.
+        endpoint (`str`):
+            Endpoint of the Hub. Default is https://huggingface.co.
+        git_reference (`str`, *optional*):
+            (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise.
+        url (`str`):
+            (property) URL of the discussion on the Hub.
     """
 
     title: str
@@ -57,6 +64,7 @@ class Discussion:
     author: str
     is_pull_request: bool
     created_at: datetime
+    endpoint: str
 
     @property
     def git_reference(self) -> Optional[str]:
@@ -68,6 +76,13 @@ class Discussion:
             return f"refs/pr/{self.num}"
         return None
 
+    @property
+    def url(self) -> str:
+        """Returns the URL of the discussion on the Hub."""
+        if self.repo_type is None or self.repo_type == REPO_TYPE_MODEL:
+            return f"{self.endpoint}/{self.repo_id}/discussions/{self.num}"
+        return f"{self.endpoint}/{self.repo_type}/{self.repo_id}/discussions/{self.num}"
+
 
 @dataclass
 class DiscussionWithDetails(Discussion):
@@ -112,6 +127,12 @@ class DiscussionWithDetails(Discussion):
             the merge commit, `None` otherwise.
         diff (`str`, *optional*):
             The git diff if this is a Pull Request , `None` otherwise.
+        endpoint (`str`):
+            Endpoint of the Hub. Default is https://huggingface.co.
+        git_reference (`str`, *optional*):
+            (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise.
+        url (`str`):
+            (property) URL of the discussion on the Hub.
     """
 
     events: List["DiscussionEvent"]

huggingface_hub/constants.py

@@ -110,6 +110,9 @@ HF_HUB_DISABLE_PROGRESS_BARS: Optional[bool] = (
 # Disable warning on machines that do not support symlinks (e.g. Windows non-developer)
 HF_HUB_DISABLE_SYMLINKS_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_SYMLINKS_WARNING"))
 
+# Disable warning when using experimental features
+HF_HUB_DISABLE_EXPERIMENTAL_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_EXPERIMENTAL_WARNING"))
+
 # Disable sending the cached token by default is all HTTP requests to the Hub
 HF_HUB_DISABLE_IMPLICIT_TOKEN: bool = _is_true(os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN"))
 

huggingface_hub/file_download.py

@@ -477,8 +477,7 @@ def http_get(
         if HF_HUB_ENABLE_HF_TRANSFER:
             try:
                 # Download file using an external Rust-based package. Download is faster
-                # (~2x speed-up) but support less features (no error handling, no retries,
-                # no progress bars).
+                # (~2x speed-up) but support less features (no progress bars).
                 from hf_transfer import download
 
                 logger.debug(f"Download {url} using HF_TRANSFER.")
@@ -539,6 +538,15 @@ def http_get(
         if chunk:  # filter out keep-alive new chunks
             progress.update(len(chunk))
             temp_file.write(chunk)
+
+    if total is not None and total != temp_file.tell():
+        raise EnvironmentError(
+            f"Consistency check failed: file should be of size {total} but has size"
+            f" {temp_file.tell()} ({displayed_name}).\nWe are sorry for the inconvenience. Please retry download and"
+            " pass `force_download=True, resume_download=False` as argument.\nIf the issue persists, please let us"
+            " know by opening an issue on https://github.com/huggingface/huggingface_hub."
+        )
+
     progress.close()
 
 
@@ -670,7 +678,7 @@ def cached_download(
                 timeout=etag_timeout,
             )
             hf_raise_for_status(r)
-            etag = r.headers.get("X-Linked-Etag") or r.headers.get("ETag")
+            etag = r.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG) or r.headers.get("ETag")
             # We favor a custom header indicating the etag of the linked resource, and
             # we fallback to the regular etag header.
             # If we don't have any of those, raise an error.
@@ -805,7 +813,8 @@ def _normalize_etag(etag: Optional[str]) -> Optional[str]:
       ETag: W/"<etag_value>"
       ETag: "<etag_value>"
 
-    The hf.co hub guarantees to only send the second form.
+    For now, we only expect the second form from the server, but we want to be future-proof so we support both. For
+    more context, see `TestNormalizeEtag` tests and https://github.com/huggingface/huggingface_hub/pull/1428.
 
     Args:
         etag (`str`, *optional*): HTTP header
@@ -816,7 +825,7 @@ def _normalize_etag(etag: Optional[str]) -> Optional[str]:
     """
     if etag is None:
         return None
-    return etag.strip('"')
+    return etag.lstrip("W/").strip('"')
 
 
 def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None:
@@ -870,7 +879,13 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
         relative_src = None
 
     try:
-        _support_symlinks = are_symlinks_supported(os.path.dirname(os.path.commonpath([abs_src, abs_dst])))
+        try:
+            commonpath = os.path.commonpath([abs_src, abs_dst])
+            _support_symlinks = are_symlinks_supported(os.path.dirname(commonpath))
+        except ValueError:
+            # Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos.
+            # See https://docs.python.org/3/library/os.path.html#os.path.commonpath
+            _support_symlinks = os.name != "nt"
     except PermissionError:
         # Permission error means src and dst are not in the same volume (e.g. destination path has been provided
         # by the user via `local_dir`. Let's test symlink support there)
@@ -892,7 +907,7 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
                 raise
     elif new_blob:
         logger.info(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}")
-        os.replace(src, dst)
+        shutil.move(src, dst)
     else:
         logger.info(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}")
         shutil.copyfile(src, dst)
@@ -1512,8 +1527,8 @@ def get_hf_file_metadata(
         etag=_normalize_etag(
             # We favor a custom header indicating the etag of the linked resource, and
             # we fallback to the regular etag header.
-            r.headers.get("ETag")
-            or r.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG)
+            r.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG)
+            or r.headers.get("ETag")
         ),
         # Either from response headers (if redirected) or defaults to request url
         # Do not use directly `url`, as `_request_wrapper` might have followed relative
@@ -1552,7 +1567,7 @@ def _chmod_and_replace(src: str, dst: str) -> None:
     finally:
         tmp_file.unlink()
 
-    os.replace(src, dst)
+    shutil.move(src, dst)
 
 
 def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str) -> str: