label-studio-sdk 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

label_studio_sdk/__init__.py

@@ -14,6 +14,8 @@ from .types import (
     BaseTaskFileUpload,
     BaseTaskUpdatedBy,
     BaseUser,
+    Comment,
+    CommentCreatedBy,
     ConvertedFormat,
     ConvertedFormatStatus,
     DataManagerTaskSerializer,
@@ -32,6 +34,16 @@ from .types import (
     GcsExportStorageStatus,
     GcsImportStorage,
     GcsImportStorageStatus,
+    InferenceRun,
+    InferenceRunCreatedBy,
+    InferenceRunOrganization,
+    InferenceRunProjectSubset,
+    InferenceRunStatus,
+    KeyIndicatorValue,
+    KeyIndicators,
+    KeyIndicatorsItem,
+    KeyIndicatorsItemAdditionalKpisItem,
+    KeyIndicatorsItemExtraKpisItem,
     LocalFilesExportStorage,
     LocalFilesExportStorageStatus,
     LocalFilesImportStorage,
@@ -39,6 +51,11 @@ from .types import (
     MlBackend,
     MlBackendAuthMethod,
     MlBackendState,
+    ModelProviderConnection,
+    ModelProviderConnectionCreatedBy,
+    ModelProviderConnectionOrganization,
+    ModelProviderConnectionProvider,
+    ModelProviderConnectionScope,
     Prediction,
     Project,
     ProjectImport,
@@ -46,10 +63,19 @@ from .types import (
     ProjectLabelConfig,
     ProjectSampling,
     ProjectSkipQueue,
+    Prompt,
+    PromptCreatedBy,
+    PromptOrganization,
+    PromptVersion,
+    PromptVersionCreatedBy,
+    PromptVersionOrganization,
+    PromptVersionProvider,
     RedisExportStorage,
     RedisExportStorageStatus,
     RedisImportStorage,
     RedisImportStorageStatus,
+    RefinedPromptResponse,
+    RefinedPromptResponseRefinementStatus,
     S3ExportStorage,
     S3ExportStorageStatus,
     S3ImportStorage,
@@ -74,12 +100,15 @@ from .errors import BadRequestError, InternalServerError
 from . import (
     actions,
     annotations,
+    comments,
     export_storage,
     files,
     import_storage,
     ml,
+    model_providers,
     predictions,
     projects,
+    prompts,
     tasks,
     users,
     views,
@@ -100,6 +129,7 @@ from .actions import (
     ActionsCreateRequestSelectedItemsExcluded,
     ActionsCreateRequestSelectedItemsIncluded,
 )
+from .annotations import AnnotationsCreateBulkResponseItem
 from .environment import LabelStudioEnvironment
 from .export_storage import ExportStorageListTypesResponseItem
 from .import_storage import ImportStorageListTypesResponseItem
@@ -112,6 +142,12 @@ from .ml import (
     MlUpdateResponseAuthMethod,
 )
 from .projects import ProjectsCreateResponse, ProjectsImportTasksResponse, ProjectsListResponse, ProjectsUpdateResponse
+from .prompts import (
+    PromptsBatchFailedPredictionsRequestFailedPredictionsItem,
+    PromptsBatchFailedPredictionsResponse,
+    PromptsBatchPredictionsRequestResultsItem,
+    PromptsBatchPredictionsResponse,
+)
 from .tasks import TasksListRequestFields, TasksListResponse
 from .users import UsersGetTokenResponse, UsersResetTokenResponse
 from .version import __version__
@@ -150,6 +186,7 @@ __all__ = [
     "Annotation",
     "AnnotationFilterOptions",
     "AnnotationLastAction",
+    "AnnotationsCreateBulkResponseItem",
     "AnnotationsDmField",
     "AnnotationsDmFieldLastAction",
     "AzureBlobExportStorage",
@@ -162,6 +199,8 @@ __all__ = [
     "BaseTaskUpdatedBy",
     "BaseUser",
     "Client",
+    "Comment",
+    "CommentCreatedBy",
     "ConvertedFormat",
     "ConvertedFormatStatus",
     "DataManagerTaskSerializer",
@@ -182,7 +221,17 @@ __all__ = [
     "GcsImportStorage",
     "GcsImportStorageStatus",
     "ImportStorageListTypesResponseItem",
+    "InferenceRun",
+    "InferenceRunCreatedBy",
+    "InferenceRunOrganization",
+    "InferenceRunProjectSubset",
+    "InferenceRunStatus",
     "InternalServerError",
+    "KeyIndicatorValue",
+    "KeyIndicators",
+    "KeyIndicatorsItem",
+    "KeyIndicatorsItemAdditionalKpisItem",
+    "KeyIndicatorsItemExtraKpisItem",
     "LabelStudioEnvironment",
     "LocalFilesExportStorage",
     "LocalFilesExportStorageStatus",
@@ -197,6 +246,11 @@ __all__ = [
     "MlUpdateRequestAuthMethod",
     "MlUpdateResponse",
     "MlUpdateResponseAuthMethod",
+    "ModelProviderConnection",
+    "ModelProviderConnectionCreatedBy",
+    "ModelProviderConnectionOrganization",
+    "ModelProviderConnectionProvider",
+    "ModelProviderConnectionScope",
     "Prediction",
     "Project",
     "ProjectImport",
@@ -208,10 +262,23 @@ __all__ = [
     "ProjectsImportTasksResponse",
     "ProjectsListResponse",
     "ProjectsUpdateResponse",
+    "Prompt",
+    "PromptCreatedBy",
+    "PromptOrganization",
+    "PromptVersion",
+    "PromptVersionCreatedBy",
+    "PromptVersionOrganization",
+    "PromptVersionProvider",
+    "PromptsBatchFailedPredictionsRequestFailedPredictionsItem",
+    "PromptsBatchFailedPredictionsResponse",
+    "PromptsBatchPredictionsRequestResultsItem",
+    "PromptsBatchPredictionsResponse",
     "RedisExportStorage",
     "RedisExportStorageStatus",
     "RedisImportStorage",
     "RedisImportStorageStatus",
+    "RefinedPromptResponse",
+    "RefinedPromptResponseRefinementStatus",
     "S3ExportStorage",
     "S3ExportStorageStatus",
     "S3ImportStorage",
@@ -255,12 +322,15 @@ __all__ = [
     "__version__",
     "actions",
     "annotations",
+    "comments",
     "export_storage",
     "files",
     "import_storage",
     "ml",
+    "model_providers",
     "predictions",
     "projects",
+    "prompts",
     "tasks",
     "users",
     "views",

label_studio_sdk/_extensions/eval/categorical.py

@@ -0,0 +1,83 @@
+import logging
+from typing import List, Dict, Iterable
+
+logger = logging.getLogger(__name__)
+
+try:
+    # TODO: after python 3.8 support is dropped (Oct'24), remove try-except block
+    from sklearn.metrics import precision_recall_fscore_support
+except ImportError:
+    logger.warning('scikit-learn is not installed. Please install scikit-learn to use this module.')
+    precision_recall_fscore_support = None
+
+
+def _get_single_choice(annotation: Dict):
+    """
+    Get the single choice from the annotation.
+
+    Args:
+        annotation: Annotation dict
+
+    Returns:
+        Single choice
+    """
+    maybe_choice = next((r['value']['choices'][0] for r in annotation['result'] if r['type'] == 'choices'), None)
+    if maybe_choice:
+        return maybe_choice
+    raise NotImplementedError('Only single choice is supported')
+
+
+def get_precision_recall_f1_per_choice(annotations: Iterable[Dict], predictions: Iterable[Dict]) -> Dict:
+    """
+    Given the iterator over annotations and predictions, calculate precision, recall, and F1 per choice.
+    Each annotation and prediction follows the format of the Label Studio output.
+
+    Args:
+        annotations: Iterable of annotation dicts
+        predictions: Iterable of prediction dicts
+
+    Returns:
+        Dict of the following format:
+        {
+            'precision': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'recall': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'f1': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'support': {
+                'choice1': 10,
+                'choice2': 20,
+                ...
+            },
+        }
+    """
+
+    if precision_recall_fscore_support is None:
+        raise ImportError('scikit-learn is not installed. Please install scikit-learn to use this module.')
+
+    annotation_choices = [_get_single_choice(annotation) for annotation in annotations]
+    prediction_choices = [_get_single_choice(prediction) for prediction in predictions]
+    # get unique choices names
+    unique_choices = sorted(set(annotation_choices + prediction_choices))
+
+    metrics_per_choice = precision_recall_fscore_support(
+        annotation_choices, prediction_choices,
+        average=None, labels=unique_choices)
+
+    results = {}
+    for metric_name, metric_per_choice in zip(['precision', 'recall', 'f1', 'support'], metrics_per_choice):
+        results[metric_name] = {}
+        for choice, metric in zip(unique_choices, metric_per_choice):
+            results[metric_name][choice] = metric
+
+    return results

label_studio_sdk/_extensions/label_studio_tools/core/utils/io.py

@@ -1,15 +1,14 @@
-import logging
-import io
-import shutil
-import urllib
 import hashlib
-import requests
+import io
+import logging
 import os
-
-from appdirs import user_cache_dir, user_data_dir
-from urllib.parse import urlparse, urljoin
+import shutil
 from contextlib import contextmanager
 from tempfile import mkdtemp
+from urllib.parse import urlparse
+
+import requests
+from appdirs import user_cache_dir, user_data_dir
 
 from label_studio_sdk._extensions.label_studio_tools.core.utils.params import get_env
 
@@ -17,6 +16,7 @@ _DIR_APP_NAME = "label-studio"
 LOCAL_FILES_DOCUMENT_ROOT = get_env(
     "LOCAL_FILES_DOCUMENT_ROOT", default=os.path.abspath(os.sep)
 )
+VERIFY_SSL = get_env("VERIFY_SSL", default=True, is_bool=True)
 
 logger = logging.getLogger(__name__)
 
@@ -80,6 +80,13 @@ def get_local_path(
             f"`localhost` is not accessible inside of docker containers. "
             f"You can check your IP with utilities like `ifconfig` and set it as LABEL_STUDIO_URL."
         )
+    if hostname and not (
+        hostname.startswith("http://") or hostname.startswith("https://")
+    ):
+        raise ValueError(
+            f"Invalid hostname in LABEL_STUDIO_URL: {hostname}. "
+            "Please provide full URL starting with protocol (http:// or https://)."
+        )
 
     # fix file upload url
     if url.startswith("upload") or url.startswith("/upload"):
@@ -180,13 +187,17 @@ def download_and_cache(
     # File specified by remote URL - download and cache it
     cache_dir = cache_dir or get_cache_dir()
     parsed_url = urlparse(url)
-    url_filename = (
-        # /data/local-files?d=dir/1.jpg => 1.jpg
-        os.path.basename(url)
-        if is_local_storage_file or is_cloud_storage_file
-        # /some/url/1.jpg?expire=xxx => 1.jpg
-        else os.path.basename(parsed_url.path)
-    )
+
+    # local storage: /data/local-files?d=dir/1.jpg => 1.jpg
+    if is_local_storage_file:
+        url_filename = os.path.basename(url.split('?d=')[1])
+    # cloud storage: s3://bucket/1.jpg => 1.jpg
+    elif is_cloud_storage_file:
+        url_filename = os.path.basename(url)
+    # all others: /some/url/1.jpg?expire=xxx => 1.jpg
+    else:
+        url_filename = os.path.basename(parsed_url.path)
+
     url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
     filepath = os.path.join(cache_dir, url_hash + "__" + url_filename)
 
@@ -205,8 +216,15 @@ def download_and_cache(
             ):
                 headers["Authorization"] = "Token " + access_token
                 logger.debug("Authorization token is used for download_and_cache")
-            r = requests.get(url, stream=True, headers=headers)
-            r.raise_for_status()
+            try:
+                r = requests.get(url, stream=True, headers=headers, verify=VERIFY_SSL)
+                r.raise_for_status()
+            except requests.exceptions.SSLError as e:
+                logger.error(
+                    f"SSL error during requests.get('{url}'): {e}\n"
+                    f"Try to set VERIFY_SSL=False in environment variables to bypass SSL verification."
+                )
+                raise e
             with io.open(filepath, mode="wb") as fout:
                 fout.write(r.content)
     return filepath

label_studio_sdk/annotations/__init__.py

@@ -1,2 +1,5 @@
 # This file was auto-generated by Fern from our API Definition.
 
+from .types import AnnotationsCreateBulkResponseItem
+
+__all__ = ["AnnotationsCreateBulkResponseItem"]

label_studio_sdk/annotations/client.py

@@ -9,6 +9,7 @@ from ..core.jsonable_encoder import jsonable_encoder
 from ..core.pydantic_utilities import pydantic_v1
 from ..core.request_options import RequestOptions
 from ..types.annotation import Annotation
+from .types.annotations_create_bulk_response_item import AnnotationsCreateBulkResponseItem
 
 # this is used as the default value for optional parameters
 OMIT = typing.cast(typing.Any, ...)
@@ -380,6 +381,60 @@ class AnnotationsClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    def create_bulk(
+        self,
+        *,
+        tasks: typing.Optional[typing.Sequence[int]] = OMIT,
+        lead_time: typing.Optional[float] = OMIT,
+        project: typing.Optional[int] = OMIT,
+        result: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[AnnotationsCreateBulkResponseItem]:
+        """
+        Create multiple annotations for specific tasks in a bulk operation.
+
+        Parameters
+        ----------
+        tasks : typing.Optional[typing.Sequence[int]]
+
+        lead_time : typing.Optional[float]
+
+        project : typing.Optional[int]
+
+        result : typing.Optional[typing.Dict[str, typing.Any]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[AnnotationsCreateBulkResponseItem]
+            Annotations created successfully
+
+        Examples
+        --------
+        from label_studio_sdk.client import LabelStudio
+
+        client = LabelStudio(
+            api_key="YOUR_API_KEY",
+        )
+        client.annotations.create_bulk()
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "api/annotations/bulk",
+            method="POST",
+            json={"tasks": tasks, "lead_time": lead_time, "project": project, "result": result},
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return pydantic_v1.parse_obj_as(typing.List[AnnotationsCreateBulkResponseItem], _response.json())  # type: ignore
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
 
 class AsyncAnnotationsClient:
     def __init__(self, *, client_wrapper: AsyncClientWrapper):
@@ -748,3 +803,57 @@ class AsyncAnnotationsClient:
         except JSONDecodeError:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def create_bulk(
+        self,
+        *,
+        tasks: typing.Optional[typing.Sequence[int]] = OMIT,
+        lead_time: typing.Optional[float] = OMIT,
+        project: typing.Optional[int] = OMIT,
+        result: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[AnnotationsCreateBulkResponseItem]:
+        """
+        Create multiple annotations for specific tasks in a bulk operation.
+
+        Parameters
+        ----------
+        tasks : typing.Optional[typing.Sequence[int]]
+
+        lead_time : typing.Optional[float]
+
+        project : typing.Optional[int]
+
+        result : typing.Optional[typing.Dict[str, typing.Any]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[AnnotationsCreateBulkResponseItem]
+            Annotations created successfully
+
+        Examples
+        --------
+        from label_studio_sdk.client import AsyncLabelStudio
+
+        client = AsyncLabelStudio(
+            api_key="YOUR_API_KEY",
+        )
+        await client.annotations.create_bulk()
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "api/annotations/bulk",
+            method="POST",
+            json={"tasks": tasks, "lead_time": lead_time, "project": project, "result": result},
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return pydantic_v1.parse_obj_as(typing.List[AnnotationsCreateBulkResponseItem], _response.json())  # type: ignore
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)

label_studio_sdk/annotations/types/__init__.py

@@ -0,0 +1,5 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from .annotations_create_bulk_response_item import AnnotationsCreateBulkResponseItem
+
+__all__ = ["AnnotationsCreateBulkResponseItem"]

label_studio_sdk/annotations/types/annotations_create_bulk_response_item.py

@@ -0,0 +1,29 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ...core.datetime_utils import serialize_datetime
+from ...core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1
+
+
+class AnnotationsCreateBulkResponseItem(pydantic_v1.BaseModel):
+    id: typing.Optional[int] = None
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults_exclude_unset: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        kwargs_with_defaults_exclude_none: typing.Any = {"by_alias": True, "exclude_none": True, **kwargs}
+
+        return deep_union_pydantic_dicts(
+            super().dict(**kwargs_with_defaults_exclude_unset), super().dict(**kwargs_with_defaults_exclude_none)
+        )
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic_v1.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

label_studio_sdk/base_client.py

@@ -7,6 +7,7 @@ import httpx
 
 from .actions.client import ActionsClient, AsyncActionsClient
 from .annotations.client import AnnotationsClient, AsyncAnnotationsClient
+from .comments.client import AsyncCommentsClient, CommentsClient
 from .core.api_error import ApiError
 from .core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
 from .environment import LabelStudioEnvironment
@@ -14,8 +15,10 @@ from .export_storage.client import AsyncExportStorageClient, ExportStorageClient
 from .files.client import AsyncFilesClient, FilesClient
 from .import_storage.client import AsyncImportStorageClient, ImportStorageClient
 from .ml.client import AsyncMlClient, MlClient
+from .model_providers.client import AsyncModelProvidersClient, ModelProvidersClient
 from .predictions.client import AsyncPredictionsClient, PredictionsClient
 from .projects.client import AsyncProjectsClient, ProjectsClient
+from .prompts.client import AsyncPromptsClient, PromptsClient
 from .tasks.client import AsyncTasksClient, TasksClient
 from .users.client import AsyncUsersClient, UsersClient
 from .views.client import AsyncViewsClient, ViewsClient
@@ -97,6 +100,9 @@ class LabelStudioBase:
         self.import_storage = ImportStorageClient(client_wrapper=self._client_wrapper)
         self.export_storage = ExportStorageClient(client_wrapper=self._client_wrapper)
         self.webhooks = WebhooksClient(client_wrapper=self._client_wrapper)
+        self.prompts = PromptsClient(client_wrapper=self._client_wrapper)
+        self.model_providers = ModelProvidersClient(client_wrapper=self._client_wrapper)
+        self.comments = CommentsClient(client_wrapper=self._client_wrapper)
         self.workspaces = WorkspacesClient(client_wrapper=self._client_wrapper)
 
 
@@ -174,6 +180,9 @@ class AsyncLabelStudioBase:
         self.import_storage = AsyncImportStorageClient(client_wrapper=self._client_wrapper)
         self.export_storage = AsyncExportStorageClient(client_wrapper=self._client_wrapper)
         self.webhooks = AsyncWebhooksClient(client_wrapper=self._client_wrapper)
+        self.prompts = AsyncPromptsClient(client_wrapper=self._client_wrapper)
+        self.model_providers = AsyncModelProvidersClient(client_wrapper=self._client_wrapper)
+        self.comments = AsyncCommentsClient(client_wrapper=self._client_wrapper)
         self.workspaces = AsyncWorkspacesClient(client_wrapper=self._client_wrapper)
 
 

label_studio_sdk/comments/__init__.py

@@ -0,0 +1,2 @@
+# This file was auto-generated by Fern from our API Definition.
+