label-studio-sdk 1.0.5__py3-none-any.whl → 1.0.8__py3-none-any.whl

label_studio_sdk/__init__.py

@@ -14,6 +14,8 @@ from .types import (
     BaseTaskFileUpload,
     BaseTaskUpdatedBy,
     BaseUser,
+    Comment,
+    CommentCreatedBy,
     ConvertedFormat,
     ConvertedFormatStatus,
     DataManagerTaskSerializer,
@@ -32,6 +34,17 @@ from .types import (
     GcsExportStorageStatus,
     GcsImportStorage,
     GcsImportStorageStatus,
+    InferenceRun,
+    InferenceRunCostEstimate,
+    InferenceRunCreatedBy,
+    InferenceRunOrganization,
+    InferenceRunProjectSubset,
+    InferenceRunStatus,
+    KeyIndicatorValue,
+    KeyIndicators,
+    KeyIndicatorsItem,
+    KeyIndicatorsItemAdditionalKpisItem,
+    KeyIndicatorsItemExtraKpisItem,
     LocalFilesExportStorage,
     LocalFilesExportStorageStatus,
     LocalFilesImportStorage,
@@ -39,6 +52,12 @@ from .types import (
     MlBackend,
     MlBackendAuthMethod,
     MlBackendState,
+    ModelProviderConnection,
+    ModelProviderConnectionBudgetResetPeriod,
+    ModelProviderConnectionCreatedBy,
+    ModelProviderConnectionOrganization,
+    ModelProviderConnectionProvider,
+    ModelProviderConnectionScope,
     Prediction,
     Project,
     ProjectImport,
@@ -46,10 +65,19 @@ from .types import (
     ProjectLabelConfig,
     ProjectSampling,
     ProjectSkipQueue,
+    Prompt,
+    PromptCreatedBy,
+    PromptOrganization,
+    PromptVersion,
+    PromptVersionCreatedBy,
+    PromptVersionOrganization,
+    PromptVersionProvider,
     RedisExportStorage,
     RedisExportStorageStatus,
     RedisImportStorage,
     RedisImportStorageStatus,
+    RefinedPromptResponse,
+    RefinedPromptResponseRefinementStatus,
     S3ExportStorage,
     S3ExportStorageStatus,
     S3ImportStorage,
@@ -61,6 +89,7 @@ from .types import (
     SerializationOptions,
     Task,
     TaskAnnotatorsItem,
+    TaskCommentAuthorsItem,
     TaskFilterOptions,
     UserSimple,
     View,
@@ -74,12 +103,15 @@ from .errors import BadRequestError, InternalServerError
 from . import (
     actions,
     annotations,
+    comments,
     export_storage,
     files,
     import_storage,
     ml,
+    model_providers,
     predictions,
     projects,
+    prompts,
     tasks,
     users,
     views,
@@ -100,6 +132,7 @@ from .actions import (
     ActionsCreateRequestSelectedItemsExcluded,
     ActionsCreateRequestSelectedItemsIncluded,
 )
+from .annotations import AnnotationsCreateBulkResponseItem
 from .environment import LabelStudioEnvironment
 from .export_storage import ExportStorageListTypesResponseItem
 from .import_storage import ImportStorageListTypesResponseItem
@@ -112,6 +145,12 @@ from .ml import (
     MlUpdateResponseAuthMethod,
 )
 from .projects import ProjectsCreateResponse, ProjectsImportTasksResponse, ProjectsListResponse, ProjectsUpdateResponse
+from .prompts import (
+    PromptsBatchFailedPredictionsRequestFailedPredictionsItem,
+    PromptsBatchFailedPredictionsResponse,
+    PromptsBatchPredictionsRequestResultsItem,
+    PromptsBatchPredictionsResponse,
+)
 from .tasks import TasksListRequestFields, TasksListResponse
 from .users import UsersGetTokenResponse, UsersResetTokenResponse
 from .version import __version__
@@ -150,6 +189,7 @@ __all__ = [
     "Annotation",
     "AnnotationFilterOptions",
     "AnnotationLastAction",
+    "AnnotationsCreateBulkResponseItem",
     "AnnotationsDmField",
     "AnnotationsDmFieldLastAction",
     "AzureBlobExportStorage",
@@ -162,6 +202,8 @@ __all__ = [
     "BaseTaskUpdatedBy",
     "BaseUser",
     "Client",
+    "Comment",
+    "CommentCreatedBy",
     "ConvertedFormat",
     "ConvertedFormatStatus",
     "DataManagerTaskSerializer",
@@ -182,7 +224,18 @@ __all__ = [
     "GcsImportStorage",
     "GcsImportStorageStatus",
     "ImportStorageListTypesResponseItem",
+    "InferenceRun",
+    "InferenceRunCostEstimate",
+    "InferenceRunCreatedBy",
+    "InferenceRunOrganization",
+    "InferenceRunProjectSubset",
+    "InferenceRunStatus",
     "InternalServerError",
+    "KeyIndicatorValue",
+    "KeyIndicators",
+    "KeyIndicatorsItem",
+    "KeyIndicatorsItemAdditionalKpisItem",
+    "KeyIndicatorsItemExtraKpisItem",
     "LabelStudioEnvironment",
     "LocalFilesExportStorage",
     "LocalFilesExportStorageStatus",
@@ -197,6 +250,12 @@ __all__ = [
     "MlUpdateRequestAuthMethod",
     "MlUpdateResponse",
     "MlUpdateResponseAuthMethod",
+    "ModelProviderConnection",
+    "ModelProviderConnectionBudgetResetPeriod",
+    "ModelProviderConnectionCreatedBy",
+    "ModelProviderConnectionOrganization",
+    "ModelProviderConnectionProvider",
+    "ModelProviderConnectionScope",
     "Prediction",
     "Project",
     "ProjectImport",
@@ -208,10 +267,23 @@ __all__ = [
     "ProjectsImportTasksResponse",
     "ProjectsListResponse",
     "ProjectsUpdateResponse",
+    "Prompt",
+    "PromptCreatedBy",
+    "PromptOrganization",
+    "PromptVersion",
+    "PromptVersionCreatedBy",
+    "PromptVersionOrganization",
+    "PromptVersionProvider",
+    "PromptsBatchFailedPredictionsRequestFailedPredictionsItem",
+    "PromptsBatchFailedPredictionsResponse",
+    "PromptsBatchPredictionsRequestResultsItem",
+    "PromptsBatchPredictionsResponse",
     "RedisExportStorage",
     "RedisExportStorageStatus",
     "RedisImportStorage",
     "RedisImportStorageStatus",
+    "RefinedPromptResponse",
+    "RefinedPromptResponseRefinementStatus",
     "S3ExportStorage",
     "S3ExportStorageStatus",
     "S3ImportStorage",
@@ -223,6 +295,7 @@ __all__ = [
     "SerializationOptions",
     "Task",
     "TaskAnnotatorsItem",
+    "TaskCommentAuthorsItem",
     "TaskFilterOptions",
     "TasksListRequestFields",
     "TasksListResponse",
@@ -255,12 +328,15 @@ __all__ = [
     "__version__",
     "actions",
     "annotations",
+    "comments",
     "export_storage",
     "files",
     "import_storage",
     "ml",
+    "model_providers",
     "predictions",
     "projects",
+    "prompts",
     "tasks",
     "users",
     "views",

label_studio_sdk/_extensions/eval/categorical.py

@@ -0,0 +1,83 @@
+import logging
+from typing import List, Dict, Iterable
+
+logger = logging.getLogger(__name__)
+
+try:
+    # TODO: after python 3.8 support is dropped (Oct'24), remove try-except block
+    from sklearn.metrics import precision_recall_fscore_support
+except ImportError:
+    logger.warning('scikit-learn is not installed. Please install scikit-learn to use this module.')
+    precision_recall_fscore_support = None
+
+
+def _get_single_choice(annotation: Dict):
+    """
+    Get the single choice from the annotation.
+
+    Args:
+        annotation: Annotation dict
+
+    Returns:
+        Single choice
+    """
+    maybe_choice = next((r['value']['choices'][0] for r in annotation['result'] if r['type'] == 'choices'), None)
+    if maybe_choice:
+        return maybe_choice
+    raise NotImplementedError('Only single choice is supported')
+
+
+def get_precision_recall_f1_per_choice(annotations: Iterable[Dict], predictions: Iterable[Dict]) -> Dict:
+    """
+    Given the iterator over annotations and predictions, calculate precision, recall, and F1 per choice.
+    Each annotation and prediction follows the format of the Label Studio output.
+
+    Args:
+        annotations: Iterable of annotation dicts
+        predictions: Iterable of prediction dicts
+
+    Returns:
+        Dict of the following format:
+        {
+            'precision': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'recall': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'f1': {
+                'choice1': 0.5,
+                'choice2': 0.7,
+                ...
+            },
+            'support': {
+                'choice1': 10,
+                'choice2': 20,
+                ...
+            },
+        }
+    """
+
+    if precision_recall_fscore_support is None:
+        raise ImportError('scikit-learn is not installed. Please install scikit-learn to use this module.')
+
+    annotation_choices = [_get_single_choice(annotation) for annotation in annotations]
+    prediction_choices = [_get_single_choice(prediction) for prediction in predictions]
+    # get unique choices names
+    unique_choices = sorted(set(annotation_choices + prediction_choices))
+
+    metrics_per_choice = precision_recall_fscore_support(
+        annotation_choices, prediction_choices,
+        average=None, labels=unique_choices)
+
+    results = {}
+    for metric_name, metric_per_choice in zip(['precision', 'recall', 'f1', 'support'], metrics_per_choice):
+        results[metric_name] = {}
+        for choice, metric in zip(unique_choices, metric_per_choice):
+            results[metric_name][choice] = metric
+
+    return results

label_studio_sdk/_extensions/label_studio_tools/core/label_config.py

@@ -82,17 +82,26 @@ def parse_config(config_string):
         elif _is_input_tag(tag):
             inputs[tag.attrib["name"]] = {
                 "type": tag.tag,
-                "value": tag.attrib["value"].lstrip("$"),
                 "valueType": tag.attrib.get("valueType"),
             }
+            if 'value' in tag.attrib:
+                inputs[tag.attrib["name"]]["value"] = tag.attrib["value"].lstrip("$")
+            elif 'valueList' in tag.attrib:
+                inputs[tag.attrib["name"]]["valueList"] = tag.attrib["valueList"].lstrip("$")
+            else:
+                raise ValueError(
+                    'Inspecting tag {tag_name}... found no "value" or "valueList" attributes.'.format(
+                        tag_name=etree.tostring(tag, encoding="unicode").strip()[:50]
+                    )
+                )
         if tag.tag not in _LABEL_TAGS:
             continue
         parent_name = _get_parent_output_tag_name(tag, outputs)
         if parent_name is not None:
-            actual_value = tag.attrib.get("alias") or tag.attrib.get("value")
+            actual_value = tag.attrib.get("alias") or tag.attrib.get("value") or tag.attrib.get("valueList")
             if not actual_value:
                 logger.debug(
-                    'Inspecting tag {tag_name}... found no "value" or "alias" attributes.'.format(
+                    'Inspecting tag {tag_name}... found no "value", "valueList", or "alias" attributes.'.format(
                         tag_name=etree.tostring(tag, encoding="unicode").strip()[:50]
                     )
                 )
@@ -137,7 +146,7 @@ def _is_input_tag(tag):
     """
     Check if tag is input
     """
-    return tag.attrib.get("name") and tag.attrib.get("value")
+    return tag.attrib.get("name") and (tag.attrib.get("value") or tag.attrib.get("valueList"))
 
 
 def _is_output_tag(tag):

label_studio_sdk/_extensions/label_studio_tools/core/utils/io.py

@@ -1,15 +1,14 @@
-import logging
-import io
-import shutil
-import urllib
 import hashlib
-import requests
+import io
+import logging
 import os
-
-from appdirs import user_cache_dir, user_data_dir
-from urllib.parse import urlparse, urljoin
+import shutil
 from contextlib import contextmanager
 from tempfile import mkdtemp
+from urllib.parse import urlparse
+
+import requests
+from appdirs import user_cache_dir, user_data_dir
 
 from label_studio_sdk._extensions.label_studio_tools.core.utils.params import get_env
 
@@ -17,6 +16,7 @@ _DIR_APP_NAME = "label-studio"
 LOCAL_FILES_DOCUMENT_ROOT = get_env(
     "LOCAL_FILES_DOCUMENT_ROOT", default=os.path.abspath(os.sep)
 )
+VERIFY_SSL = get_env("VERIFY_SSL", default=True, is_bool=True)
 
 logger = logging.getLogger(__name__)
 
@@ -80,6 +80,13 @@ def get_local_path(
             f"`localhost` is not accessible inside of docker containers. "
             f"You can check your IP with utilities like `ifconfig` and set it as LABEL_STUDIO_URL."
         )
+    if hostname and not (
+        hostname.startswith("http://") or hostname.startswith("https://")
+    ):
+        raise ValueError(
+            f"Invalid hostname in LABEL_STUDIO_URL: {hostname}. "
+            "Please provide full URL starting with protocol (http:// or https://)."
+        )
 
     # fix file upload url
     if url.startswith("upload") or url.startswith("/upload"):
@@ -180,13 +187,17 @@ def download_and_cache(
     # File specified by remote URL - download and cache it
     cache_dir = cache_dir or get_cache_dir()
     parsed_url = urlparse(url)
-    url_filename = (
-        # /data/local-files?d=dir/1.jpg => 1.jpg
-        os.path.basename(url)
-        if is_local_storage_file or is_cloud_storage_file
-        # /some/url/1.jpg?expire=xxx => 1.jpg
-        else os.path.basename(parsed_url.path)
-    )
+
+    # local storage: /data/local-files?d=dir/1.jpg => 1.jpg
+    if is_local_storage_file:
+        url_filename = os.path.basename(url.split('?d=')[1])
+    # cloud storage: s3://bucket/1.jpg => 1.jpg
+    elif is_cloud_storage_file:
+        url_filename = os.path.basename(url)
+    # all others: /some/url/1.jpg?expire=xxx => 1.jpg
+    else:
+        url_filename = os.path.basename(parsed_url.path)
+
     url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
     filepath = os.path.join(cache_dir, url_hash + "__" + url_filename)
 
@@ -205,8 +216,15 @@ def download_and_cache(
             ):
                 headers["Authorization"] = "Token " + access_token
                 logger.debug("Authorization token is used for download_and_cache")
-            r = requests.get(url, stream=True, headers=headers)
-            r.raise_for_status()
+            try:
+                r = requests.get(url, stream=True, headers=headers, verify=VERIFY_SSL)
+                r.raise_for_status()
+            except requests.exceptions.SSLError as e:
+                logger.error(
+                    f"SSL error during requests.get('{url}'): {e}\n"
+                    f"Try to set VERIFY_SSL=False in environment variables to bypass SSL verification."
+                )
+                raise e
             with io.open(filepath, mode="wb") as fout:
                 fout.write(r.content)
     return filepath

label_studio_sdk/_extensions/label_studio_tools/core/utils/json_schema.py

@@ -0,0 +1,86 @@
+import json
+import types
+import sys
+import functools
+from typing import Type, Dict, Any, Tuple, Generator
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from datamodel_code_generator import DataModelType, PythonVersion, LiteralType
+from datamodel_code_generator.model import get_data_model_types
+from datamodel_code_generator.parser.jsonschema import JsonSchemaParser
+from pydantic import BaseModel
+from contextlib import contextmanager
+
+
+@functools.lru_cache(maxsize=128)
+def _generate_model_code(json_schema_str: str, class_name: str = 'MyModel') -> str:
+
+    data_model_types = get_data_model_types(
+        DataModelType.PydanticV2BaseModel,
+        target_python_version=PythonVersion.PY_311
+    )
+
+    parser = JsonSchemaParser(
+        json_schema_str,
+        data_model_type=data_model_types.data_model,
+        data_model_root_type=data_model_types.root_model,
+        data_model_field_type=data_model_types.field_model,
+        data_type_manager_type=data_model_types.data_type_manager,
+        dump_resolve_reference_action=data_model_types.dump_resolve_reference_action,
+        enum_field_as_literal=LiteralType.All,
+        class_name=class_name
+    )
+
+    model_code = parser.parse()
+    return model_code
+
+@contextmanager
+def json_schema_to_pydantic(json_schema: dict, class_name: str = 'MyModel') -> Generator[Type[BaseModel], None, None]:
+    """
+    Convert a JSON schema to a Pydantic model and provide it as a context manager.
+
+    Args:
+        json_schema (dict): The JSON schema to convert.
+        class_name (str, optional): The name of the generated Pydantic class. Defaults to 'MyModel'.
+
+    Example:
+        ```python
+        example_schema = {
+            "type": "object",
+            "properties": {
+                "sentiment": {
+                    "type": "string",
+                    "description": "Sentiment of the text",
+                    "enum": ["Positive", "Negative", "Neutral"],
+                }
+            },
+            "required": ["sentiment"]
+        }
+        with json_schema_to_pydantic(example_schema) as ResponseModel:
+            instance = ResponseModel(sentiment='Positive')
+            print(instance.model_dump())
+        ```
+    """
+    # Convert the JSON schema dictionary to a JSON string
+    json_schema_str = json.dumps(json_schema)
+    
+    # Generate Pydantic model code from the JSON schema string
+    model_code: str = _generate_model_code(json_schema_str, class_name)
+    
+    # Create a unique module name using the id of the JSON schema string
+    module_name = f'dynamic_module_{id(json_schema_str)}'
+    
+    # Create a new module object with the unique name and execute the generated model code in the context of the new module
+    mod = types.ModuleType(module_name)
+    exec(model_code, mod.__dict__)
+    model_class = getattr(mod, class_name)
+    
+    try:
+        # Add the new module to sys.modules to make it importable
+        # This is necessary to avoid Pydantic errors related to undefined models
+        sys.modules[module_name] = mod
+        yield model_class
+    finally:
+        if module_name in sys.modules:
+            del sys.modules[module_name]
+        

label_studio_sdk/_legacy/schema/label_config_schema.json

@@ -35,22 +35,44 @@
         }
       }
     },
-    "tag_with_value_required_name": {
+    "tag_with_name_and_toname": {
       "type": "object",
-      "anyOf": [
-	{
-          "required": [
-            "@name",
-            "@valueList"
-          ]
-        },
+      "oneOf": [
         {
           "required": [
             "@name",
-            "@value"
+            "@toName"
           ]
         }
       ],
+      "properties": {
+        "@name": {
+          "$ref": "#/definitions/@name"
+        },
+        "@toName": {
+          "$ref": "#/definitions/@toName"
+        },
+        "$": {
+          "$ref": "#/definitions/$"
+        }
+      }
+    },
+    "tag_with_value_required_name": {
+      "type": "object",
+      "oneOf": [
+          {
+            "required": [
+                "@name",
+                "@value"
+            ]
+          },
+          {
+            "required": [
+                "@name",
+                "@valueList"
+            ]
+          }
+      ],
       "properties": {
         "@value": {
           "$ref": "#/definitions/@value"
@@ -80,6 +102,13 @@
         "items": {"$ref": "#/definitions/tag_with_value_required_name"}
       }, {"$ref": "#/definitions/tag_with_value_required_name"}]
     },
+    "tags_with_name_and_toname": {
+      "anyOf": [{"$ref": "#/definitions/tag_with_name_and_toname"},
+        {
+          "type": "array",
+          "items": {"$ref": "#/definitions/tag_with_name_and_toname"}
+        }]
+    },
     "View": {
       "type": "object",
       "additionalProperties": true,
@@ -92,7 +121,8 @@
         "Text": {"$ref": "#/definitions/tags_with_value_required_name"},
         "HyperText": {"$ref": "#/definitions/tags_with_value_required_name"},
         "View": {"$ref": "#/definitions/MaybeMultipleView"},
-        "TextArea": {"$ref": "#/definitions/MaybeMultipleTextAreas"}
+        "TextArea": {"$ref": "#/definitions/MaybeMultipleTextAreas"},
+        "Number": {"$ref": "#/definitions/tags_with_name_and_toname"}
       }
     },
     "MaybeMultipleView": {
@@ -219,7 +249,8 @@
         "Image": {"$ref": "#/definitions/tags_with_value_required_name"},
         "Text": {"$ref": "#/definitions/tags_with_value_required_name"},
         "HyperText": {"$ref": "#/definitions/tags_with_value_required_name"},
-        "TextArea": {"$ref": "#/definitions/MaybeMultipleTextAreas"}
+        "TextArea": {"$ref": "#/definitions/MaybeMultipleTextAreas"},
+        "Number": {"$ref": "#/definitions/tags_with_name_and_toname"}
       }
     }
   }

label_studio_sdk/annotations/__init__.py

@@ -1,2 +1,5 @@
 # This file was auto-generated by Fern from our API Definition.
 
+from .types import AnnotationsCreateBulkResponseItem
+
+__all__ = ["AnnotationsCreateBulkResponseItem"]