label-studio-sdk 0.0.32__py3-none-any.whl → 1.0.0__py3-none-any.whl

label_studio_sdk/__init__.py

@@ -1,8 +1,208 @@
-""" .. include::../docs/index.md
-"""
-from .client import Client
-from .project import Project
-from .utils import parse_config
+# This file was auto-generated by Fern from our API Definition.
 
+from .types import (
+    Annotation,
+    AnnotationFilterOptions,
+    AnnotationLastAction,
+    AzureBlobExportStorage,
+    AzureBlobExportStorageStatus,
+    AzureBlobImportStorage,
+    AzureBlobImportStorageStatus,
+    BaseTask,
+    BaseUser,
+    ConvertedFormat,
+    ConvertedFormatStatus,
+    Export,
+    ExportConvert,
+    ExportCreate,
+    ExportCreateStatus,
+    ExportStatus,
+    FileUpload,
+    Filter,
+    FilterGroup,
+    GcsExportStorage,
+    GcsExportStorageStatus,
+    GcsImportStorage,
+    GcsImportStorageStatus,
+    LocalFilesExportStorage,
+    LocalFilesExportStorageStatus,
+    LocalFilesImportStorage,
+    LocalFilesImportStorageStatus,
+    MlBackend,
+    MlBackendAuthMethod,
+    MlBackendState,
+    Prediction,
+    Project,
+    ProjectImport,
+    ProjectImportStatus,
+    ProjectLabelConfig,
+    ProjectSampling,
+    ProjectSkipQueue,
+    RedisExportStorage,
+    RedisExportStorageStatus,
+    RedisImportStorage,
+    RedisImportStorageStatus,
+    S3ExportStorage,
+    S3ExportStorageStatus,
+    S3ImportStorage,
+    S3ImportStorageStatus,
+    SerializationOption,
+    SerializationOptions,
+    Task,
+    TaskFilterOptions,
+    UserSimple,
+    View,
+    Webhook,
+    WebhookActionsItem,
+    WebhookSerializerForUpdate,
+    WebhookSerializerForUpdateActionsItem,
+)
+from .errors import BadRequestError, InternalServerError
+from . import (
+    actions,
+    annotations,
+    export_storage,
+    files,
+    import_storage,
+    ml,
+    predictions,
+    projects,
+    tasks,
+    users,
+    views,
+    webhooks,
+)
+from ._legacy import Client
+from .environment import LabelStudioEnvironment
+from .export_storage import ExportStorageListTypesResponseItem
+from .import_storage import ImportStorageListTypesResponseItem
+from .ml import (
+    MlCreateRequestAuthMethod,
+    MlCreateResponse,
+    MlCreateResponseAuthMethod,
+    MlUpdateRequestAuthMethod,
+    MlUpdateResponse,
+    MlUpdateResponseAuthMethod,
+)
+from .projects import ProjectsCreateResponse, ProjectsImportTasksResponse, ProjectsListResponse
+from .tasks import TasksListRequestFields, TasksListResponse
+from .users import UsersGetTokenResponse, UsersResetTokenResponse
+from .version import __version__
+from .views import (
+    ViewsCreateRequestData,
+    ViewsCreateRequestDataFilters,
+    ViewsCreateRequestDataFiltersConjunction,
+    ViewsCreateRequestDataFiltersItemsItem,
+    ViewsCreateRequestDataOrderingItem,
+    ViewsCreateRequestDataOrderingItemDirection,
+    ViewsUpdateRequestData,
+    ViewsUpdateRequestDataFilters,
+    ViewsUpdateRequestDataFiltersConjunction,
+    ViewsUpdateRequestDataFiltersItemsItem,
+    ViewsUpdateRequestDataOrderingItem,
+    ViewsUpdateRequestDataOrderingItemDirection,
+)
+from .webhooks import WebhooksUpdateRequestActionsItem
 
-__version__ = '0.0.32'
+__all__ = [
+    "Annotation",
+    "AnnotationFilterOptions",
+    "AnnotationLastAction",
+    "AzureBlobExportStorage",
+    "AzureBlobExportStorageStatus",
+    "AzureBlobImportStorage",
+    "AzureBlobImportStorageStatus",
+    "BadRequestError",
+    "BaseTask",
+    "BaseUser",
+    "Client",
+    "ConvertedFormat",
+    "ConvertedFormatStatus",
+    "Export",
+    "ExportConvert",
+    "ExportCreate",
+    "ExportCreateStatus",
+    "ExportStatus",
+    "ExportStorageListTypesResponseItem",
+    "FileUpload",
+    "Filter",
+    "FilterGroup",
+    "GcsExportStorage",
+    "GcsExportStorageStatus",
+    "GcsImportStorage",
+    "GcsImportStorageStatus",
+    "ImportStorageListTypesResponseItem",
+    "InternalServerError",
+    "LabelStudioEnvironment",
+    "LocalFilesExportStorage",
+    "LocalFilesExportStorageStatus",
+    "LocalFilesImportStorage",
+    "LocalFilesImportStorageStatus",
+    "MlBackend",
+    "MlBackendAuthMethod",
+    "MlBackendState",
+    "MlCreateRequestAuthMethod",
+    "MlCreateResponse",
+    "MlCreateResponseAuthMethod",
+    "MlUpdateRequestAuthMethod",
+    "MlUpdateResponse",
+    "MlUpdateResponseAuthMethod",
+    "Prediction",
+    "Project",
+    "ProjectImport",
+    "ProjectImportStatus",
+    "ProjectLabelConfig",
+    "ProjectSampling",
+    "ProjectSkipQueue",
+    "ProjectsCreateResponse",
+    "ProjectsImportTasksResponse",
+    "ProjectsListResponse",
+    "RedisExportStorage",
+    "RedisExportStorageStatus",
+    "RedisImportStorage",
+    "RedisImportStorageStatus",
+    "S3ExportStorage",
+    "S3ExportStorageStatus",
+    "S3ImportStorage",
+    "S3ImportStorageStatus",
+    "SerializationOption",
+    "SerializationOptions",
+    "Task",
+    "TaskFilterOptions",
+    "TasksListRequestFields",
+    "TasksListResponse",
+    "UserSimple",
+    "UsersGetTokenResponse",
+    "UsersResetTokenResponse",
+    "View",
+    "ViewsCreateRequestData",
+    "ViewsCreateRequestDataFilters",
+    "ViewsCreateRequestDataFiltersConjunction",
+    "ViewsCreateRequestDataFiltersItemsItem",
+    "ViewsCreateRequestDataOrderingItem",
+    "ViewsCreateRequestDataOrderingItemDirection",
+    "ViewsUpdateRequestData",
+    "ViewsUpdateRequestDataFilters",
+    "ViewsUpdateRequestDataFiltersConjunction",
+    "ViewsUpdateRequestDataFiltersItemsItem",
+    "ViewsUpdateRequestDataOrderingItem",
+    "ViewsUpdateRequestDataOrderingItemDirection",
+    "Webhook",
+    "WebhookActionsItem",
+    "WebhookSerializerForUpdate",
+    "WebhookSerializerForUpdateActionsItem",
+    "WebhooksUpdateRequestActionsItem",
+    "__version__",
+    "actions",
+    "annotations",
+    "export_storage",
+    "files",
+    "import_storage",
+    "ml",
+    "predictions",
+    "projects",
+    "tasks",
+    "users",
+    "views",
+    "webhooks",
+]

label_studio_sdk/_extensions/label_studio_tools/core/label_config.py

@@ -0,0 +1,163 @@
+import logging
+import re
+
+from collections import defaultdict
+from lxml import etree
+
+from label_studio_sdk._extensions.label_studio_tools.core.utils.exceptions import (
+    LabelStudioXMLSyntaxErrorSentryIgnored,
+)
+
+logger = logging.getLogger(__name__)
+
+_LABEL_TAGS = {"Label", "Choice", "Relation"}
+_NOT_CONTROL_TAGS = {
+    "Filter",
+}
+_DIR_APP_NAME = "label-studio"
+_VIDEO_TRACKING_TAGS = {"videorectangle"}
+
+
+def parse_config(config_string):
+    """Parse a given Label Studio labeling configuration and return a structured version of the configuration.
+    Useful for formatting results for predicted annotations and determining the type(s) of ML models that might
+    be relevant to the labeling project.
+
+    :param config_string: Label config string
+    :return: structured config of the form:
+    {
+        "<ControlTag>.name": {
+            "type": "ControlTag",
+            "to_name": ["<ObjectTag1>.name", "<ObjectTag2>.name"],
+            "inputs: [
+                {"type": "ObjectTag1", "value": "<ObjectTag1>.value"},
+                {"type": "ObjectTag2", "value": "<ObjectTag2>.value"}
+            ],
+            "labels": ["Label1", "Label2", "Label3"] // taken from "alias" if it exists, else "value"
+    }
+    """
+    if not config_string:
+        return {}
+
+    try:
+        xml_tree = etree.fromstring(config_string)
+    except etree.XMLSyntaxError as e:
+        raise LabelStudioXMLSyntaxErrorSentryIgnored(str(e))
+    inputs, outputs, labels = {}, {}, defaultdict(dict)
+    # Add variables to config (e.g. {{idx}} for index in Repeater
+    variables = []
+    for tag in xml_tree.iter():
+        if tag.attrib and "indexFlag" in tag.attrib:
+            variables.append(tag.attrib["indexFlag"])
+        if _is_output_tag(tag):
+            tag_info = {"type": tag.tag, "to_name": tag.attrib["toName"].split(",")}
+            if variables:
+                # Find variables in tag_name and regex if find it
+                for v in variables:
+                    for tag_name in tag_info["to_name"]:
+                        if v in tag_name:
+                            if "regex" not in tag_info:
+                                tag_info["regex"] = {}
+                            tag_info["regex"][v] = ".*"
+            # Grab conditionals if any
+            conditionals = {}
+            if tag.attrib.get("perRegion") == "true":
+                if tag.attrib.get("whenTagName"):
+                    conditionals = {"type": "tag", "name": tag.attrib["whenTagName"]}
+                elif tag.attrib.get("whenLabelValue"):
+                    conditionals = {
+                        "type": "label",
+                        "name": tag.attrib["whenLabelValue"],
+                    }
+                elif tag.attrib.get("whenChoiceValue"):
+                    conditionals = {
+                        "type": "choice",
+                        "name": tag.attrib["whenChoiceValue"],
+                    }
+            if conditionals:
+                tag_info["conditionals"] = conditionals
+            if has_variable(tag.attrib.get("value", "")) or tag.attrib.get("apiUrl"):
+                tag_info["dynamic_labels"] = True
+            outputs[tag.attrib["name"]] = tag_info
+        elif _is_input_tag(tag):
+            inputs[tag.attrib["name"]] = {
+                "type": tag.tag,
+                "value": tag.attrib["value"].lstrip("$"),
+                "valueType": tag.attrib.get("valueType"),
+            }
+        if tag.tag not in _LABEL_TAGS:
+            continue
+        parent_name = _get_parent_output_tag_name(tag, outputs)
+        if parent_name is not None:
+            actual_value = tag.attrib.get("alias") or tag.attrib.get("value")
+            if not actual_value:
+                logger.debug(
+                    'Inspecting tag {tag_name}... found no "value" or "alias" attributes.'.format(
+                        tag_name=etree.tostring(tag, encoding="unicode").strip()[:50]
+                    )
+                )
+            else:
+                labels[parent_name][actual_value] = dict(tag.attrib)
+    for output_tag, tag_info in outputs.items():
+        tag_info["inputs"] = []
+        for input_tag_name in tag_info["to_name"]:
+            if input_tag_name not in inputs:
+                logger.info(
+                    f"to_name={input_tag_name} is specified for output tag name={output_tag}, "
+                    "but we can't find it among input tags"
+                )
+                continue
+            tag_info["inputs"].append(inputs[input_tag_name])
+        tag_info["labels"] = list(labels[output_tag])
+        tag_info["labels_attrs"] = labels[output_tag]
+    return outputs
+
+
+def is_video_object_tracking(parsed_config):
+    for component in parsed_config:
+        if parsed_config[component]["type"].lower() in _VIDEO_TRACKING_TAGS:
+            return True
+
+
+def has_variable(actual_value):
+    """
+    Check if value has variable
+    :param actual_value: value to check
+    :return: True if value has variable
+    """
+    expression = "^\$[A-Za-z_]+$"
+    pattern = re.compile(expression)
+    full_match = pattern.fullmatch(actual_value)
+    if full_match:
+        return True
+    return False
+
+
+def _is_input_tag(tag):
+    """
+    Check if tag is input
+    """
+    return tag.attrib.get("name") and tag.attrib.get("value")
+
+
+def _is_output_tag(tag):
+    """
+    Check if tag is output
+    """
+    return (
+        tag.attrib.get("name")
+        and tag.attrib.get("toName")
+        and tag.tag not in _NOT_CONTROL_TAGS
+    )
+
+
+def _get_parent_output_tag_name(tag, outputs):
+    # Find parental <Choices> tag for nested tags like <Choices><View><View><Choice>...
+    parent = tag
+    while True:
+        parent = parent.getparent()
+        if parent is None:
+            return
+        name = parent.attrib.get("name")
+        if name in outputs:
+            return name

label_studio_sdk/_extensions/label_studio_tools/core/utils/exceptions.py

@@ -0,0 +1,2 @@
+class LabelStudioXMLSyntaxErrorSentryIgnored(Exception):
+    pass

label_studio_sdk/_extensions/label_studio_tools/core/utils/io.py

@@ -0,0 +1,228 @@
+import logging
+import io
+import shutil
+import urllib
+import hashlib
+import requests
+import os
+
+from appdirs import user_cache_dir, user_data_dir
+from urllib.parse import urlparse, urljoin
+from contextlib import contextmanager
+from tempfile import mkdtemp
+
+from label_studio_sdk._extensions.label_studio_tools.core.utils.params import get_env
+
+_DIR_APP_NAME = "label-studio"
+LOCAL_FILES_DOCUMENT_ROOT = get_env(
+    "LOCAL_FILES_DOCUMENT_ROOT", default=os.path.abspath(os.sep)
+)
+
+logger = logging.getLogger(__name__)
+
+
+def concat_urls(base_url, url):
+    return base_url.rstrip("/") + "/" + url.lstrip("/")
+
+
+def get_data_dir():
+    data_dir = user_data_dir(appname=_DIR_APP_NAME)
+    os.makedirs(data_dir, exist_ok=True)
+    return data_dir
+
+
+def get_cache_dir():
+    cache_dir = user_cache_dir(appname=_DIR_APP_NAME)
+    os.makedirs(cache_dir, exist_ok=True)
+    return cache_dir
+
+
+def get_local_path(
+    url,
+    cache_dir=None,
+    project_dir=None,
+    hostname=None,
+    image_dir=None,
+    access_token=None,
+    download_resources=True,
+    task_id=None,
+):
+    f"""This helper function is used to download (cache) url and return local path to it.
+
+    :param url: File URL to download, it can be a uploaded file, local storage, cloud storage file or just http(s) url
+    :param cache_dir: Cache directory to download or copy files
+    :param project_dir: Project directory
+    :param hostname: Label Studio Hostname, it will be used for uploaded files, local storage files and cloud storage files
+      if not provided, it will be taken from LABEL_STUDIO_URL env variable
+    :param image_dir: Image and other media upload directory
+    :param access_token: Label Studio access token, it will be used for uploaded files, local storage files and cloud storage files
+      if not provided, it will be taken from LABEL_STUDIO_API_KEY env variable
+    :param download_resources: Download and cache a file from URL
+    :param task_id: Label Studio Task ID, required for cloud storage files 
+      because the URL will be rebuilt to `{hostname}/tasks/{task_id}/presign/?fileuri={url}` 
+
+    :return: filepath
+    """
+    # get environment variables
+    hostname = (
+        hostname
+        or os.getenv("LABEL_STUDIO_URL", "")
+        or os.getenv("LABEL_STUDIO_HOST", "")
+    )
+    access_token = (
+        access_token
+        or os.getenv("LABEL_STUDIO_API_KEY", "")
+        or os.getenv("LABEL_STUDIO_ACCESS_TOKEN", "")
+    )
+    if "localhost" in hostname:
+        logger.warning(
+            f"Using `localhost` ({hostname}) in LABEL_STUDIO_URL, "
+            f"`localhost` is not accessible inside of docker containers. "
+            f"You can check your IP with utilities like `ifconfig` and set it as LABEL_STUDIO_URL."
+        )
+
+    # fix file upload url
+    if url.startswith("upload") or url.startswith("/upload"):
+        url = "/data" + ("" if url.startswith("/") else "/") + url
+
+    is_uploaded_file = url.startswith("/data/upload")
+    is_local_storage_file = url.startswith("/data/") and "?d=" in url
+    is_cloud_storage_file = (
+        url.startswith("s3:") or url.startswith("gs:") or url.startswith("azure-blob:")
+    )
+
+    # Local storage file: try to load locally otherwise download below
+    # this code allow to read Local Storage files directly from a directory
+    # instead of downloading them from LS instance
+    if is_local_storage_file:
+        filepath = url.split("?d=")[1]
+        filepath = os.path.join(LOCAL_FILES_DOCUMENT_ROOT, filepath)
+        if os.path.exists(filepath):
+            logger.debug(
+                f"Local Storage file path exists locally, use it as a local file: {filepath}"
+            )
+            return filepath
+
+    # try to get local directories
+    if image_dir is None:
+        upload_dir = os.path.join(get_data_dir(), "media", "upload")
+        image_dir = project_dir and os.path.join(project_dir, "upload") or upload_dir
+        logger.debug(
+            f"Image and upload dirs: image_dir={image_dir}, upload_dir={upload_dir}"
+        )
+
+    # Uploaded file: try to load locally otherwise download below
+    # this code allow to read Uploaded files directly from a directory
+    # instead of downloading them from LS instance
+    if is_uploaded_file and os.path.exists(image_dir):
+        project_id = url.split("/")[-2]  # To retrieve project_id
+        filepath = os.path.join(image_dir, project_id, os.path.basename(url))
+        if cache_dir and download_resources:
+            shutil.copy(filepath, cache_dir)
+        if os.path.exists(filepath):
+            logger.debug(f"Uploaded file: Path exists in image_dir: {filepath}")
+            return filepath
+
+    # Upload or Local Storage file
+    if is_uploaded_file or is_local_storage_file or is_cloud_storage_file:
+        # hostname check
+        if not hostname:
+            raise FileNotFoundError(
+                f"Can't resolve url, neither hostname or project_dir passed: {url}. "
+                "You can set LABEL_STUDIO_URL environment variable to use it as a hostname."
+            )
+        # uploaded and local storage file
+        elif is_uploaded_file or is_local_storage_file:
+            url = concat_urls(hostname, url)
+            logger.info("Resolving url using hostname [" + hostname + "]: " + url)
+        # s3, gs, azure-blob file
+        elif is_cloud_storage_file:
+            if task_id is None:
+                raise Exception(
+                    "Label Studio Task ID is required for cloud storage files"
+                )
+            url = concat_urls(hostname, f"/tasks/{task_id}/presign/?fileuri={url}")
+            logger.info(
+                "Cloud storage file: Resolving url using hostname ["
+                + hostname
+                + "]: "
+                + url
+            )
+
+        # check access token
+        if not access_token:
+            raise FileNotFoundError(
+                "To access uploaded and local storage files you have to "
+                "set LABEL_STUDIO_API_KEY environment variable."
+            )
+
+    filepath = download_and_cache(
+        url,
+        cache_dir,
+        download_resources,
+        hostname,
+        access_token,
+        is_local_storage_file,
+        is_cloud_storage_file,
+    )
+    return filepath
+
+
+def download_and_cache(
+    url,
+    cache_dir,
+    download_resources,
+    hostname,
+    access_token,
+    is_local_storage_file,
+    is_cloud_storage_file,
+):
+    # File specified by remote URL - download and cache it
+    cache_dir = cache_dir or get_cache_dir()
+    parsed_url = urlparse(url)
+    url_filename = (
+        # /data/local-files?d=dir/1.jpg => 1.jpg
+        os.path.basename(url)
+        if is_local_storage_file or is_cloud_storage_file
+        # /some/url/1.jpg?expire=xxx => 1.jpg
+        else os.path.basename(parsed_url.path)
+    )
+    url_hash = hashlib.md5(url.encode()).hexdigest()[:8]
+    filepath = os.path.join(cache_dir, url_hash + "__" + url_filename)
+
+    if not os.path.exists(filepath):
+        logger.info("Download {url} to {filepath}".format(url=url, filepath=filepath))
+        if download_resources:
+            headers = {
+                # avoid requests.exceptions.HTTPError: 403 Client Error: Forbidden. Please comply with the User-Agent policy:
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
+            }
+            # check if url matches hostname - then uses access token to this Label Studio instance
+            if (
+                access_token
+                and hostname
+                and parsed_url.netloc == urlparse(hostname).netloc
+            ):
+                headers["Authorization"] = "Token " + access_token
+                logger.debug("Authorization token is used for download_and_cache")
+            r = requests.get(url, stream=True, headers=headers)
+            r.raise_for_status()
+            with io.open(filepath, mode="wb") as fout:
+                fout.write(r.content)
+    return filepath
+
+
+@contextmanager
+def get_temp_dir():
+    dirpath = mkdtemp()
+    yield dirpath
+    shutil.rmtree(dirpath)
+
+
+def get_all_files_from_dir(d):
+    out = []
+    for name in os.listdir(d):
+        filepath = os.path.join(d, name)
+        if os.path.isfile(filepath):
+            out.append(filepath)
+    return out

label_studio_sdk/_extensions/label_studio_tools/core/utils/params.py

@@ -0,0 +1,45 @@
+import os
+
+
+def bool_from_request(params, key, default):
+    """Get boolean value from request GET, POST, etc
+    :param params: dict POST, GET, etc
+    :param key: key to find
+    :param default: default value
+    :return: boolean
+    """
+    value = params.get(key, default)
+
+    if isinstance(value, str):
+        value = cast_bool_from_str(value)
+
+    return bool(int(value))
+
+
+def cast_bool_from_str(value):
+    if isinstance(value, str):
+        if value.lower() in ["true", "yes", "on", "1"]:
+            value = True
+        elif value.lower() in ["false", "no", "not", "off", "0"]:
+            value = False
+        else:
+            raise ValueError(
+                f'Incorrect bool value "{value}". '
+                f"It should be one of [1, 0, true, false, yes, no]"
+            )
+    return value
+
+
+def get_env(name, default=None, is_bool=False):
+    for env_key in ["LABEL_STUDIO_" + name, "HEARTEX_" + name, name]:
+        value = os.environ.get(env_key)
+        if value is not None:
+            if is_bool:
+                return bool_from_request(os.environ, env_key, default)
+            else:
+                return value
+    return default
+
+
+def get_bool_env(key, default):
+    return get_env(key, default, is_bool=True)

label_studio_sdk/_extensions/label_studio_tools/etl/__init__.py

@@ -0,0 +1 @@
+from registry import transform

label_studio_sdk/_extensions/label_studio_tools/etl/beam.py

@@ -0,0 +1,34 @@
+import apache_beam as beam
+from registry import dataloader, call_dataloader
+
+
+class CallDataLoader(beam.DoFn):
+    def __init__(self, dataloader_name):
+        self.dataloader_name = dataloader_name
+
+    def setup(self):
+        print("Setup!")
+        pass
+
+    def process(self, element, *args, **kwargs):
+        return call_dataloader(self.dataloader_name, element, *args, **kwargs)
+
+    def teardown(self):
+        print("Teardown!")
+        pass
+
+
+@dataloader("test")
+def f(i, k):
+    print("I!!", k)
+    yield i
+
+
+with beam.Pipeline() as pipeline:
+    results = (
+        pipeline
+        | "Get storage keys" >> beam.io.gcp.gcsio.list_files("bucket")
+        | "Run dataloader"
+        >> beam.FlatMap(fn=CallDataLoader("test"), **dataloader_kwargs)
+        | "Load into Vector DB" >> beam.ParDo(fn=LoadDataRecordInDB)
+    )