label-studio-sdk 0.0.32__py3-none-any.whl → 1.0.0__py3-none-any.whl

label_studio_sdk/converter/exports/csv.py

@@ -0,0 +1,82 @@
+# this csv converter is not used in GUI export, see convert_to_csv function
+import pandas as pd
+import os
+import json
+
+from copy import deepcopy
+
+
+class ExportToCSV(object):
+    def __init__(self, tasks):
+        if isinstance(tasks, str) and tasks.endswith(".json"):
+            if not os.path.exists(tasks):
+                raise Exception(f"Task file not found {tasks}")
+            # input is a file
+            with open(tasks) as f:
+                self.tasks = json.load(f)
+        else:
+            # input is a JSON object
+            self.tasks = tasks
+
+    def _get_result_name(self, result):
+        return result.get("from_name")
+
+    def _minify_result(self, result):
+        value = result["value"]
+        name = self._get_result_name(result)
+        if len(value) == 1:
+            item = next(iter(value.values()))
+            if len(item) == 0:
+                return {name: None}
+            if len(item) == 1:
+                return {name: item[0]}
+            else:
+                return {name: item}
+        else:
+            return value
+
+    def _get_annotation_results(self, annotation, minify, flat_regions):
+        results = annotation["result"]
+        if not flat_regions:
+            yield {"result": results}
+
+        for result in annotation["result"]:
+            if minify:
+                yield self._minify_result(result)
+            else:
+                yield {self._get_result_name(result): result}
+
+    def _get_annotator_id(self, annotation):
+        annotator = annotation.get("completed_by", {})
+        if isinstance(annotator, int):
+            return annotator
+        elif isinstance(annotator, dict):
+            return annotator.get("email") or annotator.get("id")
+
+    def to_records(self, minify=True, flat_regions=True):
+        records = []
+        for task in self.tasks:
+            annotations = task.get("annotations")
+            if annotations is None:
+                # Temp legacy fix
+                annotations = task["completions"]
+            for annotation in annotations:
+                record = {
+                    "id": task["id"],
+                    "annotation_id": annotation.get("id"),
+                    "annotator": self._get_annotator_id(annotation),
+                }
+                record.update(task["data"])
+                for result in self._get_annotation_results(
+                    annotation, minify, flat_regions
+                ):
+                    rec = deepcopy(record)
+                    rec.update(result)
+                    records.append(rec)
+        return records
+
+    def to_dataframe(self, minify=True, flat_regions=True):
+        return pd.DataFrame.from_records(self.to_records(minify, flat_regions))
+
+    def to_file(self, file, minify=True, flat_regions=True, **kwargs):
+        return self.to_dataframe(minify, flat_regions).to_csv(file, **kwargs)

label_studio_sdk/converter/exports/csv2.py

@@ -0,0 +1,103 @@
+import os
+import csv
+import time
+import logging
+import ujson as json
+
+from copy import deepcopy, copy
+
+from label_studio_sdk.converter.utils import ensure_dir, get_annotator, prettify_result
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel("DEBUG")
+
+
+def convert(item_iterator, input_data, output_dir, **kwargs):
+    start_time = time.time()
+    logger.debug("Convert CSV started")
+    if str(output_dir).endswith(".csv"):
+        output_file = output_dir
+    else:
+        ensure_dir(output_dir)
+        output_file = os.path.join(output_dir, "result.csv")
+
+    # these keys are always presented
+    keys = {"annotator", "annotation_id", "created_at", "updated_at", "lead_time"}
+
+    # make 2 passes: the first pass is to get keys, otherwise we can't write csv without headers
+    logger.debug("Prepare column names for CSV ...")
+    for item in item_iterator(input_data):
+        record = prepare_annotation_keys(item)
+        keys.update(record)
+
+    # the second pass is to write records to csv
+    logger.debug(
+        f"Prepare done in {time.time()-start_time:0.2f} sec. Write CSV rows now ..."
+    )
+    with open(output_file, "w", encoding="utf8") as outfile:
+        writer = csv.DictWriter(
+            outfile,
+            fieldnames=sorted(list(keys)),
+            quoting=csv.QUOTE_NONNUMERIC,
+            delimiter=kwargs["sep"],
+        )
+        writer.writeheader()
+
+        for item in item_iterator(input_data):
+            record = prepare_annotation(item)
+            writer.writerow(record)
+
+    logger.debug(f"CSV conversion finished in {time.time()-start_time:0.2f} sec")
+
+
+def prepare_annotation(item):
+    record = {}
+    if item.get("id") is not None:
+        record["id"] = item["id"]
+
+    for name, value in item["output"].items():
+        pretty_value = prettify_result(value)
+        record[name] = (
+            pretty_value
+            if isinstance(pretty_value, str)
+            else json.dumps(pretty_value, ensure_ascii=False)
+        )
+
+    for name, value in item["input"].items():
+        if isinstance(value, dict) or isinstance(value, list):
+            # flat dicts and arrays from task.data to json strings
+            record[name] = json.dumps(value, ensure_ascii=False)
+        else:
+            record[name] = value
+
+    record["annotator"] = get_annotator(item)
+    record["annotation_id"] = item["annotation_id"]
+    record["created_at"] = item["created_at"]
+    record["updated_at"] = item["updated_at"]
+    record["lead_time"] = item["lead_time"]
+
+    if "agreement" in item:
+        record["agreement"] = item["agreement"]
+
+    if "history" in item and item["history"]:
+        record["history"] = json.dumps(item["history"], ensure_ascii=False)
+
+    return record
+
+
+def prepare_annotation_keys(item):
+    record = set(item["input"].keys())  # we don't need deepcopy for keys
+    if item.get("id") is not None:
+        record.add("id")
+
+    for name, value in item["output"].items():
+        record.add(name)
+
+    if "agreement" in item:
+        record.add("agreement")
+
+    if "history" in item and item["history"]:
+        record.add("history")
+
+    return record

label_studio_sdk/converter/funsd.py

@@ -0,0 +1,85 @@
+"""This code allows to export Label Studio Export JSON to FUNSD format.
+It's only the basic converter, it converts every bbox as a separate word.
+Check this github issue for more details:
+https://github.com/heartexlabs/label-studio/issues/2634#issuecomment-1251648670
+
+Usage: funsd.py export.json
+This command will export your LS OCR annotations to "./funsd/" directory.
+"""
+
+import os
+import json
+from collections import defaultdict
+
+
+def convert_annotation_to_fund(result):
+    # collect all LS results and combine labels, text, coordinates into one record
+    pre = defaultdict(dict)
+    for item in result:
+        o = pre[item["id"]]
+
+        labels = item.get("value", {}).get("labels", None)
+        if labels:
+            o["label"] = labels[0]
+
+        text = item.get("value", {}).get("text", None)
+        if text:
+            o["text"] = text[0]
+
+        if "box" not in o:
+            w, h = item["original_width"], item["original_height"]
+            v = item.get("value")
+            x1 = v["x"] / 100.0 * w
+            y1 = v["y"] / 100.0 * h
+            x2 = x1 + v["width"] / 100.0 * w
+            y2 = y1 + v["height"] / 100.0 * h
+            o["box"] = [x1, x2, y1, y2]
+
+    # make FUNSD output
+    output = []
+    counter = 0
+    for key in pre:
+        counter += 1
+        output.append(
+            {
+                "id": counter,
+                "box": pre[key]["box"],
+                "text": pre[key]["text"],
+                "label": pre[key]["label"],
+                "words": [{"box": pre[key]["box"], "text": pre[key]["text"]}],
+                "linking": [],
+            }
+        )
+
+    return {"form": output}
+
+
+def ls_to_funsd_converter(
+    ls_export_path="export.json", funsd_dir="funsd", data_key="ocr"
+):
+    with open(ls_export_path) as f:
+        tasks = json.load(f)
+
+    os.makedirs(funsd_dir, exist_ok=True)
+
+    for task in tasks:
+        for annotation in task["annotations"]:
+            output = convert_annotation_to_fund(annotation["result"])
+            filename = task["data"][data_key]
+            filename = os.path.basename(filename)
+            filename = (
+                f'{funsd_dir}/task-{task["id"]}-annotation-{annotation["id"]}-'
+                f"{filename}.json"
+            )
+
+            with open(filename, "w") as f:
+                json.dump(output, f)
+
+
+if __name__ == "__main__":
+    import sys
+
+    print("Usage:", sys.argv[0], "export.json")
+    print('This command will export your LS OCR annotations to "./funsd/" directory')
+
+    ls_to_funsd_converter(sys.argv[1])

label_studio_sdk/converter/imports/coco.py

@@ -0,0 +1,314 @@
+import os
+import json  # better to use "imports ujson as json" for the best performance
+import uuid
+import logging
+from PIL import Image
+
+from label_studio_sdk.converter.utils import ExpandFullPath
+from label_studio_sdk.converter.imports.label_config import generate_label_config
+
+logger = logging.getLogger("root")
+
+
+def new_task(out_type, root_url, file_name):
+    return {
+        "data": {"image": os.path.join(root_url, file_name)},
+        # 'annotations' or 'predictions'
+        out_type: [
+            {
+                "result": [],
+                "ground_truth": False,
+            }
+        ],
+    }
+
+
+def create_bbox(annotation, categories, from_name, image_height, image_width, to_name):
+    label = categories[int(annotation["category_id"])]
+    x, y, width, height = annotation["bbox"]
+    x, y, width, height = float(x), float(y), float(width), float(height)
+    item = {
+        "id": uuid.uuid4().hex[0:10],
+        "type": "rectanglelabels",
+        "value": {
+            "x": x / image_width * 100.0,
+            "y": y / image_height * 100.0,
+            "width": width / image_width * 100.0,
+            "height": height / image_height * 100.0,
+            "rotation": 0,
+            "rectanglelabels": [label],
+        },
+        "to_name": to_name,
+        "from_name": from_name,
+        "image_rotation": 0,
+        "original_width": image_width,
+        "original_height": image_height,
+    }
+    return item
+
+
+def create_segmentation(
+    annotation, categories, from_name, image_height, image_width, to_name
+):
+    label = categories[int(annotation["category_id"])]
+    segmentation = annotation["segmentation"][0]
+    points = [list(x) for x in zip(*[iter(segmentation)] * 2)]
+
+    for i in range(len(points)):
+        points[i][0] = points[i][0] / image_width * 100.0
+        points[i][1] = points[i][1] / image_height * 100.0
+
+    item = {
+        "id": uuid.uuid4().hex[0:10],
+        "type": "polygonlabels",
+        "value": {"points": points, "polygonlabels": [label]},
+        "to_name": to_name,
+        "from_name": from_name,
+        "image_rotation": 0,
+        "original_width": image_width,
+        "original_height": image_height,
+    }
+    return item
+
+
+def create_keypoints(
+    annotation, categories, from_name, to_name, image_height, image_width, point_width
+):
+    label = categories[int(annotation["category_id"])]
+    points = annotation["keypoints"]
+    items = []
+
+    for i in range(0, len(points), 3):
+        x, y, v = points[i : i + 3]  # x, y, visibility
+        x, y, v = float(x), float(y), int(v)
+        item = {
+            "id": uuid.uuid4().hex[0:10],
+            "type": "keypointlabels",
+            "value": {
+                "x": x / image_width * 100.0,
+                "y": y / image_height * 100.0,
+                "width": point_width,
+                "keypointlabels": [label],
+            },
+            "to_name": to_name,
+            "from_name": from_name,
+            "image_rotation": 0,
+            "original_width": image_width,
+            "original_height": image_height,
+        }
+
+        # visibility
+        if v < 2:
+            item["value"]["hidden"] = True
+
+        items.append(item)
+    return items
+
+
+def convert_coco_to_ls(
+    input_file,
+    out_file,
+    to_name="image",
+    from_name="label",
+    out_type="annotations",
+    image_root_url="/data/local-files/?d=",
+    use_super_categories=False,
+    point_width=1.0,
+):
+    """Convert COCO labeling to Label Studio JSON
+
+    :param input_file: file with COCO json
+    :param out_file: output file with Label Studio JSON tasks
+    :param to_name: object name from Label Studio labeling config
+    :param from_name: control tag name from Label Studio labeling config
+    :param out_type: annotation type - "annotations" or "predictions"
+    :param image_root_url: root URL path where images will be hosted, e.g.: http://example.com/images
+    :param use_super_categories: use super categories from categories if they are presented
+    :param point_width: key point width
+    """
+
+    tasks = {}  # image_id => task
+    logger.info("Reading COCO notes and categories from %s", input_file)
+
+    with open(input_file, encoding="utf8") as f:
+        coco = json.load(f)
+
+    # build categories => labels dict
+    new_categories = {}
+    # list to dict conversion: [...] => {category_id: category_item}
+    categories = {int(category["id"]): category for category in coco["categories"]}
+    ids = sorted(categories.keys())  # sort labels by their origin ids
+
+    for i in ids:
+        name = categories[i]["name"]
+        if use_super_categories and "supercategory" in categories[i]:
+            name = categories[i]["supercategory"] + ":" + name
+        new_categories[i] = name
+
+    # mapping: id => category name
+    categories = new_categories
+
+    # mapping: image id => image
+    images = {item["id"]: item for item in coco["images"]}
+
+    logger.info(
+        f'Found {len(categories)} categories, {len(images)} images and {len(coco["annotations"])} annotations'
+    )
+
+    # flags for labeling config composing
+    segmentation = bbox = keypoints = rle = False
+    segmentation_once = bbox_once = keypoints_once = rle_once = False
+    rectangles_from_name, keypoints_from_name = (
+        from_name + "_rectangles",
+        from_name + "_keypoints",
+    )
+    segmentation_from_name = from_name + "polygons"
+    tags = {}
+
+    # create tasks
+    for image in coco["images"]:
+        image_id, image_file_name = image["id"], image["file_name"]
+        tasks[image_id] = new_task(out_type, image_root_url, image_file_name)
+
+    for i, annotation in enumerate(coco["annotations"]):
+        segmentation |= "segmentation" in annotation
+        bbox |= "bbox" in annotation
+        keypoints |= "keypoints" in annotation
+        rle |= (
+            annotation.get("iscrowd") == 1
+        )  # 0 - polygons are in segmentation, otherwise rle
+
+        if rle and not rle_once:  # not supported
+            logger.error("RLE in segmentation is not yet supported in COCO")
+            rle_once = True
+        if keypoints and not keypoints_once:
+            logger.warning("Keypoints are partially supported without skeletons")
+            tags.update({keypoints_from_name: "KeyPointLabels"})
+            keypoints_once = True
+        if segmentation and not segmentation_once:  # not supported
+            logger.warning("Segmentation in COCO is experimental")
+            tags.update({segmentation_from_name: "PolygonLabels"})
+            segmentation_once = True
+        if bbox and not bbox_once:
+            tags.update({rectangles_from_name: "RectangleLabels"})
+            bbox_once = True
+
+        # read image sizes
+        image_id = annotation["image_id"]
+        image = images[image_id]
+        image_file_name, image_width, image_height = (
+            image["file_name"],
+            image["width"],
+            image["height"],
+        )
+
+        task = tasks[image_id]
+
+        if "bbox" in annotation:
+            item = create_bbox(
+                annotation,
+                categories,
+                rectangles_from_name,
+                image_height,
+                image_width,
+                to_name,
+            )
+            task[out_type][0]["result"].append(item)
+
+        if "segmentation" in annotation and len(annotation["segmentation"]):
+            item = create_segmentation(
+                annotation,
+                categories,
+                segmentation_from_name,
+                image_height,
+                image_width,
+                to_name,
+            )
+            task[out_type][0]["result"].append(item)
+
+        if "keypoints" in annotation:
+            items = create_keypoints(
+                annotation,
+                categories,
+                keypoints_from_name,
+                to_name,
+                image_height,
+                image_width,
+                point_width,
+            )
+            task[out_type][0]["result"] += items
+
+        tasks[image_id] = task
+
+    # generate and save labeling config
+    label_config_file = out_file.replace(".json", "") + ".label_config.xml"
+    generate_label_config(categories, tags, to_name, from_name, label_config_file)
+
+    if len(tasks) > 0:
+        tasks = [tasks[key] for key in sorted(tasks.keys())]
+        logger.info("Saving Label Studio JSON to %s", out_file)
+        with open(out_file, "w") as out:
+            json.dump(tasks, out)
+
+        print(
+            "\n"
+            f"  1. Create a new project in Label Studio\n"
+            f'  2. Use Labeling Config from "{label_config_file}"\n'
+            f"  3. Setup serving for images [e.g. you can use Local Storage (or others):\n"
+            f"     https://labelstud.io/guide/storage.html#Local-storage]\n"
+            f'  4. Import "{out_file}" to the project\n'
+        )
+    else:
+        logger.error("No labels converted")
+
+
+def add_parser(subparsers):
+    coco = subparsers.add_parser("coco")
+
+    coco.add_argument(
+        "-i",
+        "--input",
+        dest="input",
+        required=True,
+        help="directory with COCO where images, labels, notes.json are located",
+        action=ExpandFullPath,
+    )
+    coco.add_argument(
+        "-o",
+        "--output",
+        dest="output",
+        help="output file with Label Studio JSON tasks",
+        default="output.json",
+        action=ExpandFullPath,
+    )
+    coco.add_argument(
+        "--to-name",
+        dest="to_name",
+        help="object name from Label Studio labeling config",
+        default="image",
+    )
+    coco.add_argument(
+        "--from-name",
+        dest="from_name",
+        help="control tag name from Label Studio labeling config",
+        default="label",
+    )
+    coco.add_argument(
+        "--out-type",
+        dest="out_type",
+        help='annotation type - "annotations" or "predictions"',
+        default="annotations",
+    )
+    coco.add_argument(
+        "--image-root-url",
+        dest="image_root_url",
+        help="root URL path where images will be hosted, e.g.: http://example.com/images",
+        default="/data/local-files/?d=",
+    )
+    coco.add_argument(
+        "--point-width",
+        dest="point_width",
+        help="key point width (size)",
+        default=1.0,
+        type=float,
+    )