labelr 0.8.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

labelr/sample/llm.py

@@ -0,0 +1,75 @@
+import typing
+from collections.abc import Iterator
+from pathlib import Path
+
+import datasets
+import orjson
+from PIL import Image
+from pydantic import BaseModel, Field
+
+from labelr.sample.common import SampleMeta
+from labelr.utils import download_image
+
+
+class LLMImageExtractionSample(BaseModel):
+    class Config:
+        # required to allow PIL Image type
+        arbitrary_types_allowed = True
+
+    image_id: str = Field(
+        ...,
+        description="unique ID for the image. For Open Food Facts images, it follows the "
+        "format `barcode:imgid`",
+    )
+    image: Image.Image = Field(..., description="Image to extract information from")
+    output: str | None = Field(..., description="Expected response of the LLM")
+    meta: SampleMeta = Field(..., description="Metadata associated with the sample")
+
+
+HF_DS_LLM_IMAGE_EXTRACTION_FEATURES = datasets.Features(
+    {
+        "image_id": datasets.Value("string"),
+        "image": datasets.features.Image(),
+        "output": datasets.features.Value("string"),
+        "meta": {
+            "barcode": datasets.Value("string"),
+            "off_image_id": datasets.Value("string"),
+            "image_url": datasets.Value("string"),
+        },
+    }
+)
+
+
+def load_llm_image_extraction_dataset_from_jsonl(
+    dataset_path: Path, **kwargs
+) -> Iterator[LLMImageExtractionSample]:
+    """Load a Hugging Face dataset for LLM image extraction from a JSONL file.
+
+    Args:
+        dataset_path (Path): Path to the JSONL dataset file.
+        **kwargs: Additional keyword arguments to pass to the image downloader.
+    Yields:
+        Iterator[LLMImageExtractionSample]: Iterator of LLM image extraction
+            samples.
+    """
+    with dataset_path.open("r") as f:
+        for line in f:
+            item = orjson.loads(line)
+            image_id = item["image_id"]
+            image_url = item["image_url"]
+            image = typing.cast(Image.Image, download_image(image_url, **kwargs))
+            barcode = item.pop("barcode", None)
+            off_image_id = item.pop("off_image_id", None)
+            output = item.pop("output", None)
+            meta = SampleMeta(
+                barcode=barcode,
+                off_image_id=off_image_id,
+                image_url=image_url,
+            )
+            sample = LLMImageExtractionSample(
+                image_id=image_id,
+                image=image,
+                output=output,
+                meta=meta,
+            )
+            yield sample

labelr/{sample.py → sample/object_detection.py}

@@ -249,20 +249,3 @@ HF_DS_OBJECT_DETECTION_FEATURES = datasets.Features(
         },
     }
 )
-
-
-HF_DS_CLASSIFICATION_FEATURES = datasets.Features(
-    {
-        "image_id": datasets.Value("string"),
-        "image": datasets.features.Image(),
-        "width": datasets.Value("int64"),
-        "height": datasets.Value("int64"),
-        "meta": {
-            "barcode": datasets.Value("string"),
-            "off_image_id": datasets.Value("string"),
-            "image_url": datasets.Value("string"),
-        },
-        "category_id": datasets.Value("int64"),
-        "category_name": datasets.Value("string"),
-    }
-)

labelr/utils.py

@@ -1,3 +1,12 @@
+import io
+from pathlib import Path
+
+from google.cloud import storage
+from openfoodfacts.images import download_image as _download_image
+from openfoodfacts.utils import ImageDownloadItem
+from PIL import Image
+
+
 def parse_hf_repo_id(hf_repo_id: str) -> tuple[str, str]:
     """Parse the repo_id and the revision from a hf_repo_id in the format:
     `org/repo-name@revision`.
@@ -11,3 +20,79 @@ def parse_hf_repo_id(hf_repo_id: str) -> tuple[str, str]:
         revision = "main"
 
     return hf_repo_id, revision
+
+
+def download_image(
+    image: str | tuple[str, str],
+    *,
+    error_raise: bool = True,
+    return_struct: bool = False,
+    **kwargs,
+) -> Image.Image | ImageDownloadItem | None:
+    """Download an image from a URL or GCS URI and return it as a PIL Image.
+    Args:
+        image (str | tuple[str, str]): The URL or GCS URI of the image.
+        error_raise (bool): Whether to raise an error if the image cannot be
+            downloaded.
+        return_struct (bool): Whether to return an ImageDownloadItem struct
+            instead of a PIL Image.
+        **kwargs: Additional arguments to pass to the download function.
+    Returns:
+        Image.Image | ImageDownloadItem: The downloaded image as a PIL Image
+            or an ImageDownloadItem struct.
+    """
+    if isinstance(image, str) and image.startswith("gs://"):
+        return download_image_from_gcs(image, return_struct=return_struct, **kwargs)
+    return _download_image(
+        image,
+        error_raise=error_raise,
+        return_struct=return_struct,
+        **kwargs,
+    )
+
+
+def download_image_from_gcs(
+    image_uri: str, client: storage.Client | None = None, return_struct: bool = False
+) -> Image.Image | ImageDownloadItem:
+    """Download an image from a Google Cloud Storage URI and return it as a
+    PIL Image.
+
+    Args:
+        image_uri (str): The GCS URI of the image
+            (e.g., gs://bucket_name/path/to/image.jpg).
+        client (storage.Client | None): An optional Google Cloud Storage
+            client. If not provided, a new client will be created.
+    """
+    if client is None:
+        client = storage.Client()
+
+    bucket_name, blob_name = image_uri.replace("gs://", "").split("/", 1)
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(blob_name)
+    image_data = blob.download_as_bytes()
+    pil_image = Image.open(io.BytesIO(image_data))
+
+    if return_struct:
+        return ImageDownloadItem(
+            url=image_uri,
+            image=pil_image,
+            error=None,
+        )
+    return pil_image
+
+
+class PathWithContext:
+    """A context manager that yields a Path object.
+
+    This is useful to have a common interface with tempfile.TemporaryDirectory
+    without actually creating a temporary directory.
+    """
+
+    def __init__(self, path: Path):
+        self.path = path
+
+    def __enter__(self) -> Path:
+        return self.path
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        pass

{labelr-0.8.0.dist-info → labelr-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: labelr
-Version: 0.8.0
+Version: 0.10.0
 Summary: A command-line tool to manage labeling tasks with Label Studio.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -13,6 +13,14 @@ Requires-Dist: openfoodfacts>=2.9.0
 Requires-Dist: typer>=0.15.1
 Requires-Dist: google-cloud-batch==0.18.0
 Requires-Dist: huggingface-hub
+Requires-Dist: deepdiff>=8.6.1
+Requires-Dist: rapidfuzz>=3.14.3
+Requires-Dist: aiohttp
+Requires-Dist: aiofiles
+Requires-Dist: orjson
+Requires-Dist: google-cloud-storage
+Requires-Dist: gcloud-aio-storage
+Requires-Dist: google-genai>=1.56.0
 Provides-Extra: ultralytics
 Requires-Dist: ultralytics==8.3.223; extra == "ultralytics"
 Provides-Extra: fiftyone

labelr-0.10.0.dist-info/RECORD

@@ -0,0 +1,36 @@
+labelr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+labelr/__main__.py,sha256=G4e95-IfhI-lOmkOBP6kQ8wl1x_Fl7dZlLOYr90K83c,66
+labelr/annotate.py,sha256=3fJ9FYbcozcOoKuhNtzPHV8sSnp-45FsNnMc8UeBHGU,3503
+labelr/check.py,sha256=3wK6mE0UsKvoBNm0_lyWhCMq7gxkv5r50pvO70damXY,2476
+labelr/config.py,sha256=3RXF_NdkSuHvfVMGMlYmjlw45fU77zQkLX7gmZq7NxM,64
+labelr/dataset_features.py,sha256=ZC9QAUw9oKHqyUPla2h3xQFaRT9sHq8hkPNN4RDDwmo,1257
+labelr/google_genai.py,sha256=x5p98eYoI887QMBDgziFxEW9WNdZ8Cw0EHjAFQ71SaE,14728
+labelr/main.py,sha256=OTiJSkD_TrzQmQQm291FhknD-HQQTWfBEBgImxqL0KM,2634
+labelr/project_config.py,sha256=CIHEcgSOfXb53naHWEBkTDm2V9m3abAu8C54VSzHjAs,1260
+labelr/types.py,sha256=8CHfLyifF_N94OYDhG-7IcWboOh9o0Z_0LBtQapT8TQ,313
+labelr/utils.py,sha256=8Yp0L2MCIdUYSjvmF4U5iiaBpaZJbYw4rHJOMhCCudE,3075
+labelr/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+labelr/apps/datasets.py,sha256=tAD6TZSnwh7uhkleSfDP0PFqztXC1S3Vx2aMSVCFfRU,12725
+labelr/apps/evaluate.py,sha256=UC4CuSKa4vgR5xTBZ-dFgp_1pYnkM55s2IJgix0YtkI,1157
+labelr/apps/google_batch.py,sha256=Mlz5jRVcR1XzRJg2HLte3rIhiOk4xQQjjLAJsc3lJjo,9572
+labelr/apps/hugging_face.py,sha256=B0GaDZeUZj2A7nEeC1OtCANb0DqvBkhWwFWM_9Nm2kU,1608
+labelr/apps/label_studio.py,sha256=lQ7K16noA4Mnr1hc0oxya1sgGgABWnpIIJTM5ENp7so,16869
+labelr/apps/train.py,sha256=wmOSpO9JsrwCXYMgRg2srMbV5B5TvnlfhAKPqUt6wSg,7328
+labelr/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+labelr/evaluate/object_detection.py,sha256=QJIwrDY-Vsy0-It6tZSkN3qgAlmIu2W1-kGdmibiPSQ,3349
+labelr/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+labelr/export/classification.py,sha256=rnm99vGMJy1UkdXiZ8t_TgFe3CyLBBYowWwzaZeniIs,4699
+labelr/export/common.py,sha256=lJ-ZDOMKGpC48fCuEnIrA8sZBhXGZOcghBbsLM1h66o,1252
+labelr/export/llm.py,sha256=Jlopi0EQ4YUWLe_s-kTFcISTzO1QmdX-qXQxayO6E-k,3186
+labelr/export/object_detection.py,sha256=91ywkPago7WgbY2COQKpwjFLYAAsXeGOu7TkGHi17OU,12338
+labelr/sample/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+labelr/sample/classification.py,sha256=7Z5hvxG6q6wfJMYj00JWbRBhfjOyhjaL8fpJjgBi9N8,539
+labelr/sample/common.py,sha256=f0XDS6s0z6Vw4G2FDELJ1VQSe5Tsh0q3-3VU9unK9eY,431
+labelr/sample/llm.py,sha256=zAsI3TmfGCbBPv4_hNtYR4Np3yAmUDzXGAvlQLF6V6w,2474
+labelr/sample/object_detection.py,sha256=XZasR_k4AxzsiWdVMC2ZnyjfA14PKJPrx1U-XPr5tWQ,8427
+labelr-0.10.0.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+labelr-0.10.0.dist-info/METADATA,sha256=pS2Ipq-aICU3TluuqSNocGP5-V8ztLk6X_udwwnECPk,7243
+labelr-0.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+labelr-0.10.0.dist-info/entry_points.txt,sha256=OACukVeR_2z54i8yQuWqqk_jdEHlyTwmTFOFBmxPp1k,43
+labelr-0.10.0.dist-info/top_level.txt,sha256=bjZo50aGZhXIcZYpYOX4sdAQcamxh8nwfEh7A9RD_Ag,7
+labelr-0.10.0.dist-info/RECORD,,

labelr-0.8.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-labelr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-labelr/__main__.py,sha256=G4e95-IfhI-lOmkOBP6kQ8wl1x_Fl7dZlLOYr90K83c,66
-labelr/annotate.py,sha256=3fJ9FYbcozcOoKuhNtzPHV8sSnp-45FsNnMc8UeBHGU,3503
-labelr/check.py,sha256=3wK6mE0UsKvoBNm0_lyWhCMq7gxkv5r50pvO70damXY,2476
-labelr/config.py,sha256=3RXF_NdkSuHvfVMGMlYmjlw45fU77zQkLX7gmZq7NxM,64
-labelr/dataset_features.py,sha256=ZC9QAUw9oKHqyUPla2h3xQFaRT9sHq8hkPNN4RDDwmo,1257
-labelr/export.py,sha256=HpsPT3MjWqOHbkNG8hHVns21t-2Ej2nXQXxXOKc1TFA,17771
-labelr/main.py,sha256=hVaaCU1voUZwr681d7wkA7HPyaDB2MaXaw20Hw2_Kz8,2439
-labelr/project_config.py,sha256=CIHEcgSOfXb53naHWEBkTDm2V9m3abAu8C54VSzHjAs,1260
-labelr/sample.py,sha256=Q1Itfwsis1VivaqsmtKfcwojaY0xKJNZ_88M5zOGTY0,8951
-labelr/types.py,sha256=8CHfLyifF_N94OYDhG-7IcWboOh9o0Z_0LBtQapT8TQ,313
-labelr/utils.py,sha256=e0R15jePWBzRdN8LB6kBSH5Dl_P0MNEtRmeqB9eu5d8,415
-labelr/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-labelr/apps/datasets.py,sha256=kJQWwm3mjA2uWIA8O_DslM7OS5ht5mgWqcFC_zF4gCo,11187
-labelr/apps/evaluate.py,sha256=UC4CuSKa4vgR5xTBZ-dFgp_1pYnkM55s2IJgix0YtkI,1157
-labelr/apps/hugging_face.py,sha256=B0GaDZeUZj2A7nEeC1OtCANb0DqvBkhWwFWM_9Nm2kU,1608
-labelr/apps/label_studio.py,sha256=su9shoi0K9PmI8RBLipV2KQf_MRjkF5vy5-JUcbXr5A,16852
-labelr/apps/train.py,sha256=wmOSpO9JsrwCXYMgRg2srMbV5B5TvnlfhAKPqUt6wSg,7328
-labelr/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-labelr/evaluate/llm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-labelr/evaluate/object_detection.py,sha256=QJIwrDY-Vsy0-It6tZSkN3qgAlmIu2W1-kGdmibiPSQ,3349
-labelr-0.8.0.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-labelr-0.8.0.dist-info/METADATA,sha256=FNA9oU3P_BbEZ8PRidRBxvthfm8ywLt1wwo2qn2cOKo,7003
-labelr-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-labelr-0.8.0.dist-info/entry_points.txt,sha256=OACukVeR_2z54i8yQuWqqk_jdEHlyTwmTFOFBmxPp1k,43
-labelr-0.8.0.dist-info/top_level.txt,sha256=bjZo50aGZhXIcZYpYOX4sdAQcamxh8nwfEh7A9RD_Ag,7
-labelr-0.8.0.dist-info/RECORD,,