darwin-py 3.6.2__tar.gz → 3.6.4__tar.gz

{darwin_py-3.6.2 → darwin_py-3.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: darwin-py
-Version: 3.6.2
+Version: 3.6.4
 Summary: Library and command line interface for darwin.v7labs.com
 License: MIT
 License-File: LICENSE
@@ -265,6 +265,23 @@ $ darwin dataset pull test:0.1
 Dataset example-team/test:0.1 downloaded at /directory/choosen/at/authentication/time .
 ```
 
+To indent native Darwin JSON files after download, add `--pretty-print`.
+This formats local annotation files and metadata with two-space indentation.
+It keeps the remote export unchanged and preserves all JSON values.
+The flag is off by default.
+
+```sh
+darwin dataset pull example-team/test:0.1 --only-annotations --pretty-print
+```
+
+Local formatting uses extra CPU time and temporary disk space for one formatted
+file. The formatter processes one file at a time with bounded buffers. Media
+downloads run sequentially with this option. They format affected annotations
+again when they record local file paths.
+Other pull steps still parse annotation files and can use more RAM.
+If a formatting step fails, the affected file keeps the bytes it had before that
+step. The command reports an error.
+
 ---
 
 ## Usage as a Python library

{darwin_py-3.6.2 → darwin_py-3.6.4}/README.md

@@ -192,6 +192,23 @@ $ darwin dataset pull test:0.1
 Dataset example-team/test:0.1 downloaded at /directory/choosen/at/authentication/time .
 ```
 
+To indent native Darwin JSON files after download, add `--pretty-print`.
+This formats local annotation files and metadata with two-space indentation.
+It keeps the remote export unchanged and preserves all JSON values.
+The flag is off by default.
+
+```sh
+darwin dataset pull example-team/test:0.1 --only-annotations --pretty-print
+```
+
+Local formatting uses extra CPU time and temporary disk space for one formatted
+file. The formatter processes one file at a time with bounded buffers. Media
+downloads run sequentially with this option. They format affected annotations
+again when they record local file paths.
+Other pull steps still parse annotation files and can use more RAM.
+If a formatting step fails, the affected file keeps the bytes it had before that
+step. The command reports an error.
+
 ---
 
 ## Usage as a Python library

{darwin_py-3.6.2 → darwin_py-3.6.4}/darwin/cli.py

@@ -176,6 +176,7 @@ def _run(args: Namespace, parser: ArgumentParser) -> None:
                 args.retry_interval,
                 args.team,
                 multi_processed=not args.no_multi_processed,
+                pretty_print=args.pretty_print,
             )
         elif args.action == "import":
             f.dataset_import(

{darwin_py-3.6.2 → darwin_py-3.6.4}/darwin/cli_functions.py

@@ -31,6 +31,7 @@ from darwin.client import Client
 from darwin.config import Config
 from darwin.dataset import RemoteDataset
 from darwin.dataset.identifier import DatasetIdentifier
+from darwin.dataset.json_formatter import PrettyPrintError
 from darwin.dataset.release import Release
 from darwin.dataset.split_manager import split_dataset
 from darwin.dataset.upload_manager import LocalFile
@@ -365,6 +366,7 @@ def pull_dataset(
     retry_interval: int = 10,
     team: Optional[str] = None,
     multi_processed: bool = True,
+    pretty_print: bool = False,
 ) -> None:
     """
     Downloads a remote dataset (images and annotations) in the datasets directory.
@@ -396,6 +398,8 @@ def pull_dataset(
     multi_processed: bool
         If True (default), uses multiprocessing to download files in parallel.
         If False, the dataset is downloaded on a single process.
+    pretty_print: bool
+        Indents downloaded annotation JSON locally. Defaults to False.
     """
     identifier: DatasetIdentifier = DatasetIdentifier.parse(dataset_slug)
     if team:
@@ -432,6 +436,7 @@ def pull_dataset(
             retry_timeout=retry_timeout,
             retry_interval=retry_interval,
             multi_processed=multi_processed,
+            pretty_print=pretty_print,
         )
         print_new_version_info(client)
     except NotFound:
@@ -444,6 +449,8 @@ def pull_dataset(
             f"Version '{dataset.identifier}:{version}' is of format '{uef.format}', "
             f"only the darwin formats ('json', 'darwin_json_2') are supported for `darwin dataset pull`"
         )
+    except PrettyPrintError as error:
+        _error(str(error))
     except MissingDependency as e:
         _error(str(e))
 

{darwin_py-3.6.2 → darwin_py-3.6.4}/darwin/dataset/download_manager.py

@@ -7,6 +7,7 @@ import os
 import time
 import urllib
 from collections import Counter
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import (
@@ -27,6 +28,7 @@ from PIL import Image
 from rich.console import Console
 
 import darwin.datatypes as dt
+from darwin.dataset.json_formatter import PrettyPrintError, pretty_print_json
 from darwin.dataset.utils import (
     sanitize_filename,
     SUPPORTED_IMAGE_EXTENSIONS,
@@ -57,6 +59,7 @@ def download_all_images_from_annotations(
     video_frames: bool = False,
     force_slots: bool = False,
     ignore_slots: bool = False,
+    pretty_print: bool = False,
 ) -> Tuple[Callable[[], Iterable[Any]], int]:
     """
     Downloads all the images corresponding to a project.
@@ -82,13 +85,16 @@ def download_all_images_from_annotations(
     force_slots: bool, default: False
         Pulls all slots of items into deeper file structure ({prefix}/{item_name}/{slot_name}/{file_name})
         If False, all multi-slotted items and items with slots containing multiple source files will be downloaded as the deeper file structure
+    pretty_print: bool, default: False
+        Preserves JSON tokens when adding local media paths and indents the result.
 
     Returns
     -------
     generator : function
-        Generator for doing the actual downloads
+        Factory for download callables. With ``pretty_print=True``, returns at most
+        one callable that downloads all media sequentially.
     count : int
-        The files count
+        The number of media files, which can exceed the number of callables.
 
     Raises
     ------
@@ -112,6 +118,14 @@ def download_all_images_from_annotations(
         if annotation is None:
             continue
 
+        source_indices: Dict[Optional[str], List[Tuple[int, int]]] = {}
+        if pretty_print:
+            for slot_index, slot in enumerate(annotation.slots):
+                for source_index, source in enumerate(slot.source_files):
+                    source_indices.setdefault(source.url, []).append(
+                        (slot_index, source_index)
+                    )
+
         if not force_replace:
             # Check the planned path for the image against the existing images
             planned_image_paths = _get_planned_image_paths(
@@ -131,7 +145,9 @@ def download_all_images_from_annotations(
                 len(slot.source_files) > 1 for slot in annotation.slots
             )
 
-        annotations_to_download_path.append((annotation_path, force_slots_for_item))
+        annotations_to_download_path.append(
+            (annotation_path, force_slots_for_item, source_indices)
+        )
 
     if remove_extra:
         annotations = (
@@ -155,23 +171,100 @@ def download_all_images_from_annotations(
 
     # Create the generator with the partial functions
     download_functions: List = []
-    for annotation_path, force_slots in annotations_to_download_path:
-        file_download_functions = lazy_download_image_from_annotation(
-            client,
-            annotation_path,
-            images_path,
-            annotation_format,
-            use_folders,
-            video_frames,
-            force_slots,
-            ignore_slots,
+    grouped_functions: List = []
+    for annotation_path, force_slots, source_indices in annotations_to_download_path:
+        file_download_functions = list(
+            lazy_download_image_from_annotation(
+                client,
+                annotation_path,
+                images_path,
+                annotation_format,
+                use_folders,
+                video_frames,
+                force_slots,
+                ignore_slots,
+            )
         )
         download_functions.extend(file_download_functions)
+        if pretty_print:
+            trace_functions = []
+            for function in file_download_functions:
+                if (
+                    isinstance(function, functools.partial)
+                    and function.func is _download_image_with_trace
+                ):
+                    trace_functions.append(function)
+                else:
+                    grouped_functions.append(function)
+            if trace_functions:
+                grouped_functions.append(
+                    _DownloadAnnotationMedia(
+                        annotation_path, trace_functions, source_indices
+                    )
+                )
 
     if not use_folders:
         _check_for_duplicate_local_filepaths(download_functions)
 
-    return lambda: download_functions, len(download_functions)
+    if pretty_print and grouped_functions:
+        grouped_functions = [_DownloadMediaSequentially(grouped_functions)]
+
+    return (
+        lambda: grouped_functions if pretty_print else download_functions,
+        len(download_functions),
+    )
+
+
+def _raise_media_errors(errors: List[Exception]) -> None:
+    if not errors:
+        return
+    if len(errors) == 1:
+        raise errors[0]
+    message = "\n".join(str(error) for error in errors)
+    for error in errors:
+        if isinstance(error, PrettyPrintError):
+            raise PrettyPrintError(message) from error
+    raise RuntimeError(message) from errors[0]
+
+
+@dataclass
+class _DownloadMediaSequentially:
+    downloads: List[Callable[[], None]]
+
+    def __call__(self) -> None:
+        errors: List[Exception] = []
+        for download in self.downloads:
+            try:
+                download()
+            except Exception as error:
+                errors.append(error)
+        _raise_media_errors(errors)
+
+
+@dataclass
+class _DownloadAnnotationMedia:
+    annotation_path: Path
+    downloads: List[functools.partial]
+    source_indices: Dict[Optional[str], List[Tuple[int, int]]]
+
+    def __call__(self) -> None:
+        local_paths: Dict[Tuple[int, int], str] = {}
+        errors: List[Exception] = []
+        for download in self.downloads:
+            annotation, url, path, client = download.args
+            try:
+                _download_image(url, path, client)
+            except Exception as error:
+                errors.append(error)
+            else:
+                if annotation.version.major != 1:
+                    for index in self.source_indices.get(url, []):
+                        local_paths[index] = str(path)
+        try:
+            pretty_print_json(self.annotation_path, local_paths=local_paths)
+        except PrettyPrintError as error:
+            errors.append(error)
+        _raise_media_errors(errors)
 
 
 def lazy_download_image_from_annotation(

darwin_py-3.6.4/darwin/dataset/json_formatter.py

@@ -0,0 +1,353 @@
+"""Format JSON with bounded buffers without changing string or number tokens."""
+
+import json
+import os
+import re
+import stat
+import tempfile
+from contextlib import suppress
+from dataclasses import dataclass
+from enum import Enum, auto
+from pathlib import Path
+from typing import IO, Mapping, Optional, Protocol
+
+
+class PrettyPrintError(Exception):
+    """A local JSON file could not be formatted and retains its original bytes."""
+
+
+_CHUNK_SIZE = 64 * 1024
+_STRING_SPECIAL = re.compile(r'["\\\x00-\x1f]')
+_WHITESPACE = re.compile(r"[ \t\r\n]*")
+_DIGITS = re.compile(r"[0-9]*")
+
+
+class _Writer(Protocol):
+    def write(self, value: str) -> object: ...
+
+
+class _Reader:
+    def __init__(self, source: IO[str]) -> None:
+        self.source = source
+        self.buffer = ""
+        self.offset = 0
+
+    def peek(self) -> str:
+        if self.offset == len(self.buffer):
+            self.buffer = self.source.read(_CHUNK_SIZE)
+            self.offset = 0
+        return self.buffer[self.offset : self.offset + 1]
+
+    def take(self) -> str:
+        char = self.peek()
+        self.offset += bool(char)
+        return char
+
+    def skip_whitespace(self) -> None:
+        while self.peek():
+            match = _WHITESPACE.match(self.buffer, self.offset)
+            assert match is not None
+            self.offset = match.end()
+            if self.offset < len(self.buffer):
+                return
+
+    def copy_digits(self, output: _Writer) -> None:
+        while self.peek():
+            match = _DIGITS.match(self.buffer, self.offset)
+            assert match is not None
+            end = match.end()
+            output.write(self.buffer[self.offset : end])
+            self.offset = end
+            if end < len(self.buffer):
+                return
+
+
+def _copy_string(reader: _Reader, output: _Writer) -> None:
+    output.write(reader.take())
+    while reader.peek():
+        match = _STRING_SPECIAL.search(reader.buffer, reader.offset)
+        end = match.start() if match else len(reader.buffer)
+        output.write(reader.buffer[reader.offset : end])
+        reader.offset = end
+        if match is None:
+            continue
+        char = reader.take()
+        if char == '"':
+            output.write(char)
+            return
+        if char != "\\":
+            raise ValueError("Unescaped control character in string")
+        escape = reader.take()
+        if escape == "u":
+            output.write("\\u")
+            for _ in range(4):
+                digit = reader.take()
+                if not digit or digit not in "0123456789abcdefABCDEF":
+                    raise ValueError("Invalid Unicode escape in string")
+                output.write(digit)
+        elif escape and escape in '"\\/bfnrt':
+            output.write("\\" + escape)
+        else:
+            raise ValueError("Invalid escape in string")
+    raise ValueError("Unterminated string")
+
+
+def _copy_number(reader: _Reader, output: _Writer) -> None:
+    if reader.peek() == "-":
+        output.write(reader.take())
+    char = reader.peek()
+    if char == "0":
+        output.write(reader.take())
+    elif char and char in "123456789":
+        reader.copy_digits(output)
+    else:
+        raise ValueError("Expected a digit in number")
+    if reader.peek() == ".":
+        output.write(reader.take())
+        if not reader.peek() or reader.peek() not in "0123456789":
+            raise ValueError("Expected a digit after decimal point")
+        reader.copy_digits(output)
+    if reader.peek() and reader.peek() in "eE":
+        output.write(reader.take())
+        if reader.peek() and reader.peek() in "+-":
+            output.write(reader.take())
+        if not reader.peek() or reader.peek() not in "0123456789":
+            raise ValueError("Expected a digit in exponent")
+        reader.copy_digits(output)
+
+
+class _State(Enum):
+    VALUE = auto()
+    ARRAY_FIRST = auto()
+    ARRAY_VALUE = auto()
+    OBJECT_FIRST = auto()
+    OBJECT_KEY = auto()
+    COLON = auto()
+    OBJECT_VALUE = auto()
+    COMMA_OR_END = auto()
+    COMPLETE = auto()
+
+
+class _Location(Enum):
+    OTHER = auto()
+    ROOT = auto()
+    ITEM = auto()
+    SLOTS = auto()
+    SLOT = auto()
+    SOURCES = auto()
+    SOURCE = auto()
+
+
+@dataclass
+class _Container:
+    closing: str
+    state: _State
+    location: _Location = _Location.OTHER
+    slot_index: int = -1
+    index: int = -1
+    key: str = ""
+    has_local_path: bool = False
+    discard: bool = False
+
+
+class _KeyWriter:
+    def __init__(self, output: _Writer) -> None:
+        self.output = output
+        self.token = ""
+        self.too_long = False
+
+    def write(self, value: str) -> None:
+        self.output.write(value)
+        if not self.too_long:
+            if len(self.token) + len(value) > 80:
+                self.token = ""
+                self.too_long = True
+            else:
+                self.token += value
+
+
+class _DiscardWriter:
+    def write(self, value: str) -> None:
+        pass
+
+
+def _child_location(current: _Container, opening: str) -> _Location:
+    if not current.closing and opening == "{":
+        return _Location.ROOT
+    if current.location == _Location.OTHER:
+        return _Location.OTHER
+    if current.location == _Location.SLOTS and opening == "{":
+        return _Location.SLOT
+    if current.location == _Location.SOURCES and opening == "{":
+        return _Location.SOURCE
+    return {
+        (_Location.ROOT, "item", "{"): _Location.ITEM,
+        (_Location.ITEM, "slots", "["): _Location.SLOTS,
+        (_Location.SLOT, "source_files", "["): _Location.SOURCES,
+    }.get((current.location, current.key, opening), _Location.OTHER)
+
+
+def _format_json(
+    source: IO[str],
+    destination: _Writer,
+    local_paths: Mapping[tuple[int, int], str],
+) -> None:
+    reader = _Reader(source)
+    containers = [_Container("", _State.VALUE)]
+    discard = _DiscardWriter()
+    while containers:
+        reader.skip_whitespace()
+        char = reader.peek()
+        current = containers[-1]
+        output = discard if current.discard else destination
+        state = current.state
+        local_path = (
+            local_paths.get((current.slot_index, current.index))
+            if current.location == _Location.SOURCE
+            else None
+        )
+        if state == _State.COMPLETE:
+            if char:
+                raise ValueError("Unexpected content after JSON value")
+            output.write("\n")
+            return
+        if not char:
+            raise ValueError("Unexpected end of JSON")
+
+        if (
+            local_path is not None
+            and not current.has_local_path
+            and char == "}"
+            and state in (_State.OBJECT_FIRST, _State.COMMA_OR_END)
+        ):
+            if state == _State.COMMA_OR_END:
+                output.write(",")
+            output.write("\n" + "  " * (len(containers) - 1))
+            output.write('"local_path": ' + json.dumps(local_path))
+            current.has_local_path = True
+            state = current.state = _State.COMMA_OR_END
+
+        if state == _State.COMMA_OR_END:
+            if char == ",":
+                output.write(reader.take())
+                current.state = (
+                    _State.OBJECT_KEY if current.closing == "}" else _State.ARRAY_VALUE
+                )
+                continue
+            if char != current.closing:
+                raise ValueError("Expected a comma or the end of a container")
+            output.write("\n" + "  " * (len(containers) - 2))
+        elif state in (_State.ARRAY_FIRST, _State.OBJECT_FIRST):
+            if char != current.closing:
+                current.state = (
+                    _State.OBJECT_KEY if current.closing == "}" else _State.ARRAY_VALUE
+                )
+                continue
+        elif state == _State.COLON:
+            if char != ":":
+                raise ValueError("Expected a colon after object key")
+            reader.take()
+            output.write(": ")
+            current.state = _State.OBJECT_VALUE
+            continue
+        elif state == _State.OBJECT_KEY:
+            if char != '"':
+                raise ValueError("Expected a quoted object key")
+            output.write("\n" + "  " * (len(containers) - 1))
+            if local_paths and current.location in (
+                _Location.ROOT,
+                _Location.ITEM,
+                _Location.SLOT,
+                _Location.SOURCE,
+            ):
+                key_writer = _KeyWriter(output)
+                _copy_string(reader, key_writer)
+                current.key = (
+                    json.loads(key_writer.token) if not key_writer.too_long else ""
+                )
+            else:
+                _copy_string(reader, output)
+            current.state = _State.COLON
+            continue
+        else:
+            if state == _State.ARRAY_VALUE:
+                output.write("\n" + "  " * (len(containers) - 1))
+                current.index += 1
+            replace_path = local_path is not None and current.key == "local_path"
+            if replace_path:
+                output.write(json.dumps(local_path))
+                current.has_local_path = True
+                output = discard
+            current.state = _State.COMMA_OR_END if current.closing else _State.COMPLETE
+            if char in "[{":
+                output.write(reader.take())
+                containers.append(
+                    _Container(
+                        "}" if char == "{" else "]",
+                        _State.OBJECT_FIRST if char == "{" else _State.ARRAY_FIRST,
+                        (
+                            _child_location(current, char)
+                            if local_paths
+                            else _Location.OTHER
+                        ),
+                        (
+                            current.index
+                            if current.location == _Location.SLOTS
+                            else current.slot_index
+                        ),
+                        current.index if current.location == _Location.SOURCES else -1,
+                        discard=current.discard or replace_path,
+                    )
+                )
+            elif char == '"':
+                _copy_string(reader, output)
+            elif char in "-0123456789":
+                _copy_number(reader, output)
+            elif char in "tfn":
+                literal = {"t": "true", "f": "false", "n": "null"}[char]
+                for expected in literal:
+                    if reader.take() != expected:
+                        raise ValueError("Invalid JSON literal")
+                output.write(literal)
+            else:
+                raise ValueError("Expected a JSON value")
+            continue
+
+        output.write(reader.take())
+        containers.pop()
+
+
+def pretty_print_json(
+    path: Path, *, local_paths: Optional[Mapping[tuple[int, int], str]] = None
+) -> None:
+    """Atomically indent one UTF-8 JSON file while preserving all value tokens.
+
+    Uses fixed input/output buffers plus container nesting state. The temporary
+    output requires space next to the original file until replacement succeeds.
+    ``local_paths`` updates only indexed ``item.slots[].source_files[]`` paths.
+    """
+    temporary_path = None
+    try:
+        mode = stat.S_IMODE(path.stat().st_mode)
+        with path.open("r", encoding="utf-8", newline="") as source:
+            with tempfile.NamedTemporaryFile(
+                mode="w",
+                encoding="utf-8",
+                newline="\n",
+                dir=path.parent,
+                prefix=".darwin-pretty-print-",
+                suffix=".tmp",
+                delete=False,
+            ) as output:
+                temporary_path = Path(output.name)
+                _format_json(source, output.file, local_paths or {})
+        temporary_path.chmod(mode)
+        os.replace(temporary_path, path)
+    except (OSError, UnicodeError, ValueError) as error:
+        raise PrettyPrintError(
+            f"Could not pretty-print '{path}': {error}. The original file is unchanged."
+        ) from error
+    finally:
+        if temporary_path is not None:
+            with suppress(OSError):
+                temporary_path.unlink(missing_ok=True)