orca-sdk 0.0.92__py3-none-any.whl → 0.0.94__py3-none-any.whl

orca_sdk/_generated_api_client/models/http_validation_error.py

@@ -0,0 +1,86 @@
+"""
+This file is generated by the openapi-python-client tool via the generate_api_client.py script
+
+It is a customized template from the openapi-python-client tool's default template:
+https://github.com/openapi-generators/openapi-python-client/blob/861ef5622f10fc96d240dc9becb0edf94e61446c/openapi_python_client/templates/model.py.jinja
+
+The main change is:
+- Fix typing issues
+"""
+
+# flake8: noqa: C901
+
+from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..types import UNSET, Unset
+
+if TYPE_CHECKING:
+    from ..models.validation_error import ValidationError
+
+
+T = TypeVar("T", bound="HTTPValidationError")
+
+
+@_attrs_define
+class HTTPValidationError:
+    """
+    Attributes:
+        detail (Union[Unset, List['ValidationError']]):
+    """
+
+    detail: Union[Unset, List["ValidationError"]] = UNSET
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        detail: Union[Unset, List[Dict[str, Any]]] = UNSET
+        if not isinstance(self.detail, Unset):
+            detail = []
+            for detail_item_data in self.detail:
+                detail_item = detail_item_data.to_dict()
+                detail.append(detail_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+        if detail is not UNSET:
+            field_dict["detail"] = detail
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: dict[str, Any]) -> T:
+        from ..models.validation_error import ValidationError
+
+        d = src_dict.copy()
+        detail = []
+        _detail = d.pop("detail", UNSET)
+        for detail_item_data in _detail or []:
+            detail_item = ValidationError.from_dict(detail_item_data)
+
+            detail.append(detail_item)
+
+        http_validation_error = cls(
+            detail=detail,
+        )
+
+        http_validation_error.additional_properties = d
+        return http_validation_error
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

orca_sdk/_generated_api_client/models/label_prediction_memory_lookup.py

@@ -38,6 +38,7 @@ class LabelPredictionMemoryLookup:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (MemoryMetrics):
         label (int):
         label_name (Union[None, str]):
@@ -54,6 +55,7 @@ class LabelPredictionMemoryLookup:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "MemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -81,6 +83,8 @@ class LabelPredictionMemoryLookup:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -106,6 +110,7 @@ class LabelPredictionMemoryLookup:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -148,6 +153,8 @@ class LabelPredictionMemoryLookup:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = MemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -174,6 +181,7 @@ class LabelPredictionMemoryLookup:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memory.py

@@ -38,6 +38,7 @@ class LabeledMemory:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (LabeledMemoryMetrics): Metrics computed for a labeled memory.
         label (int):
         label_name (Union[None, str]):
@@ -51,6 +52,7 @@ class LabeledMemory:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "LabeledMemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -75,6 +77,8 @@ class LabeledMemory:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -94,6 +98,7 @@ class LabeledMemory:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -133,6 +138,8 @@ class LabeledMemory:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = LabeledMemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -153,6 +160,7 @@ class LabeledMemory:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memory_lookup.py

@@ -38,6 +38,7 @@ class LabeledMemoryLookup:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (MemoryMetrics):
         label (int):
         label_name (Union[None, str]):
@@ -52,6 +53,7 @@ class LabeledMemoryLookup:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "MemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -77,6 +79,8 @@ class LabeledMemoryLookup:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -98,6 +102,7 @@ class LabeledMemoryLookup:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -138,6 +143,8 @@ class LabeledMemoryLookup:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = MemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -160,6 +167,7 @@ class LabeledMemoryLookup:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/labeled_memory_with_feedback_metrics.py

@@ -40,6 +40,7 @@ class LabeledMemoryWithFeedbackMetrics:
         memory_version (int):
         created_at (datetime.datetime):
         updated_at (datetime.datetime):
+        edited_at (datetime.datetime):
         metrics (LabeledMemoryMetrics): Metrics computed for a labeled memory.
         label (int):
         label_name (Union[None, str]):
@@ -55,6 +56,7 @@ class LabeledMemoryWithFeedbackMetrics:
     memory_version: int
     created_at: datetime.datetime
     updated_at: datetime.datetime
+    edited_at: datetime.datetime
     metrics: "LabeledMemoryMetrics"
     label: int
     label_name: Union[None, str]
@@ -81,6 +83,8 @@ class LabeledMemoryWithFeedbackMetrics:
 
         updated_at = self.updated_at.isoformat()
 
+        edited_at = self.edited_at.isoformat()
+
         metrics = self.metrics.to_dict()
 
         label = self.label
@@ -104,6 +108,7 @@ class LabeledMemoryWithFeedbackMetrics:
                 "memory_version": memory_version,
                 "created_at": created_at,
                 "updated_at": updated_at,
+                "edited_at": edited_at,
                 "metrics": metrics,
                 "label": label,
                 "label_name": label_name,
@@ -148,6 +153,8 @@ class LabeledMemoryWithFeedbackMetrics:
 
         updated_at = isoparse(d.pop("updated_at"))
 
+        edited_at = isoparse(d.pop("edited_at"))
+
         metrics = LabeledMemoryMetrics.from_dict(d.pop("metrics"))
 
         label = d.pop("label")
@@ -172,6 +179,7 @@ class LabeledMemoryWithFeedbackMetrics:
             memory_version=memory_version,
             created_at=created_at,
             updated_at=updated_at,
+            edited_at=edited_at,
             metrics=metrics,
             label=label,
             label_name=label_name,

orca_sdk/_generated_api_client/models/list_predictions_request.py

@@ -10,11 +10,13 @@ The main change is:
 
 # flake8: noqa: C901
 
+import datetime
 from enum import Enum
 from typing import Any, List, Type, TypeVar, Union, cast
 
 from attrs import define as _attrs_define
 from attrs import field as _attrs_field
+from dateutil.parser import isoparse
 
 from ..models.prediction_sort_item_item_type_0 import PredictionSortItemItemType0
 from ..models.prediction_sort_item_item_type_1 import PredictionSortItemItemType1
@@ -30,6 +32,8 @@ class ListPredictionsRequest:
         model_id (Union[None, Unset, str]):
         tag (Union[None, Unset, str]):
         prediction_ids (Union[List[str], None, Unset]):
+        start_timestamp (Union[None, Unset, datetime.datetime]):
+        end_timestamp (Union[None, Unset, datetime.datetime]):
         limit (Union[None, Unset, int]):
         offset (Union[None, Unset, int]):  Default: 0.
         sort (Union[Unset, List[List[Union[PredictionSortItemItemType0, PredictionSortItemItemType1]]]]):
@@ -39,6 +43,8 @@ class ListPredictionsRequest:
     model_id: Union[None, Unset, str] = UNSET
     tag: Union[None, Unset, str] = UNSET
     prediction_ids: Union[List[str], None, Unset] = UNSET
+    start_timestamp: Union[None, Unset, datetime.datetime] = UNSET
+    end_timestamp: Union[None, Unset, datetime.datetime] = UNSET
     limit: Union[None, Unset, int] = UNSET
     offset: Union[None, Unset, int] = 0
     sort: Union[Unset, List[List[Union[PredictionSortItemItemType0, PredictionSortItemItemType1]]]] = UNSET
@@ -67,6 +73,22 @@ class ListPredictionsRequest:
         else:
             prediction_ids = self.prediction_ids
 
+        start_timestamp: Union[None, Unset, str]
+        if isinstance(self.start_timestamp, Unset):
+            start_timestamp = UNSET
+        elif isinstance(self.start_timestamp, datetime.datetime):
+            start_timestamp = self.start_timestamp.isoformat()
+        else:
+            start_timestamp = self.start_timestamp
+
+        end_timestamp: Union[None, Unset, str]
+        if isinstance(self.end_timestamp, Unset):
+            end_timestamp = UNSET
+        elif isinstance(self.end_timestamp, datetime.datetime):
+            end_timestamp = self.end_timestamp.isoformat()
+        else:
+            end_timestamp = self.end_timestamp
+
         limit: Union[None, Unset, int]
         if isinstance(self.limit, Unset):
             limit = UNSET
@@ -118,6 +140,10 @@ class ListPredictionsRequest:
             field_dict["tag"] = tag
         if prediction_ids is not UNSET:
             field_dict["prediction_ids"] = prediction_ids
+        if start_timestamp is not UNSET:
+            field_dict["start_timestamp"] = start_timestamp
+        if end_timestamp is not UNSET:
+            field_dict["end_timestamp"] = end_timestamp
         if limit is not UNSET:
             field_dict["limit"] = limit
         if offset is not UNSET:
@@ -168,6 +194,40 @@ class ListPredictionsRequest:
 
         prediction_ids = _parse_prediction_ids(d.pop("prediction_ids", UNSET))
 
+        def _parse_start_timestamp(data: object) -> Union[None, Unset, datetime.datetime]:
+            if data is None:
+                return data
+            if isinstance(data, Unset):
+                return data
+            try:
+                if not isinstance(data, str):
+                    raise TypeError()
+                start_timestamp_type_0 = isoparse(data)
+
+                return start_timestamp_type_0
+            except:  # noqa: E722
+                pass
+            return cast(Union[None, Unset, datetime.datetime], data)
+
+        start_timestamp = _parse_start_timestamp(d.pop("start_timestamp", UNSET))
+
+        def _parse_end_timestamp(data: object) -> Union[None, Unset, datetime.datetime]:
+            if data is None:
+                return data
+            if isinstance(data, Unset):
+                return data
+            try:
+                if not isinstance(data, str):
+                    raise TypeError()
+                end_timestamp_type_0 = isoparse(data)
+
+                return end_timestamp_type_0
+            except:  # noqa: E722
+                pass
+            return cast(Union[None, Unset, datetime.datetime], data)
+
+        end_timestamp = _parse_end_timestamp(d.pop("end_timestamp", UNSET))
+
         def _parse_limit(data: object) -> Union[None, Unset, int]:
             if data is None:
                 return data
@@ -231,6 +291,8 @@ class ListPredictionsRequest:
             model_id=model_id,
             tag=tag,
             prediction_ids=prediction_ids,
+            start_timestamp=start_timestamp,
+            end_timestamp=end_timestamp,
             limit=limit,
             offset=offset,
             sort=sort,

orca_sdk/_generated_api_client/models/memoryset_analysis_configs.py

@@ -13,7 +13,6 @@ The main change is:
 from typing import TYPE_CHECKING, Any, Dict, Type, TypeVar, Union, cast
 
 from attrs import define as _attrs_define
-from attrs import field as _attrs_field
 
 from ..types import UNSET, Unset
 
@@ -44,7 +43,6 @@ class MemorysetAnalysisConfigs:
     duplicate: Union["MemorysetDuplicateAnalysisConfig", None, Unset] = UNSET
     projection: Union["MemorysetProjectionAnalysisConfig", None, Unset] = UNSET
     cluster: Union["MemorysetClusterAnalysisConfig", None, Unset] = UNSET
-    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> dict[str, Any]:
         from ..models.memoryset_cluster_analysis_config import MemorysetClusterAnalysisConfig
@@ -94,7 +92,6 @@ class MemorysetAnalysisConfigs:
             cluster = self.cluster
 
         field_dict: dict[str, Any] = {}
-        field_dict.update(self.additional_properties)
         field_dict.update({})
         if neighbor is not UNSET:
             field_dict["neighbor"] = neighbor
@@ -212,21 +209,4 @@ class MemorysetAnalysisConfigs:
             cluster=cluster,
         )
 
-        memoryset_analysis_configs.additional_properties = d
         return memoryset_analysis_configs
-
-    @property
-    def additional_keys(self) -> list[str]:
-        return list(self.additional_properties.keys())
-
-    def __getitem__(self, key: str) -> Any:
-        return self.additional_properties[key]
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        self.additional_properties[key] = value
-
-    def __delitem__(self, key: str) -> None:
-        del self.additional_properties[key]
-
-    def __contains__(self, key: str) -> bool:
-        return key in self.additional_properties

orca_sdk/_generated_api_client/models/prediction_request.py

@@ -28,14 +28,16 @@ class PredictionRequest:
         expected_labels (Union[List[int], None, Unset]):
         tags (Union[Unset, List[str]]):
         memoryset_override_id (Union[None, Unset, str]):
-        disable_telemetry (Union[Unset, bool]):  Default: False.
+        save_telemetry (Union[Unset, bool]):  Default: True.
+        save_telemetry_synchronously (Union[Unset, bool]):  Default: False.
     """
 
     input_values: List[str]
     expected_labels: Union[List[int], None, Unset] = UNSET
     tags: Union[Unset, List[str]] = UNSET
     memoryset_override_id: Union[None, Unset, str] = UNSET
-    disable_telemetry: Union[Unset, bool] = False
+    save_telemetry: Union[Unset, bool] = True
+    save_telemetry_synchronously: Union[Unset, bool] = False
     additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> dict[str, Any]:
@@ -62,7 +64,9 @@ class PredictionRequest:
         else:
             memoryset_override_id = self.memoryset_override_id
 
-        disable_telemetry = self.disable_telemetry
+        save_telemetry = self.save_telemetry
+
+        save_telemetry_synchronously = self.save_telemetry_synchronously
 
         field_dict: dict[str, Any] = {}
         field_dict.update(self.additional_properties)
@@ -77,8 +81,10 @@ class PredictionRequest:
             field_dict["tags"] = tags
         if memoryset_override_id is not UNSET:
             field_dict["memoryset_override_id"] = memoryset_override_id
-        if disable_telemetry is not UNSET:
-            field_dict["disable_telemetry"] = disable_telemetry
+        if save_telemetry is not UNSET:
+            field_dict["save_telemetry"] = save_telemetry
+        if save_telemetry_synchronously is not UNSET:
+            field_dict["save_telemetry_synchronously"] = save_telemetry_synchronously
 
         return field_dict
 
@@ -156,14 +162,17 @@ class PredictionRequest:
 
         memoryset_override_id = _parse_memoryset_override_id(d.pop("memoryset_override_id", UNSET))
 
-        disable_telemetry = d.pop("disable_telemetry", UNSET)
+        save_telemetry = d.pop("save_telemetry", UNSET)
+
+        save_telemetry_synchronously = d.pop("save_telemetry_synchronously", UNSET)
 
         prediction_request = cls(
             input_values=input_values,
             expected_labels=expected_labels,
             tags=tags,
             memoryset_override_id=memoryset_override_id,
-            disable_telemetry=disable_telemetry,
+            save_telemetry=save_telemetry,
+            save_telemetry_synchronously=save_telemetry_synchronously,
         )
 
         prediction_request.additional_properties = d

orca_sdk/_generated_api_client/models/pretrained_embedding_model_name.py

@@ -2,11 +2,16 @@ from enum import Enum
 
 
 class PretrainedEmbeddingModelName(str, Enum):
+    BGE_BASE = "BGE_BASE"
     CDE_SMALL = "CDE_SMALL"
     CLIP_BASE = "CLIP_BASE"
     DISTILBERT = "DISTILBERT"
+    E5_LARGE = "E5_LARGE"
+    GIST_LARGE = "GIST_LARGE"
     GTE_BASE = "GTE_BASE"
     GTE_SMALL = "GTE_SMALL"
+    MXBAI_LARGE = "MXBAI_LARGE"
+    QWEN2_1_5B = "QWEN2_1_5B"
 
     def __str__(self) -> str:
         return str(self.value)

orca_sdk/_generated_api_client/models/validation_error.py

@@ -0,0 +1,99 @@
+"""
+This file is generated by the openapi-python-client tool via the generate_api_client.py script
+
+It is a customized template from the openapi-python-client tool's default template:
+https://github.com/openapi-generators/openapi-python-client/blob/861ef5622f10fc96d240dc9becb0edf94e61446c/openapi_python_client/templates/model.py.jinja
+
+The main change is:
+- Fix typing issues
+"""
+
+# flake8: noqa: C901
+
+from typing import Any, List, Type, TypeVar, Union, cast
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+T = TypeVar("T", bound="ValidationError")
+
+
+@_attrs_define
+class ValidationError:
+    """
+    Attributes:
+        loc (List[Union[int, str]]):
+        msg (str):
+        type (str):
+    """
+
+    loc: List[Union[int, str]]
+    msg: str
+    type: str
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        loc = []
+        for loc_item_data in self.loc:
+            loc_item: Union[int, str]
+            loc_item = loc_item_data
+            loc.append(loc_item)
+
+        msg = self.msg
+
+        type = self.type
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "loc": loc,
+                "msg": msg,
+                "type": type,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: dict[str, Any]) -> T:
+        d = src_dict.copy()
+        loc = []
+        _loc = d.pop("loc")
+        for loc_item_data in _loc:
+
+            def _parse_loc_item(data: object) -> Union[int, str]:
+                return cast(Union[int, str], data)
+
+            loc_item = _parse_loc_item(loc_item_data)
+
+            loc.append(loc_item)
+
+        msg = d.pop("msg")
+
+        type = d.pop("type")
+
+        validation_error = cls(
+            loc=loc,
+            msg=msg,
+            type=type,
+        )
+
+        validation_error.additional_properties = d
+        return validation_error
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

orca_sdk/_utils/data_parsing.py

@@ -1,12 +1,16 @@
+import logging
 import pickle
 from dataclasses import asdict, is_dataclass
 from os import PathLike
+from tempfile import TemporaryDirectory
 from typing import Any, cast
 
 from datasets import Dataset
 from torch.utils.data import DataLoader as TorchDataLoader
 from torch.utils.data import Dataset as TorchDataset
 
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
 
 def parse_dict_like(item: Any, column_names: list[str] | None = None) -> dict:
     if isinstance(item, dict):
@@ -40,7 +44,24 @@ def parse_batch(batch: Any, column_names: list[str] | None = None) -> list[dict]
     return [{key: batch[key][idx] for key in keys} for idx in range(batch_size)]
 
 
-def hf_dataset_from_torch(torch_data: TorchDataLoader | TorchDataset, column_names: list[str] | None = None) -> Dataset:
+def hf_dataset_from_torch(
+    torch_data: TorchDataLoader | TorchDataset, column_names: list[str] | None = None, ignore_cache=False
+) -> Dataset:
+    """
+    Create a HuggingFace Dataset from a PyTorch DataLoader or Dataset.
+
+    NOTE:  It's important to ignore the cached files when testing (i.e., ignore_cache=Ture), because
+    cached results can ignore changes you've made to tests. This can make a test appear to succeed
+    when it's actually broken or vice versa.
+
+    Params:
+        torch_data: A PyTorch DataLoader or Dataset object to create the HuggingFace Dataset from.
+        column_names: Optional list of column names to use for the dataset. If not provided,
+            the column names will be inferred from the data.
+        ignore_cache: If True, the dataset will not be cached on disk.
+    Returns:
+        A HuggingFace Dataset object containing the data from the PyTorch DataLoader or Dataset.
+    """
     if isinstance(torch_data, TorchDataLoader):
         dataloader = torch_data
     else:
@@ -50,7 +71,15 @@ def hf_dataset_from_torch(torch_data: TorchDataLoader | TorchDataset, column_nam
         for batch in dataloader:
             yield from parse_batch(batch, column_names=column_names)
 
-    return cast(Dataset, Dataset.from_generator(generator))
+    if ignore_cache:
+        with TemporaryDirectory() as temp_dir:
+            ds = Dataset.from_generator(generator, cache_dir=temp_dir)
+    else:
+        ds = Dataset.from_generator(generator)
+
+    if not isinstance(ds, Dataset):
+        raise ValueError(f"Failed to create dataset from generator: {type(ds)}")
+    return ds
 
 
 def hf_dataset_from_disk(file_path: str | PathLike) -> Dataset: