datachain 0.7.3__tar.gz → 0.7.4__tar.gz

{datachain-0.7.3/src/datachain.egg-info → datachain-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.3
+Version: 0.7.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.7.3 → datachain-0.7.4}/src/datachain/data_storage/metastore.py

@@ -685,13 +685,6 @@ class AbstractDBMetastore(AbstractMetastore):
             return None
         return reduce(lambda ds, version: ds.merge_versions(version), versions)
 
-    def _parse_datasets(self, rows) -> Iterator["DatasetRecord"]:
-        # grouping rows by dataset id
-        for _, g in groupby(rows, lambda r: r[0]):
-            dataset = self._parse_dataset(list(g))
-            if dataset:
-                yield dataset
-
     def _parse_list_dataset(self, rows) -> Optional[DatasetListRecord]:
         versions = [self.dataset_list_class.parse(*r) for r in rows]
         if not versions:

{datachain-0.7.3 → datachain-0.7.4}/src/datachain/dataset.py

@@ -2,6 +2,7 @@ import builtins
 import json
 from dataclasses import dataclass, fields
 from datetime import datetime
+from functools import cached_property
 from typing import (
     Any,
     NewType,
@@ -11,6 +12,8 @@ from typing import (
 )
 from urllib.parse import urlparse
 
+import orjson
+
 from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
@@ -178,7 +181,7 @@ class DatasetVersion:
     schema: dict[str, Union[SQLType, type[SQLType]]]
     num_objects: Optional[int]
     size: Optional[int]
-    preview: Optional[list[dict]]
+    _preview_data: Optional[Union[str, list[dict]]]
     sources: str = ""
     query_script: str = ""
     job_id: Optional[str] = None
@@ -199,7 +202,7 @@ class DatasetVersion:
         script_output: str,
         num_objects: Optional[int],
         size: Optional[int],
-        preview: Optional[str],
+        preview: Optional[Union[str, list[dict]]],
         schema: dict[str, Union[SQLType, type[SQLType]]],
         sources: str = "",
         query_script: str = "",
@@ -220,7 +223,7 @@ class DatasetVersion:
             schema,
             num_objects,
             size,
-            json.loads(preview) if preview else None,
+            preview,
             sources,
             query_script,
             job_id,
@@ -260,9 +263,17 @@ class DatasetVersion:
             for c_name, c_type in self.schema.items()
         }
 
+    @cached_property
+    def preview(self) -> Optional[list[dict]]:
+        if isinstance(self._preview_data, str):
+            return orjson.loads(self._preview_data)
+        return self._preview_data if self._preview_data else None
+
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetVersion":
         kwargs = {f.name: d[f.name] for f in fields(cls) if f.name in d}
+        if not hasattr(kwargs, "_preview_data"):
+            kwargs["_preview_data"] = d.get("preview")
         return cls(**kwargs)
 
 

{datachain-0.7.3 → datachain-0.7.4/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.3
+Version: 0.7.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.7.3 → datachain-0.7.4}/tests/unit/test_dataset.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime, timezone
 
 import pytest
@@ -6,7 +7,7 @@ from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect
 from sqlalchemy.schema import CreateTable
 
 from datachain.data_storage.schema import DataTable
-from datachain.dataset import DatasetDependency, DatasetDependencyType
+from datachain.dataset import DatasetDependency, DatasetDependencyType, DatasetVersion
 from datachain.sql.types import (
     JSON,
     Array,
@@ -106,3 +107,34 @@ def test_dataset_dependency_dataset_name(dep_name, dep_type, expected):
     )
 
     assert dep.dataset_name == expected
+
+
+@pytest.mark.parametrize(
+    "use_string",
+    [True, False],
+)
+def test_dataset_version_from_dict(use_string):
+    preview = [{"id": 1, "thing": "a"}, {"id": 2, "thing": "b"}]
+
+    preview_data = json.dumps(preview) if use_string else preview
+
+    data = {
+        "id": 1,
+        "uuid": "98928be4-b6e8-4b7b-a7c5-2ce3b33130d8",
+        "dataset_id": 40,
+        "version": 2,
+        "status": 1,
+        "feature_schema": {},
+        "created_at": datetime.fromisoformat("2023-10-01T12:00:00"),
+        "finished_at": None,
+        "error_message": "",
+        "error_stack": "",
+        "script_output": "",
+        "schema": {},
+        "num_objects": 100,
+        "size": 1000000,
+        "preview": preview_data,
+    }
+
+    dataset_version = DatasetVersion.from_dict(data)
+    assert dataset_version.preview == preview