datachain 0.2.13__tar.gz → 0.2.14__tar.gz

{datachain-0.2.13/src/datachain.egg-info → datachain-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.13
+Version: 0.2.14
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -36,7 +36,7 @@ Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: dill==0.3.8
 Requires-Dist: cloudpickle
-Requires-Dist: ujson>=5.9.0
+Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic<3,>=2
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
@@ -81,7 +81,6 @@ Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
-Requires-Dist: types-ujson; extra == "dev"
 
 |PyPI| |Python Version| |Codecov| |Tests|
 

{datachain-0.2.13 → datachain-0.2.14}/docs/index.md

@@ -58,8 +58,8 @@ def trim_text(text):
     match = re.search(r'[A-Z][^.]*\.', text)
     return match.group(0) if match else ''
 
-images = chain.collect_one("file")
-captions = chain.collect_one("scene")
+images = chain.collect("file")
+captions = chain.collect("scene")
 _ , axes = plt.subplots(1, len(captions), figsize=(15, 5))
 
 for ax, img, caption in zip(axes, images, captions):

{datachain-0.2.13 → datachain-0.2.14}/pyproject.toml

@@ -39,7 +39,7 @@ dependencies = [
   "multiprocess==0.70.16",
   "dill==0.3.8",
   "cloudpickle",
-  "ujson>=5.9.0",
+  "orjson>=3.10.5",
   "pydantic>=2,<3",
   "jmespath>=1.0",
   "datamodel-code-generator>=0.25",
@@ -91,8 +91,7 @@ dev = [
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
-  "types-requests",
-  "types-ujson"
+  "types-requests"
 ]
 
 [project.urls]
@@ -118,6 +117,11 @@ markers = [
 ]
 asyncio_mode = "auto"
 filterwarnings = [
+  "error::pandas.errors.PerformanceWarning",
+  "error::pydantic.warnings.PydanticDeprecatedSince20",
+  "error::pytest_mock.PytestMockWarning",
+  "error::pytest.PytestCollectionWarning",
+  "error::sqlalchemy.exc.SADeprecationWarning",
   "ignore:Field name .* shadows an attribute in parent:UserWarning"  # datachain.lib.feature
 ]
 

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/data_storage/metastore.py

@@ -421,10 +421,6 @@ class AbstractMetastore(ABC, Serializable):
     ) -> None:
         """Set the status of the given job and dataset."""
 
-    @abstractmethod
-    def get_possibly_stale_jobs(self) -> list[tuple[str, str, int]]:
-        """Returns the possibly stale jobs."""
-
 
 class AbstractDBMetastore(AbstractMetastore):
     """

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/data_storage/schema.py

@@ -19,8 +19,12 @@ from datachain.sql.types import Int, SQLType, UInt64
 if TYPE_CHECKING:
     from sqlalchemy import Engine
     from sqlalchemy.engine.interfaces import Dialect
-    from sqlalchemy.sql.base import Executable, ReadOnlyColumnCollection
-    from sqlalchemy.sql.elements import KeyedColumnElement
+    from sqlalchemy.sql.base import (
+        ColumnCollection,
+        Executable,
+        ReadOnlyColumnCollection,
+    )
+    from sqlalchemy.sql.elements import ColumnElement
 
 
 def dedup_columns(columns: Iterable[sa.Column]) -> list[sa.Column]:
@@ -43,7 +47,7 @@ def dedup_columns(columns: Iterable[sa.Column]) -> list[sa.Column]:
 
 
 def convert_rows_custom_column_types(
-    columns: "ReadOnlyColumnCollection[str, KeyedColumnElement[Any]]",
+    columns: "ColumnCollection[str, ColumnElement[Any]]",
     rows: Iterator[tuple[Any, ...]],
     dialect: "Dialect",
 ):

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/data_storage/sqlite.py

@@ -496,9 +496,6 @@ class SQLiteMetastore(AbstractDBMetastore):
     def _jobs_insert(self) -> "Insert":
         return sqlite.insert(self._jobs)
 
-    def get_possibly_stale_jobs(self) -> list[tuple[str, str, int]]:
-        raise NotImplementedError("get_possibly_stale_jobs not implemented for SQLite")
-
 
 class SQLiteWarehouse(AbstractWarehouse):
     """
@@ -594,7 +591,7 @@ class SQLiteWarehouse(AbstractWarehouse):
     ):
         rows = self.db.execute(select_query, **kwargs)
         yield from convert_rows_custom_column_types(
-            select_query.columns, rows, sqlite_dialect
+            select_query.selected_columns, rows, sqlite_dialect
         )
 
     def get_dataset_sources(

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/data_storage/warehouse.py

@@ -494,7 +494,7 @@ class AbstractWarehouse(ABC, Serializable):
         This gets nodes based on the provided query, and should be used sparingly,
         as it will be slow on any OLAP database systems.
         """
-        columns = [c.name for c in query.columns]
+        columns = [c.name for c in query.selected_columns]
         for row in self.db.execute(query):
             d = dict(zip(columns, row))
             yield Node(**d)
@@ -912,29 +912,6 @@ class AbstractWarehouse(ABC, Serializable):
         for name in names:
             self.db.drop_table(Table(name, self.db.metadata), if_exists=True)
 
-    def subtract_query(
-        self,
-        source_query: sa.sql.selectable.Select,
-        target_query: sa.sql.selectable.Select,
-    ) -> sa.sql.selectable.Select:
-        sq = source_query.alias("source_query")
-        tq = target_query.alias("target_query")
-
-        source_target_join = sa.join(
-            sq,
-            tq,
-            (sq.c.source == tq.c.source)
-            & (sq.c.parent == tq.c.parent)
-            & (sq.c.name == tq.c.name),
-            isouter=True,
-        )
-
-        return (
-            select(*sq.c)
-            .select_from(source_target_join)
-            .where((tq.c.name == None) | (tq.c.name == ""))  # noqa: E711
-        )
-
     def changed_query(
         self,
         source_query: sa.sql.selectable.Select,

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/lib/convert/flatten.py

@@ -48,10 +48,10 @@ def _flatten_fields_values(fields, obj: BaseModel):
         value = getattr(obj, name)
 
         if isinstance(value, list):
-            yield [
-                val.model_dump() if ModelStore.is_pydantic(type(val)) else val
-                for val in value
-            ]
+            if value and ModelStore.is_pydantic(type(value[0])):
+                yield [val.model_dump() for val in value]
+            else:
+                yield value
         elif isinstance(value, dict):
             yield {
                 key: val.model_dump() if ModelStore.is_pydantic(type(val)) else val

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/lib/convert/values_to_tuples.py

@@ -71,7 +71,10 @@ def values_to_tuples(  # noqa: C901, PLR0912
                     f"signal '{k}' has unsupported type '{typ.__name__}'."
                     f" Please use DataModel types: {DataTypeNames}",
                 )
-            types_map[k] = typ
+            if typ is list:
+                types_map[k] = list[type(v[0][0])]  # type: ignore[misc]
+            else:
+                types_map[k] = typ
 
         if length < 0:
             length = len_

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/lib/dc.py

@@ -342,7 +342,7 @@ class DataChain(DatasetQuery):
         spec: Optional[DataType] = None,
         schema_from: Optional[str] = "auto",
         jmespath: Optional[str] = None,
-        object_name: str = "",
+        object_name: Optional[str] = "",
         model_name: Optional[str] = None,
         show_schema: Optional[bool] = False,
         meta_type: Optional[str] = "json",
@@ -364,12 +364,12 @@ class DataChain(DatasetQuery):
             nrows : optional row limit for jsonl and JSON arrays
 
         Example:
-            infer JSON schema from data, reduce using JMESPATH, print schema
+            infer JSON schema from data, reduce using JMESPATH
             ```py
             chain = DataChain.from_json("gs://json", jmespath="key1.key2")
             ```
 
-            infer JSON schema from a particular path, print data model
+            infer JSON schema from a particular path
             ```py
             chain = DataChain.from_json("gs://json_ds", schema_from="gs://json/my.json")
             ```
@@ -384,7 +384,7 @@ class DataChain(DatasetQuery):
         if (not object_name) and jmespath:
             object_name = jmespath_to_name(jmespath)
         if not object_name:
-            object_name = "json"
+            object_name = meta_type
         chain = DataChain.from_storage(path=path, type=type, **kwargs)
         signal_dict = {
             object_name: read_meta(
@@ -397,7 +397,67 @@ class DataChain(DatasetQuery):
                 nrows=nrows,
             )
         }
-        return chain.gen(**signal_dict)  # type: ignore[arg-type]
+        return chain.gen(**signal_dict)  # type: ignore[misc, arg-type]
+
+    @classmethod
+    def from_jsonl(
+        cls,
+        path,
+        type: Literal["binary", "text", "image"] = "text",
+        spec: Optional[DataType] = None,
+        schema_from: Optional[str] = "auto",
+        jmespath: Optional[str] = None,
+        object_name: Optional[str] = "",
+        model_name: Optional[str] = None,
+        show_schema: Optional[bool] = False,
+        meta_type: Optional[str] = "jsonl",
+        nrows=None,
+        **kwargs,
+    ) -> "DataChain":
+        """Get data from JSON lines. It returns the chain itself.
+
+        Parameters:
+            path : storage URI with directory. URI must start with storage prefix such
+                as `s3://`, `gs://`, `az://` or "file:///"
+            type : read file as "binary", "text", or "image" data. Default is "binary".
+            spec : optional Data Model
+            schema_from : path to sample to infer spec (if schema not provided)
+            object_name : generated object column name
+            model_name : optional generated model name
+            show_schema : print auto-generated schema
+            jmespath : optional JMESPATH expression to reduce JSON
+            nrows : optional row limit for jsonl and JSON arrays
+
+        Example:
+            infer JSONl schema from data, limit parsing to 1 row
+            ```py
+            chain = DataChain.from_jsonl("gs://myjsonl", nrows=1)
+            ```
+        """
+        if schema_from == "auto":
+            schema_from = path
+
+        def jmespath_to_name(s: str):
+            name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
+            return s[:name_end]
+
+        if (not object_name) and jmespath:
+            object_name = jmespath_to_name(jmespath)
+        if not object_name:
+            object_name = meta_type
+        chain = DataChain.from_storage(path=path, type=type, **kwargs)
+        signal_dict = {
+            object_name: read_meta(
+                schema_from=schema_from,
+                meta_type=meta_type,
+                spec=spec,
+                model_name=model_name,
+                show_schema=show_schema,
+                jmespath=jmespath,
+                nrows=nrows,
+            )
+        }
+        return chain.gen(**signal_dict)  # type: ignore[misc, arg-type]
 
     @classmethod
     def datasets(
@@ -951,6 +1011,41 @@ class DataChain(DatasetQuery):
 
         return ds
 
+    def subtract(  # type: ignore[override]
+        self,
+        other: "DataChain",
+        on: Optional[Union[str, Sequence[str]]] = None,
+    ) -> "Self":
+        """Remove rows that appear in another chain.
+
+        Parameters:
+            other: chain whose rows will be removed from `self`
+            on: columns to consider for determining row equality. If unspecified,
+                defaults to all common columns between `self` and `other`.
+        """
+        if isinstance(on, str):
+            on = [on]
+        if on is None:
+            other_columns = set(other._effective_signals_schema.db_signals())
+            signals = [
+                c
+                for c in self._effective_signals_schema.db_signals()
+                if c in other_columns
+            ]
+            if not signals:
+                raise DataChainParamsError("subtract(): no common columns")
+        elif not isinstance(on, Sequence):
+            raise TypeError(
+                f"'on' must be 'str' or 'Sequence' object but got type '{type(on)}'",
+            )
+        elif not on:
+            raise DataChainParamsError(
+                "'on' cannot be empty",
+            )
+        else:
+            signals = self.signals_schema.resolve(*on).db_signals()
+        return super()._subtract(other, signals)
+
     @classmethod
     def from_values(
         cls,

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/lib/file.py

@@ -12,7 +12,6 @@ from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
 
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
-from fsspec.implementations.local import LocalFileSystem
 from PIL import Image
 from pydantic import Field, field_validator
 
@@ -283,9 +282,8 @@ class File(DataModel):
     def get_path(self) -> str:
         """Returns file path."""
         path = unquote(self.get_uri())
-        fs = self.get_fs()
-        if isinstance(fs, LocalFileSystem):
-            # Drop file:// protocol
+        source = urlparse(self.source)
+        if source.scheme == "file":
             path = urlparse(path).path
             path = url2pathname(path)
         return path
@@ -300,13 +298,10 @@ class File(DataModel):
         elif placement == "etag":
             path = f"{self.etag}{self.get_file_suffix()}"
         elif placement == "fullpath":
-            fs = self.get_fs()
-            if isinstance(fs, LocalFileSystem):
-                path = unquote(self.get_full_name())
-            else:
-                path = (
-                    Path(urlparse(self.source).netloc) / unquote(self.get_full_name())
-                ).as_posix()
+            path = unquote(self.get_full_name())
+            source = urlparse(self.source)
+            if source.scheme and source.scheme != "file":
+                path = posixpath.join(source.netloc, path)
         elif placement == "checksum":
             raise NotImplementedError("Checksum placement not implemented yet")
         else:

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/lib/meta_formats.py

@@ -11,9 +11,9 @@ from collections.abc import Iterator
 from typing import Any, Callable
 
 import jmespath as jsp
-from pydantic import ValidationError
+from pydantic import Field, ValidationError  # noqa: F401
 
-from datachain.lib.data_model import ModelStore  # noqa: F401
+from datachain.lib.data_model import DataModel  # noqa: F401
 from datachain.lib.file import File
 
 
@@ -87,7 +87,8 @@ def read_schema(source_file, data_type="csv", expr=None, model_name=None):
     except subprocess.CalledProcessError as e:
         model_output = f"An error occurred in datamodel-codegen: {e.stderr}"
     print(f"{model_output}")
-    print("\n" + f"ModelStore.register({model_name})" + "\n")
+    print("\n" + "from datachain.lib.data_model import DataModel" + "\n")
+    print("\n" + f"DataModel.register({model_name})" + "\n")
     print("\n" + f"spec={model_name}" + "\n")
     return model_output
 
@@ -147,7 +148,7 @@ def read_meta(  # noqa: C901
 
     def parse_data(
         file: File,
-        DataModel=spec,  # noqa: N803
+        data_model=spec,
         meta_type=meta_type,
         jmespath=jmespath,
         nrows=nrows,
@@ -155,7 +156,7 @@ def read_meta(  # noqa: C901
         def validator(json_object: dict) -> spec:
             json_string = json.dumps(json_object)
             try:
-                data_instance = DataModel.model_validate_json(json_string)
+                data_instance = data_model.model_validate_json(json_string)
                 yield data_instance
             except ValidationError as e:
                 print(f"Validation error occurred in file {file.name}:", e)

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/query/dataset.py

@@ -25,6 +25,7 @@ from typing import (
 
 import attrs
 import sqlalchemy
+import sqlalchemy as sa
 from attrs import frozen
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
 from sqlalchemy import Column
@@ -250,7 +251,7 @@ class DatasetDiffOperation(Step):
         self,
         source_query: Select,
         target_query: Select,
-    ) -> Select:
+    ) -> sa.Selectable:
         """
         Should return select query that calculates desired diff between dataset queries
         """
@@ -268,7 +269,7 @@ class DatasetDiffOperation(Step):
 
         columns = [
             c if isinstance(c, Column) else Column(c.name, c.type)
-            for c in source_query.columns
+            for c in source_query.selected_columns
         ]
         temp_table = self.catalog.warehouse.create_dataset_rows_table(
             temp_table_name,
@@ -292,23 +293,16 @@ class DatasetDiffOperation(Step):
 
 @frozen
 class Subtract(DatasetDiffOperation):
-    """
-    Calculates rows that are in a source query but are not in target query (diff)
-    This can be used to do delta updates (calculate UDF only on newly added rows)
-    Example:
-        >>> ds = DatasetQuery(name="dogs_cats") # some older dataset with embeddings
-        >>> ds_updated = (
-                DatasetQuery("gs://dvcx-datalakes/dogs-and-cats")
-                .filter(C.size > 1000) # we can also filter out source query
-                .subtract(ds)
-                .add_signals(calc_embeddings) # calculae embeddings only on new rows
-                .union(ds) # union with old dataset that's missing new rows
-                .save("dogs_cats_updated")
-            )
-    """
+    on: Sequence[str]
 
-    def query(self, source_query: Select, target_query: Select) -> Select:
-        return self.catalog.warehouse.subtract_query(source_query, target_query)
+    def query(self, source_query: Select, target_query: Select) -> sa.Selectable:
+        sq = source_query.alias("source_query")
+        tq = target_query.alias("target_query")
+        where_clause = sa.and_(
+            getattr(sq.c, col_name).is_not_distinct_from(getattr(tq.c, col_name))
+            for col_name in self.on
+        )  # type: ignore[arg-type]
+        return sq.select().except_(sq.select().where(where_clause))
 
 
 @frozen
@@ -1260,7 +1254,7 @@ class DatasetQuery:
     def as_iterable(self, **kwargs) -> Iterator[ResultIter]:
         try:
             query = self.apply_steps().select()
-            selected_columns = [c.name for c in query.columns]
+            selected_columns = [c.name for c in query.selected_columns]
             yield ResultIter(
                 self.catalog.warehouse.dataset_rows_select(query, **kwargs),
                 selected_columns,
@@ -1564,8 +1558,12 @@ class DatasetQuery:
 
     @detach
     def subtract(self, dq: "DatasetQuery") -> "Self":
+        return self._subtract(dq, on=["source", "parent", "name"])
+
+    @detach
+    def _subtract(self, dq: "DatasetQuery", on: Sequence[str]) -> "Self":
         query = self.clone()
-        query.steps.append(Subtract(dq, self.catalog))
+        query.steps.append(Subtract(dq, self.catalog, on=on))
         return query
 
     @detach
@@ -1684,7 +1682,7 @@ class DatasetQuery:
                     f.row_number().over(order_by=q._order_by_clauses).label("sys__id")
                 )
 
-            cols = tuple(c.name for c in q.columns)
+            cols = tuple(c.name for c in q.selected_columns)
             insert_q = sqlalchemy.insert(dr.get_table()).from_select(cols, q)
             self.catalog.warehouse.db.execute(insert_q, **kwargs)
             self.catalog.metastore.update_dataset_status(

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/sql/sqlite/base.py

@@ -5,8 +5,8 @@ from datetime import MAXYEAR, MINYEAR, datetime, timezone
 from types import MappingProxyType
 from typing import Callable, Optional
 
+import orjson
 import sqlalchemy as sa
-import ujson
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.elements import literal
@@ -149,7 +149,7 @@ def missing_vector_function(name, exc):
 
 
 def sqlite_string_split(string: str, sep: str, maxsplit: int = -1) -> str:
-    return ujson.dumps(string.split(sep, maxsplit))
+    return orjson.dumps(string.split(sep, maxsplit)).decode("utf-8")
 
 
 def register_user_defined_sql_functions() -> None:
@@ -274,7 +274,7 @@ def compile_euclidean_distance(element, compiler, **kwargs):
 
 
 def py_json_array_length(arr):
-    return len(ujson.loads(arr))
+    return len(orjson.loads(arr))
 
 
 def compile_array_length(element, compiler, **kwargs):

{datachain-0.2.13 → datachain-0.2.14}/src/datachain/sql/sqlite/types.py

@@ -1,7 +1,6 @@
-import json
 import sqlite3
 
-import ujson
+import orjson
 from sqlalchemy import types
 
 from datachain.sql.types import TypeConverter, TypeReadConverter
@@ -29,22 +28,15 @@ class Array(types.UserDefinedType):
 
 
 def adapt_array(arr):
-    return ujson.dumps(arr)
+    return orjson.dumps(arr).decode("utf-8")
 
 
 def convert_array(arr):
-    return ujson.loads(arr)
+    return orjson.loads(arr)
 
 
 def adapt_np_array(arr):
-    def _json_serialize(obj):
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return obj
-
-    if np.issubdtype(arr.dtype, np.object_):
-        return json.dumps(arr.tolist(), default=_json_serialize)
-    return ujson.dumps(arr.tolist())
+    return orjson.dumps(arr, option=orjson.OPT_SERIALIZE_NUMPY).decode("utf-8")
 
 
 def adapt_np_generic(val):
@@ -70,5 +62,5 @@ class SQLiteTypeConverter(TypeConverter):
 class SQLiteTypeReadConverter(TypeReadConverter):
     def array(self, value, item_type, dialect):
         if isinstance(value, str):
-            value = ujson.loads(value)
+            value = orjson.loads(value)
         return super().array(value, item_type, dialect)

{datachain-0.2.13 → datachain-0.2.14/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.13
+Version: 0.2.14
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -36,7 +36,7 @@ Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
 Requires-Dist: dill==0.3.8
 Requires-Dist: cloudpickle
-Requires-Dist: ujson>=5.9.0
+Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic<3,>=2
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
@@ -81,7 +81,6 @@ Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
-Requires-Dist: types-ujson; extra == "dev"
 
 |PyPI| |Python Version| |Codecov| |Tests|
 

{datachain-0.2.13 → datachain-0.2.14}/src/datachain.egg-info/requires.txt

@@ -17,7 +17,7 @@ sqlalchemy>=2
 multiprocess==0.70.16
 dill==0.3.8
 cloudpickle
-ujson>=5.9.0
+orjson>=3.10.5
 pydantic<3,>=2
 jmespath>=1.0
 datamodel-code-generator>=0.25
@@ -33,7 +33,6 @@ types-python-dateutil
 types-pytz
 types-PyYAML
 types-requests
-types-ujson
 
 [docs]
 mkdocs>=1.5.2

{datachain-0.2.13 → datachain-0.2.14}/tests/examples/test_wds_e2e.py

@@ -90,7 +90,7 @@ def test_wds(catalog, webdataset_tars):
         assert laion_wds.file.parent
         assert laion_wds.file.name == f"{idx}.jpg"
         assert laion_wds.file.location
-        assert laion_wds.json.dict() == Laion(**data).dict()
+        assert laion_wds.json.model_dump() == Laion(**data).model_dump()
 
     assert num_rows == len(WDS_TAR_SHARDS)
 

{datachain-0.2.13 → datachain-0.2.14}/tests/func/test_datachain.py

@@ -17,12 +17,8 @@ from tests.utils import images_equal
 
 
 @pytest.mark.parametrize("anon", [True, False])
-def test_catalog_anon(catalog, anon):
-    chain = (
-        DataChain.from_storage("gs://dvcx-datalakes/dogs-and-cats/", anon=anon)
-        .limit(5)
-        .save("test_catalog_anon")
-    )
+def test_catalog_anon(tmp_dir, catalog, anon):
+    chain = DataChain.from_storage(tmp_dir.as_uri(), anon=anon)
     assert chain.catalog.client_config.get("anon", False) is anon
 
 

{datachain-0.2.13 → datachain-0.2.14}/tests/func/test_datasets.py

@@ -210,15 +210,16 @@ def test_create_dataset_from_sources_failed(listed_bucket, cloud_test_catalog, m
     dataset_name = uuid.uuid4().hex
     src_uri = cloud_test_catalog.src_uri
     catalog = cloud_test_catalog.catalog
-    with mocker.patch.object(
+    # Mocks are automatically undone at the end of a test.
+    mocker.patch.object(
         catalog.warehouse.__class__,
         "create_dataset_rows_table",
         side_effect=RuntimeError("Error"),
-    ) as _:
-        with pytest.raises(RuntimeError):
-            catalog.create_dataset_from_sources(
-                dataset_name, [f"{src_uri}/dogs/*"], recursive=True
-            )
+    )
+    with pytest.raises(RuntimeError):
+        catalog.create_dataset_from_sources(
+            dataset_name, [f"{src_uri}/dogs/*"], recursive=True
+        )
 
     dataset = catalog.get_dataset(dataset_name)
     dataset_version = dataset.get_version(dataset.latest_version)