datachain 0.3.20__py3-none-any.whl → 0.5.0__py3-none-any.whl

datachain/catalog/catalog.py

@@ -979,7 +979,6 @@ class Catalog:
         script_output="",
         create_rows_table=True,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ) -> DatasetRecord:
         """
         Creates dataset version if it doesn't exist.
@@ -1001,7 +1000,6 @@ class Catalog:
             script_output=script_output,
             schema=schema,
             job_id=job_id,
-            is_job_result=is_job_result,
             ignore_if_exists=True,
         )
 
@@ -1211,7 +1209,6 @@ class Catalog:
             size=dataset_version.size,
             preview=dataset_version.preview,
             job_id=dataset_version.job_id,
-            is_job_result=dataset_version.is_job_result,
         )
         # to avoid re-creating rows table, we are just renaming it for a new version
         # of target dataset

datachain/cli.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 
 import shtab
 
-from datachain import utils
+from datachain import Session, utils
 from datachain.cli_utils import BooleanOptionalAction, CommaSeparatedArgs, KeyValueArgs
 from datachain.lib.dc import DataChain
 from datachain.telemetry import telemetry
@@ -770,7 +770,8 @@ def show(
     show_records(records, collapse_columns=not no_collapse)
     if schema and dataset_version.feature_schema:
         print("\nSchema:")
-        dc = DataChain(name=name, version=version, catalog=catalog)
+        session = Session.get(catalog=catalog)
+        dc = DataChain.from_dataset(name=name, version=version, session=session)
         dc.print_schema()
 
 

datachain/data_storage/metastore.py

@@ -15,7 +15,6 @@ from uuid import uuid4
 from sqlalchemy import (
     JSON,
     BigInteger,
-    Boolean,
     Column,
     DateTime,
     ForeignKey,
@@ -228,7 +227,7 @@ class AbstractMetastore(ABC, Serializable):
         self,
         dataset: DatasetRecord,
         version: int,
-        status: int = DatasetStatus.CREATED,
+        status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
         query_script: str = "",
@@ -243,7 +242,6 @@ class AbstractMetastore(ABC, Serializable):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ) -> DatasetRecord:
         """Creates new dataset version."""
 
@@ -449,7 +447,6 @@ class AbstractDBMetastore(AbstractMetastore):
             Column("name", Text, nullable=False),
             Column("description", Text),
             Column("labels", JSON, nullable=True),
-            Column("shadow", Boolean, nullable=False),
             Column("status", Integer, nullable=False),
             Column("feature_schema", JSON, nullable=True),
             Column("created_at", DateTime(timezone=True)),
@@ -482,8 +479,11 @@ class AbstractDBMetastore(AbstractMetastore):
                 nullable=False,
             ),
             Column("version", Integer, nullable=False),
-            # adding default for now until we fully remove shadow datasets
-            Column("status", Integer, nullable=False, default=DatasetStatus.COMPLETE),
+            Column(
+                "status",
+                Integer,
+                nullable=False,
+            ),
             Column("feature_schema", JSON, nullable=True),
             Column("created_at", DateTime(timezone=True)),
             Column("finished_at", DateTime(timezone=True)),
@@ -497,7 +497,6 @@ class AbstractDBMetastore(AbstractMetastore):
             Column("query_script", Text, nullable=False, default=""),
             Column("schema", JSON, nullable=True),
             Column("job_id", Text, nullable=True),
-            Column("is_job_result", Boolean, nullable=False, default=False),
             UniqueConstraint("dataset_id", "version"),
         ]
 
@@ -971,7 +970,6 @@ class AbstractDBMetastore(AbstractMetastore):
         # TODO abstract this method and add registered = True based on kwargs
         query = self._datasets_insert().values(
             name=name,
-            shadow=False,
             status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=datetime.now(timezone.utc),
@@ -994,7 +992,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self,
         dataset: DatasetRecord,
         version: int,
-        status: int = DatasetStatus.CREATED,
+        status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
         query_script: str = "",
@@ -1009,7 +1007,6 @@ class AbstractDBMetastore(AbstractMetastore):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
         conn=None,
     ) -> DatasetRecord:
         """Creates new dataset version."""
@@ -1021,7 +1018,7 @@ class AbstractDBMetastore(AbstractMetastore):
         query = self._datasets_versions_insert().values(
             dataset_id=dataset.id,
             version=version,
-            status=status,  # for now until we remove shadow datasets
+            status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=created_at or datetime.now(timezone.utc),
             finished_at=finished_at,
@@ -1035,7 +1032,6 @@ class AbstractDBMetastore(AbstractMetastore):
             size=size,
             preview=json.dumps(preview or []),
             job_id=job_id or os.getenv("DATACHAIN_JOB_ID"),
-            is_job_result=is_job_result,
         )
         if ignore_if_exists and hasattr(query, "on_conflict_do_nothing"):
             # SQLite and PostgreSQL both support 'on_conflict_do_nothing',

datachain/data_storage/warehouse.py

@@ -919,9 +919,7 @@ class AbstractWarehouse(ABC, Serializable):
     def is_temp_table_name(self, name: str) -> bool:
         """Returns if the given table name refers to a temporary
         or no longer needed table."""
-        return name.startswith(
-            (self.TMP_TABLE_NAME_PREFIX, self.UDF_TABLE_NAME_PREFIX, "ds_shadow_")
-        ) or name.endswith("_shadow")
+        return name.startswith((self.TMP_TABLE_NAME_PREFIX, self.UDF_TABLE_NAME_PREFIX))
 
     def get_temp_table_names(self) -> list[str]:
         return [

datachain/dataset.py

@@ -179,7 +179,6 @@ class DatasetVersion:
     sources: str = ""
     query_script: str = ""
     job_id: Optional[str] = None
-    is_job_result: bool = False
 
     @classmethod
     def parse(  # noqa: PLR0913
@@ -201,7 +200,6 @@ class DatasetVersion:
         sources: str = "",
         query_script: str = "",
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ):
         return cls(
             id,
@@ -221,7 +219,6 @@ class DatasetVersion:
             sources,
             query_script,
             job_id,
-            is_job_result,
         )
 
     def __eq__(self, other):
@@ -270,7 +267,6 @@ class DatasetRecord:
     name: str
     description: Optional[str]
     labels: list[str]
-    shadow: bool
     schema: dict[str, Union[SQLType, type[SQLType]]]
     feature_schema: dict
     versions: list[DatasetVersion]
@@ -299,7 +295,6 @@ class DatasetRecord:
         name: str,
         description: Optional[str],
         labels: str,
-        shadow: int,
         status: int,
         feature_schema: Optional[str],
         created_at: datetime,
@@ -327,7 +322,6 @@ class DatasetRecord:
         version_query_script: Optional[str],
         version_schema: str,
         version_job_id: Optional[str] = None,
-        version_is_job_result: bool = False,
     ) -> "DatasetRecord":
         labels_lst: list[str] = json.loads(labels) if labels else []
         schema_dct: dict[str, Any] = json.loads(schema) if schema else {}
@@ -353,7 +347,6 @@ class DatasetRecord:
             version_sources,  # type: ignore[arg-type]
             version_query_script,  # type: ignore[arg-type]
             version_job_id,
-            version_is_job_result,
         )
 
         return cls(
@@ -361,7 +354,6 @@ class DatasetRecord:
             name,
             description,
             labels_lst,
-            bool(shadow),
             cls.parse_schema(schema_dct),  # type: ignore[arg-type]
             json.loads(feature_schema) if feature_schema else {},
             [dataset_version],

datachain/lib/dc.py

@@ -54,7 +54,6 @@ from datachain.query import Session
 from datachain.query.dataset import (
     DatasetQuery,
     PartitionByType,
-    detach,
 )
 from datachain.query.schema import DEFAULT_DELIMITER, Column, DatasetRow
 from datachain.sql.functions import path as pathfunc
@@ -159,7 +158,7 @@ class Sys(DataModel):
     rand: int
 
 
-class DataChain(DatasetQuery):
+class DataChain:
     """DataChain - a data structure for batch data processing and evaluation.
 
     It represents a sequence of data manipulation steps such as reading data from
@@ -238,33 +237,20 @@ class DataChain(DatasetQuery):
         "size": 0,
     }
 
-    def __init__(self, *args, settings: Optional[dict] = None, **kwargs):
-        """This method needs to be redefined as a part of Dataset and DataChain
-        decoupling.
-        """
-        super().__init__(  # type: ignore[misc]
-            *args,
-            **kwargs,
-            indexing_column_types=File._datachain_column_types,
-        )
-
-        telemetry.send_event_once("class", "datachain_init", **kwargs)
-
-        if settings:
-            self._settings = Settings(**settings)
-        else:
-            self._settings = Settings()
-        self._setup: dict = {}
-
-        self.signals_schema = SignalSchema({"sys": Sys})
-        if self.feature_schema:
-            self.signals_schema |= SignalSchema.deserialize(self.feature_schema)
-        else:
-            self.signals_schema |= SignalSchema.from_column_types(
-                self.column_types or {}
-            )
-
-        self._sys = False
+    def __init__(
+        self,
+        query: DatasetQuery,
+        settings: Settings,
+        signal_schema: SignalSchema,
+        setup: Optional[dict] = None,
+        _sys: bool = False,
+    ) -> None:
+        """Don't instantiate this directly, use one of the from_XXX constructors."""
+        self._query = query
+        self._settings = settings
+        self.signals_schema = signal_schema
+        self._setup: dict = setup or {}
+        self._sys = _sys
 
     @property
     def schema(self) -> dict[str, DataType]:
@@ -290,18 +276,55 @@ class DataChain(DatasetQuery):
     def c(self, column: Union[str, Column]) -> Column:
         """Returns Column instance attached to the current chain."""
         c = self.column(column) if isinstance(column, str) else self.column(column.name)
-        c.table = self.table
+        c.table = self._query.table
         return c
 
+    @property
+    def session(self) -> Session:
+        """Session of the chain."""
+        return self._query.session
+
+    @property
+    def name(self) -> Optional[str]:
+        """Name of the underlying dataset, if there is one."""
+        return self._query.name
+
+    @property
+    def version(self) -> Optional[int]:
+        """Version of the underlying dataset, if there is one."""
+        return self._query.version
+
+    def __or__(self, other: "Self") -> "Self":
+        """Return `self.union(other)`."""
+        return self.union(other)
+
     def print_schema(self) -> None:
         """Print schema of the chain."""
         self._effective_signals_schema.print_tree()
 
-    def clone(self, new_table: bool = True) -> "Self":
+    def clone(self) -> "Self":
         """Make a copy of the chain in a new table."""
-        obj = super().clone(new_table=new_table)
-        obj.signals_schema = copy.deepcopy(self.signals_schema)
-        return obj
+        return self._evolve(query=self._query.clone(new_table=True))
+
+    def _evolve(
+        self,
+        *,
+        query: Optional[DatasetQuery] = None,
+        settings: Optional[Settings] = None,
+        signal_schema=None,
+        _sys=None,
+    ) -> "Self":
+        if query is None:
+            query = self._query.clone(new_table=False)
+        if settings is None:
+            settings = self._settings
+        if signal_schema is None:
+            signal_schema = copy.deepcopy(self.signals_schema)
+        if _sys is None:
+            _sys = self._sys
+        return type(self)(
+            query, settings, signal_schema=signal_schema, setup=self._setup, _sys=_sys
+        )
 
     def settings(
         self,
@@ -332,11 +355,11 @@ class DataChain(DatasetQuery):
             )
             ```
         """
-        chain = self.clone()
-        if sys is not None:
-            chain._sys = sys
-        chain._settings.add(Settings(cache, parallel, workers, min_task_size))
-        return chain
+        if sys is None:
+            sys = self._sys
+        settings = copy.copy(self._settings)
+        settings.add(Settings(cache, parallel, workers, min_task_size))
+        return self._evolve(settings=settings, _sys=sys)
 
     def reset_settings(self, settings: Optional[Settings] = None) -> "Self":
         """Reset all settings to default values."""
@@ -434,7 +457,7 @@ class DataChain(DatasetQuery):
         version: Optional[int] = None,
         session: Optional[Session] = None,
         settings: Optional[dict] = None,
-    ) -> "DataChain":
+    ) -> "Self":
         """Get data from a saved Dataset. It returns the chain itself.
 
         Parameters:
@@ -446,7 +469,24 @@ class DataChain(DatasetQuery):
             chain = DataChain.from_dataset("my_cats")
             ```
         """
-        return DataChain(name=name, version=version, session=session, settings=settings)
+        query = DatasetQuery(
+            name=name,
+            version=version,
+            session=session,
+            indexing_column_types=File._datachain_column_types,
+        )
+        telemetry.send_event_once("class", "datachain_init", name=name, version=version)
+        if settings:
+            _settings = Settings(**settings)
+        else:
+            _settings = Settings()
+
+        signals_schema = SignalSchema({"sys": Sys})
+        if query.feature_schema:
+            signals_schema |= SignalSchema.deserialize(query.feature_schema)
+        else:
+            signals_schema |= SignalSchema.from_column_types(query.column_types or {})
+        return cls(query, _settings, signals_schema)
 
     @classmethod
     def from_json(
@@ -699,7 +739,11 @@ class DataChain(DatasetQuery):
             version : version of a dataset. Default - the last version that exist.
         """
         schema = self.signals_schema.clone_without_sys_signals().serialize()
-        return super().save(name=name, version=version, feature_schema=schema, **kwargs)
+        return self._evolve(
+            query=self._query.save(
+                name=name, version=version, feature_schema=schema, **kwargs
+            )
+        )
 
     def apply(self, func, *args, **kwargs):
         """Apply any function to the chain.
@@ -765,13 +809,14 @@ class DataChain(DatasetQuery):
         """
         udf_obj = self._udf_to_obj(Mapper, func, params, output, signal_map)
 
-        chain = self.add_signals(
-            udf_obj.to_udf_wrapper(),
-            **self._settings.to_dict(),
+        return self._evolve(
+            query=self._query.add_signals(
+                udf_obj.to_udf_wrapper(),
+                **self._settings.to_dict(),
+            ),
+            signal_schema=self.signals_schema | udf_obj.output,
         )
 
-        return chain.add_schema(udf_obj.output).reset_settings(self._settings)
-
     def gen(
         self,
         func: Optional[Callable] = None,
@@ -800,14 +845,14 @@ class DataChain(DatasetQuery):
             ```
         """
         udf_obj = self._udf_to_obj(Generator, func, params, output, signal_map)
-        chain = DatasetQuery.generate(
-            self,
-            udf_obj.to_udf_wrapper(),
-            **self._settings.to_dict(),
+        return self._evolve(
+            query=self._query.generate(
+                udf_obj.to_udf_wrapper(),
+                **self._settings.to_dict(),
+            ),
+            signal_schema=udf_obj.output,
         )
 
-        return chain.reset_schema(udf_obj.output).reset_settings(self._settings)
-
     def agg(
         self,
         func: Optional[Callable] = None,
@@ -840,15 +885,15 @@ class DataChain(DatasetQuery):
             ```
         """
         udf_obj = self._udf_to_obj(Aggregator, func, params, output, signal_map)
-        chain = DatasetQuery.generate(
-            self,
-            udf_obj.to_udf_wrapper(),
-            partition_by=partition_by,
-            **self._settings.to_dict(),
+        return self._evolve(
+            query=self._query.generate(
+                udf_obj.to_udf_wrapper(),
+                partition_by=partition_by,
+                **self._settings.to_dict(),
+            ),
+            signal_schema=udf_obj.output,
         )
 
-        return chain.reset_schema(udf_obj.output).reset_settings(self._settings)
-
     def batch_map(
         self,
         func: Optional[Callable] = None,
@@ -876,14 +921,14 @@ class DataChain(DatasetQuery):
             ```
         """
         udf_obj = self._udf_to_obj(BatchMapper, func, params, output, signal_map)
-        chain = DatasetQuery.add_signals(
-            self,
-            udf_obj.to_udf_wrapper(batch),
-            **self._settings.to_dict(),
+        return self._evolve(
+            query=self._query.add_signals(
+                udf_obj.to_udf_wrapper(batch),
+                **self._settings.to_dict(),
+            ),
+            signal_schema=self.signals_schema | udf_obj.output,
         )
 
-        return chain.add_schema(udf_obj.output).reset_settings(self._settings)
-
     def _udf_to_obj(
         self,
         target_class: type[UDFBase],
@@ -907,17 +952,12 @@ class DataChain(DatasetQuery):
         return target_class._create(sign, params_schema)
 
     def _extend_to_data_model(self, method_name, *args, **kwargs):
-        super_func = getattr(super(), method_name)
+        query_func = getattr(self._query, method_name)
 
         new_schema = self.signals_schema.resolve(*args)
         columns = [C(col) for col in new_schema.db_signals()]
-        res = super_func(*columns, **kwargs)
-        if isinstance(res, DataChain):
-            res.signals_schema = new_schema
-
-        return res
+        return query_func(*columns, **kwargs)
 
-    @detach
     @resolve_columns
     def order_by(self, *args, descending: bool = False) -> "Self":
         """Orders by specified set of signals.
@@ -928,9 +968,8 @@ class DataChain(DatasetQuery):
         if descending:
             args = tuple(sqlalchemy.desc(a) for a in args)
 
-        return super().order_by(*args)
+        return self._evolve(query=self._query.order_by(*args))
 
-    @detach
     def distinct(self, arg: str, *args: str) -> "Self":  # type: ignore[override]
         """Removes duplicate rows based on uniqueness of some input column(s)
         i.e if rows are found with the same value of input column(s), only one
@@ -942,29 +981,30 @@ class DataChain(DatasetQuery):
         )
         ```
         """
-        return super().distinct(*self.signals_schema.resolve(arg, *args).db_signals())
+        return self._evolve(
+            query=self._query.distinct(
+                *self.signals_schema.resolve(arg, *args).db_signals()
+            )
+        )
 
-    @detach
     def select(self, *args: str, _sys: bool = True) -> "Self":
         """Select only a specified set of signals."""
         new_schema = self.signals_schema.resolve(*args)
         if _sys:
             new_schema = SignalSchema({"sys": Sys}) | new_schema
         columns = new_schema.db_signals()
-        chain = super().select(*columns)
-        chain.signals_schema = new_schema
-        return chain
+        return self._evolve(
+            query=self._query.select(*columns), signal_schema=new_schema
+        )
 
-    @detach
     def select_except(self, *args: str) -> "Self":
         """Select all the signals expect the specified signals."""
         new_schema = self.signals_schema.select_except_signals(*args)
         columns = new_schema.db_signals()
-        chain = super().select(*columns)
-        chain.signals_schema = new_schema
-        return chain
+        return self._evolve(
+            query=self._query.select(*columns), signal_schema=new_schema
+        )
 
-    @detach
     def mutate(self, **kwargs) -> "Self":
         """Create new signals based on existing signals.
 
@@ -1029,9 +1069,9 @@ class DataChain(DatasetQuery):
                 # adding new signal
                 mutated[name] = value
 
-        chain = super().mutate(**mutated)
-        chain.signals_schema = schema.mutate(kwargs)
-        return chain
+        return self._evolve(
+            query=self._query.mutate(**mutated), signal_schema=schema.mutate(kwargs)
+        )
 
     @property
     def _effective_signals_schema(self) -> "SignalSchema":
@@ -1058,7 +1098,7 @@ class DataChain(DatasetQuery):
                           a tuple of row values.
         """
         db_signals = self._effective_signals_schema.db_signals()
-        with super().select(*db_signals).as_iterable() as rows:
+        with self._query.select(*db_signals).as_iterable() as rows:
             if row_factory:
                 rows = (row_factory(db_signals, r) for r in rows)
             yield from rows
@@ -1126,7 +1166,7 @@ class DataChain(DatasetQuery):
         chain = self.select(*cols) if cols else self
         signals_schema = chain._effective_signals_schema
         db_signals = signals_schema.db_signals()
-        with super().select(*db_signals).as_iterable() as rows:
+        with self._query.select(*db_signals).as_iterable() as rows:
             for row in rows:
                 ret = signals_schema.row_to_features(
                     row, catalog=chain.session.catalog, cache=chain._settings.cache
@@ -1156,7 +1196,7 @@ class DataChain(DatasetQuery):
         """
         from datachain.torch import PytorchDataset
 
-        if self.attached:
+        if self._query.attached:
             chain = self
         else:
             chain = self.save()
@@ -1164,7 +1204,7 @@ class DataChain(DatasetQuery):
         return PytorchDataset(
             chain.name,
             chain.version,
-            catalog=self.catalog,
+            catalog=self.session.catalog,
             transform=transform,
             tokenizer=tokenizer,
             tokenizer_kwargs=tokenizer_kwargs,
@@ -1175,7 +1215,6 @@ class DataChain(DatasetQuery):
         schema = self.signals_schema.clone_without_file_signals()
         return self.select(*schema.values.keys())
 
-    @detach
     def merge(
         self,
         right_ds: "DataChain",
@@ -1240,7 +1279,7 @@ class DataChain(DatasetQuery):
                 )
 
         if self == right_ds:
-            right_ds = right_ds.clone(new_table=True)
+            right_ds = right_ds.clone()
 
         errors = []
 
@@ -1266,9 +1305,11 @@ class DataChain(DatasetQuery):
                 on, right_on, f"Could not resolve {', '.join(errors)}"
             )
 
-        ds = self.join(right_ds, sqlalchemy.and_(*ops), inner, rname + "{name}")
-
-        ds.feature_schema = None
+        query = self._query.join(
+            right_ds._query, sqlalchemy.and_(*ops), inner, rname + "{name}"
+        )
+        query.feature_schema = None
+        ds = self._evolve(query=query)
 
         signals_schema = self.signals_schema.clone_without_sys_signals()
         right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
@@ -1278,6 +1319,14 @@ class DataChain(DatasetQuery):
 
         return ds
 
+    def union(self, other: "Self") -> "Self":
+        """Return the set union of the two datasets.
+
+        Parameters:
+            other: chain whose rows will be added to `self`.
+        """
+        return self._evolve(query=self._query.union(other._query))
+
     def subtract(  # type: ignore[override]
         self,
         other: "DataChain",
@@ -1341,7 +1390,7 @@ class DataChain(DatasetQuery):
                     other.signals_schema.resolve(*right_on).db_signals(),
                 )  # type: ignore[arg-type]
             )
-        return super().subtract(other, signals)  # type: ignore[arg-type]
+        return self._evolve(query=self._query.subtract(other._query, signals))  # type: ignore[arg-type]
 
     @classmethod
     def from_values(
@@ -1449,7 +1498,7 @@ class DataChain(DatasetQuery):
             transpose : Whether to transpose rows and columns.
             truncate : Whether or not to truncate the contents of columns.
         """
-        dc = self.limit(limit) if limit > 0 else self
+        dc = self.limit(limit) if limit > 0 else self  # type: ignore[misc]
         df = dc.to_pandas(flatten)
 
         if df.empty:
@@ -1782,7 +1831,7 @@ class DataChain(DatasetQuery):
         settings: Optional[dict] = None,
         in_memory: bool = False,
         schema: Optional[dict[str, DataType]] = None,
-    ) -> "DataChain":
+    ) -> "Self":
         """Create a DataChain from the provided records. This method can be used for
         programmatically generating a chain in contrast of reading data from storages
         or other sources.
@@ -1837,7 +1886,7 @@ class DataChain(DatasetQuery):
         insert_q = dr.get_table().insert()
         for record in to_insert:
             db.execute(insert_q.values(**record))
-        return DataChain(name=dsr.name, settings=settings)
+        return cls.from_dataset(name=dsr.name, session=session, settings=settings)
 
     def sum(self, fr: DataType):  # type: ignore[override]
         """Compute the sum of a column."""
@@ -1898,8 +1947,8 @@ class DataChain(DatasetQuery):
     ) -> None:
         """Method that exports all files from chain to some folder."""
         if placement == "filename" and (
-            super().distinct(pathfunc.name(C(f"{signal}__path"))).count()
-            != self.count()
+            self._query.distinct(pathfunc.name(C(f"{signal}__path"))).count()
+            != self._query.count()
         ):
             raise ValueError("Files with the same name found")
 
@@ -1919,10 +1968,9 @@ class DataChain(DatasetQuery):
         NOTE: Samples are not deterministic, and streamed/paginated queries or
         multiple workers will draw samples with replacement.
         """
-        return super().sample(n)
+        return self._evolve(query=self._query.sample(n))
 
-    @detach
-    def filter(self, *args) -> "Self":
+    def filter(self, *args: Any) -> "Self":
         """Filter the chain according to conditions.
 
         Example:
@@ -1955,14 +2003,50 @@ class DataChain(DatasetQuery):
             )
             ```
         """
-        return super().filter(*args)
+        return self._evolve(query=self._query.filter(*args))
 
-    @detach
     def limit(self, n: int) -> "Self":
-        """Return the first n rows of the chain."""
-        return super().limit(n)
+        """Return the first `n` rows of the chain.
+
+        If the chain is unordered, which rows are returned is undefined.
+        If the chain has less than `n` rows, the whole chain is returned.
+
+        Parameters:
+            n (int): Number of rows to return.
+        """
+        return self._evolve(query=self._query.limit(n))
 
-    @detach
     def offset(self, offset: int) -> "Self":
-        """Return the results starting with the offset row."""
-        return super().offset(offset)
+        """Return the results starting with the offset row.
+
+        If the chain is unordered, which rows are skipped in undefined.
+        If the chain has less than `offset` rows, the result is an empty chain.
+
+        Parameters:
+            offset (int): Number of rows to skip.
+        """
+        return self._evolve(query=self._query.offset(offset))
+
+    def count(self) -> int:
+        """Return the number of rows in the chain."""
+        return self._query.count()
+
+    def exec(self) -> "Self":
+        """Execute the chain."""
+        return self._evolve(query=self._query.exec())
+
+    def chunk(self, index: int, total: int) -> "Self":
+        """Split a chain into smaller chunks for e.g. parallelization.
+
+        Example:
+            ```py
+            chain = DataChain.from_storage(...)
+            chunk_1 = query._chunk(0, 2)
+            chunk_2 = query._chunk(1, 2)
+            ```
+
+        Note:
+            Bear in mind that `index` is 0-indexed but `total` isn't.
+            Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
+        """
+        return self._evolve(query=self._query.chunk(index, total))

datachain/lib/listing.py

@@ -1,7 +1,7 @@
 import posixpath
 from collections.abc import Iterator
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Callable, Optional
+from typing import TYPE_CHECKING, Callable, Optional, TypeVar
 
 from fsspec.asyn import get_loop
 from sqlalchemy.sql.expression import true
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
 LISTING_TTL = 4 * 60 * 60  # cached listing lasts 4 hours
 LISTING_PREFIX = "lst__"  # listing datasets start with this name
 
+D = TypeVar("D", bound="DataChain")
+
 
 def list_bucket(uri: str, cache, client_config=None) -> Callable:
     """
@@ -38,11 +40,11 @@ def list_bucket(uri: str, cache, client_config=None) -> Callable:
 
 
 def ls(
-    dc: "DataChain",
+    dc: D,
     path: str,
     recursive: Optional[bool] = True,
     object_name="file",
-):
+) -> D:
     """
     Return files by some path from DataChain instance which contains bucket listing.
     Path can have globs.

datachain/lib/pytorch.py

@@ -9,6 +9,7 @@ from torch.utils.data import IterableDataset, get_worker_info
 from torchvision.transforms import v2
 from tqdm import tqdm
 
+from datachain import Session
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
 from datachain.lib.text import convert_text
@@ -87,8 +88,11 @@ class PytorchDataset(IterableDataset):
     def __iter__(self) -> Iterator[Any]:
         if self.catalog is None:
             self.catalog = self._get_catalog()
+        session = Session.get(catalog=self.catalog)
         total_rank, total_workers = self.get_rank_and_workers()
-        ds = DataChain(name=self.name, version=self.version, catalog=self.catalog)
+        ds = DataChain.from_dataset(
+            name=self.name, version=self.version, session=session
+        )
         ds = ds.remove_file_signals()
 
         if self.num_samples > 0:

datachain/query/dataset.py

@@ -1037,7 +1037,7 @@ class DatasetQuery:
         session: Optional[Session] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
         in_memory: bool = False,
-    ):
+    ) -> None:
         self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
         self.steps: list[Step] = []

{datachain-0.3.20.dist-info → datachain-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.20
+Version: 0.5.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.3.20.dist-info → datachain-0.5.0.dist-info}/RECORD

@@ -2,10 +2,10 @@ datachain/__init__.py,sha256=ofPJ6B-d-ybSDRrE7J6wqF_ZRAB2W9U8l-eeuBtqPLg,865
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=Lg3Ck1PQLjQziMx9KU4atzbEnJXTE0924WMYkhgWtGU,8247
 datachain/cache.py,sha256=s0YHN7qurmQv-eC265TjeureK84TebWWAnL07cxchZQ,2997
-datachain/cli.py,sha256=TQ1OKMulAcsJndKLCyxJpfNqbMWQgOa4Aeihnu36cR8,30095
+datachain/cli.py,sha256=vVK7hNEyF7p5bUTmixkbgS7JYyTSpXeyRZJkWfpYUOw,30164
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=PfC7W5yO6HFO6-iMB4YB-0RR88LPiGmD6sS_SfVbGso,1979
-datachain/dataset.py,sha256=2NCQU9ZSgNGhA01SP5ON18VhMohXif-btOB4Lz-Uvds,14911
+datachain/dataset.py,sha256=w7qqJP7xYrm9CmBSmSezSxUQHZDsHKkwviF8AYUob7o,14671
 datachain/error.py,sha256=vbIbamnFMIojh1UpmxWoA6Omup7WFAFNJnf8xAkGWwI,1146
 datachain/job.py,sha256=Jt4sNutMHJReaGsj3r3scueN5aESLGfhimAa8pUP7Is,1271
 datachain/listing.py,sha256=TkMmBzCiru26x4RaZiagWJTmTGbiy6yGrAsSJMr8cFE,8213
@@ -18,7 +18,7 @@ datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=KeFSRHsiYthnTu4a6bH-rw04mX1m8krTX0f2NqfQGFI,12114
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=MC8qxu5r0eWtVSWBxPmnYsc-0sUnkzGUZZxgwFQDhH0,64002
+datachain/catalog/catalog.py,sha256=FuKuIiCwPgN5Ea25hnFe_ZFZH9YEUZ2ma9k_Lczk-JU,63867
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
 datachain/client/__init__.py,sha256=T4wiYL9KIM0ZZ_UqIyzV8_ufzYlewmizlV4iymHNluE,86
@@ -33,25 +33,25 @@ datachain/data_storage/__init__.py,sha256=cEOJpyu1JDZtfUupYucCDNFI6e5Wmp_Oyzq6rZ
 datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kTUCaru4,3406
 datachain/data_storage/id_generator.py,sha256=lCEoU0BM37Ai2aRpSbwo5oQT0GqZnSpYwwvizathRMQ,4292
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=BePe3bVxo-Zuuccok8TLRo4cMHVnAIa8hfZMadbxzqM,52649
+datachain/data_storage/metastore.py,sha256=NV4FJ_W16Q19Sx70i5Qtre-n4DC2kMD0qw0vBz3j7Ks,52228
 datachain/data_storage/schema.py,sha256=AGbjyEir5UmRZXI3m0jChZogUh5wd8csj6-YlUWaAxQ,8383
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=EBKJncuzcyQfcKFm2mUjvHjHRTODsteM-k_zndunBrw,28834
-datachain/data_storage/warehouse.py,sha256=Vwhu_OfcNAoTtg1BHui80VCzlPeTUjZQL0QWziu8awY,32186
+datachain/data_storage/warehouse.py,sha256=fXhVfao3NfWFGbbG5uJ-Ga4bX1FiKVfcbDyQgECYfk8,32122
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/arrow.py,sha256=aUsoQmxDmuSnB8Ik9p57Y66gc_dgx6NBqkDDIfLsvno,7630
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=gHIjlow84GMRDa78yLL1Ud-N18or21fnTyPEwsatpXY,2045
 datachain/lib/dataset_info.py,sha256=srPPhI2UHf6hFPBecyFEVw2SS5aPisIIMsvGgKqi7ss,2366
-datachain/lib/dc.py,sha256=kabEHnqbcoat7gd-yl0PvmuC6SyKbRa8r7NWKcN6GEQ,68978
+datachain/lib/dc.py,sha256=yTyHrKIswCzdlvl2n-wdEVZEEF5VQpkLJPzPfUL9CTU,72054
 datachain/lib/file.py,sha256=LjTW_-PDAnoUhvyB4bJ8Y8n__XGqrxvmd9mDOF0Gir8,14875
 datachain/lib/hf.py,sha256=cPnmLuprr0pYABH7KqA5FARQ1JGlywdDwD3yDzVAm4k,5920
 datachain/lib/image.py,sha256=AMXYwQsmarZjRbPCZY3M1jDsM2WAB_b3cTY4uOIuXNU,2675
-datachain/lib/listing.py,sha256=cHPN5-Fq8yb0gP6DARImhmZWxykDDNqhhJujDxEp53A,4104
+datachain/lib/listing.py,sha256=_2oQXh03RAOydeyW3G4OSXCncZaapMGlyGCYcvuUPhc,4145
 datachain/lib/listing_info.py,sha256=36NZ-tXY5Y118wurkajuWWbcE8UCjkRwZlacDtN9F3g,954
 datachain/lib/meta_formats.py,sha256=3f-0vpMTesagS9iMd3y9-u9r-7g0eqYsxmK4fVfNWlw,6635
 datachain/lib/model_store.py,sha256=DNIv8Y6Jtk1_idNLzIpsThOsdW2BMAudyUCbPUcgcxk,2515
-datachain/lib/pytorch.py,sha256=8LNyFaBrx8zws--MEsFg5g3pb8oLnaQAUlgGvtjKxX4,5960
+datachain/lib/pytorch.py,sha256=W-ARi2xH1f1DUkVfRuerW-YWYgSaJASmNCxtz2lrJGI,6072
 datachain/lib/settings.py,sha256=39thOpYJw-zPirzeNO6pmRC2vPrQvt4eBsw1xLWDFsw,2344
 datachain/lib/signal_schema.py,sha256=iqgubjCBRiUJB30miv05qFX4uU04dA_Pzi3DCUsHZGs,24177
 datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
@@ -70,7 +70,7 @@ datachain/lib/convert/unflatten.py,sha256=Ogvh_5wg2f38_At_1lN0D_e2uZOOpYEvwvB2xd
 datachain/lib/convert/values_to_tuples.py,sha256=YOdbjzHq-uj6-cV2Qq43G72eN2avMNDGl4x5t6yQMl8,3931
 datachain/query/__init__.py,sha256=0NBOZVgIDpCcj1Ci883dQ9A0iiwe03xzmotkOCFbxYc,293
 datachain/query/batch.py,sha256=-vlpINJiertlnaoUVv1C95RatU0F6zuhpIYRufJRo1M,3660
-datachain/query/dataset.py,sha256=F9WEVhDuFm6NQT6l-Vi3PMU-mQVpqwKHMgZIA4eWB18,53602
+datachain/query/dataset.py,sha256=tLCTaj4K93BY93GgOPv9PknZByEF89zpHc7y9s8ZF_w,53610
 datachain/query/dispatch.py,sha256=CFAc09O6UllcyUSSEY1GUlEMPzeO8RYhXinNN4HBl9M,12405
 datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
@@ -97,9 +97,9 @@ datachain/sql/sqlite/base.py,sha256=WLPHBhZbXbiqPoRV1VgDrXJqku4UuvJpBhYeQ0k5rI8,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.20.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.20.dist-info/METADATA,sha256=zFk_QWL3Ag3kxLdQPqYAFEXnTD2WkxrvJmLLGOxXpsE,17157
-datachain-0.3.20.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-datachain-0.3.20.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.20.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.20.dist-info/RECORD,,
+datachain-0.5.0.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.5.0.dist-info/METADATA,sha256=tKSZNiHZY0WJ_w6irkpSF7qDfuOTfiYNEQ6St3eBs-M,17156
+datachain-0.5.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+datachain-0.5.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.5.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.5.0.dist-info/RECORD,,