datachain 0.14.1__py3-none-any.whl → 0.14.3__py3-none-any.whl

datachain/__init__.py

@@ -5,16 +5,16 @@ from datachain.lib.dc import (
     DataChain,
     Sys,
     datasets,
-    from_csv,
-    from_dataset,
-    from_hf,
-    from_json,
-    from_pandas,
-    from_parquet,
-    from_records,
-    from_storage,
-    from_values,
     listings,
+    read_csv,
+    read_dataset,
+    read_hf,
+    read_json,
+    read_pandas,
+    read_parquet,
+    read_records,
+    read_storage,
+    read_values,
 )
 from datachain.lib.file import (
     ArrowRow,
@@ -61,17 +61,17 @@ __all__ = [
     "VideoFragment",
     "VideoFrame",
     "datasets",
-    "from_csv",
-    "from_dataset",
-    "from_hf",
-    "from_json",
-    "from_pandas",
-    "from_parquet",
-    "from_records",
-    "from_storage",
-    "from_values",
     "is_chain_type",
     "listings",
     "metrics",
     "param",
+    "read_csv",
+    "read_dataset",
+    "read_hf",
+    "read_json",
+    "read_pandas",
+    "read_parquet",
+    "read_records",
+    "read_storage",
+    "read_values",
 ]

datachain/catalog/catalog.py

@@ -583,10 +583,10 @@ class Catalog:
         object_name="file",
         skip_indexing=False,
     ) -> tuple[Optional["Listing"], "Client", str]:
-        from datachain import from_storage
+        from datachain import read_storage
         from datachain.listing import Listing
 
-        from_storage(
+        read_storage(
             source, session=self.session, update=update, object_name=object_name
         ).exec()
 
@@ -994,14 +994,14 @@ class Catalog:
         if not sources:
             raise ValueError("Sources needs to be non empty list")
 
-        from datachain import from_dataset, from_storage
+        from datachain import read_dataset, read_storage
 
         chains = []
         for source in sources:
             if source.startswith(DATASET_PREFIX):
-                dc = from_dataset(source[len(DATASET_PREFIX) :], session=self.session)
+                dc = read_dataset(source[len(DATASET_PREFIX) :], session=self.session)
             else:
-                dc = from_storage(source, session=self.session, recursive=recursive)
+                dc = read_storage(source, session=self.session, recursive=recursive)
 
             chains.append(dc)
 

datachain/catalog/loader.py

@@ -7,6 +7,7 @@ from datachain.utils import get_envs_by_prefix
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
     from datachain.data_storage import AbstractMetastore, AbstractWarehouse
+    from datachain.query.udf import AbstractUDFDistributor
 
 METASTORE_SERIALIZED = "DATACHAIN__METASTORE"
 METASTORE_IMPORT_PATH = "DATACHAIN_METASTORE"
@@ -15,7 +16,6 @@ WAREHOUSE_SERIALIZED = "DATACHAIN__WAREHOUSE"
 WAREHOUSE_IMPORT_PATH = "DATACHAIN_WAREHOUSE"
 WAREHOUSE_ARG_PREFIX = "DATACHAIN_WAREHOUSE_ARG_"
 DISTRIBUTED_IMPORT_PATH = "DATACHAIN_DISTRIBUTED"
-DISTRIBUTED_ARG_PREFIX = "DATACHAIN_DISTRIBUTED_ARG_"
 
 IN_MEMORY_ERROR_MESSAGE = "In-memory is only supported on SQLite"
 
@@ -100,27 +100,22 @@ def get_warehouse(in_memory: bool = False) -> "AbstractWarehouse":
     return warehouse_class(**warehouse_args)
 
 
-def get_distributed_class(**kwargs):
+def get_udf_distributor_class() -> type["AbstractUDFDistributor"]:
     distributed_import_path = os.environ.get(DISTRIBUTED_IMPORT_PATH)
-    distributed_arg_envs = get_envs_by_prefix(DISTRIBUTED_ARG_PREFIX)
-    # Convert env variable names to keyword argument names by lowercasing them
-    distributed_args = {k.lower(): v for k, v in distributed_arg_envs.items()}
 
     if not distributed_import_path:
         raise RuntimeError(
             f"{DISTRIBUTED_IMPORT_PATH} import path is required "
             "for distributed UDF processing."
         )
-    # Distributed class paths are specified as (for example):
-    # module.classname
+    # Distributed class paths are specified as (for example): module.classname
     if "." not in distributed_import_path:
         raise RuntimeError(
             f"Invalid {DISTRIBUTED_IMPORT_PATH} import path: {distributed_import_path}"
         )
     module_name, _, class_name = distributed_import_path.rpartition(".")
     distributed = import_module(module_name)
-    distributed_class = getattr(distributed, class_name)
-    return distributed_class(**distributed_args | kwargs)
+    return getattr(distributed, class_name)
 
 
 def get_catalog(

datachain/cli/commands/show.py

@@ -18,7 +18,7 @@ def show(
     schema: bool = False,
     include_hidden: bool = False,
 ) -> None:
-    from datachain import Session, from_dataset
+    from datachain import Session, read_dataset
     from datachain.query.dataset import DatasetQuery
     from datachain.utils import show_records
 
@@ -51,5 +51,5 @@ def show(
     if schema and dataset_version.feature_schema:
         print("\nSchema:")
         session = Session.get(catalog=catalog)
-        dc = from_dataset(name=name, version=version, session=session)
+        dc = read_dataset(name=name, version=version, session=session)
         dc.print_schema()

datachain/data_storage/warehouse.py

@@ -199,6 +199,15 @@ class AbstractWarehouse(ABC, Serializable):
     # Query Execution
     #
 
+    def query_count(self, query: sa.sql.selectable.Select) -> int:
+        """Count the number of rows in a query."""
+        count_query = sa.select(func.count(1)).select_from(query.subquery())
+        return next(self.db.execute(count_query))[0]
+
+    def table_rows_count(self, table) -> int:
+        count_query = sa.select(func.count(1)).select_from(table)
+        return next(self.db.execute(count_query))[0]
+
     def dataset_select_paginated(
         self,
         query,

datachain/lib/dc/__init__.py

@@ -1,15 +1,15 @@
-from .csv import from_csv
+from .csv import read_csv
 from .datachain import C, Column, DataChain
-from .datasets import datasets, from_dataset
-from .hf import from_hf
-from .json import from_json
+from .datasets import datasets, read_dataset
+from .hf import read_hf
+from .json import read_json
 from .listings import listings
-from .pandas import from_pandas
-from .parquet import from_parquet
-from .records import from_records
-from .storage import from_storage
+from .pandas import read_pandas
+from .parquet import read_parquet
+from .records import read_records
+from .storage import read_storage
 from .utils import DatasetMergeError, DatasetPrepareError, Sys
-from .values import from_values
+from .values import read_values
 
 __all__ = [
     "C",
@@ -19,14 +19,14 @@ __all__ = [
     "DatasetPrepareError",
     "Sys",
     "datasets",
-    "from_csv",
-    "from_dataset",
-    "from_hf",
-    "from_json",
-    "from_pandas",
-    "from_parquet",
-    "from_records",
-    "from_storage",
-    "from_values",
     "listings",
+    "read_csv",
+    "read_dataset",
+    "read_hf",
+    "read_json",
+    "read_pandas",
+    "read_parquet",
+    "read_records",
+    "read_storage",
+    "read_values",
 ]

datachain/lib/dc/csv.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
     from .datachain import DataChain
 
 
-def from_csv(
+def read_csv(
     path,
     delimiter: Optional[str] = None,
     header: bool = True,
@@ -58,13 +58,13 @@ def from_csv(
         Reading a csv file:
         ```py
         import datachain as dc
-        chain = dc.from_csv("s3://mybucket/file.csv")
+        chain = dc.read_csv("s3://mybucket/file.csv")
         ```
 
         Reading csv files from a directory as a combined dataset:
         ```py
         import datachain as dc
-        chain = dc.from_csv("s3://mybucket/dir")
+        chain = dc.read_csv("s3://mybucket/dir")
         ```
     """
     from pandas.io.parsers.readers import STR_NA_VALUES
@@ -72,7 +72,7 @@ def from_csv(
     from pyarrow.dataset import CsvFileFormat
     from pyarrow.lib import type_for_alias
 
-    from .storage import from_storage
+    from .storage import read_storage
 
     parse_options = parse_options or {}
     if "delimiter" not in parse_options:
@@ -88,7 +88,7 @@ def from_csv(
     else:
         column_types = {}
 
-    chain = from_storage(path, session=session, settings=settings, **kwargs)
+    chain = read_storage(path, session=session, settings=settings, **kwargs)
 
     column_names = None
     if not header:

datachain/lib/dc/datachain.py

@@ -84,22 +84,22 @@ class DataChain:
     underlyind library `Pydantic`.
 
     See Also:
-        `from_storage("s3://my-bucket/my-dir/")` - reading unstructured
+        `read_storage("s3://my-bucket/my-dir/")` - reading unstructured
             data files from storages such as S3, gs or Azure ADLS.
 
         `DataChain.save("name")` - saving to a dataset.
 
-        `from_dataset("name")` - reading from a dataset.
+        `read_dataset("name")` - reading from a dataset.
 
-        `from_values(fib=[1, 2, 3, 5, 8])` - generating from values.
+        `read_values(fib=[1, 2, 3, 5, 8])` - generating from values.
 
-        `from_pandas(pd.DataFrame(...))` - generating from pandas.
+        `read_pandas(pd.DataFrame(...))` - generating from pandas.
 
-        `from_json("file.json")` - generating from json.
+        `read_json("file.json")` - generating from json.
 
-        `from_csv("file.csv")` - generating from csv.
+        `read_csv("file.csv")` - generating from csv.
 
-        `from_parquet("file.parquet")` - generating from parquet.
+        `read_parquet("file.parquet")` - generating from parquet.
 
     Example:
         ```py
@@ -118,7 +118,7 @@ class DataChain:
         api_key = os.environ["MISTRAL_API_KEY"]
 
         chain = (
-            dc.from_storage("gs://datachain-demo/chatbot-KiT/")
+            dc.read_storage("gs://datachain-demo/chatbot-KiT/")
             .limit(5)
             .settings(cache=True, parallel=5)
             .map(
@@ -315,27 +315,27 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .storage import from_storage
+        from .storage import read_storage
 
         warnings.warn(
             "Class method `from_storage` is deprecated. "
-            "Use `from_storage` function instead from top_module.",
+            "Use `read_storage` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_storage(*args, **kwargs)
+        return read_storage(*args, **kwargs)
 
     @classmethod
     def from_dataset(cls, *args, **kwargs) -> "DataChain":
-        from .datasets import from_dataset
+        from .datasets import read_dataset
 
         warnings.warn(
             "Class method `from_dataset` is deprecated. "
-            "Use `from_dataset` function instead from top_module.",
+            "Use `read_dataset` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_dataset(*args, **kwargs)
+        return read_dataset(*args, **kwargs)
 
     @classmethod
     def from_json(
@@ -343,15 +343,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .json import from_json
+        from .json import read_json
 
         warnings.warn(
             "Class method `from_json` is deprecated. "
-            "Use `from_json` function instead from top_module.",
+            "Use `read_json` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_json(*args, **kwargs)
+        return read_json(*args, **kwargs)
 
     def explode(
         self,
@@ -487,7 +487,7 @@ class DataChain:
                 )
 
             chain = (
-                dc.from_storage("s3://my-bucket")
+                dc.read_storage("s3://my-bucket")
                 .apply(parse_stem)
                 .filter(C("stem").glob("*cat*"))
             )
@@ -727,7 +727,7 @@ class DataChain:
 
         Note:
             Order is not guaranteed when steps are added after an `order_by` statement.
-            I.e. when using `from_dataset` an `order_by` statement should be used if
+            I.e. when using `read_dataset` an `order_by` statement should be used if
             the order of the records in the chain is important.
             Using `order_by` directly before `limit`, `collect` and `collect_flatten`
             will give expected results.
@@ -1466,15 +1466,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .values import from_values
+        from .values import read_values
 
         warnings.warn(
             "Class method `from_values` is deprecated. "
-            "Use `from_values` function instead from top_module.",
+            "Use `read_values` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_values(*args, **kwargs)
+        return read_values(*args, **kwargs)
 
     @classmethod
     def from_pandas(
@@ -1482,15 +1482,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .pandas import from_pandas
+        from .pandas import read_pandas
 
         warnings.warn(
             "Class method `from_pandas` is deprecated. "
-            "Use `from_pandas` function instead from top_module.",
+            "Use `read_pandas` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_pandas(*args, **kwargs)
+        return read_pandas(*args, **kwargs)
 
     def to_pandas(self, flatten=False, include_hidden=True) -> "pd.DataFrame":
         """Return a pandas DataFrame from the chain.
@@ -1575,15 +1575,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .hf import from_hf
+        from .hf import read_hf
 
         warnings.warn(
             "Class method `from_hf` is deprecated. "
-            "Use `from_hf` function instead from top_module.",
+            "Use `read_hf` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_hf(*args, **kwargs)
+        return read_hf(*args, **kwargs)
 
     def parse_tabular(
         self,
@@ -1610,7 +1610,7 @@ class DataChain:
             Reading a json lines file:
             ```py
             import datachain as dc
-            chain = dc.from_storage("s3://mybucket/file.jsonl")
+            chain = dc.read_storage("s3://mybucket/file.jsonl")
             chain = chain.parse_tabular(format="json")
             ```
 
@@ -1618,7 +1618,7 @@ class DataChain:
             ```py
             import datachain as dc
 
-            chain = dc.from_storage("s3://mybucket")
+            chain = dc.read_storage("s3://mybucket")
             chain = chain.filter(dc.C("file.name").glob("*.jsonl"))
             chain = chain.parse_tabular(format="json")
             ```
@@ -1680,15 +1680,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .csv import from_csv
+        from .csv import read_csv
 
         warnings.warn(
             "Class method `from_csv` is deprecated. "
-            "Use `from_csv` function instead from top_module.",
+            "Use `read_csv` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_csv(*args, **kwargs)
+        return read_csv(*args, **kwargs)
 
     @classmethod
     def from_parquet(
@@ -1696,15 +1696,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .parquet import from_parquet
+        from .parquet import read_parquet
 
         warnings.warn(
             "Class method `from_parquet` is deprecated. "
-            "Use `from_parquet` function instead from top_module.",
+            "Use `read_parquet` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_parquet(*args, **kwargs)
+        return read_parquet(*args, **kwargs)
 
     def to_parquet(
         self,
@@ -1930,15 +1930,15 @@ class DataChain:
         *args,
         **kwargs,
     ) -> "DataChain":
-        from .records import from_records
+        from .records import read_records
 
         warnings.warn(
             "Class method `from_records` is deprecated. "
-            "Use `from_records` function instead from top_module.",
+            "Use `read_records` function instead from top_module.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return from_records(*args, **kwargs)
+        return read_records(*args, **kwargs)
 
     def sum(self, fr: DataType):  # type: ignore[override]
         """Compute the sum of a column."""
@@ -1969,7 +1969,7 @@ class DataChain:
             import datachain as dc
 
             (
-                dc.from_storage(DATA, type="text")
+                dc.read_storage(DATA, type="text")
                 .settings(parallel=4, cache=True)
                 .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
                 .map(
@@ -2021,7 +2021,7 @@ class DataChain:
             ```py
             import datachain as dc
 
-            ds = dc.from_storage("s3://mybucket")
+            ds = dc.read_storage("s3://mybucket")
             ds.to_storage("gs://mybucket", placement="filename")
             ```
         """
@@ -2139,7 +2139,7 @@ class DataChain:
             ```py
             import datachain as dc
 
-            chain = dc.from_storage(...)
+            chain = dc.read_storage(...)
             chunk_1 = query._chunk(0, 2)
             chunk_2 = query._chunk(1, 2)
             ```

datachain/lib/dc/datasets.py

@@ -13,7 +13,7 @@ from datachain.query import Session
 from datachain.query.dataset import DatasetQuery
 
 from .utils import Sys
-from .values import from_values
+from .values import read_values
 
 if TYPE_CHECKING:
     from typing_extensions import ParamSpec
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     P = ParamSpec("P")
 
 
-def from_dataset(
+def read_dataset(
     name: str,
     version: Optional[int] = None,
     session: Optional[Session] = None,
@@ -44,15 +44,15 @@ def from_dataset(
     Example:
         ```py
         import datachain as dc
-        chain = dc.from_dataset("my_cats")
+        chain = dc.read_dataset("my_cats")
         ```
 
         ```py
-        chain = dc.from_dataset("my_cats", fallback_to_studio=False)
+        chain = dc.read_dataset("my_cats", fallback_to_studio=False)
         ```
 
         ```py
-        chain = dc.from_dataset("my_cats", version=1)
+        chain = dc.read_dataset("my_cats", version=1)
         ```
 
         ```py
@@ -64,7 +64,7 @@ def from_dataset(
             "min_task_size": 1000,
             "prefetch": 10,
         }
-        chain = dc.from_dataset(
+        chain = dc.read_dataset(
             name="my_cats",
             version=1,
             session=session,
@@ -140,7 +140,7 @@ def datasets(
         )
     ]
 
-    return from_values(
+    return read_values(
         session=session,
         settings=settings,
         in_memory=in_memory,

datachain/lib/dc/hf.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
     P = ParamSpec("P")
 
 
-def from_hf(
+def read_hf(
     dataset: Union[str, "HFDatasetType"],
     *args,
     session: Optional[Session] = None,
@@ -42,7 +42,7 @@ def from_hf(
         Load from Hugging Face Hub:
         ```py
         import datachain as dc
-        chain = dc.from_hf("beans", split="train")
+        chain = dc.read_hf("beans", split="train")
         ```
 
         Generate chain from loaded dataset:
@@ -50,12 +50,12 @@ def from_hf(
         from datasets import load_dataset
         ds = load_dataset("beans", split="train")
         import datachain as dc
-        chain = dc.from_hf(ds)
+        chain = dc.read_hf(ds)
         ```
     """
     from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits
 
-    from .values import from_values
+    from .values import read_values
 
     output: dict[str, DataType] = {}
     ds_dict = stream_splits(dataset, *args, **kwargs)
@@ -69,5 +69,5 @@ def from_hf(
     if object_name:
         output = {object_name: model}
 
-    chain = from_values(split=list(ds_dict.keys()), session=session, settings=settings)
+    chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
     return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)

datachain/lib/dc/json.py

@@ -22,7 +22,7 @@ if TYPE_CHECKING:
     P = ParamSpec("P")
 
 
-def from_json(
+def read_json(
     path: Union[str, os.PathLike[str]],
     type: FileType = "text",
     spec: Optional[DataType] = None,
@@ -52,16 +52,16 @@ def from_json(
         infer JSON schema from data, reduce using JMESPATH
         ```py
         import datachain as dc
-        chain = dc.from_json("gs://json", jmespath="key1.key2")
+        chain = dc.read_json("gs://json", jmespath="key1.key2")
         ```
 
         infer JSON schema from a particular path
         ```py
         import datachain as dc
-        chain = dc.from_json("gs://json_ds", schema_from="gs://json/my.json")
+        chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")
         ```
     """
-    from .storage import from_storage
+    from .storage import read_storage
 
     if schema_from == "auto":
         schema_from = os.fspath(path)
@@ -74,7 +74,7 @@ def from_json(
         object_name = jmespath_to_name(jmespath)
     if not object_name:
         object_name = format
-    chain = from_storage(uri=path, type=type, **kwargs)
+    chain = read_storage(uri=path, type=type, **kwargs)
     signal_dict = {
         object_name: read_meta(
             schema_from=schema_from,

datachain/lib/dc/listings.py

@@ -6,7 +6,7 @@ from typing import (
 from datachain.lib.listing_info import ListingInfo
 from datachain.query import Session
 
-from .values import from_values
+from .values import read_values
 
 if TYPE_CHECKING:
     from typing_extensions import ParamSpec
@@ -35,7 +35,7 @@ def listings(
     session = Session.get(session, in_memory=in_memory)
     catalog = kwargs.get("catalog") or session.catalog
 
-    return from_values(
+    return read_values(
         session=session,
         in_memory=in_memory,
         output={object_name: ListingInfo},