datachain 0.13.1__py3-none-any.whl → 0.14.1__py3-none-any.whl

datachain/__init__.py

@@ -1,5 +1,21 @@
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
-from datachain.lib.dc import C, Column, DataChain, Sys
+from datachain.lib.dc import (
+    C,
+    Column,
+    DataChain,
+    Sys,
+    datasets,
+    from_csv,
+    from_dataset,
+    from_hf,
+    from_json,
+    from_pandas,
+    from_parquet,
+    from_records,
+    from_storage,
+    from_values,
+    listings,
+)
 from datachain.lib.file import (
     ArrowRow,
     File,
@@ -44,7 +60,18 @@ __all__ = [
     "VideoFile",
     "VideoFragment",
     "VideoFrame",
+    "datasets",
+    "from_csv",
+    "from_dataset",
+    "from_hf",
+    "from_json",
+    "from_pandas",
+    "from_parquet",
+    "from_records",
+    "from_storage",
+    "from_values",
     "is_chain_type",
+    "listings",
     "metrics",
     "param",
 ]

datachain/catalog/catalog.py

@@ -583,12 +583,12 @@ class Catalog:
         object_name="file",
         skip_indexing=False,
     ) -> tuple[Optional["Listing"], "Client", str]:
-        from datachain.lib.dc import DataChain
+        from datachain import from_storage
         from datachain.listing import Listing
 
-        DataChain.from_storage(
+        from_storage(
             source, session=self.session, update=update, object_name=object_name
-        )
+        ).exec()
 
         list_ds_name, list_uri, list_path, _ = get_listing(
             source, self.session, update=update
@@ -994,18 +994,14 @@ class Catalog:
         if not sources:
             raise ValueError("Sources needs to be non empty list")
 
-        from datachain.lib.dc import DataChain
+        from datachain import from_dataset, from_storage
 
         chains = []
         for source in sources:
             if source.startswith(DATASET_PREFIX):
-                dc = DataChain.from_dataset(
-                    source[len(DATASET_PREFIX) :], session=self.session
-                )
+                dc = from_dataset(source[len(DATASET_PREFIX) :], session=self.session)
             else:
-                dc = DataChain.from_storage(
-                    source, session=self.session, recursive=recursive
-                )
+                dc = from_storage(source, session=self.session, recursive=recursive)
 
             chains.append(dc)
 

datachain/cli/commands/ls.py

@@ -36,7 +36,7 @@ def ls_local(
     client_config=None,
     **kwargs,
 ):
-    from datachain import DataChain
+    from datachain import listings
 
     if sources:
         if catalog is None:
@@ -63,7 +63,7 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = list(DataChain.listings().collect("listing"))
+        listing = list(listings().collect("listing"))
         for ls in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 

datachain/cli/commands/show.py

@@ -18,8 +18,7 @@ def show(
     schema: bool = False,
     include_hidden: bool = False,
 ) -> None:
-    from datachain import Session
-    from datachain.lib.dc import DataChain
+    from datachain import Session, from_dataset
     from datachain.query.dataset import DatasetQuery
     from datachain.utils import show_records
 
@@ -52,5 +51,5 @@ def show(
     if schema and dataset_version.feature_schema:
         print("\nSchema:")
         session = Session.get(catalog=catalog)
-        dc = DataChain.from_dataset(name=name, version=version, session=session)
+        dc = from_dataset(name=name, version=version, session=session)
         dc.print_schema()

datachain/client/fsspec.py

@@ -89,9 +89,9 @@ class Client(ABC):
         from .local import FileClient
         from .s3 import ClientS3
 
-        protocol = urlparse(str(url)).scheme
+        protocol = urlparse(os.fspath(url)).scheme
 
-        if not protocol or _is_win_local_path(str(url)):
+        if not protocol or _is_win_local_path(os.fspath(url)):
             return FileClient
         if protocol == ClientS3.protocol:
             return ClientS3
@@ -122,7 +122,7 @@ class Client(ABC):
         source: Union[str, os.PathLike[str]], cache: Cache, **kwargs
     ) -> "Client":
         cls = Client.get_implementation(source)
-        storage_url, _ = cls.split_url(str(source))
+        storage_url, _ = cls.split_url(os.fspath(source))
         if os.name == "nt":
             storage_url = storage_url.removeprefix("/")
 

datachain/lib/dc/__init__.py

@@ -0,0 +1,32 @@
+from .csv import from_csv
+from .datachain import C, Column, DataChain
+from .datasets import datasets, from_dataset
+from .hf import from_hf
+from .json import from_json
+from .listings import listings
+from .pandas import from_pandas
+from .parquet import from_parquet
+from .records import from_records
+from .storage import from_storage
+from .utils import DatasetMergeError, DatasetPrepareError, Sys
+from .values import from_values
+
+__all__ = [
+    "C",
+    "Column",
+    "DataChain",
+    "DatasetMergeError",
+    "DatasetPrepareError",
+    "Sys",
+    "datasets",
+    "from_csv",
+    "from_dataset",
+    "from_hf",
+    "from_json",
+    "from_pandas",
+    "from_parquet",
+    "from_records",
+    "from_storage",
+    "from_values",
+    "listings",
+]

datachain/lib/dc/csv.py

@@ -0,0 +1,127 @@
+from collections.abc import Sequence
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Optional,
+    Union,
+)
+
+from datachain.lib.dc.utils import DatasetPrepareError, OutputType
+from datachain.lib.model_store import ModelStore
+from datachain.query import Session
+
+if TYPE_CHECKING:
+    from pyarrow import DataType as ArrowDataType
+
+    from .datachain import DataChain
+
+
+def from_csv(
+    path,
+    delimiter: Optional[str] = None,
+    header: bool = True,
+    output: OutputType = None,
+    object_name: str = "",
+    model_name: str = "",
+    source: bool = True,
+    nrows=None,
+    session: Optional[Session] = None,
+    settings: Optional[dict] = None,
+    column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
+    parse_options: Optional[dict[str, "Union[str, Union[bool, Callable]]"]] = None,
+    **kwargs,
+) -> "DataChain":
+    """Generate chain from csv files.
+
+    Parameters:
+        path : Storage URI with directory. URI must start with storage prefix such
+            as `s3://`, `gs://`, `az://` or "file:///".
+        delimiter : Character for delimiting columns. Takes precedence if also
+            specified in `parse_options`. Defaults to ",".
+        header : Whether the files include a header row.
+        output : Dictionary or feature class defining column names and their
+            corresponding types. List of column names is also accepted, in which
+            case types will be inferred.
+        object_name : Created object column name.
+        model_name : Generated model name.
+        source : Whether to include info about the source file.
+        nrows : Optional row limit.
+        session : Session to use for the chain.
+        settings : Settings to use for the chain.
+        column_types : Dictionary of column names and their corresponding types.
+            It is passed to CSV reader and for each column specified type auto
+            inference is disabled.
+        parse_options: Tells the parser how to process lines.
+            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html
+
+    Example:
+        Reading a csv file:
+        ```py
+        import datachain as dc
+        chain = dc.from_csv("s3://mybucket/file.csv")
+        ```
+
+        Reading csv files from a directory as a combined dataset:
+        ```py
+        import datachain as dc
+        chain = dc.from_csv("s3://mybucket/dir")
+        ```
+    """
+    from pandas.io.parsers.readers import STR_NA_VALUES
+    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
+    from pyarrow.dataset import CsvFileFormat
+    from pyarrow.lib import type_for_alias
+
+    from .storage import from_storage
+
+    parse_options = parse_options or {}
+    if "delimiter" not in parse_options:
+        parse_options["delimiter"] = ","
+    if delimiter:
+        parse_options["delimiter"] = delimiter
+
+    if column_types:
+        column_types = {
+            name: type_for_alias(typ) if isinstance(typ, str) else typ
+            for name, typ in column_types.items()
+        }
+    else:
+        column_types = {}
+
+    chain = from_storage(path, session=session, settings=settings, **kwargs)
+
+    column_names = None
+    if not header:
+        if not output:
+            msg = "error parsing csv - provide output if no header"
+            raise DatasetPrepareError(chain.name, msg)
+        if isinstance(output, Sequence):
+            column_names = output  # type: ignore[assignment]
+        elif isinstance(output, dict):
+            column_names = list(output.keys())
+        elif (fr := ModelStore.to_pydantic(output)) is not None:
+            column_names = list(fr.model_fields.keys())
+        else:
+            msg = f"error parsing csv - incompatible output type {type(output)}"
+            raise DatasetPrepareError(chain.name, msg)
+
+    parse_options = ParseOptions(**parse_options)
+    read_options = ReadOptions(column_names=column_names)
+    convert_options = ConvertOptions(
+        strings_can_be_null=True,
+        null_values=STR_NA_VALUES,
+        column_types=column_types,
+    )
+    format = CsvFileFormat(
+        parse_options=parse_options,
+        read_options=read_options,
+        convert_options=convert_options,
+    )
+    return chain.parse_tabular(
+        output=output,
+        object_name=object_name,
+        model_name=model_name,
+        source=source,
+        nrows=nrows,
+        format=format,
+    )