datachain 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

datachain/__init__.py

@@ -1,5 +1,21 @@
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
-from datachain.lib.dc import C, Column, DataChain, Sys
+from datachain.lib.dc import (
+    C,
+    Column,
+    DataChain,
+    Sys,
+    datasets,
+    from_csv,
+    from_dataset,
+    from_hf,
+    from_json,
+    from_pandas,
+    from_parquet,
+    from_records,
+    from_storage,
+    from_values,
+    listings,
+)
 from datachain.lib.file import (
     ArrowRow,
     File,
@@ -44,7 +60,18 @@ __all__ = [
     "VideoFile",
     "VideoFragment",
     "VideoFrame",
+    "datasets",
+    "from_csv",
+    "from_dataset",
+    "from_hf",
+    "from_json",
+    "from_pandas",
+    "from_parquet",
+    "from_records",
+    "from_storage",
+    "from_values",
     "is_chain_type",
+    "listings",
     "metrics",
     "param",
 ]

datachain/catalog/catalog.py

@@ -583,10 +583,10 @@ class Catalog:
         object_name="file",
         skip_indexing=False,
     ) -> tuple[Optional["Listing"], "Client", str]:
-        from datachain.lib.dc import DataChain
+        from datachain import from_storage
         from datachain.listing import Listing
 
-        DataChain.from_storage(
+        from_storage(
             source, session=self.session, update=update, object_name=object_name
         )
 
@@ -795,6 +795,19 @@ class Catalog:
         try:
             dataset = self.get_dataset(name)
             default_version = dataset.next_version
+
+            if (description or labels) and (
+                dataset.description != description or dataset.labels != labels
+            ):
+                description = description or dataset.description
+                labels = labels or dataset.labels
+
+                self.update_dataset(
+                    dataset,
+                    description=description,
+                    labels=labels,
+                )
+
         except DatasetNotFoundError:
             schema = {
                 c.name: c.type.to_dict() for c in columns if isinstance(c.type, SQLType)
@@ -981,18 +994,14 @@ class Catalog:
         if not sources:
             raise ValueError("Sources needs to be non empty list")
 
-        from datachain.lib.dc import DataChain
+        from datachain import from_dataset, from_storage
 
         chains = []
         for source in sources:
             if source.startswith(DATASET_PREFIX):
-                dc = DataChain.from_dataset(
-                    source[len(DATASET_PREFIX) :], session=self.session
-                )
+                dc = from_dataset(source[len(DATASET_PREFIX) :], session=self.session)
             else:
-                dc = DataChain.from_storage(
-                    source, session=self.session, recursive=recursive
-                )
+                dc = from_storage(source, session=self.session, recursive=recursive)
 
             chains.append(dc)
 

datachain/cli/commands/ls.py

@@ -36,7 +36,7 @@ def ls_local(
     client_config=None,
     **kwargs,
 ):
-    from datachain import DataChain
+    from datachain import listings
 
     if sources:
         if catalog is None:
@@ -63,7 +63,7 @@ def ls_local(
                     print(format_ls_entry(entry))
     else:
         # Collect results in a list here to prevent interference from `tqdm` and `print`
-        listing = list(DataChain.listings().collect("listing"))
+        listing = list(listings().collect("listing"))
         for ls in listing:
             print(format_ls_entry(f"{ls.uri}@v{ls.version}"))  # type: ignore[union-attr]
 

datachain/cli/commands/show.py

@@ -18,8 +18,7 @@ def show(
     schema: bool = False,
     include_hidden: bool = False,
 ) -> None:
-    from datachain import Session
-    from datachain.lib.dc import DataChain
+    from datachain import Session, from_dataset
     from datachain.query.dataset import DatasetQuery
     from datachain.utils import show_records
 
@@ -52,5 +51,5 @@ def show(
     if schema and dataset_version.feature_schema:
         print("\nSchema:")
         session = Session.get(catalog=catalog)
-        dc = DataChain.from_dataset(name=name, version=version, session=session)
+        dc = from_dataset(name=name, version=version, session=session)
         dc.print_schema()

datachain/diff/__init__.py

@@ -74,6 +74,7 @@ def _compare(  # noqa: C901
     # all left and right columns
     cols = left.signals_schema.clone_without_sys_signals().db_signals()
     right_cols = right.signals_schema.clone_without_sys_signals().db_signals()
+    cols_select = list(left.signals_schema.clone_without_sys_signals().values.keys())
 
     # getting correct on and right_on column names
     on = left.signals_schema.resolve(*on).db_signals()  # type: ignore[assignment]
@@ -131,10 +132,12 @@ def _compare(  # noqa: C901
         # when the row is deleted, we need to take column values from the right chain
         .mutate(
             **{
-                f"{c}": ifelse(
-                    C(diff_col) == CompareStatus.DELETED, C(f"{rname}{c}"), C(c)
+                f"{l_on}": ifelse(
+                    C(diff_col) == CompareStatus.DELETED,
+                    C(f"{rname + l_on if on == right_on else r_on}"),
+                    C(l_on),
                 )
-                for c in [c for c in cols if c in right_cols]
+                for l_on, r_on in zip(on, right_on)  # type: ignore[arg-type]
             }
         )
         .select_except(ldiff_col, rdiff_col)
@@ -150,9 +153,9 @@ def _compare(  # noqa: C901
         dc_diff = dc_diff.filter(C(diff_col) != CompareStatus.DELETED)
 
     if status_col:
-        cols.append(diff_col)  # type: ignore[arg-type]
+        cols_select.append(diff_col)
 
-    dc_diff = dc_diff.select(*cols)
+    dc_diff = dc_diff.select(*cols_select)
 
     # final schema is schema from the left chain with status column added if needed
     dc_diff.signals_schema = (

datachain/lib/dc/__init__.py

@@ -0,0 +1,32 @@
+from .csv import from_csv
+from .datachain import C, Column, DataChain
+from .datasets import datasets, from_dataset
+from .hf import from_hf
+from .json import from_json
+from .listings import listings
+from .pandas import from_pandas
+from .parquet import from_parquet
+from .records import from_records
+from .storage import from_storage
+from .utils import DatasetMergeError, DatasetPrepareError, Sys
+from .values import from_values
+
+__all__ = [
+    "C",
+    "Column",
+    "DataChain",
+    "DatasetMergeError",
+    "DatasetPrepareError",
+    "Sys",
+    "datasets",
+    "from_csv",
+    "from_dataset",
+    "from_hf",
+    "from_json",
+    "from_pandas",
+    "from_parquet",
+    "from_records",
+    "from_storage",
+    "from_values",
+    "listings",
+]

datachain/lib/dc/csv.py

@@ -0,0 +1,127 @@
+from collections.abc import Sequence
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Optional,
+    Union,
+)
+
+from datachain.lib.dc.utils import DatasetPrepareError, OutputType
+from datachain.lib.model_store import ModelStore
+from datachain.query import Session
+
+if TYPE_CHECKING:
+    from pyarrow import DataType as ArrowDataType
+
+    from .datachain import DataChain
+
+
+def from_csv(
+    path,
+    delimiter: Optional[str] = None,
+    header: bool = True,
+    output: OutputType = None,
+    object_name: str = "",
+    model_name: str = "",
+    source: bool = True,
+    nrows=None,
+    session: Optional[Session] = None,
+    settings: Optional[dict] = None,
+    column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
+    parse_options: Optional[dict[str, "Union[str, Union[bool, Callable]]"]] = None,
+    **kwargs,
+) -> "DataChain":
+    """Generate chain from csv files.
+
+    Parameters:
+        path : Storage URI with directory. URI must start with storage prefix such
+            as `s3://`, `gs://`, `az://` or "file:///".
+        delimiter : Character for delimiting columns. Takes precedence if also
+            specified in `parse_options`. Defaults to ",".
+        header : Whether the files include a header row.
+        output : Dictionary or feature class defining column names and their
+            corresponding types. List of column names is also accepted, in which
+            case types will be inferred.
+        object_name : Created object column name.
+        model_name : Generated model name.
+        source : Whether to include info about the source file.
+        nrows : Optional row limit.
+        session : Session to use for the chain.
+        settings : Settings to use for the chain.
+        column_types : Dictionary of column names and their corresponding types.
+            It is passed to CSV reader and for each column specified type auto
+            inference is disabled.
+        parse_options: Tells the parser how to process lines.
+            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html
+
+    Example:
+        Reading a csv file:
+        ```py
+        import datachain as dc
+        chain = dc.from_csv("s3://mybucket/file.csv")
+        ```
+
+        Reading csv files from a directory as a combined dataset:
+        ```py
+        import datachain as dc
+        chain = dc.from_csv("s3://mybucket/dir")
+        ```
+    """
+    from pandas.io.parsers.readers import STR_NA_VALUES
+    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
+    from pyarrow.dataset import CsvFileFormat
+    from pyarrow.lib import type_for_alias
+
+    from .storage import from_storage
+
+    parse_options = parse_options or {}
+    if "delimiter" not in parse_options:
+        parse_options["delimiter"] = ","
+    if delimiter:
+        parse_options["delimiter"] = delimiter
+
+    if column_types:
+        column_types = {
+            name: type_for_alias(typ) if isinstance(typ, str) else typ
+            for name, typ in column_types.items()
+        }
+    else:
+        column_types = {}
+
+    chain = from_storage(path, session=session, settings=settings, **kwargs)
+
+    column_names = None
+    if not header:
+        if not output:
+            msg = "error parsing csv - provide output if no header"
+            raise DatasetPrepareError(chain.name, msg)
+        if isinstance(output, Sequence):
+            column_names = output  # type: ignore[assignment]
+        elif isinstance(output, dict):
+            column_names = list(output.keys())
+        elif (fr := ModelStore.to_pydantic(output)) is not None:
+            column_names = list(fr.model_fields.keys())
+        else:
+            msg = f"error parsing csv - incompatible output type {type(output)}"
+            raise DatasetPrepareError(chain.name, msg)
+
+    parse_options = ParseOptions(**parse_options)
+    read_options = ReadOptions(column_names=column_names)
+    convert_options = ConvertOptions(
+        strings_can_be_null=True,
+        null_values=STR_NA_VALUES,
+        column_types=column_types,
+    )
+    format = CsvFileFormat(
+        parse_options=parse_options,
+        read_options=read_options,
+        convert_options=convert_options,
+    )
+    return chain.parse_tabular(
+        output=output,
+        object_name=object_name,
+        model_name=model_name,
+        source=source,
+        nrows=nrows,
+        format=format,
+    )