datachain 0.14.4__py3-none-any.whl → 0.15.0__py3-none-any.whl

datachain/lib/dc/datasets.py

@@ -1,7 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-)
+from typing import TYPE_CHECKING, Optional, get_origin, get_type_hints
 
 from datachain.lib.dataset_info import DatasetInfo
 from datachain.lib.file import (
@@ -102,7 +99,7 @@ def datasets(
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     in_memory: bool = False,
-    object_name: str = "dataset",
+    column: Optional[str] = None,
     include_listing: bool = False,
     studio: bool = False,
 ) -> "DataChain":
@@ -112,7 +109,8 @@ def datasets(
         session: Optional session instance. If not provided, uses default session.
         settings: Optional dictionary of settings to configure the chain.
         in_memory: If True, creates an in-memory session. Defaults to False.
-        object_name: Name of the output object in the chain. Defaults to "dataset".
+        column: Name of the output column in the chain. Defaults to None which
+            means no top level column will be created.
         include_listing: If True, includes listing datasets. Defaults to False.
         studio: If True, returns datasets from Studio only,
             otherwise returns all local datasets. Defaults to False.
@@ -124,7 +122,7 @@ def datasets(
         ```py
         import datachain as dc
 
-        chain = dc.datasets()
+        chain = dc.datasets(column="dataset")
         for ds in chain.collect("dataset"):
             print(f"{ds.name}@v{ds.version}")
         ```
@@ -139,13 +137,75 @@ def datasets(
             include_listing=include_listing, studio=studio
         )
     ]
-
     datasets_values = [d for d in datasets_values if not d.is_temp]
 
+    if not column:
+        # flattening dataset fields
+        schema = {
+            k: get_origin(v) if get_origin(v) is dict else v
+            for k, v in get_type_hints(DatasetInfo).items()
+            if k in DatasetInfo.model_fields
+        }
+        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
+        for d in [d.model_dump() for d in datasets_values]:
+            for field, value in d.items():
+                data[field].append(value)
+
+        return read_values(
+            session=session,
+            settings=settings,
+            in_memory=in_memory,
+            output=schema,
+            **data,  # type: ignore[arg-type]
+        )
+
     return read_values(
         session=session,
         settings=settings,
         in_memory=in_memory,
-        output={object_name: DatasetInfo},
-        **{object_name: datasets_values},  # type: ignore[arg-type]
+        output={column: DatasetInfo},
+        **{column: datasets_values},  # type: ignore[arg-type]
     )
+
+
+def delete_dataset(
+    name: str,
+    version: Optional[int] = None,
+    force: Optional[bool] = False,
+    studio: Optional[bool] = False,
+    session: Optional[Session] = None,
+    in_memory: bool = False,
+) -> None:
+    """Removes specific dataset version or all dataset versions, depending on
+    a force flag.
+
+    Args:
+        name : Dataset name
+        version : Optional dataset version
+        force: If true, all datasets versions will be removed. Defaults to False.
+        studio: If True, removes dataset from Studio only,
+            otherwise remove from local. Defaults to False.
+        session: Optional session instance. If not provided, uses default session.
+        in_memory: If True, creates an in-memory session. Defaults to False.
+
+    Returns: None
+
+    Example:
+        ```py
+        import datachain as dc
+        dc.delete_dataset("cats")
+        ```
+
+        ```py
+        import datachain as dc
+        dc.delete_dataset("cats", version=1)
+        ```
+    """
+
+    session = Session.get(session, in_memory=in_memory)
+    catalog = session.catalog
+    if not force:
+        version = version or catalog.get_dataset(name).latest_version
+    else:
+        version = None
+    catalog.remove_dataset(name, version=version, force=force, studio=studio)

datachain/lib/dc/hf.py

@@ -23,7 +23,7 @@ def read_hf(
     *args,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
-    object_name: str = "",
+    column: str = "",
     model_name: str = "",
     **kwargs,
 ) -> "DataChain":
@@ -34,7 +34,7 @@ def read_hf(
             or an instance of `datasets.Dataset`-like object.
         session : Session to use for the chain.
         settings : Settings to use for the chain.
-        object_name : Generated object column name.
+        column : Generated object column name.
         model_name : Generated model name.
         kwargs : Parameters to pass to datasets.load_dataset.
 
@@ -62,12 +62,12 @@ def read_hf(
     if len(ds_dict) > 1:
         output = {"split": str}
 
-    model_name = model_name or object_name or ""
+    model_name = model_name or column or ""
     hf_features = next(iter(ds_dict.values())).features
     output = output | get_output_schema(hf_features)
     model = dict_to_data_model(model_name, output)
-    if object_name:
-        output = {object_name: model}
+    if column:
+        output = {column: model}
 
     chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
     return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)

datachain/lib/dc/json.py

@@ -28,7 +28,7 @@ def read_json(
     spec: Optional[DataType] = None,
     schema_from: Optional[str] = "auto",
     jmespath: Optional[str] = None,
-    object_name: Optional[str] = "",
+    column: Optional[str] = "",
     model_name: Optional[str] = None,
     format: Optional[str] = "json",
     nrows=None,
@@ -42,7 +42,7 @@ def read_json(
         type : read file as "binary", "text", or "image" data. Default is "text".
         spec : optional Data Model
         schema_from : path to sample to infer spec (if schema not provided)
-        object_name : generated object column name
+        column : generated column name
         model_name : optional generated model name
         format: "json", "jsonl"
         jmespath : optional JMESPATH expression to reduce JSON
@@ -70,13 +70,13 @@ def read_json(
         name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
         return s[:name_end]
 
-    if (not object_name) and jmespath:
-        object_name = jmespath_to_name(jmespath)
-    if not object_name:
-        object_name = format
+    if (not column) and jmespath:
+        column = jmespath_to_name(jmespath)
+    if not column:
+        column = format
     chain = read_storage(uri=path, type=type, **kwargs)
     signal_dict = {
-        object_name: read_meta(
+        column: read_meta(
             schema_from=schema_from,
             format=format,
             spec=spec,

datachain/lib/dc/listings.py

@@ -19,7 +19,7 @@ if TYPE_CHECKING:
 def listings(
     session: Optional[Session] = None,
     in_memory: bool = False,
-    object_name: str = "listing",
+    column: str = "listing",
     **kwargs,
 ) -> "DataChain":
     """Generate chain with list of cached listings.
@@ -38,6 +38,6 @@ def listings(
     return read_values(
         session=session,
         in_memory=in_memory,
-        output={object_name: ListingInfo},
-        **{object_name: catalog.listings()},  # type: ignore[arg-type]
+        output={column: ListingInfo},
+        **{column: catalog.listings()},  # type: ignore[arg-type]
     )

datachain/lib/dc/pandas.py

@@ -22,7 +22,7 @@ def read_pandas(  # type: ignore[override]
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     in_memory: bool = False,
-    object_name: str = "",
+    column: str = "",
 ) -> "DataChain":
     """Generate chain from pandas data-frame.
 
@@ -37,20 +37,27 @@ def read_pandas(  # type: ignore[override]
     """
     from .utils import DatasetPrepareError
 
-    fr_map = {col.lower(): df[col].tolist() for col in df.columns}
+    def get_col_name(col):
+        if isinstance(col, tuple):
+            # Join tuple elements with underscore for MultiIndex columns
+            return "_".join(map(str, col)).lower()
+        # Handle regular string column names
+        return str(col).lower()
 
-    for column in fr_map:
-        if not column.isidentifier():
+    fr_map = {get_col_name(col): df[col].tolist() for col in df.columns}
+
+    for c in fr_map:
+        if not c.isidentifier():
             raise DatasetPrepareError(
                 name,
-                f"import from pandas error - '{column}' cannot be a column name",
+                f"import from pandas error - '{c}' cannot be a column name",
             )
 
     return read_values(
         name,
         session,
         settings=settings,
-        object_name=object_name,
+        column=column,
         in_memory=in_memory,
         **fr_map,
     )

datachain/lib/dc/parquet.py

@@ -19,7 +19,7 @@ def read_parquet(
     path,
     partitioning: Any = "hive",
     output: Optional[dict[str, DataType]] = None,
-    object_name: str = "",
+    column: str = "",
     model_name: str = "",
     source: bool = True,
     session: Optional[Session] = None,
@@ -33,7 +33,7 @@ def read_parquet(
             as `s3://`, `gs://`, `az://` or "file:///".
         partitioning : Any pyarrow partitioning schema.
         output : Dictionary defining column names and their corresponding types.
-        object_name : Created object column name.
+        column : Created column name.
         model_name : Generated model name.
         source : Whether to include info about the source file.
         session : Session to use for the chain.
@@ -57,7 +57,7 @@ def read_parquet(
     chain = read_storage(path, session=session, settings=settings, **kwargs)
     return chain.parse_tabular(
         output=output,
-        object_name=object_name,
+        column=column,
         model_name=model_name,
         source=source,
         format="parquet",

datachain/lib/dc/records.py

@@ -1,8 +1,5 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-    Union,
-)
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Optional, Union
 
 import sqlalchemy
 
@@ -12,6 +9,7 @@ from datachain.lib.file import (
 )
 from datachain.lib.signal_schema import SignalSchema
 from datachain.query import Session
+from datachain.query.schema import Column
 
 if TYPE_CHECKING:
     from typing_extensions import ParamSpec
@@ -22,7 +20,7 @@ if TYPE_CHECKING:
 
 
 def read_records(
-    to_insert: Optional[Union[dict, list[dict]]],
+    to_insert: Optional[Union[dict, Iterable[dict]]],
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     in_memory: bool = False,
@@ -54,10 +52,11 @@ def read_records(
 
     if schema:
         signal_schema = SignalSchema(schema)
-        columns = [
-            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
-            for c in signal_schema.db_signals(as_columns=True)  # type: ignore[assignment]
-        ]
+        columns = []
+        for c in signal_schema.db_signals(as_columns=True):
+            assert isinstance(c, Column)
+            kw = {"nullable": c.nullable} if c.nullable is not None else {}
+            columns.append(sqlalchemy.Column(c.name, c.type, **kw))
     else:
         columns = [
             sqlalchemy.Column(name, typ)
@@ -83,8 +82,7 @@ def read_records(
 
     warehouse = catalog.warehouse
     dr = warehouse.dataset_rows(dsr)
-    db = warehouse.db
-    insert_q = dr.get_table().insert()
-    for record in to_insert:
-        db.execute(insert_q.values(**record))
+    table = dr.get_table()
+    warehouse.insert_rows(table, to_insert)
+    warehouse.insert_rows_done(table)
     return read_dataset(name=dsr.name, session=session, settings=settings)

datachain/lib/dc/storage.py

@@ -29,7 +29,7 @@ def read_storage(
     settings: Optional[dict] = None,
     in_memory: bool = False,
     recursive: Optional[bool] = True,
-    object_name: str = "file",
+    column: str = "file",
     update: bool = False,
     anon: bool = False,
     client_config: Optional[dict] = None,
@@ -43,7 +43,7 @@ def read_storage(
             as `s3://`, `gs://`, `az://` or "file:///"
         type : read file as "binary", "text", or "image" data. Default is "binary".
         recursive : search recursively for the given path.
-        object_name : Created object column name.
+        column : Created column name.
         update : force storage reindexing. Default is False.
         anon : If True, we will treat cloud bucket as public one
         client_config : Optional client configuration for the storage client.
@@ -124,7 +124,7 @@ def read_storage(
 
         dc = read_dataset(list_ds_name, session=session, settings=settings)
         dc._query.update = update
-        dc.signals_schema = dc.signals_schema.mutate({f"{object_name}": file_type})
+        dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})
 
         if update or not list_ds_exists:
 
@@ -140,7 +140,7 @@ def read_storage(
                     .settings(prefetch=0)
                     .gen(
                         list_bucket(lst_uri, cache, client_config=client_config),
-                        output={f"{object_name}": file_type},
+                        output={f"{column}": file_type},
                     )
                     .save(ds_name, listing=True)
                 )
@@ -149,7 +149,7 @@ def read_storage(
                 lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
             )
 
-        chain = ls(dc, list_path, recursive=recursive, object_name=object_name)
+        chain = ls(dc, list_path, recursive=recursive, column=column)
 
         storage_chain = storage_chain.union(chain) if storage_chain else chain
         listed_ds_name.add(list_ds_name)
@@ -162,7 +162,7 @@ def read_storage(
             file=file_values,
         )
         file_chain.signals_schema = file_chain.signals_schema.mutate(
-            {f"{object_name}": file_type}
+            {f"{column}": file_type}
         )
         storage_chain = storage_chain.union(file_chain) if storage_chain else file_chain
 

datachain/lib/dc/values.py

@@ -24,7 +24,7 @@ def read_values(
     settings: Optional[dict] = None,
     in_memory: bool = False,
     output: OutputType = None,
-    object_name: str = "",
+    column: str = "",
     **fr_map,
 ) -> "DataChain":
     """Generate chain from list of values.
@@ -48,6 +48,6 @@ def read_values(
         settings=settings,
         in_memory=in_memory,
     )
-    if object_name:
-        output = {object_name: dict_to_data_model(object_name, output)}  # type: ignore[arg-type]
+    if column:
+        output = {column: dict_to_data_model(column, output)}  # type: ignore[arg-type]
     return chain.gen(_func_fr, output=output)

datachain/lib/listing.py

@@ -72,7 +72,7 @@ def ls(
     dc: D,
     path: str,
     recursive: Optional[bool] = True,
-    object_name="file",
+    column="file",
 ) -> D:
     """
     Return files by some path from DataChain instance which contains bucket listing.
@@ -82,7 +82,7 @@ def ls(
     """
 
     def _file_c(name: str) -> Column:
-        return Column(f"{object_name}.{name}")
+        return Column(f"{column}.{name}")
 
     dc = dc.filter(_file_c("is_latest") == true())
 

datachain/lib/signal_schema.py

@@ -87,6 +87,12 @@ class SignalResolvingTypeError(SignalResolvingError):
         )
 
 
+class SignalRemoveError(SignalSchemaError):
+    def __init__(self, path: Optional[list[str]], msg: str):
+        name = " '" + ".".join(path) + "'" if path else ""
+        super().__init__(f"cannot remove signal name{name}: {msg}")
+
+
 class CustomType(BaseModel):
     schema_version: int = Field(ge=1, le=2, strict=True)
     name: str
@@ -575,7 +581,11 @@ class SignalSchema:
         signals = [
             DEFAULT_DELIMITER.join(path)
             if not as_columns
-            else Column(DEFAULT_DELIMITER.join(path), python_to_sql(_type))
+            else Column(
+                DEFAULT_DELIMITER.join(path),
+                python_to_sql(_type),
+                nullable=is_optional(_type),
+            )
             for path, _type, has_subtree, _ in self.get_flat_tree(
                 include_hidden=include_hidden
             )
@@ -620,18 +630,27 @@ class SignalSchema:
         return curr_type
 
     def select_except_signals(self, *args: str) -> "SignalSchema":
-        schema = copy.deepcopy(self.values)
-        for field in args:
-            if not isinstance(field, str):
-                raise SignalResolvingTypeError("select_except()", field)
+        def has_signal(signal: str):
+            signal = signal.replace(".", DEFAULT_DELIMITER)
+            return any(signal == s for s in self.db_signals())
 
-            if field not in self.values:
+        schema = copy.deepcopy(self.values)
+        for signal in args:
+            if not isinstance(signal, str):
+                raise SignalResolvingTypeError("select_except()", signal)
+
+            if signal not in self.values:
+                if has_signal(signal):
+                    raise SignalRemoveError(
+                        signal.split("."),
+                        "select_except() error - removing nested signal would"
+                        " break parent schema, which isn't supported.",
+                    )
                 raise SignalResolvingError(
-                    field.split("."),
-                    "select_except() error - the feature name does not exist or "
-                    "inside of feature (not supported)",
+                    signal.split("."),
+                    "select_except() error - the signal does not exist",
                 )
-            del schema[field]
+            del schema[signal]
 
         return SignalSchema(schema)
 
@@ -975,3 +994,8 @@ class SignalSchema:
             }
 
         return SignalSchema.deserialize(schema)
+
+
+def is_optional(type_: Any) -> bool:
+    """Check if a type is Optional."""
+    return get_origin(type_) is Union and type(None) in get_args(type_)

datachain/listing.py

@@ -27,14 +27,14 @@ class Listing:
         client: "Client",
         dataset_name: Optional["str"] = None,
         dataset_version: Optional[int] = None,
-        object_name: str = "file",
+        column: str = "file",
     ):
         self.metastore = metastore
         self.warehouse = warehouse
         self.client = client
         self.dataset_name = dataset_name  # dataset representing bucket listing
         self.dataset_version = dataset_version  # dataset representing bucket listing
-        self.object_name = object_name
+        self.column = column
 
     def clone(self) -> "Listing":
         return self.__class__(
@@ -43,7 +43,7 @@ class Listing:
             self.client,
             self.dataset_name,
             self.dataset_version,
-            self.object_name,
+            self.column,
         )
 
     def __enter__(self) -> "Listing":
@@ -74,7 +74,7 @@ class Listing:
         return self.warehouse.dataset_rows(
             dataset,
             self.dataset_version or dataset.latest_version,
-            object_name=self.object_name,
+            column=self.column,
         )
 
     def expand_path(self, path, use_glob=True) -> list[Node]:

datachain/query/dataset.py

@@ -437,9 +437,17 @@ class UDFStep(Step, ABC):
                             "distributed processing."
                         )
 
-                    from datachain.catalog.loader import get_udf_distributor_class
+                    from datachain.catalog.loader import (
+                        DISTRIBUTED_IMPORT_PATH,
+                        get_udf_distributor_class,
+                    )
+
+                    if not (udf_distributor_class := get_udf_distributor_class()):
+                        raise RuntimeError(
+                            f"{DISTRIBUTED_IMPORT_PATH} import path is required "
+                            "for distributed UDF processing."
+                        )
 
-                    udf_distributor_class = get_udf_distributor_class()
                     udf_distributor = udf_distributor_class(
                         catalog=catalog,
                         table=udf_table,
@@ -1162,16 +1170,6 @@ class DatasetQuery:
         )
         return sqlalchemy.table(table_name)
 
-    @staticmethod
-    def delete(
-        name: str, version: Optional[int] = None, catalog: Optional["Catalog"] = None
-    ) -> None:
-        from datachain.catalog import get_catalog
-
-        catalog = catalog or get_catalog()
-        version = version or catalog.get_dataset(name).latest_version
-        catalog.remove_dataset(name, version)
-
     @property
     def attached(self) -> bool:
         """

datachain/query/dispatch.py

@@ -13,7 +13,7 @@ from multiprocess import get_context
 
 from datachain.catalog import Catalog
 from datachain.catalog.catalog import clone_catalog_with_cache
-from datachain.catalog.loader import get_udf_distributor_class
+from datachain.catalog.loader import DISTRIBUTED_IMPORT_PATH, get_udf_distributor_class
 from datachain.lib.udf import _get_cache
 from datachain.query.batch import RowsOutput, RowsOutputBatch
 from datachain.query.dataset import (
@@ -91,7 +91,12 @@ def udf_entrypoint() -> int:
 
 
 def udf_worker_entrypoint() -> int:
-    return get_udf_distributor_class().run_worker()
+    if not (udf_distributor_class := get_udf_distributor_class()):
+        raise RuntimeError(
+            f"{DISTRIBUTED_IMPORT_PATH} import path is required "
+            "for distributed UDF processing."
+        )
+    return udf_distributor_class.run_worker()
 
 
 class UDFDispatcher:

datachain/query/schema.py

@@ -40,12 +40,15 @@ class ColumnMeta(type):
 class Column(sa.ColumnClause, metaclass=ColumnMeta):
     inherit_cache: Optional[bool] = True
 
-    def __init__(self, text, type_=None, is_literal=False, _selectable=None):
+    def __init__(
+        self, text, type_=None, is_literal=False, nullable=None, _selectable=None
+    ):
         """Dataset column."""
         self.name = ColumnMeta.to_db_name(text)
         super().__init__(
             self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
         )
+        self.nullable = nullable
 
     def __getattr__(self, name: str):
         return Column(self.name + DEFAULT_DELIMITER + name)

{datachain-0.14.4.dist-info → datachain-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.14.4
+Version: 0.15.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -171,7 +171,7 @@ high confidence scores.
 
     import datachain as dc
 
-    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", column="meta", anon=True)
     images = dc.read_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
 
     images_id = images.map(id=lambda file: file.path.split('.')[-2])
@@ -213,7 +213,7 @@ Python code:
          return result.lower().startswith("success")
 
     chain = (
-       dc.read_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       dc.read_storage("gs://datachain-demo/chatbot-KiT/", column="file", anon=True)
        .settings(parallel=4, cache=True)
        .map(is_success=eval_dialogue)
        .save("mistral_files")