datachain 0.14.4__tar.gz → 0.14.5__tar.gz

{datachain-0.14.4 → datachain-0.14.5}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.11.2'
+    rev: 'v0.11.4'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.14.4/src/datachain.egg-info → datachain-0.14.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.14.4
+Version: 0.14.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -171,7 +171,7 @@ high confidence scores.
 
     import datachain as dc
 
-    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", column="meta", anon=True)
     images = dc.read_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
 
     images_id = images.map(id=lambda file: file.path.split('.')[-2])
@@ -213,7 +213,7 @@ Python code:
          return result.lower().startswith("success")
 
     chain = (
-       dc.read_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       dc.read_storage("gs://datachain-demo/chatbot-KiT/", column="file", anon=True)
        .settings(parallel=4, cache=True)
        .map(is_success=eval_dialogue)
        .save("mistral_files")

{datachain-0.14.4 → datachain-0.14.5}/README.rst

@@ -60,7 +60,7 @@ high confidence scores.
 
     import datachain as dc
 
-    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+    meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", column="meta", anon=True)
     images = dc.read_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
 
     images_id = images.map(id=lambda file: file.path.split('.')[-2])
@@ -102,7 +102,7 @@ Python code:
          return result.lower().startswith("success")
 
     chain = (
-       dc.read_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+       dc.read_storage("gs://datachain-demo/chatbot-KiT/", column="file", anon=True)
        .settings(parallel=4, cache=True)
        .map(is_success=eval_dialogue)
        .save("mistral_files")

{datachain-0.14.4 → datachain-0.14.5}/docs/quick-start.md

@@ -39,7 +39,7 @@ using JSON metadata:
 ``` py
 import datachain as dc
 
-meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta", anon=True)
+meta = dc.read_json("gs://datachain-demo/dogs-and-cats/*json", column="meta", anon=True)
 images = dc.read_storage("gs://datachain-demo/dogs-and-cats/*jpg", anon=True)
 
 images_id = images.map(id=lambda file: file.path.split('.')[-2])
@@ -78,7 +78,7 @@ def is_positive_dialogue_ending(file) -> bool:
 
 chain = (
    dc.read_storage("gs://datachain-demo/chatbot-KiT/",
-                          object_name="file", type="text", anon=True)
+                          column="file", type="text", anon=True)
    .settings(parallel=8, cache=True)
    .map(is_positive=is_positive_dialogue_ending)
    .save("file_response")
@@ -132,7 +132,7 @@ def eval_dialogue(file: dc.File) -> bool:
      return result.lower().startswith("success")
 
 chain = (
-   dc.read_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+   dc.read_storage("gs://datachain-demo/chatbot-KiT/", column="file", anon=True)
    .map(is_success=eval_dialogue)
    .save("mistral_files")
 )
@@ -177,7 +177,7 @@ def eval_dialog(file: dc.File) -> ChatCompletionResponse:
                    {"role": "user", "content": file.read()}])
 
 chain = (
-   dc.read_storage("gs://datachain-demo/chatbot-KiT/", object_name="file", anon=True)
+   dc.read_storage("gs://datachain-demo/chatbot-KiT/", column="file", anon=True)
    .settings(parallel=4, cache=True)
    .map(response=eval_dialog)
    .map(status=lambda response: response.choices[0].message.content.lower()[:7])

{datachain-0.14.4 → datachain-0.14.5}/examples/get_started/json-csv-reader.py

@@ -63,13 +63,13 @@ def main():
 
     # Static CSV with header schema test parsing 3.5K objects
     uri = "gs://datachain-demo/chatbot-csv/"
-    static_csv_ds = dc.read_csv(uri, output=ChatDialog, object_name="chat", anon="True")
+    static_csv_ds = dc.read_csv(uri, output=ChatDialog, column="chat", anon="True")
     static_csv_ds.print_schema()
     static_csv_ds.show()
 
     # Dynamic CSV with header schema test parsing 3/3M objects
     uri = "gs://datachain-demo/laion-aesthetics-csv/laion_aesthetics_1024_33M_1.csv"
-    dynamic_csv_ds = dc.read_csv(uri, object_name="laion", nrows=3, anon="True")
+    dynamic_csv_ds = dc.read_csv(uri, column="laion", nrows=3, anon="True")
     dynamic_csv_ds.print_schema()
     dynamic_csv_ds.show()
 

{datachain-0.14.4 → datachain-0.14.5}/examples/multimodal/hf_pipeline.py

@@ -93,7 +93,7 @@ if __name__ == "__main__":
 
     print("\nAudio emotion classification:")
     try:
-        subprocess.run(["ffmpeg", "-L"], check=True)  # noqa: S603, S607
+        subprocess.run(["ffmpeg", "-L"], check=True)  # noqa: S607
         (
             dc.read_storage(
                 audio_source,

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/catalog/catalog.py

@@ -580,15 +580,13 @@ class Catalog:
         source: str,
         update=False,
         client_config=None,
-        object_name="file",
+        column="file",
         skip_indexing=False,
     ) -> tuple[Optional["Listing"], "Client", str]:
         from datachain import read_storage
         from datachain.listing import Listing
 
-        read_storage(
-            source, session=self.session, update=update, object_name=object_name
-        ).exec()
+        read_storage(source, session=self.session, update=update, column=column).exec()
 
         list_ds_name, list_uri, list_path, _ = get_listing(
             source, self.session, update=update
@@ -602,7 +600,7 @@ class Catalog:
                 self.warehouse.clone(),
                 client,
                 dataset_name=list_ds_name,
-                object_name=object_name,
+                column=column,
             )
 
         return lst, client, list_path

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/data_storage/schema.py

@@ -30,8 +30,8 @@ if TYPE_CHECKING:
 DEFAULT_DELIMITER = "__"
 
 
-def col_name(name: str, object_name: str = "file") -> str:
-    return f"{object_name}{DEFAULT_DELIMITER}{name}"
+def col_name(name: str, column: str = "file") -> str:
+    return f"{column}{DEFAULT_DELIMITER}{name}"
 
 
 def dedup_columns(columns: Iterable[sa.Column]) -> list[sa.Column]:
@@ -84,19 +84,19 @@ def convert_rows_custom_column_types(
 
 
 class DirExpansion:
-    def __init__(self, object_name: str):
-        self.object_name = object_name
+    def __init__(self, column: str):
+        self.column = column
 
-    def col_name(self, name: str, object_name: Optional[str] = None) -> str:
-        object_name = object_name or self.object_name
-        return col_name(name, object_name)
+    def col_name(self, name: str, column: Optional[str] = None) -> str:
+        column = column or self.column
+        return col_name(name, column)
 
-    def c(self, query, name: str, object_name: Optional[str] = None) -> str:
-        return getattr(query.c, self.col_name(name, object_name=object_name))
+    def c(self, query, name: str, column: Optional[str] = None) -> str:
+        return getattr(query.c, self.col_name(name, column=column))
 
     def base_select(self, q):
         return sa.select(
-            self.c(q, "id", object_name="sys"),
+            self.c(q, "id", column="sys"),
             false().label(self.col_name("is_dir")),
             self.c(q, "source"),
             self.c(q, "path"),
@@ -153,12 +153,12 @@ class DataTable:
         name: str,
         engine: "DatabaseEngine",
         column_types: Optional[dict[str, SQLType]] = None,
-        object_name: str = "file",
+        column: str = "file",
     ):
         self.name: str = name
         self.engine = engine
         self.column_types: dict[str, SQLType] = column_types or {}
-        self.object_name = object_name
+        self.column = column
 
     @staticmethod
     def copy_column(
@@ -224,18 +224,16 @@ class DataTable:
     def columns(self) -> "ReadOnlyColumnCollection[str, sa.Column[Any]]":
         return self.table.columns
 
-    def col_name(self, name: str, object_name: Optional[str] = None) -> str:
-        object_name = object_name or self.object_name
-        return col_name(name, object_name)
+    def col_name(self, name: str, column: Optional[str] = None) -> str:
+        column = column or self.column
+        return col_name(name, column)
 
-    def without_object(
-        self, column_name: str, object_name: Optional[str] = None
-    ) -> str:
-        object_name = object_name or self.object_name
-        return column_name.removeprefix(f"{object_name}{DEFAULT_DELIMITER}")
+    def without_object(self, column_name: str, column: Optional[str] = None) -> str:
+        column = column or self.column
+        return column_name.removeprefix(f"{column}{DEFAULT_DELIMITER}")
 
-    def c(self, name: str, object_name: Optional[str] = None):
-        return getattr(self.columns, self.col_name(name, object_name=object_name))
+    def c(self, name: str, column: Optional[str] = None):
+        return getattr(self.columns, self.col_name(name, column=column))
 
     @property
     def table(self) -> "sa.Table":
@@ -275,7 +273,7 @@ class DataTable:
         ]
 
     def dir_expansion(self):
-        return DirExpansion(self.object_name)
+        return DirExpansion(self.column)
 
 
 PARTITION_COLUMN_ID = "partition_id"

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/data_storage/sqlite.py

@@ -489,7 +489,7 @@ class SQLiteWarehouse(AbstractWarehouse):
         self, dataset: DatasetRecord, version: int
     ) -> list[StorageURI]:
         dr = self.dataset_rows(dataset, version)
-        query = dr.select(dr.c("source", object_name="file")).distinct()
+        query = dr.select(dr.c("source", column="file")).distinct()
         cur = self.db.cursor()
         cur.row_factory = sqlite3.Row  # type: ignore[assignment]
 

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/data_storage/warehouse.py

@@ -179,7 +179,7 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         dataset: DatasetRecord,
         version: Optional[int] = None,
-        object_name: str = "file",
+        column: str = "file",
     ):
         version = version or dataset.latest_version
 
@@ -188,7 +188,7 @@ class AbstractWarehouse(ABC, Serializable):
             table_name,
             self.db,
             dataset.get_schema(version),
-            object_name=object_name,
+            column=column,
         )
 
     @property
@@ -487,7 +487,7 @@ class AbstractWarehouse(ABC, Serializable):
         dataset_rows: "DataTable",
         path_list: list[str],
         glob_name: str,
-        object_name="file",
+        column="file",
     ) -> Iterator[Node]:
         """Finds all Nodes that correspond to GLOB like path pattern."""
         dr = dataset_rows
@@ -521,7 +521,7 @@ class AbstractWarehouse(ABC, Serializable):
         de = dr.dir_expansion()
         q = de.query(
             dr.select().where(dr.c("is_latest") == true()).subquery(),
-            object_name=dr.object_name,
+            column=dr.column,
         ).subquery()
         q = self.expand_query(de, q, dr)
 
@@ -597,12 +597,10 @@ class AbstractWarehouse(ABC, Serializable):
             with_default(dr.c("is_latest")),
             dr.c("last_modified"),
             with_default(dr.c("size")),
-            with_default(dr.c("rand", object_name="sys")),
+            with_default(dr.c("rand", column="sys")),
             dr.c("location"),
             de.c(q, "source"),
-        ).select_from(
-            q.outerjoin(dr.table, q.c.sys__id == dr.c("id", object_name="sys"))
-        )
+        ).select_from(q.outerjoin(dr.table, q.c.sys__id == dr.c("id", column="sys")))
 
     def get_node_by_path(self, dataset_rows: "DataTable", path: str) -> Node:
         """Gets node that corresponds to some path"""

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/csv.py

@@ -21,7 +21,7 @@ def read_csv(
     delimiter: Optional[str] = None,
     header: bool = True,
     output: OutputType = None,
-    object_name: str = "",
+    column: str = "",
     model_name: str = "",
     source: bool = True,
     nrows=None,
@@ -42,7 +42,7 @@ def read_csv(
         output : Dictionary or feature class defining column names and their
             corresponding types. List of column names is also accepted, in which
             case types will be inferred.
-        object_name : Created object column name.
+        column : Created column name.
         model_name : Generated model name.
         source : Whether to include info about the source file.
         nrows : Optional row limit.
@@ -119,7 +119,7 @@ def read_csv(
     )
     return chain.parse_tabular(
         output=output,
-        object_name=object_name,
+        column=column,
         model_name=model_name,
         source=source,
         nrows=nrows,

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/datachain.py

@@ -357,7 +357,7 @@ class DataChain:
         self,
         col: str,
         model_name: Optional[str] = None,
-        object_name: Optional[str] = None,
+        column: Optional[str] = None,
         schema_sample_size: int = 1,
     ) -> "DataChain":
         """Explodes a column containing JSON objects (dict or str DataChain type) into
@@ -368,7 +368,7 @@ class DataChain:
             col: the name of the column containing JSON to be exploded.
             model_name: optional generated model name.  By default generates the name
                 automatically.
-            object_name: optional generated object column name. By default generates the
+            column: optional generated column name. By default generates the
                 name automatically.
             schema_sample_size: the number of rows to use for inferring the schema of
                 the JSON (in case some fields are optional and it's not enough to
@@ -406,10 +406,10 @@ class DataChain:
             )
             return model.model_validate(json_dict)
 
-        if not object_name:
-            object_name = f"{col}_expl"
+        if not column:
+            column = f"{col}_expl"
 
-        return self.map(json_to_model, params=col, output={object_name: model})
+        return self.map(json_to_model, params=col, output={column: model})
 
     @classmethod
     def datasets(
@@ -1588,7 +1588,7 @@ class DataChain:
     def parse_tabular(
         self,
         output: OutputType = None,
-        object_name: str = "",
+        column: str = "",
         model_name: str = "",
         source: bool = True,
         nrows: Optional[int] = None,
@@ -1600,7 +1600,7 @@ class DataChain:
             output : Dictionary or feature class defining column names and their
                 corresponding types. List of column names is also accepted, in which
                 case types will be inferred.
-            object_name : Generated object column name.
+            column : Generated column name.
             model_name : Generated model name.
             source : Whether to include info about the source file.
             nrows : Optional row limit.
@@ -1651,14 +1651,14 @@ class DataChain:
                 raise DatasetPrepareError(self.name, e) from e
 
         if isinstance(output, dict):
-            model_name = model_name or object_name or ""
+            model_name = model_name or column or ""
             model = dict_to_data_model(model_name, output)
             output = model
         else:
             model = output  # type: ignore[assignment]
 
-        if object_name:
-            output = {object_name: model}  # type: ignore[dict-item]
+        if column:
+            output = {column: model}  # type: ignore[dict-item]
         elif isinstance(output, type(BaseModel)):
             output = {
                 name: info.annotation  # type: ignore[misc]

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/datasets.py

@@ -1,7 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    Optional,
-)
+from typing import TYPE_CHECKING, Optional, get_origin, get_type_hints
 
 from datachain.lib.dataset_info import DatasetInfo
 from datachain.lib.file import (
@@ -102,7 +99,7 @@ def datasets(
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     in_memory: bool = False,
-    object_name: str = "dataset",
+    column: Optional[str] = None,
     include_listing: bool = False,
     studio: bool = False,
 ) -> "DataChain":
@@ -112,7 +109,8 @@ def datasets(
         session: Optional session instance. If not provided, uses default session.
         settings: Optional dictionary of settings to configure the chain.
         in_memory: If True, creates an in-memory session. Defaults to False.
-        object_name: Name of the output object in the chain. Defaults to "dataset".
+        column: Name of the output column in the chain. Defaults to None which
+            means no top level column will be created.
         include_listing: If True, includes listing datasets. Defaults to False.
         studio: If True, returns datasets from Studio only,
             otherwise returns all local datasets. Defaults to False.
@@ -124,7 +122,7 @@ def datasets(
         ```py
         import datachain as dc
 
-        chain = dc.datasets()
+        chain = dc.datasets(column="dataset")
         for ds in chain.collect("dataset"):
             print(f"{ds.name}@v{ds.version}")
         ```
@@ -139,13 +137,32 @@ def datasets(
             include_listing=include_listing, studio=studio
         )
     ]
-
     datasets_values = [d for d in datasets_values if not d.is_temp]
 
+    if not column:
+        # flattening dataset fields
+        schema = {
+            k: get_origin(v) if get_origin(v) is dict else v
+            for k, v in get_type_hints(DatasetInfo).items()
+            if k in DatasetInfo.model_fields
+        }
+        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
+        for d in [d.model_dump() for d in datasets_values]:
+            for field, value in d.items():
+                data[field].append(value)
+
+        return read_values(
+            session=session,
+            settings=settings,
+            in_memory=in_memory,
+            output=schema,
+            **data,  # type: ignore[arg-type]
+        )
+
     return read_values(
         session=session,
         settings=settings,
         in_memory=in_memory,
-        output={object_name: DatasetInfo},
-        **{object_name: datasets_values},  # type: ignore[arg-type]
+        output={column: DatasetInfo},
+        **{column: datasets_values},  # type: ignore[arg-type]
     )

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/hf.py

@@ -23,7 +23,7 @@ def read_hf(
     *args,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
-    object_name: str = "",
+    column: str = "",
     model_name: str = "",
     **kwargs,
 ) -> "DataChain":
@@ -34,7 +34,7 @@ def read_hf(
             or an instance of `datasets.Dataset`-like object.
         session : Session to use for the chain.
         settings : Settings to use for the chain.
-        object_name : Generated object column name.
+        column : Generated object column name.
         model_name : Generated model name.
         kwargs : Parameters to pass to datasets.load_dataset.
 
@@ -62,12 +62,12 @@ def read_hf(
     if len(ds_dict) > 1:
         output = {"split": str}
 
-    model_name = model_name or object_name or ""
+    model_name = model_name or column or ""
     hf_features = next(iter(ds_dict.values())).features
     output = output | get_output_schema(hf_features)
     model = dict_to_data_model(model_name, output)
-    if object_name:
-        output = {object_name: model}
+    if column:
+        output = {column: model}
 
     chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
     return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/json.py

@@ -28,7 +28,7 @@ def read_json(
     spec: Optional[DataType] = None,
     schema_from: Optional[str] = "auto",
     jmespath: Optional[str] = None,
-    object_name: Optional[str] = "",
+    column: Optional[str] = "",
     model_name: Optional[str] = None,
     format: Optional[str] = "json",
     nrows=None,
@@ -42,7 +42,7 @@ def read_json(
         type : read file as "binary", "text", or "image" data. Default is "text".
         spec : optional Data Model
         schema_from : path to sample to infer spec (if schema not provided)
-        object_name : generated object column name
+        column : generated column name
         model_name : optional generated model name
         format: "json", "jsonl"
         jmespath : optional JMESPATH expression to reduce JSON
@@ -70,13 +70,13 @@ def read_json(
         name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
         return s[:name_end]
 
-    if (not object_name) and jmespath:
-        object_name = jmespath_to_name(jmespath)
-    if not object_name:
-        object_name = format
+    if (not column) and jmespath:
+        column = jmespath_to_name(jmespath)
+    if not column:
+        column = format
     chain = read_storage(uri=path, type=type, **kwargs)
     signal_dict = {
-        object_name: read_meta(
+        column: read_meta(
             schema_from=schema_from,
             format=format,
             spec=spec,

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/listings.py

@@ -19,7 +19,7 @@ if TYPE_CHECKING:
 def listings(
     session: Optional[Session] = None,
     in_memory: bool = False,
-    object_name: str = "listing",
+    column: str = "listing",
     **kwargs,
 ) -> "DataChain":
     """Generate chain with list of cached listings.
@@ -38,6 +38,6 @@ def listings(
     return read_values(
         session=session,
         in_memory=in_memory,
-        output={object_name: ListingInfo},
-        **{object_name: catalog.listings()},  # type: ignore[arg-type]
+        output={column: ListingInfo},
+        **{column: catalog.listings()},  # type: ignore[arg-type]
     )

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/pandas.py

@@ -22,7 +22,7 @@ def read_pandas(  # type: ignore[override]
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     in_memory: bool = False,
-    object_name: str = "",
+    column: str = "",
 ) -> "DataChain":
     """Generate chain from pandas data-frame.
 
@@ -39,18 +39,18 @@ def read_pandas(  # type: ignore[override]
 
     fr_map = {col.lower(): df[col].tolist() for col in df.columns}
 
-    for column in fr_map:
-        if not column.isidentifier():
+    for c in fr_map:
+        if not c.isidentifier():
             raise DatasetPrepareError(
                 name,
-                f"import from pandas error - '{column}' cannot be a column name",
+                f"import from pandas error - '{c}' cannot be a column name",
             )
 
     return read_values(
         name,
         session,
         settings=settings,
-        object_name=object_name,
+        column=column,
         in_memory=in_memory,
         **fr_map,
     )

{datachain-0.14.4 → datachain-0.14.5}/src/datachain/lib/dc/parquet.py

@@ -19,7 +19,7 @@ def read_parquet(
     path,
     partitioning: Any = "hive",
     output: Optional[dict[str, DataType]] = None,
-    object_name: str = "",
+    column: str = "",
     model_name: str = "",
     source: bool = True,
     session: Optional[Session] = None,
@@ -33,7 +33,7 @@ def read_parquet(
             as `s3://`, `gs://`, `az://` or "file:///".
         partitioning : Any pyarrow partitioning schema.
         output : Dictionary defining column names and their corresponding types.
-        object_name : Created object column name.
+        column : Created column name.
         model_name : Generated model name.
         source : Whether to include info about the source file.
         session : Session to use for the chain.
@@ -57,7 +57,7 @@ def read_parquet(
     chain = read_storage(path, session=session, settings=settings, **kwargs)
     return chain.parse_tabular(
         output=output,
-        object_name=object_name,
+        column=column,
         model_name=model_name,
         source=source,
         format="parquet",