datachain 0.16.0__tar.gz → 0.16.1__tar.gz

{datachain-0.16.0/src/datachain.egg-info → datachain-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.16.0
+Version: 0.16.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.16.0 → datachain-0.16.1}/docs/examples.md

@@ -94,7 +94,7 @@ dc.DataModel.register(MistralModel)
 chain = (
     dc
     .read_storage("gs://datachain-demo/chatbot-KiT/", type="text")
-    .filter(dc.Column("file.name").glob("*.txt"))
+    .filter(dc.Column("file.path").glob("*.txt"))
     .limit(5)
     .settings(parallel=4, cache=True)
     .map(
@@ -228,7 +228,7 @@ Here is an example from MS COCO “captions” JSON which employs separate secti
 
 Note how complicated the setup is. Every image is references by the name, and the metadata for this file is keyed by the “id” field. This same field is references later in the “annotations” array, which is present in JSON files describing captions and the detected instances. The categories for the instances are stored in the “categories” array.
 
-However, Datachain can easily parse the entire COCO structure via several reading and merging operators:
+However, DataChain can easily parse the entire COCO structure via several reading and merging operators:
 
 ```python
 import datachain as dc
@@ -240,7 +240,7 @@ images = dc.read_storage(images_uri)
 meta = dc.read_json(captions_uri, jmespath="images")
 captions = dc.read_json(captions_uri, jmespath="annotations")
 
-images_meta = images.merge(meta, on="file.name", right_on="images.file_name")
+images_meta = images.merge(meta, on="file.path", right_on="images.file_name")
 captioned_images = images_meta.merge(captions, on="images.id", right_on="annotations.image_id")
 ```
 
@@ -248,12 +248,12 @@ The resulting dataset has image entries as files decorated with all the metadata
 
 ```python
 images_with_dogs = captioned_images.filter(dc.Column("annotations.caption").glob("*dog*"))
-images_with_dogs.select("annotations", "file.name").show()
+images_with_dogs.select("annotations", "file.path").show()
 ```
 
 ```
    captions captions                                           captions              file
-   image_id       id                                            caption              name
+   image_id       id                                            caption              path
 0     17029   778902         a dog jumping to catch a frisbee in a yard  000000017029.jpg
 1     17029   779838   A dog jumping to catch a red frisbee in a garden  000000017029.jpg
 2     17029   781941  The dog is catching the Frisbee in mid air in ...  000000017029.jpg

{datachain-0.16.0 → datachain-0.16.1}/docs/quick-start.md

@@ -184,7 +184,7 @@ chain = (
    .save("response")
 )
 
-chain.select("file.name", "status", "response.usage").show(5)
+chain.select("file.path", "status", "response.usage").show(5)
 
 success_rate = chain.filter(dc.Column("status") == "success").count() / chain.count()
 print(f"{100*success_rate:.1f}% dialogs were successful")
@@ -194,7 +194,7 @@ Output:
 
 ``` shell
 file   status      response     response          response
-name                  usage        usage             usage
+path                  usage        usage             usage
               prompt_tokens total_tokens completion_tokens
 0   1.txt  success           547          548                 1
 1  10.txt  failure          3576         3578                 2
@@ -277,7 +277,7 @@ processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
 
 chain = (
     dc.read_storage("gs://datachain-demo/dogs-and-cats/", type="image", anon=True)
-    .map(label=lambda name: name.split(".")[0], params=["file.name"])
+    .map(label=lambda name: name.split(".")[0], params=["file.path"])
     .select("file", "label").to_pytorch(
         transform=processor.image_processor,
         tokenizer=processor.tokenizer,

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/func/aggregate.py

@@ -165,7 +165,7 @@ def any_value(col: str) -> Func:
     Example:
         ```py
         dc.group_by(
-            file_example=func.any_value("file.name"),
+            file_example=func.any_value("file.path"),
             partition_by="signal.category",
         )
         ```
@@ -227,7 +227,7 @@ def concat(col: str, separator="") -> Func:
     Example:
         ```py
         dc.group_by(
-            files=func.concat("file.name", separator=", "),
+            files=func.concat("file.path", separator=", "),
             partition_by="signal.category",
         )
         ```
@@ -343,7 +343,7 @@ def first(col: str) -> Func:
         ```py
         window = func.window(partition_by="signal.category", order_by="created_at")
         dc.mutate(
-            first_file=func.first("file.name").over(window),
+            first_file=func.first("file.path").over(window),
         )
         ```
 

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/lib/convert/values_to_tuples.py

@@ -1,6 +1,6 @@
 import itertools
 from collections.abc import Sequence
-from typing import Any, Optional, Union
+from typing import Any, Union
 
 from datachain.lib.data_model import (
     DataType,
@@ -71,14 +71,13 @@ def values_to_tuples(  # noqa: C901, PLR0912
             # If a non-None value appears early, it won't check the remaining items for
             # `None` values.
             try:
-                pos, first_not_none_element = next(
-                    itertools.dropwhile(lambda pair: pair[1] is None, enumerate(v))
+                first_not_none_element = next(
+                    itertools.dropwhile(lambda i: i is None, v)
                 )
             except StopIteration:
-                typ = str  # default to str if all values are None or has length 0
-                nullable = True
+                # set default type to `str` if column is empty or all values are `None`
+                typ = str
             else:
-                nullable = pos > 0
                 typ = type(first_not_none_element)  # type: ignore[assignment]
                 if not is_chain_type(typ):
                     raise ValuesToTupleError(
@@ -88,8 +87,7 @@ def values_to_tuples(  # noqa: C901, PLR0912
                     )
                 if isinstance(first_not_none_element, list):
                     typ = list[type(first_not_none_element[0])]  # type: ignore[assignment, misc]
-
-            types_map[k] = Optional[typ] if nullable else typ  # type: ignore[assignment]
+            types_map[k] = typ
 
         if length < 0:
             length = len_

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/lib/dc/datachain.py

@@ -756,7 +756,7 @@ class DataChain:
 
         Example:
             ```py
-            dc.distinct("file.parent", "file.name")
+            dc.distinct("file.path")
             ```
         """
         return self._evolve(
@@ -882,7 +882,7 @@ class DataChain:
         ```py
          dc.mutate(
             area=Column("image.height") * Column("image.width"),
-            extension=file_ext(Column("file.name")),
+            extension=file_ext(Column("file.path")),
             dist=cosine_distance(embedding_text, embedding_image)
         )
         ```
@@ -1071,13 +1071,13 @@ class DataChain:
 
             Iterating over all rows with selected columns:
             ```py
-            for name, size in dc.collect("file.name", "file.size"):
+            for name, size in dc.collect("file.path", "file.size"):
                 print(name, size)
             ```
 
             Iterating over a single column:
             ```py
-            for file in dc.collect("file.name"):
+            for file in dc.collect("file.path"):
                 print(file)
             ```
         """
@@ -1630,7 +1630,7 @@ class DataChain:
             import datachain as dc
 
             chain = dc.read_storage("s3://mybucket")
-            chain = chain.filter(dc.C("file.name").glob("*.jsonl"))
+            chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
             chain = chain.parse_tabular(format="json")
             ```
         """
@@ -2089,25 +2089,31 @@ class DataChain:
 
             Using glob to match patterns
             ```py
-            dc.filter(C("file.name").glob("*.jpg"))
+            dc.filter(C("file.path").glob("*.jpg"))
+            ```
+
+            Using in to match lists
+            ```py
+            ids = [1,2,3]
+            dc.filter(C("experiment_id").in_(ids))
             ```
 
             Using `datachain.func`
             ```py
             from datachain.func import string
-            dc.filter(string.length(C("file.name")) > 5)
+            dc.filter(string.length(C("file.path")) > 5)
             ```
 
             Combining filters with "or"
             ```py
-            dc.filter(C("file.name").glob("cat*") | C("file.name").glob("dog*))
+            dc.filter(C("file.path").glob("cat*") | C("file.path").glob("dog*))
             ```
 
             Combining filters with "and"
             ```py
             dc.filter(
-                C("file.name").glob("*.jpg) &
-                (string.length(C("file.name")) > 5)
+                C("file.path").glob("*.jpg) &
+                (string.length(C("file.path")) > 5)
             )
             ```
         """

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/lib/dc/records.py

@@ -4,12 +4,9 @@ from typing import TYPE_CHECKING, Optional, Union
 import sqlalchemy
 
 from datachain.lib.data_model import DataType
-from datachain.lib.file import (
-    File,
-)
+from datachain.lib.file import File
 from datachain.lib.signal_schema import SignalSchema
 from datachain.query import Session
-from datachain.query.schema import Column
 
 if TYPE_CHECKING:
     from typing_extensions import ParamSpec
@@ -41,6 +38,9 @@ def read_records(
         single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
         ```
     """
+    from datachain.query.dataset import adjust_outputs, get_col_types
+    from datachain.sql.types import SQLType
+
     from .datasets import read_dataset
 
     session = Session.get(session, in_memory=in_memory)
@@ -52,11 +52,10 @@ def read_records(
 
     if schema:
         signal_schema = SignalSchema(schema)
-        columns = []
-        for c in signal_schema.db_signals(as_columns=True):
-            assert isinstance(c, Column)
-            kw = {"nullable": c.nullable} if c.nullable is not None else {}
-            columns.append(sqlalchemy.Column(c.name, c.type, **kw))
+        columns = [
+            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
+            for c in signal_schema.db_signals(as_columns=True)
+        ]
     else:
         columns = [
             sqlalchemy.Column(name, typ)
@@ -83,6 +82,13 @@ def read_records(
     warehouse = catalog.warehouse
     dr = warehouse.dataset_rows(dsr)
     table = dr.get_table()
-    warehouse.insert_rows(table, to_insert)
+
+    # Optimization: Compute row types once, rather than for every row.
+    col_types = get_col_types(
+        warehouse,
+        {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
+    )
+    records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
+    warehouse.insert_rows(table, records)
     warehouse.insert_rows_done(table)
     return read_dataset(name=dsr.name, session=session, settings=settings)

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/lib/dc/utils.py

@@ -31,8 +31,8 @@ def resolve_columns(
 ) -> "Callable[Concatenate[D, P], D]":
     """Decorator that resolvs input column names to their actual DB names. This is
     specially important for nested columns as user works with them by using dot
-    notation e.g (file.name) but are actually defined with default delimiter
-    in DB, e.g file__name.
+    notation e.g (file.path) but are actually defined with default delimiter
+    in DB, e.g file__path.
     If there are any sql functions in arguments, they will just be transferred as is
     to a method.
     """

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/lib/signal_schema.py

@@ -581,11 +581,7 @@ class SignalSchema:
         signals = [
             DEFAULT_DELIMITER.join(path)
             if not as_columns
-            else Column(
-                DEFAULT_DELIMITER.join(path),
-                python_to_sql(_type),
-                nullable=is_optional(_type),
-            )
+            else Column(DEFAULT_DELIMITER.join(path), python_to_sql(_type))
             for path, _type, has_subtree, _ in self.get_flat_tree(
                 include_hidden=include_hidden
             )
@@ -994,8 +990,3 @@ class SignalSchema:
             }
 
         return SignalSchema.deserialize(schema)
-
-
-def is_optional(type_: Any) -> bool:
-    """Check if a type is Optional."""
-    return get_origin(type_) is Union and type(None) in get_args(type_)

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/query/dataset.py

@@ -57,6 +57,7 @@ from datachain.query.schema import C, UDFParamSpec, normalize_param
 from datachain.query.session import Session
 from datachain.query.udf import UdfInfo
 from datachain.sql.functions.random import rand
+from datachain.sql.types import SQLType
 from datachain.utils import (
     batched,
     determine_processes,
@@ -67,6 +68,8 @@ from datachain.utils import (
 )
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from sqlalchemy.sql.elements import ClauseElement
     from sqlalchemy.sql.schema import Table
     from sqlalchemy.sql.selectable import GenerativeSelect
@@ -273,7 +276,9 @@ class Subtract(DatasetDiffOperation):
 
 
 def adjust_outputs(
-    warehouse: "AbstractWarehouse", row: dict[str, Any], udf_col_types: list[tuple]
+    warehouse: "AbstractWarehouse",
+    row: dict[str, Any],
+    col_types: list[tuple[str, SQLType, type, str, Any]],
 ) -> dict[str, Any]:
     """
     This function does a couple of things to prepare a row for inserting into the db:
@@ -289,7 +294,7 @@ def adjust_outputs(
         col_python_type,
         col_type_name,
         default_value,
-    ) in udf_col_types:
+    ) in col_types:
         row_val = row.get(col_name)
 
         # Fill None or missing values with defaults (get returns None if not in the row)
@@ -304,8 +309,10 @@ def adjust_outputs(
     return row
 
 
-def get_udf_col_types(warehouse: "AbstractWarehouse", udf: "UDFAdapter") -> list[tuple]:
-    """Optimization: Precompute UDF column types so these don't have to be computed
+def get_col_types(
+    warehouse: "AbstractWarehouse", output: "Mapping[str, Any]"
+) -> list[tuple]:
+    """Optimization: Precompute column types so these don't have to be computed
     in the convert_type function for each row in a loop."""
     dialect = warehouse.db.dialect
     return [
@@ -317,7 +324,7 @@ def get_udf_col_types(warehouse: "AbstractWarehouse", udf: "UDFAdapter") -> list
             type(col_type_inst).__name__,
             col_type.default_value(dialect),
         )
-        for col_name, col_type in udf.output.items()
+        for col_name, col_type in output.items()
     ]
 
 
@@ -333,7 +340,7 @@ def process_udf_outputs(
 
     rows: list[UDFResult] = []
     # Optimization: Compute row types once, rather than for every row.
-    udf_col_types = get_udf_col_types(warehouse, udf)
+    udf_col_types = get_col_types(warehouse, udf.output)
 
     for udf_output in udf_results:
         if not udf_output:

{datachain-0.16.0 → datachain-0.16.1}/src/datachain/query/schema.py

@@ -40,15 +40,12 @@ class ColumnMeta(type):
 class Column(sa.ColumnClause, metaclass=ColumnMeta):
     inherit_cache: Optional[bool] = True
 
-    def __init__(
-        self, text, type_=None, is_literal=False, nullable=None, _selectable=None
-    ):
+    def __init__(self, text, type_=None, is_literal=False, _selectable=None):
         """Dataset column."""
         self.name = ColumnMeta.to_db_name(text)
         super().__init__(
             self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
         )
-        self.nullable = nullable
 
     def __getattr__(self, name: str):
         return Column(self.name + DEFAULT_DELIMITER + name)

{datachain-0.16.0 → datachain-0.16.1/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.16.0
+Version: 0.16.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.16.0 → datachain-0.16.1}/tests/func/test_read_database.py

@@ -1,15 +1,15 @@
+import json
 import os
 import sqlite3
 from contextlib import closing
-from typing import Optional
 
 import pytest
 import sqlalchemy
 from sqlalchemy.orm import Session
 
 from datachain import read_database
+from datachain.data_storage.sqlite import SQLiteWarehouse
 from datachain.lib.dc import database
-from tests.utils import skip_if_not_sqlite
 
 
 @pytest.fixture
@@ -81,14 +81,7 @@ def test(sqlite3_connection, connection, test_session):
     ]
 
 
-# FIXME: `clickhouse` requires wrapping column types in `Nullable` to make the column
-# nullable, setting `nullable=True` is not enough.
-# https://github.com/xzkostyan/clickhouse-sqlalchemy/issues/189#issuecomment-1274736713
-# Also, was not able to figure out how to read nullable columns back from clickhouse.
-
-
-@skip_if_not_sqlite
-def test_nullable(sqlite3_connection, test_session):
+def test_nullable(sqlite3_connection, test_session, warehouse):
     """
     Verify that a column containing a sequence of NULL values is handled correctly
     when the number of leading NULLs is less than `infer_schema_length`.
@@ -101,14 +94,14 @@ def test_nullable(sqlite3_connection, test_session):
     sqlite3_connection.commit()
 
     chain = read_database("select * from tbl", sqlite3_connection, session=test_session)
-    assert chain.schema == {"id": int, "value": Optional[str]}
+    assert chain.schema == {"id": int, "value": str}
+    default_value = None if isinstance(warehouse, SQLiteWarehouse) else ""
     assert sorted(chain.to_records(), key=lambda r: r["id"]) == [
-        {"id": i, "value": None if i < 50 else str(i)} for i in range(1, 1000)
+        {"id": i, "value": default_value if i < 50 else str(i)} for i in range(1, 1000)
     ]
 
 
-@skip_if_not_sqlite
-def test_all_null_values(sqlite3_connection, test_session):
+def test_all_null_values(sqlite3_connection, test_session, warehouse):
     sqlite3_connection.execute("CREATE TABLE tbl (id INTEGER PRIMARY KEY, num INTEGER)")
     sqlite3_connection.executemany(
         "INSERT INTO tbl(num) VALUES(?)", [(None,) for _ in range(1, 1000)]
@@ -117,9 +110,10 @@ def test_all_null_values(sqlite3_connection, test_session):
 
     chain = read_database("select * from tbl", sqlite3_connection, session=test_session)
     # if all values are null, the column type defaults to str
-    assert chain.schema == {"id": int, "num": Optional[str]}
+    assert chain.schema == {"id": int, "num": str}
+    default_value = None if isinstance(warehouse, SQLiteWarehouse) else ""
     assert sorted(chain.to_records(), key=lambda r: r["id"]) == [
-        {"id": i, "num": None} for i in range(1, 1000)
+        {"id": i, "num": default_value} for i in range(1, 1000)
     ]
 
 
@@ -128,7 +122,7 @@ def test_empty(sqlite3_connection, test_session):
 
     chain = read_database("select * from tbl", sqlite3_connection, session=test_session)
     # if the table is empty, the column type defaults to str
-    assert chain.schema == {"id": Optional[str], "value": Optional[str]}
+    assert chain.schema == {"id": str, "value": str}
     assert chain.to_records() == []
 
 
@@ -173,3 +167,23 @@ def test_schema_is_not_inferred_when_all_types_are_provided(
     )
     spy.assert_called_once_with(mocker.ANY, [], 100)
     assert chain.schema == {"id": int, "value": int}
+
+
+def test_json_type(sqlite3_connection, test_session):
+    sqlite3_connection.execute("CREATE TABLE tbl (id INTEGER PRIMARY KEY, value TEXT)")
+    sqlite3_connection.executemany(
+        "INSERT INTO tbl(value) VALUES(?)",
+        [(json.dumps({"i": i}),) for i in range(1, 10)],
+    )
+    sqlite3_connection.commit()
+
+    chain = read_database(
+        "select * from tbl",
+        sqlite3_connection,
+        output={"value": dict},
+        session=test_session,
+    )
+    assert chain.schema == {"id": int, "value": dict}
+    assert sorted(chain.to_records(), key=lambda r: r["id"]) == [
+        {"id": i, "value": {"i": i}} for i in range(1, 10)
+    ]