datachain 0.6.5__tar.gz → 0.6.7__tar.gz

{datachain-0.6.5 → datachain-0.6.7}/.github/workflows/tests-studio.yml

@@ -52,14 +52,11 @@ jobs:
           STUDIO_READ_ACCESS_TOKEN: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
         run: |
           echo "DataChain branch: $BRANCH"
-          if [[ "$BRANCH" == "main" ]]
-          then
-              STUDIO_BRANCH=develop
-          elif git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/iterative/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null
+          if git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/iterative/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null
           then
               STUDIO_BRANCH="$BRANCH"
           else
-              STUDIO_BRANCH=develop
+              STUDIO_BRANCH=main
           fi
           echo "STUDIO_BRANCH=$STUDIO_BRANCH" >> $GITHUB_ENV
           echo "Studio branch: $STUDIO_BRANCH"

{datachain-0.6.5 → datachain-0.6.7}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.1'
+    rev: 'v0.7.2'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.6.5/src/datachain.egg-info → datachain-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.5
+Version: 0.6.7
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.6.5 → datachain-0.6.7}/overrides/main.html

@@ -4,6 +4,7 @@
 
 {{ super() }}
 
+<script data-domain="docs.datachain.ai" src="https://plausible.io/js/script.outbound-links.js"></script>
 <script type="text/javascript">
   !function () { var e, t, n; e = "14ffd92a6cbf5f2", t = function () { Reo.init({ clientID: "14ffd92a6cbf5f2" }) }, (n = document.createElement("script")).src = "https://static.reo.dev/" + e + "/reo.js", n.async = !0, n.onload = t, document.head.appendChild(n) }();
 </script>

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/catalog/catalog.py

@@ -58,7 +58,7 @@ from datachain.listing import Listing
 from datachain.node import DirType, Node, NodeWithPath
 from datachain.nodes_thread_pool import NodesThreadPool
 from datachain.remote.studio import StudioClient
-from datachain.sql.types import DateTime, SQLType, String
+from datachain.sql.types import DateTime, SQLType
 from datachain.utils import (
     DataChainDir,
     batched,
@@ -196,11 +196,6 @@ class DatasetRowsFetcher(NodesThreadPool):
         for c in [c for c, t in self.schema.items() if t == DateTime]:
             df[c] = pd.to_datetime(df[c], unit="s")
 
-        # strings are represented as binaries in parquet export so need to
-        # decode it back to strings
-        for c in [c for c, t in self.schema.items() if t == String]:
-            df[c] = df[c].str.decode("utf-8")
-
     def do_task(self, urls):
         import lz4.frame
         import pandas as pd
@@ -1403,6 +1398,7 @@ class Catalog:
             query_script=remote_dataset_version.query_script,
             create_rows=True,
             columns=columns,
+            feature_schema=remote_dataset_version.feature_schema,
             validate_version=False,
         )
 

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/data_storage/schema.py

@@ -145,6 +145,8 @@ class DirExpansion:
 
 
 class DataTable:
+    MAX_RANDOM = 2**63 - 1
+
     def __init__(
         self,
         name: str,
@@ -269,8 +271,8 @@ class DataTable:
     def delete(self):
         return self.apply_conditions(self.table.delete())
 
-    @staticmethod
-    def sys_columns():
+    @classmethod
+    def sys_columns(cls):
         return [
             sa.Column("sys__id", Int, primary_key=True),
             sa.Column(

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/lib/dc.py

@@ -981,10 +981,23 @@ class DataChain:
 
     @resolve_columns
     def order_by(self, *args, descending: bool = False) -> "Self":
-        """Orders by specified set of signals.
+        """Orders by specified set of columns.
 
         Parameters:
             descending (bool): Whether to sort in descending order or not.
+
+        Example:
+            ```py
+            dc.order_by("similarity_score", descending=True).limit(10)
+            ```
+
+        Note:
+            Order is not guaranteed when steps are added after an `order_by` statement.
+            I.e. when using `from_dataset` an `order_by` statement should be used if
+            the order of the records in the chain is important.
+            Using `order_by` directly before `limit`, `collect` and `collect_flatten`
+            will give expected results.
+            See https://github.com/iterative/datachain/issues/477 for further details.
         """
         if descending:
             args = tuple(sqlalchemy.desc(a) for a in args)
@@ -1179,7 +1192,7 @@ class DataChain:
                           a tuple of row values.
         """
         db_signals = self._effective_signals_schema.db_signals()
-        with self._query.select(*db_signals).as_iterable() as rows:
+        with self._query.ordered_select(*db_signals).as_iterable() as rows:
             if row_factory:
                 rows = (row_factory(db_signals, r) for r in rows)
             yield from rows
@@ -1270,7 +1283,7 @@ class DataChain:
         chain = self.select(*cols) if cols else self
         signals_schema = chain._effective_signals_schema
         db_signals = signals_schema.db_signals()
-        with self._query.select(*db_signals).as_iterable() as rows:
+        with self._query.ordered_select(*db_signals).as_iterable() as rows:
             for row in rows:
                 ret = signals_schema.row_to_features(
                     row, catalog=chain.session.catalog, cache=chain._settings.cache
@@ -1678,7 +1691,7 @@ class DataChain:
 
         model_name = model_name or object_name or ""
         hf_features = next(iter(ds_dict.values())).features
-        output = output | get_output_schema(hf_features, model_name)
+        output = output | get_output_schema(hf_features)
         model = dict_to_data_model(model_name, output)
         if object_name:
             output = {object_name: model}

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/lib/hf.py

@@ -138,17 +138,15 @@ def convert_feature(val: Any, feat: Any, anno: Any) -> Any:  # noqa: PLR0911
         return HFAudio(**val)
 
 
-def get_output_schema(
-    features: Features, model_name: str = "", stream: bool = True
-) -> dict[str, DataType]:
+def get_output_schema(features: Features) -> dict[str, DataType]:
     """Generate UDF output schema from huggingface datasets features."""
     fields_dict = {}
     for name, val in features.items():
-        fields_dict[name] = _feature_to_chain_type(name, val)  # type: ignore[assignment]
-    return fields_dict  # type: ignore[return-value]
+        fields_dict[name] = _feature_to_chain_type(name, val)
+    return fields_dict
 
 
-def _feature_to_chain_type(name: str, val: Any) -> type:  # noqa: PLR0911
+def _feature_to_chain_type(name: str, val: Any) -> DataType:  # noqa: PLR0911
     if isinstance(val, Value):
         return arrow_type_mapper(val.pa_type)
     if isinstance(val, ClassLabel):

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/query/dataset.py

@@ -1276,6 +1276,27 @@ class DatasetQuery:
         query.steps.append(SQLSelect((*args, *named_args)))
         return query
 
+    @detach
+    def ordered_select(self, *args, **kwargs) -> "Self":
+        """
+        Select the given columns or expressions using a subquery whilst
+        maintaining query ordering (only applicable if last step was order_by).
+
+        If used with no arguments, this simply creates a subquery and
+        select all columns from it.
+
+        Example:
+            >>> ds.ordered_select(C.name, C.size * 10)
+            >>> ds.ordered_select(C.name, size10x=C.size * 10)
+        """
+        named_args = [v.label(k) for k, v in kwargs.items()]
+        query = self.clone()
+        order_by = query.last_step if query.is_ordered else None
+        query.steps.append(SQLSelect((*args, *named_args)))
+        if order_by:
+            query.steps.append(order_by)
+        return query
+
     @detach
     def select_except(self, *args) -> "Self":
         """
@@ -1338,7 +1359,7 @@ class DatasetQuery:
         query = self.clone(new_table=False)
         if (
             query.steps
-            and (last_step := query.steps[-1])
+            and (last_step := query.last_step)
             and isinstance(last_step, SQLLimit)
         ):
             query.steps[-1] = SQLLimit(min(n, last_step.n))
@@ -1591,3 +1612,11 @@ class DatasetQuery:
         finally:
             self.cleanup()
         return self.__class__(name=name, version=version, catalog=self.catalog)
+
+    @property
+    def is_ordered(self) -> bool:
+        return isinstance(self.last_step, SQLOrderBy)
+
+    @property
+    def last_step(self) -> Optional[Step]:
+        return self.steps[-1] if self.steps else None

{datachain-0.6.5 → datachain-0.6.7}/src/datachain/sql/types.py

@@ -187,6 +187,22 @@ class Int32(Int):
         return read_converter(dialect).int32(value)
 
 
+class UInt32(Int):
+    def load_dialect_impl(self, dialect):
+        return converter(dialect).uint32()
+
+    @staticmethod
+    def default_value(dialect):
+        return type_defaults(dialect).uint32()
+
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).uint32()
+
+    def on_read_convert(self, value, dialect):
+        return read_converter(dialect).uint32(value)
+
+
 class Int64(Int):
     def load_dialect_impl(self, dialect):
         return converter(dialect).int64()
@@ -395,6 +411,9 @@ class TypeReadConverter:
     def int32(self, value):
         return value
 
+    def uint32(self, value):
+        return value
+
     def int64(self, value):
         return value
 
@@ -421,6 +440,8 @@ class TypeReadConverter:
 
     def json(self, value):
         if isinstance(value, str):
+            if value == "":
+                return {}
             return orjson.loads(value)
         return value
 
@@ -446,6 +467,9 @@ class TypeConverter:
     def int32(self):
         return self.int()
 
+    def uint32(self):
+        return self.int()
+
     def int64(self):
         return self.int()
 
@@ -487,6 +511,9 @@ class TypeDefaults:
     def int32(self):
         return None
 
+    def uint32(self):
+        return None
+
     def int64(self):
         return None
 
@@ -528,6 +555,9 @@ class DBDefaults:
     def int32(self):
         return self.int()
 
+    def uint32(self):
+        return self.int()
+
     def int64(self):
         return self.int()
 
@@ -561,6 +591,7 @@ TYPES = [
     Boolean,
     Int,
     Int32,
+    UInt32,
     Int64,
     UInt64,
     Float,

{datachain-0.6.5 → datachain-0.6.7/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.5
+Version: 0.6.7
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.6.5 → datachain-0.6.7}/tests/examples/test_examples.py

@@ -19,8 +19,12 @@ llm_and_nlp_examples = sorted(
     [
         filename
         for filename in glob.glob("examples/llm_and_nlp/**/*.py", recursive=True)
-        # no anthropic token
+        # no anthropic token, HF runs against actual API - thus run it only once
         if "claude" not in filename
+        and (
+            "hf-" not in filename
+            or (sys.platform == "darwin" and sys.version_info >= (3, 12))
+        )
     ]
 )
 

{datachain-0.6.5 → datachain-0.6.7}/tests/func/test_datachain.py

@@ -448,6 +448,30 @@ def test_show_no_truncate(capsys, test_session):
         assert details[i] in normalized_output
 
 
+@pytest.mark.parametrize("ordered_by", ["letter", "number"])
+def test_show_ordered(capsys, test_session, ordered_by):
+    numbers = [6, 2, 3, 1, 5, 7, 4]
+    letters = ["u", "y", "x", "z", "v", "t", "w"]
+
+    DataChain.from_values(
+        number=numbers, letter=letters, session=test_session
+    ).order_by(ordered_by).show()
+
+    captured = capsys.readouterr()
+    normalized_lines = [
+        re.sub(r"\s+", " ", line).strip() for line in captured.out.strip().split("\n")
+    ]
+
+    ordered_entries = sorted(
+        zip(numbers, letters), key=lambda x: x[0 if ordered_by == "number" else 1]
+    )
+
+    assert normalized_lines[0].strip() == "number letter"
+    for i, line in enumerate(normalized_lines[1:]):
+        number, letter = ordered_entries[i]
+        assert line == f"{i} {number} {letter}"
+
+
 def test_from_storage_dataset_stats(tmp_dir, test_session):
     for i in range(4):
         (tmp_dir / f"file{i}.txt").write_text(f"file{i}")

{datachain-0.6.5 → datachain-0.6.7}/tests/func/test_datasets.py

@@ -5,7 +5,7 @@ from unittest.mock import ANY
 import pytest
 import sqlalchemy as sa
 
-from datachain.data_storage.sqlite import SQLiteWarehouse
+from datachain.data_storage.schema import DataTable
 from datachain.dataset import DatasetDependencyType, DatasetStatus
 from datachain.error import (
     DatasetInvalidVersionError,
@@ -827,10 +827,7 @@ def test_row_random(cloud_test_catalog):
     # Random values are unique
     assert len(set(random_values)) == len(random_values)
 
-    if isinstance(catalog.warehouse, SQLiteWarehouse):
-        RAND_MAX = 2**63  # noqa: N806
-    else:
-        RAND_MAX = 2**64  # noqa: N806
+    RAND_MAX = DataTable.MAX_RANDOM  # noqa: N806
 
     # Values are drawn uniformly from range(2**63)
     assert 0 <= min(random_values) < 0.4 * RAND_MAX

{datachain-0.6.5 → datachain-0.6.7}/tests/func/test_listing.py

@@ -17,7 +17,7 @@ def test_listing_generator(cloud_test_catalog, cloud_type):
     entries = sorted(
         [e for e in ENTRIES if e.path.startswith("cats/")], key=lambda e: e.path
     )
-    files = sorted(dc.collect("file"), key=lambda f: f.path)
+    files = dc.order_by("file.path").collect("file")
 
     for cat_file, cat_entry in zip(files, entries):
         assert cat_file.source == ctc.src_uri

{datachain-0.6.5 → datachain-0.6.7}/tests/func/test_pull.py

@@ -49,19 +49,17 @@ def dog_entries_parquet_lz4(dog_entries) -> bytes:
         """
         adapted = {}
         for k, v in row.items():
-            if isinstance(v, str):
-                adapted[k] = v.encode("utf-8")
-            elif isinstance(v, datetime):
+            if isinstance(v, datetime):
                 adapted[k] = v.timestamp()
             elif v is None:
-                adapted[k] = b""
+                adapted[k] = ""
             else:
                 adapted[k] = v
 
         adapted["sys__id"] = 1
         adapted["sys__rand"] = 1
-        adapted["file__location"] = b""
-        adapted["file__source"] = b"s3://dogs"
+        adapted["file__location"] = ""
+        adapted["file__source"] = "s3://dogs"
         return adapted
 
     dog_entries = [_adapt_row(e) for e in dog_entries]

{datachain-0.6.5 → datachain-0.6.7}/tests/unit/lib/test_datachain.py

@@ -1824,6 +1824,32 @@ def test_order_by_with_nested_columns(test_session, with_function):
     ]
 
 
+def test_order_by_collect(test_session):
+    numbers = [6, 2, 3, 1, 5, 7, 4]
+    letters = ["u", "y", "x", "z", "v", "t", "w"]
+
+    dc = DataChain.from_values(number=numbers, letter=letters, session=test_session)
+    assert list(dc.order_by("number").collect()) == [
+        (1, "z"),
+        (2, "y"),
+        (3, "x"),
+        (4, "w"),
+        (5, "v"),
+        (6, "u"),
+        (7, "t"),
+    ]
+
+    assert list(dc.order_by("letter").collect()) == [
+        (7, "t"),
+        (6, "u"),
+        (5, "v"),
+        (4, "w"),
+        (3, "x"),
+        (2, "y"),
+        (1, "z"),
+    ]
+
+
 @pytest.mark.parametrize("with_function", [True, False])
 def test_order_by_descending(test_session, with_function):
     names = ["a.txt", "c.txt", "d.txt", "a.txt", "b.txt"]
@@ -1852,7 +1878,7 @@ def test_union(test_session):
     chain2 = DataChain.from_values(value=[3, 4], session=test_session)
     chain3 = chain1 | chain2
     assert chain3.count() == 4
-    assert sorted(chain3.collect("value")) == [1, 2, 3, 4]
+    assert list(chain3.order_by("value").collect("value")) == [1, 2, 3, 4]
 
 
 def test_union_different_columns(test_session):
@@ -1887,7 +1913,7 @@ def test_union_different_column_order(test_session):
     chain2 = DataChain.from_values(
         name=["different", "order"], value=[9, 10], session=test_session
     )
-    assert sorted(chain1.union(chain2).collect()) == [
+    assert list(chain1.union(chain2).order_by("value").collect()) == [
         (1, "chain"),
         (2, "more"),
         (9, "different"),

{datachain-0.6.5 → datachain-0.6.7}/tests/unit/lib/test_signal_schema.py

@@ -27,6 +27,7 @@ from datachain.sql.types import (
     Int32,
     Int64,
     String,
+    UInt32,
     UInt64,
 )
 
@@ -721,6 +722,7 @@ def test_mutate_change_type():
         [Boolean, bool],
         [Int, int],
         [Int32, int],
+        [UInt32, int],
         [Int64, int],
         [UInt64, int],
         [Float, float],

{datachain-0.6.5 → datachain-0.6.7}/tests/unit/test_data_storage.py

@@ -18,6 +18,7 @@ from datachain.sql.types import (
     Int32,
     Int64,
     String,
+    UInt32,
     UInt64,
 )
 from tests.utils import (
@@ -173,6 +174,7 @@ def test_convert_type(cloud_test_catalog):
         [Boolean(), False],
         [Int(), 0],
         [Int32(), 0],
+        [UInt32(), 0],
         [Int64(), 0],
         [UInt64(), 0],
         [Float(), lambda val: math.isnan(val)],