datachain 0.3.5__tar.gz → 0.3.6__tar.gz

{datachain-0.3.5/src/datachain.egg-info → datachain-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.5
+Version: 0.3.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.3.5 → datachain-0.3.6}/src/datachain/catalog/catalog.py

@@ -120,13 +120,25 @@ def noop(_: str):
 
 @contextmanager
 def print_and_capture(
-    stream: "IO[str]", callback: Callable[[str], None] = noop
+    stream: "IO[bytes]|IO[str]", callback: Callable[[str], None] = noop
 ) -> "Iterator[list[str]]":
     lines: list[str] = []
     append = lines.append
 
     def loop() -> None:
-        for line in iter(stream.readline, ""):
+        buffer = b""
+        while byt := stream.read(1):  # Read one byte at a time
+            buffer += byt.encode("utf-8") if isinstance(byt, str) else byt
+
+            if byt in (b"\n", b"\r"):  # Check for newline or carriage return
+                line = buffer.decode("utf-8")
+                print(line, end="")
+                callback(line)
+                append(line)
+                buffer = b""  # Clear buffer for next line
+
+        if buffer:  # Handle any remaining data in the buffer
+            line = buffer.decode("utf-8")
             print(line, end="")
             callback(line)
             append(line)
@@ -2128,7 +2140,7 @@ class Catalog:
             stdout=subprocess.PIPE if capture_output else None,
             stderr=subprocess.STDOUT if capture_output else None,
             bufsize=1,
-            text=True,
+            text=False,
             **kwargs,
         ) as proc:
             os.close(w)

{datachain-0.3.5 → datachain-0.3.6}/src/datachain/data_storage/sqlite.py

@@ -209,6 +209,7 @@ class SQLiteDatabaseEngine(DatabaseEngine):
             return cursor.executemany(self.compile(query).string, params)
         return self.db.executemany(self.compile(query).string, params)
 
+    @retry_sqlite_locks
     def execute_str(self, sql: str, parameters=None) -> sqlite3.Cursor:
         if parameters is None:
             return self.db.execute(sql)

{datachain-0.3.5 → datachain-0.3.6}/src/datachain/lib/dc.py

@@ -839,6 +839,10 @@ class DataChain(DatasetQuery):
     def mutate(self, **kwargs) -> "Self":
         """Create new signals based on existing signals.
 
+        This method cannot modify existing columns. If you need to modify an
+        existing column, use a different name for the new column and then use
+        `select()` to choose which columns to keep.
+
         This method is vectorized and more efficient compared to map(), and it does not
         extract or download any data from the internal database. However, it can only
         utilize predefined built-in functions and their combinations.
@@ -859,7 +863,26 @@ class DataChain(DatasetQuery):
                 dist=cosine_distance(embedding_text, embedding_image)
         )
         ```
+
+        This method can be also used to rename signals. If the Column("name") provided
+        as value for the new signal - the old column will be dropped. Otherwise a new
+        column is created.
+
+        Example:
+        ```py
+         dc.mutate(
+                newkey=Column("oldkey")
+        )
+        ```
         """
+        existing_columns = set(self.signals_schema.values.keys())
+        for col_name in kwargs:
+            if col_name in existing_columns:
+                raise DataChainColumnError(
+                    col_name,
+                    "Cannot modify existing column with mutate(). "
+                    "Use a different name for the new column.",
+                )
         for col_name, expr in kwargs.items():
             if not isinstance(expr, Column) and isinstance(expr.type, NullType):
                 raise DataChainColumnError(

{datachain-0.3.5 → datachain-0.3.6/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.5
+Version: 0.3.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.3.5 → datachain-0.3.6}/tests/func/test_datachain.py

@@ -8,10 +8,11 @@ import pandas as pd
 import pytest
 import pytz
 from PIL import Image
+from sqlalchemy import Column
 
 from datachain.data_storage.sqlite import SQLiteWarehouse
 from datachain.dataset import DatasetStats
-from datachain.lib.dc import DataChain
+from datachain.lib.dc import DataChain, DataChainColumnError
 from datachain.lib.file import File, ImageFile
 from tests.utils import images_equal
 
@@ -314,3 +315,16 @@ def test_from_storage_check_rows(tmp_dir, test_session):
             location=None,
             vtype="",
         )
+
+
+def test_mutate_existing_column(catalog):
+    ds = DataChain.from_values(ids=[1, 2, 3])
+
+    with pytest.raises(DataChainColumnError) as excinfo:
+        ds.mutate(ids=Column("ids") + 1)
+
+    assert (
+        str(excinfo.value)
+        == "Error for column ids: Cannot modify existing column with mutate()."
+        " Use a different name for the new column."
+    )