datachain 0.36.1__tar.gz → 0.36.3__tar.gz

{datachain-0.36.1 → datachain-0.36.3}/.github/workflows/tests.yml

@@ -110,7 +110,7 @@ jobs:
 
       - name: Install FFmpeg on Windows
         if: runner.os == 'Windows'
-        run: choco install ffmpeg
+        run: choco install ffmpeg --version 7.0
 
       - name: Install FFmpeg on macOS
         if: runner.os == 'macOS'

{datachain-0.36.1 → datachain-0.36.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.36.1
+Version: 0.36.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -76,6 +76,7 @@ Requires-Dist: datasets[vision]>=4.0.0; extra == "hf"
 Requires-Dist: datasets[audio]>=4.0.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Requires-Dist: torch<2.9.0; extra == "hf"
+Requires-Dist: torchcodec<0.8.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"
 Requires-Dist: imageio[ffmpeg,pyav]>=2.37.0; extra == "video"

{datachain-0.36.1 → datachain-0.36.3}/pyproject.toml

@@ -91,7 +91,8 @@ hf = [
   # Until datasets solve the issue, run test_hf_audio test to see if this can be removed
   # https://github.com/meta-pytorch/torchcodec/issues/912
   # https://github.com/huggingface/transformers/pull/41610
-  "torch<2.9.0"
+  "torch<2.9.0",
+  "torchcodec<0.8.0 ; (sys_platform == 'linux' or sys_platform == 'darwin')"
 ]
 video = [
   "ffmpeg-python",
@@ -139,6 +140,8 @@ examples = [
   "open_clip_torch",
   "openai",
   # Transformers still require it
+  # Remove this after HF tests and example work w/o these restrictions,
+  # make them >= probably
   "torchaudio<2.9.0"
 ]
 

{datachain-0.36.1 → datachain-0.36.3}/src/datachain/data_storage/metastore.py

@@ -56,13 +56,15 @@ from datachain.project import Project
 from datachain.utils import JSONSerialize
 
 if TYPE_CHECKING:
-    from sqlalchemy import Delete, Insert, Select, Update
+    from sqlalchemy import CTE, Delete, Insert, Select, Subquery, Update
     from sqlalchemy.schema import SchemaItem
+    from sqlalchemy.sql.elements import ColumnElement
 
     from datachain.data_storage import schema
     from datachain.data_storage.db_engine import DatabaseEngine
 
 logger = logging.getLogger("datachain")
+DEPTH_LIMIT_DEFAULT = 100
 
 
 class AbstractMetastore(ABC, Serializable):
@@ -1463,6 +1465,18 @@ class AbstractDBMetastore(AbstractMetastore):
         Returns a list of columns to select in a query for fetching dataset dependencies
         """
 
+    @abstractmethod
+    def _dataset_dependency_nodes_select_columns(
+        self,
+        namespaces_subquery: "Subquery",
+        dependency_tree_cte: "CTE",
+        datasets_subquery: "Subquery",
+    ) -> list["ColumnElement"]:
+        """
+        Returns a list of columns to select in a query for fetching
+        dataset dependency nodes.
+        """
+
     def get_direct_dataset_dependencies(
         self, dataset: DatasetRecord, version: str
     ) -> list[DatasetDependency | None]:
@@ -1493,7 +1507,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return [self.dependency_class.parse(*r) for r in self.db.execute(query)]
 
     def get_dataset_dependency_nodes(
-        self, dataset_id: int, version_id: int
+        self, dataset_id: int, version_id: int, depth_limit: int = DEPTH_LIMIT_DEFAULT
     ) -> list[DatasetDependencyNode | None]:
         n = self._namespaces_select().subquery()
         p = self._projects
@@ -1522,33 +1536,31 @@ class AbstractDBMetastore(AbstractMetastore):
         cte = base_query.cte(name="dependency_tree", recursive=True)
 
         # Recursive case: dependencies of dependencies
-        recursive_query = select(
-            *dep_fields,
-            (cte.c.depth + 1).label("depth"),
-        ).select_from(
-            cte.join(
-                dd,
-                (cte.c.dataset_id == dd.c.source_dataset_id)
-                & (cte.c.dataset_version_id == dd.c.source_dataset_version_id),
+        # Limit depth to 100 to prevent infinite loops in case of circular dependencies
+        recursive_query = (
+            select(
+                *dep_fields,
+                (cte.c.depth + 1).label("depth"),
             )
+            .select_from(
+                cte.join(
+                    dd,
+                    (cte.c.dataset_id == dd.c.source_dataset_id)
+                    & (cte.c.dataset_version_id == dd.c.source_dataset_version_id),
+                )
+            )
+            .where(cte.c.depth < depth_limit)
         )
 
         cte = cte.union(recursive_query)
 
         # Fetch all with full details
-        final_query = select(
-            n.c.name,
-            p.c.name,
-            cte.c.id,
-            cte.c.dataset_id,
-            cte.c.dataset_version_id,
-            d.c.name,
-            dv.c.version,
-            dv.c.created_at,
-            cte.c.source_dataset_id,
-            cte.c.source_dataset_version_id,
-            cte.c.depth,
-        ).select_from(
+        select_cols = self._dataset_dependency_nodes_select_columns(
+            namespaces_subquery=n,
+            dependency_tree_cte=cte,
+            datasets_subquery=d,
+        )
+        final_query = self._datasets_dependencies_select(*select_cols).select_from(
             # Use outer joins to handle cases where dependent datasets have been
             # physically deleted. This allows us to return dependency records with
             # None values instead of silently omitting them, making broken

{datachain-0.36.1 → datachain-0.36.3}/src/datachain/data_storage/sqlite.py

@@ -20,7 +20,10 @@ from sqlalchemy import (
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.schema import CreateIndex, CreateTable, DropTable
 from sqlalchemy.sql import func
-from sqlalchemy.sql.elements import BinaryExpression, BooleanClauseList
+from sqlalchemy.sql.elements import (
+    BinaryExpression,
+    BooleanClauseList,
+)
 from sqlalchemy.sql.expression import bindparam, cast
 from sqlalchemy.sql.selectable import Select
 from tqdm.auto import tqdm
@@ -41,6 +44,7 @@ from datachain.sql.types import SQLType
 from datachain.utils import DataChainDir, batched, batched_it
 
 if TYPE_CHECKING:
+    from sqlalchemy import CTE, Subquery
     from sqlalchemy.dialects.sqlite import Insert
     from sqlalchemy.engine.base import Engine
     from sqlalchemy.schema import SchemaItem
@@ -539,6 +543,26 @@ class SQLiteMetastore(AbstractDBMetastore):
             self._datasets_versions.c.created_at,
         ]
 
+    def _dataset_dependency_nodes_select_columns(
+        self,
+        namespaces_subquery: "Subquery",
+        dependency_tree_cte: "CTE",
+        datasets_subquery: "Subquery",
+    ) -> list["ColumnElement"]:
+        return [
+            namespaces_subquery.c.name,
+            self._projects.c.name,
+            dependency_tree_cte.c.id,
+            dependency_tree_cte.c.dataset_id,
+            dependency_tree_cte.c.dataset_version_id,
+            datasets_subquery.c.name,
+            self._datasets_versions.c.version,
+            self._datasets_versions.c.created_at,
+            dependency_tree_cte.c.source_dataset_id,
+            dependency_tree_cte.c.source_dataset_version_id,
+            dependency_tree_cte.c.depth,
+        ]
+
     #
     # Jobs
     #

{datachain-0.36.1 → datachain-0.36.3}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.36.1
+Version: 0.36.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -76,6 +76,7 @@ Requires-Dist: datasets[vision]>=4.0.0; extra == "hf"
 Requires-Dist: datasets[audio]>=4.0.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Requires-Dist: torch<2.9.0; extra == "hf"
+Requires-Dist: torchcodec<0.8.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"
 Requires-Dist: imageio[ffmpeg,pyav]>=2.37.0; extra == "video"

{datachain-0.36.1 → datachain-0.36.3}/src/datachain.egg-info/requires.txt

@@ -76,6 +76,7 @@ torch<2.9.0
 
 [hf:sys_platform == "linux" or sys_platform == "darwin"]
 datasets[audio]>=4.0.0
+torchcodec<0.8.0
 
 [postgres]
 psycopg2-binary>=2.9.0

{datachain-0.36.1 → datachain-0.36.3}/tests/func/test_hf.py

@@ -27,7 +27,7 @@ def require_torchcodec(test_case):
     """
     if not importlib.util.find_spec("torchcodec"):
         test_case = pytest.mark.skip(
-            "test requires torchcoded, not available on Windows yet"
+            "test requires torchcodec, not available on Windows yet"
         )(test_case)
     return test_case
 

{datachain-0.36.1 → datachain-0.36.3}/tests/unit/lib/test_datachain.py

@@ -1072,6 +1072,19 @@ def test_select_except(test_session):
     assert n == len(features_nested)
 
 
+def test_select_except_after_gen(test_session):
+    # https://github.com/iterative/datachain/issues/1359
+    # fixed by https://github.com/iterative/datachain/pull/1400
+    chain = dc.read_values(id=range(10), session=test_session)
+
+    chain = chain.gen(lambda id: [(id, 0)], output={"id": int, "x": int})
+    chain = chain.select_except("x")
+    chain = chain.merge(chain, on="id")
+    chain = chain.select_except("right_id")
+
+    assert set(chain.to_values("id")) == set(range(10))
+
+
 def test_select_wrong_type(test_session):
     chain = dc.read_values(fr1=features_nested, fr2=features, session=test_session)