datachain 0.2.6__tar.gz → 0.2.8__tar.gz

{datachain-0.2.6/src/datachain.egg-info → datachain-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.6
+Version: 0.2.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.6 → datachain-0.2.8}/src/datachain/__init__.py

@@ -2,8 +2,6 @@ from datachain.lib.dc import C, DataChain
 from datachain.lib.feature import Feature
 from datachain.lib.feature_utils import pydantic_to_feature
 from datachain.lib.file import File, FileError, FileFeature, IndexedFile, TarVFile
-from datachain.lib.image import ImageFile, convert_images
-from datachain.lib.text import convert_text
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
 from datachain.query.dataset import UDF as BaseUDF  # noqa: N811
@@ -23,12 +21,9 @@ __all__ = [
     "FileError",
     "FileFeature",
     "Generator",
-    "ImageFile",
     "IndexedFile",
     "Mapper",
     "Session",
     "TarVFile",
-    "convert_images",
-    "convert_text",
     "pydantic_to_feature",
 ]

datachain-0.2.8/src/datachain/image/__init__.py

@@ -0,0 +1,3 @@
+from datachain.lib.image import ImageFile, convert_images
+
+__all__ = ["ImageFile", "convert_images"]

datachain-0.2.8/src/datachain/text/__init__.py

@@ -0,0 +1,3 @@
+from datachain.lib.text import convert_text
+
+__all__ = ["convert_text"]

{datachain-0.2.6 → datachain-0.2.8/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.6
+Version: 0.2.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.6 → datachain-0.2.8}/src/datachain.egg-info/SOURCES.txt

@@ -125,6 +125,7 @@ src/datachain/data_storage/schema.py
 src/datachain/data_storage/serializer.py
 src/datachain/data_storage/sqlite.py
 src/datachain/data_storage/warehouse.py
+src/datachain/image/__init__.py
 src/datachain/lib/__init__.py
 src/datachain/lib/arrow.py
 src/datachain/lib/cached_stream.py
@@ -181,6 +182,7 @@ src/datachain/sql/sqlite/__init__.py
 src/datachain/sql/sqlite/base.py
 src/datachain/sql/sqlite/types.py
 src/datachain/sql/sqlite/vector.py
+src/datachain/text/__init__.py
 tests/__init__.py
 tests/conftest.py
 tests/data.py
@@ -234,6 +236,7 @@ tests/unit/test_udf.py
 tests/unit/test_utils.py
 tests/unit/test_warehouse.py
 tests/unit/lib/__init__.py
+tests/unit/lib/conftest.py
 tests/unit/lib/test_arrow.py
 tests/unit/lib/test_clip.py
 tests/unit/lib/test_datachain.py

{datachain-0.2.6 → datachain-0.2.8}/tests/func/test_pytorch.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import open_clip
 import pytest
 from torch import Size, Tensor
@@ -8,20 +10,23 @@ from datachain.lib.dc import DataChain
 from datachain.lib.pytorch import PytorchDataset
 
 
-@pytest.fixture
-def fake_dataset(tmp_path, catalog):
+@pytest.fixture(scope="module")
+def fake_dir(tmpdir_factory):
     # Create fake images in labeled dirs
-    data_path = tmp_path / "data" / ""
+    data_path = Path(tmpdir_factory.mktemp("data"))
     for i, (img, label) in enumerate(FakeData()):
         label = str(label)
         (data_path / label).mkdir(parents=True, exist_ok=True)
         img.save(data_path / label / f"{i}.jpg")
 
     # Create dataset from images
-    uri = data_path.as_uri()
+    return data_path.as_uri()
 
+
+@pytest.fixture
+def fake_dataset(fake_dir):
     return (
-        DataChain.from_storage(uri, type="image")
+        DataChain.from_storage(fake_dir, type="image")
         .map(text=lambda file: file.parent.split("/")[-1], output=str)
         .map(label=lambda text: int(text), output=int)
         .save("fake")
@@ -37,11 +42,11 @@ def test_pytorch_dataset(fake_dataset):
         transform=transform,
         tokenizer=tokenizer,
     )
-    for img, text, label in pt_dataset:
-        assert isinstance(img, Tensor)
-        assert isinstance(text, Tensor)
-        assert isinstance(label, int)
-        assert img.size() == Size([3, 64, 64])
+    img, text, label = next(iter(pt_dataset))
+    assert isinstance(img, Tensor)
+    assert isinstance(text, Tensor)
+    assert isinstance(label, int)
+    assert img.size() == Size([3, 64, 64])
 
 
 def test_pytorch_dataset_sample(fake_dataset):
@@ -62,8 +67,8 @@ def test_to_pytorch(fake_dataset):
     tokenizer = open_clip.get_tokenizer("ViT-B-32")
     pt_dataset = fake_dataset.to_pytorch(transform=transform, tokenizer=tokenizer)
     assert isinstance(pt_dataset, IterableDataset)
-    for img, text, label in pt_dataset:
-        assert isinstance(img, Tensor)
-        assert isinstance(text, Tensor)
-        assert isinstance(label, int)
-        assert img.size() == Size([3, 64, 64])
+    img, text, label = next(iter(pt_dataset))
+    assert isinstance(img, Tensor)
+    assert isinstance(text, Tensor)
+    assert isinstance(label, int)
+    assert img.size() == Size([3, 64, 64])

datachain-0.2.8/tests/unit/lib/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+import torch
+from torch import float32
+from torchvision.transforms import v2
+
+
+@pytest.fixture(scope="session")
+def fake_clip_model():
+    class Model:
+        def encode_image(self, tensor):
+            return torch.randn(len(tensor), 512)
+
+        def encode_text(self, tensor):
+            return torch.randn(len(tensor), 512)
+
+    def tokenizer(tensor, context_length=77):
+        return torch.randn(len(tensor), context_length)
+
+    model = Model()
+    preprocess = v2.ToDtype(float32, scale=True)
+    return model, preprocess, tokenizer

{datachain-0.2.6 → datachain-0.2.8}/tests/unit/lib/test_clip.py

@@ -1,4 +1,3 @@
-import open_clip
 import pytest
 from PIL import Image
 from transformers import CLIPModel, CLIPProcessor
@@ -7,10 +6,6 @@ from datachain.lib.clip import similarity_scores
 
 IMAGES = [Image.new(mode="RGB", size=(64, 64)), Image.new(mode="RGB", size=(32, 32))]
 TEXTS = ["text1", "text2"]
-MODEL, _, PREPROCESS = open_clip.create_model_and_transforms(
-    "ViT-B-32", pretrained="laion2b_s34b_b79k"
-)
-TOKENIZER = open_clip.get_tokenizer("ViT-B-32")
 
 
 @pytest.mark.parametrize(
@@ -20,15 +15,16 @@ TOKENIZER = open_clip.get_tokenizer("ViT-B-32")
 @pytest.mark.parametrize("text", [None, "text", TEXTS])
 @pytest.mark.parametrize("prob", [True, False])
 @pytest.mark.parametrize("image_to_text", [True, False])
-def test_similarity_scores(images, text, prob, image_to_text):
+def test_similarity_scores(fake_clip_model, images, text, prob, image_to_text):
+    model, preprocess, tokenizer = fake_clip_model
     if not (images or text):
         with pytest.raises(ValueError):
             scores = similarity_scores(
-                images, text, MODEL, PREPROCESS, TOKENIZER, prob, image_to_text
+                images, text, model, preprocess, tokenizer, prob, image_to_text
             )
     else:
         scores = similarity_scores(
-            images, text, MODEL, PREPROCESS, TOKENIZER, prob, image_to_text
+            images, text, model, preprocess, tokenizer, prob, image_to_text
         )
         assert isinstance(scores, list)
         if not images:

{datachain-0.2.6 → datachain-0.2.8}/tests/unit/lib/test_text.py

@@ -1,4 +1,3 @@
-import open_clip
 import torch
 from transformers import CLIPModel, CLIPProcessor
 
@@ -6,10 +5,9 @@ from datachain.lib.file import TextFile
 from datachain.lib.text import convert_text
 
 
-def test_convert_text():
+def test_convert_text(fake_clip_model):
     text = "thisismytext"
-    tokenizer_model = "ViT-B-32"
-    tokenizer = open_clip.get_tokenizer(tokenizer_model)
+    model, _, tokenizer = fake_clip_model
     converted_text = convert_text(text, tokenizer=tokenizer)
     assert isinstance(converted_text, torch.Tensor)
 
@@ -22,7 +20,6 @@ def test_convert_text():
     converted_text = convert_text(
         text, tokenizer=tokenizer, tokenizer_kwargs=tokenizer_kwargs
     )
-    model, _, _ = open_clip.create_model_and_transforms(tokenizer_model)
     converted_text = convert_text(text, tokenizer=tokenizer, encoder=model.encode_text)
     assert converted_text.dtype == torch.float32
 

{datachain-0.2.6 → datachain-0.2.8}/tests/unit/test_module_exports.py

@@ -18,14 +18,13 @@ def test_module_exports():
             FileError,
             FileFeature,
             Generator,
-            ImageFile,
             IndexedFile,
             Mapper,
             Session,
             TarVFile,
-            convert_images,
-            convert_text,
             pydantic_to_feature,
         )
+        from datachain.image import ImageFile, convert_images
+        from datachain.text import convert_text
     except Exception as e:  # noqa: BLE001
         pytest.fail(f"Importing raised an exception: {e}")