datachain 0.6.2__tar.gz → 0.6.3__tar.gz

{datachain-0.6.2 → datachain-0.6.3}/.github/workflows/tests-studio.yml

@@ -101,6 +101,6 @@ jobs:
           pytest
           --config-file=pyproject.toml -rs
           --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
-          -m 'not benchmark'
+          --benchmark-skip
           tests ../datachain/tests
         working-directory: backend/datachain_server

{datachain-0.6.2 → datachain-0.6.3}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.0'
+    rev: 'v0.7.1'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.6.2/src/datachain.egg-info → datachain-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.2
+Version: 0.6.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -82,7 +82,7 @@ Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.12.1; extra == "dev"
+Requires-Dist: mypy==1.13.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"

{datachain-0.6.2 → datachain-0.6.3}/noxfile.py

@@ -22,8 +22,7 @@ def bench(session: nox.Session) -> None:
     session.install(".[tests]")
     session.run(
         "pytest",
-        "-m",
-        "benchmark",
+        "--benchmark-only",
         "--benchmark-group-by",
         "func",
         *session.posargs,

{datachain-0.6.2 → datachain-0.6.3}/pyproject.toml

@@ -94,7 +94,7 @@ tests = [
 ]
 dev = [
   "datachain[docs,tests]",
-  "mypy==1.12.1",
+  "mypy==1.13.0",
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
@@ -127,9 +127,8 @@ namespaces = false
 [tool.setuptools_scm]
 
 [tool.pytest.ini_options]
-addopts = "-rfEs -m 'not benchmark and not examples'"
+addopts = "-rfEs -m 'not examples' --benchmark-skip"
 markers = [
-  "benchmark: benchmarks.",
   "e2e: End-to-end tests",
   "examples: All examples",
   "computer_vision: Computer vision examples",
@@ -214,6 +213,7 @@ ignore = [
 select = [
   "B",  # flake8-bugbear
   "C4",  # flake8-comprehensions
+  "C420",  # unnecessary-dict-comprehension-for-iterable
   "C90",  # mccabe
   "W",  # pycodestyle - Warning
   "E",  # pycodestyle - Error
@@ -252,11 +252,10 @@ select = [
   "NPY",  # numpy
   "TRY004",  # type-check-without-type-error
   "TRY201",  # verbose-raise
-  "TRY302",  # useless-try-except
+  "TRY203",  # useless-try-except
   "TRY401",  # verbose-log-message
   "RUF022",  # unsorted-dunder-all
   "RUF023",  # unsorted-dunder-slots
-  "RUF025",  # unnecessary-dict-comprehension-for-iterable
   "RUF027",  # missing-f-string-syntax
   "RUF030",  # assert-with-print-message
   "RUF101",  # redirected-noqa

{datachain-0.6.2 → datachain-0.6.3}/src/datachain/lib/arrow.py

@@ -1,4 +1,3 @@
-import re
 from collections.abc import Sequence
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Optional
@@ -13,6 +12,7 @@ from datachain.lib.file import ArrowRow, File
 from datachain.lib.model_store import ModelStore
 from datachain.lib.signal_schema import SignalSchema
 from datachain.lib.udf import Generator
+from datachain.lib.utils import normalize_col_names
 
 if TYPE_CHECKING:
     from datasets.features.features import Features
@@ -128,7 +128,7 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
     signal_schema = _get_datachain_schema(schema)
     if signal_schema:
         return signal_schema.values
-    columns = _convert_col_names(col_names)  # type: ignore[arg-type]
+    columns = list(normalize_col_names(col_names).keys())  # type: ignore[arg-type]
     hf_schema = _get_hf_schema(schema)
     if hf_schema:
         return {
@@ -143,19 +143,6 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
     return output
 
 
-def _convert_col_names(col_names: Sequence[str]) -> list[str]:
-    default_column = 0
-    converted_col_names = []
-    for column in col_names:
-        column = column.lower()
-        column = re.sub("[^0-9a-z_]+", "", column)
-        if not column:
-            column = f"c{default_column}"
-            default_column += 1
-        converted_col_names.append(column)
-    return converted_col_names
-
-
 def arrow_type_mapper(col_type: pa.DataType, column: str = "") -> type:  # noqa: PLR0911
     """Convert pyarrow types to basic types."""
     from datetime import datetime

{datachain-0.6.2 → datachain-0.6.3}/src/datachain/lib/data_model.py

@@ -2,9 +2,10 @@ from collections.abc import Sequence
 from datetime import datetime
 from typing import ClassVar, Union, get_args, get_origin
 
-from pydantic import BaseModel, create_model
+from pydantic import BaseModel, Field, create_model
 
 from datachain.lib.model_store import ModelStore
+from datachain.lib.utils import normalize_col_names
 
 StandardType = Union[
     type[int],
@@ -60,7 +61,14 @@ def is_chain_type(t: type) -> bool:
 
 
 def dict_to_data_model(name: str, data_dict: dict[str, DataType]) -> type[BaseModel]:
-    fields = {name: (anno, ...) for name, anno in data_dict.items()}
+    # Gets a map of a normalized_name -> original_name
+    columns = normalize_col_names(list(data_dict.keys()))
+    # We reverse if for convenience to original_name -> normalized_name
+    columns = {v: k for k, v in columns.items()}
+
+    fields = {
+        columns[name]: (anno, Field(alias=name)) for name, anno in data_dict.items()
+    }
     return create_model(
         name,
         __base__=(DataModel,),  # type: ignore[call-overload]

datachain-0.6.3/src/datachain/lib/utils.py

@@ -0,0 +1,60 @@
+import re
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+
+
+class AbstractUDF(ABC):
+    @abstractmethod
+    def process(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def setup(self):
+        pass
+
+    @abstractmethod
+    def teardown(self):
+        pass
+
+
+class DataChainError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class DataChainParamsError(DataChainError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class DataChainColumnError(DataChainParamsError):
+    def __init__(self, col_name, msg):
+        super().__init__(f"Error for column {col_name}: {msg}")
+
+
+def normalize_col_names(col_names: Sequence[str]) -> dict[str, str]:
+    gen_col_counter = 0
+    new_col_names = {}
+    org_col_names = set(col_names)
+
+    for org_column in col_names:
+        new_column = org_column.lower()
+        new_column = re.sub("[^0-9a-z]+", "_", new_column)
+        new_column = new_column.strip("_")
+
+        generated_column = new_column
+
+        while (
+            not generated_column.isidentifier()
+            or generated_column in new_col_names
+            or (generated_column != org_column and generated_column in org_col_names)
+        ):
+            if new_column:
+                generated_column = f"c{gen_col_counter}_{new_column}"
+            else:
+                generated_column = f"c{gen_col_counter}"
+            gen_col_counter += 1
+
+        new_col_names[generated_column] = org_column
+
+    return new_col_names

{datachain-0.6.2 → datachain-0.6.3/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.2
+Version: 0.6.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -82,7 +82,7 @@ Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.12.1; extra == "dev"
+Requires-Dist: mypy==1.13.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"

{datachain-0.6.2 → datachain-0.6.3}/src/datachain.egg-info/requires.txt

@@ -30,7 +30,7 @@ dvc-studio-client<1,>=0.21
 
 [dev]
 datachain[docs,tests]
-mypy==1.12.1
+mypy==1.13.0
 types-python-dateutil
 types-pytz
 types-PyYAML

datachain-0.6.3/tests/benchmarks/conftest.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def datasets():
+    return Path(__file__).parent / "datasets"

{datachain-0.6.2 → datachain-0.6.3}/tests/benchmarks/test_datachain.py

@@ -1,10 +1,7 @@
-import pytest
-
 from datachain.lib.dc import DataChain
 from datachain.lib.webdataset_laion import process_laion_meta
 
 
-@pytest.mark.benchmark
 def test_datachain(tmp_dir, test_session, datasets, benchmark):
     def run_script(uri, **kwargs):
         DataChain.from_storage(uri, session=test_session, **kwargs).gen(

datachain-0.6.3/tests/benchmarks/test_ls.py

@@ -0,0 +1,6 @@
+from datachain.cli import ls
+
+
+def test_ls(benchmark, tmp_dir):
+    bucket = "s3://noaa-bathymetry-pds/"
+    benchmark.pedantic(ls, args=([bucket],), kwargs={"client_config": {"anon": True}})

datachain-0.6.3/tests/benchmarks/test_version.py

@@ -0,0 +1,7 @@
+import shutil
+import subprocess
+
+
+def test_version(benchmark):
+    bin = shutil.which("datachain")
+    benchmark(subprocess.check_call, [bin, "--help"])

{datachain-0.6.2 → datachain-0.6.3}/tests/unit/lib/test_arrow.py

@@ -168,13 +168,21 @@ def test_parquet_convert_column_names():
             ("dot.notation.col", pa.int32()),
             ("with-dashes", pa.int32()),
             ("with spaces", pa.int32()),
+            ("with-multiple--dashes", pa.int32()),
+            ("with__underscores", pa.int32()),
+            ("__leading__underscores", pa.int32()),
+            ("trailing__underscores__", pa.int32()),
         ]
     )
     assert list(schema_to_output(schema)) == [
         "uppercasecol",
-        "dotnotationcol",
-        "withdashes",
-        "withspaces",
+        "dot_notation_col",
+        "with_dashes",
+        "with_spaces",
+        "with_multiple_dashes",
+        "with_underscores",
+        "leading_underscores",
+        "trailing_underscores",
     ]
 
 

{datachain-0.6.2 → datachain-0.6.3}/tests/unit/lib/test_datachain.py

@@ -36,6 +36,18 @@ DF_DATA = {
     "city": ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"],
 }
 
+DF_DATA_NESTED_NOT_NORMALIZED = {
+    "nAmE": [
+        {"first-SELECT": "Alice", "l--as@t": "Smith"},
+        {"l--as@t": "Jones", "first-SELECT": "Bob"},
+        {"first-SELECT": "Charlie", "l--as@t": "Brown"},
+        {"first-SELECT": "David", "l--as@t": "White"},
+        {"first-SELECT": "Eva", "l--as@t": "Black"},
+    ],
+    "AgE": [25, 30, 35, 40, 45],
+    "citY": ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"],
+}
+
 DF_OTHER_DATA = {
     "last_name": ["Smith", "Jones"],
     "country": ["USA", "Russia"],
@@ -272,7 +284,9 @@ def test_listings(test_session, tmp_dir):
     assert listing.expires
     assert listing.version == 1
     assert listing.num_objects == 1
-    assert listing.size == 2912
+    # Exact number if unreliable here since it depends on the PyArrow version
+    assert listing.size > 1000
+    assert listing.size < 5000
     assert listing.status == 4
 
 
@@ -988,6 +1002,25 @@ def test_parse_tabular_format(tmp_dir, test_session):
     assert df1.equals(df)
 
 
+def test_parse_nested_json(tmp_dir, test_session):
+    df = pd.DataFrame(DF_DATA_NESTED_NOT_NORMALIZED)
+    path = tmp_dir / "test.jsonl"
+    path.write_text(df.to_json(orient="records", lines=True))
+    dc = DataChain.from_storage(path.as_uri(), session=test_session).parse_tabular(
+        format="json"
+    )
+    # Field names are normalized, values are preserved
+    # E.g. nAmE -> name, l--as@t -> l_as_t, etc
+    df1 = dc.select("name", "age", "city").to_pandas()
+
+    assert df1["name"]["first_select"].to_list() == [
+        d["first-SELECT"] for d in df["nAmE"].to_list()
+    ]
+    assert df1["name"]["l_as_t"].to_list() == [
+        d["l--as@t"] for d in df["nAmE"].to_list()
+    ]
+
+
 def test_parse_tabular_partitions(tmp_dir, test_session):
     df = pd.DataFrame(DF_DATA)
     path = tmp_dir / "test.parquet"

datachain-0.6.3/tests/unit/lib/test_utils.py

@@ -0,0 +1,128 @@
+from collections.abc import Iterable, Mapping
+from typing import Literal, Optional, Union
+
+import pytest
+from pydantic import BaseModel
+
+from datachain.lib.convert.python_to_sql import python_to_sql
+from datachain.lib.utils import normalize_col_names
+from datachain.sql.types import JSON, Array, String
+
+
+class MyModel(BaseModel):
+    val1: str
+
+
+class MyFeature(BaseModel):
+    val1: str
+
+
+@pytest.mark.parametrize(
+    "typ,expected",
+    (
+        (str, String),
+        (String, String),
+        (Literal["text"], String),
+        (dict[str, int], JSON),
+        (Mapping[str, int], JSON),
+        (Optional[str], String),
+        (Union[dict, list[dict]], JSON),
+    ),
+)
+def test_convert_type_to_datachain(typ, expected):
+    assert python_to_sql(typ) == expected
+
+
+@pytest.mark.parametrize(
+    "typ,expected",
+    (
+        (list[str], Array(String())),
+        (Iterable[str], Array(String())),
+        (list[list[str]], Array(Array(String()))),
+    ),
+)
+def test_convert_type_to_datachain_array(typ, expected):
+    assert python_to_sql(typ).to_dict() == expected.to_dict()
+
+
+@pytest.mark.parametrize(
+    "typ",
+    (
+        Union[str, int],
+        list[Union[str, int]],
+        MyFeature,
+        MyModel,
+    ),
+)
+def test_convert_type_to_datachain_error(typ):
+    with pytest.raises(TypeError):
+        python_to_sql(typ)
+
+
+def test_normalize_column_names():
+    res = normalize_col_names(
+        [
+            "UpperCase",
+            "_underscore_start",
+            "double__underscore",
+            "1start_with_number",
+            "не_ascii_start",
+            "  space_start",
+            "space_end  ",
+            "dash-end-",
+            "-dash-start",
+            "--multiple--dash--",
+            "-_ mix_  -dash_ -",
+            "__2digit_after_uderscore",
+            "",
+            "_-_-  _---_ _",
+            "_-_-  _---_ _1",
+        ]
+    )
+    assert list(res.keys()) == [
+        "uppercase",
+        "underscore_start",
+        "double_underscore",
+        "c0_1start_with_number",
+        "ascii_start",
+        "space_start",
+        "space_end",
+        "dash_end",
+        "dash_start",
+        "multiple_dash",
+        "mix_dash",
+        "c1_2digit_after_uderscore",
+        "c2",
+        "c3",
+        "c4_1",
+    ]
+
+
+def test_normalize_column_names_case_repeat():
+    res = normalize_col_names(["UpperCase", "UpPerCase"])
+
+    assert list(res.keys()) == ["uppercase", "c0_uppercase"]
+
+
+def test_normalize_column_names_exists_after_normalize():
+    res = normalize_col_names(["1digit", "c0_1digit"])
+
+    assert list(res.keys()) == ["c1_1digit", "c0_1digit"]
+
+
+def test_normalize_column_names_normalized_repeat():
+    res = normalize_col_names(["column", "_column"])
+
+    assert list(res.keys()) == ["column", "c0_column"]
+
+
+def test_normalize_column_names_normalized_case_repeat():
+    res = normalize_col_names(["CoLuMn", "_column"])
+
+    assert res == {"column": "CoLuMn", "c0_column": "_column"}
+
+
+def test_normalize_column_names_repeat_generated_after_normalize():
+    res = normalize_col_names(["c0_CoLuMn", "_column", "column"])
+
+    assert res == {"c0_column": "c0_CoLuMn", "c1_column": "_column", "column": "column"}

datachain-0.6.2/src/datachain/lib/utils.py

@@ -1,30 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class AbstractUDF(ABC):
-    @abstractmethod
-    def process(self, *args, **kwargs):
-        pass
-
-    @abstractmethod
-    def setup(self):
-        pass
-
-    @abstractmethod
-    def teardown(self):
-        pass
-
-
-class DataChainError(Exception):
-    def __init__(self, message):
-        super().__init__(message)
-
-
-class DataChainParamsError(DataChainError):
-    def __init__(self, message):
-        super().__init__(message)
-
-
-class DataChainColumnError(DataChainParamsError):
-    def __init__(self, col_name, msg):
-        super().__init__(f"Error for column {col_name}: {msg}")

datachain-0.6.2/tests/benchmarks/conftest.py

@@ -1,137 +0,0 @@
-import os
-import shutil
-from pathlib import Path
-from subprocess import check_output
-
-import pytest
-import virtualenv
-from dulwich.porcelain import clone
-from packaging import version
-
-
-@pytest.fixture
-def bucket():
-    return "s3://noaa-bathymetry-pds/"
-
-
-def pytest_generate_tests(metafunc):
-    str_revs = metafunc.config.getoption("--datachain-revs")
-    revs = str_revs.split(",") if str_revs else [None]
-    if "datachain_rev" in metafunc.fixturenames:
-        metafunc.parametrize("datachain_rev", revs, scope="session")
-
-
-class VirtualEnv:
-    def __init__(self, path) -> None:
-        self.path = path
-        self.bin = self.path / ("Scripts" if os.name == "nt" else "bin")
-
-    def create(self) -> None:
-        virtualenv.cli_run([os.fspath(self.path)])
-
-    def run(self, cmd: str, *args: str, env=None) -> None:
-        exe = self.which(cmd)
-        check_output([exe, *args], env=env)  # noqa: S603
-
-    def which(self, cmd: str) -> str:
-        assert self.bin.exists()
-        return shutil.which(cmd, path=self.bin) or cmd
-
-
-@pytest.fixture(scope="session", name="make_datachain_venv")
-def fixture_make_datachain_venv(tmp_path_factory):
-    def _make_datachain_venv(name):
-        venv_dir = tmp_path_factory.mktemp(f"datachain-venv-{name}")
-        venv = VirtualEnv(venv_dir)
-        venv.create()
-        return venv
-
-    return _make_datachain_venv
-
-
-@pytest.fixture(scope="session", name="datachain_venvs")
-def fixture_datachain_venvs():
-    return {}
-
-
-@pytest.fixture(scope="session", name="datachain_git_repo")
-def fixture_datachain_git_repo(tmp_path_factory, test_config):
-    url = test_config.datachain_git_repo
-
-    if os.path.isdir(url):
-        return url
-
-    tmp_path = os.fspath(tmp_path_factory.mktemp("datachain-git-repo"))
-    clone(url, tmp_path)
-
-    return tmp_path
-
-
-@pytest.fixture(scope="session", name="datachain_bin")
-def fixture_datachain_bin(
-    datachain_rev,
-    datachain_venvs,
-    make_datachain_venv,
-    datachain_git_repo,
-    test_config,
-):
-    if datachain_rev:
-        venv = datachain_venvs.get(datachain_rev)
-        if not venv:
-            venv = make_datachain_venv(datachain_rev)
-            venv.run("pip", "install", "-U", "pip")
-            venv.run(
-                "pip", "install", f"git+file://{datachain_git_repo}@{datachain_rev}"
-            )
-            datachain_venvs[datachain_rev] = venv
-        datachain_bin = venv.which("datachain")
-    else:
-        datachain_bin = test_config.datachain_bin
-
-    def _datachain_bin(*args):
-        return check_output([datachain_bin, *args], text=True)  # noqa: S603
-
-    actual = version.parse(_datachain_bin("--version"))
-    _datachain_bin.version = (actual.major, actual.minor, actual.micro)
-
-    return _datachain_bin
-
-
-@pytest.fixture(scope="function", name="make_bench")
-def fixture_make_bench(request):
-    def _make_bench(name):
-        import pytest_benchmark.plugin
-
-        # hack from https://github.com/ionelmc/pytest-benchmark/issues/166
-        bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request)
-
-        suffix = f"-{name}"
-
-        def add_suffix(_name):
-            start, sep, end = _name.partition("[")
-            return start + suffix + sep + end
-
-        bench.name = add_suffix(bench.name)
-        bench.fullname = add_suffix(bench.fullname)
-
-        return bench
-
-    return _make_bench
-
-
-@pytest.fixture(
-    scope="function", params=[pytest.param(None, marks=pytest.mark.benchmark)]
-)
-def bench_datachain(datachain_bin, make_bench):
-    def _bench_datachain(*args, **kwargs):
-        name = kwargs.pop("name", None)
-        name = f"-{name}" if name else ""
-        bench = make_bench(args[0] + name)
-        return bench.pedantic(datachain_bin, args=args, **kwargs)
-
-    return _bench_datachain
-
-
-@pytest.fixture
-def datasets():
-    return Path(__file__).parent / "datasets"

datachain-0.6.2/tests/benchmarks/test_ls.py

@@ -1,2 +0,0 @@
-def test_ls(bench_datachain, tmp_dir, bucket):
-    bench_datachain("ls", bucket, "--anon")

datachain-0.6.2/tests/benchmarks/test_version.py

@@ -1,2 +0,0 @@
-def test_version(bench_datachain):
-    bench_datachain("--help", rounds=100)