datachain 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

datachain/lib/listing_info.py

@@ -0,0 +1,32 @@
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+
+from datachain.client import Client
+from datachain.lib.dataset_info import DatasetInfo
+from datachain.lib.listing import LISTING_PREFIX, LISTING_TTL
+
+
+class ListingInfo(DatasetInfo):
+    @property
+    def uri(self) -> str:
+        return self.name.removeprefix(LISTING_PREFIX)
+
+    @property
+    def storage_uri(self) -> str:
+        client, _ = Client.parse_url(self.uri, None)  # type: ignore[arg-type]
+        return client.uri
+
+    @property
+    def expires(self) -> Optional[datetime]:
+        if not self.finished_at:
+            return None
+        return self.finished_at + timedelta(seconds=LISTING_TTL)
+
+    @property
+    def is_expired(self) -> bool:
+        return datetime.now(timezone.utc) > self.expires if self.expires else False
+
+    @property
+    def last_inserted_at(self):
+        # TODO we need to add updated_at to dataset version or explicit last_inserted_at
+        raise NotImplementedError

datachain/lib/meta_formats.py

@@ -2,14 +2,14 @@
 # pip install jmespath
 #
 import csv
-import io
 import json
-import subprocess
-import sys
+import tempfile
 import uuid
 from collections.abc import Iterator
+from pathlib import Path
 from typing import Any, Callable
 
+import datamodel_code_generator
 import jmespath as jsp
 from pydantic import BaseModel, ConfigDict, Field, ValidationError  # noqa: F401
 
@@ -47,17 +47,16 @@ def read_schema(source_file, data_type="csv", expr=None, model_name=None):
     data_string = ""
     # using uiid to get around issue #1617
     if not model_name:
-        uid_str = str(generate_uuid()).replace(
-            "-", ""
-        )  # comply with Python class names
+        # comply with Python class names
+        uid_str = str(generate_uuid()).replace("-", "")
         model_name = f"Model{data_type}{uid_str}"
     try:
         with source_file.open() as fd:  # CSV can be larger than memory
             if data_type == "csv":
-                data_string += fd.readline().decode("utf-8", "ignore").replace("\r", "")
-                data_string += fd.readline().decode("utf-8", "ignore").replace("\r", "")
+                data_string += fd.readline().replace("\r", "")
+                data_string += fd.readline().replace("\r", "")
             elif data_type == "jsonl":
-                data_string = fd.readline().decode("utf-8", "ignore").replace("\r", "")
+                data_string = fd.readline().replace("\r", "")
             else:
                 data_string = fd.read()  # other meta must fit into RAM
     except OSError as e:
@@ -70,33 +69,27 @@ def read_schema(source_file, data_type="csv", expr=None, model_name=None):
         if data_type == "jsonl":
             data_type = "json"  # treat json line as plain JSON in auto-schema
         data_string = json.dumps(json_object)
-    command = [
-        "datamodel-codegen",
-        "--input-file-type",
-        data_type,
-        "--class-name",
-        model_name,
-        "--base-class",
-        "datachain.lib.meta_formats.UserModel",
-    ]
-    try:
-        result = subprocess.run(  # noqa: S603
-            command,
-            input=data_string,
-            text=True,
-            capture_output=True,
-            check=True,
+
+    input_file_types = {i.value: i for i in datamodel_code_generator.InputFileType}
+    input_file_type = input_file_types[data_type]
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output = Path(tmpdir) / "model.py"
+        datamodel_code_generator.generate(
+            data_string,
+            input_file_type=input_file_type,
+            output=output,
+            target_python_version=datamodel_code_generator.PythonVersion.PY_39,
+            base_class="datachain.lib.meta_formats.UserModel",
+            class_name=model_name,
+            additional_imports=["datachain.lib.data_model.DataModel"],
+            use_standard_collections=True,
         )
-        model_output = (
-            result.stdout
-        )  # This will contain the output from datamodel-codegen
-    except subprocess.CalledProcessError as e:
-        model_output = f"An error occurred in datamodel-codegen: {e.stderr}"
-    print(f"{model_output}")
-    print("from datachain.lib.data_model import DataModel")
-    print("\n" + f"DataModel.register({model_name})" + "\n")
-    print("\n" + f"spec={model_name}" + "\n")
-    return model_output
+        epilogue = f"""
+{model_name}.model_rebuild()
+DataModel.register({model_name})
+spec = {model_name}
+"""
+        return output.read_text() + epilogue
 
 
 #
@@ -113,34 +106,24 @@ def read_meta(  # noqa: C901
 ) -> Callable:
     from datachain.lib.dc import DataChain
 
-    # ugly hack: datachain is run redirecting printed outputs to a variable
     if schema_from:
-        captured_output = io.StringIO()
-        current_stdout = sys.stdout
-        sys.stdout = captured_output
-        try:
-            chain = (
-                DataChain.from_storage(schema_from)
-                .limit(1)
-                .map(  # dummy column created (#1615)
-                    meta_schema=lambda file: read_schema(
-                        file, data_type=meta_type, expr=jmespath, model_name=model_name
-                    ),
-                    output=str,
-                )
+        chain = (
+            DataChain.from_storage(schema_from, type="text")
+            .limit(1)
+            .map(  # dummy column created (#1615)
+                meta_schema=lambda file: read_schema(
+                    file, data_type=meta_type, expr=jmespath, model_name=model_name
+                ),
+                output=str,
             )
-            chain.exec()
-        finally:
-            sys.stdout = current_stdout
-        model_output = captured_output.getvalue()
-        captured_output.close()
-
+        )
+        (model_output,) = chain.collect("meta_schema")
         if print_schema:
             print(f"{model_output}")
         # Below 'spec' should be a dynamically converted DataModel from Pydantic
         if not spec:
             local_vars: dict[str, Any] = {}
-            exec(model_output, globals(), local_vars)  # noqa: S102
+            exec(model_output, globals(), local_vars)  # type: ignore[arg-type] # noqa: S102
             spec = local_vars["spec"]
 
     if not (spec) and not (schema_from):

datachain/lib/signal_schema.py

@@ -16,7 +16,6 @@ from typing import (
     get_origin,
 )
 
-import sqlalchemy as sa
 from pydantic import BaseModel, create_model
 from typing_extensions import Literal as LiteralEx
 
@@ -341,7 +340,7 @@ class SignalSchema:
         signals = [
             DEFAULT_DELIMITER.join(path)
             if not as_columns
-            else sa.Column(DEFAULT_DELIMITER.join(path), python_to_sql(_type))
+            else Column(DEFAULT_DELIMITER.join(path), python_to_sql(_type))
             for path, _type, has_subtree, _ in self.get_flat_tree()
             if not has_subtree
         ]
@@ -415,6 +414,10 @@ class SignalSchema:
                 # renaming existing signal
                 del new_values[value.name]
                 new_values[name] = self.values[value.name]
+            elif name in self.values:
+                # changing the type of existing signal, e.g File -> ImageFile
+                del new_values[name]
+                new_values[name] = args_map[name]
             else:
                 # adding new signal
                 new_values.update(sql_to_python({name: value}))

datachain/node.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Optional
 import attrs
 
 from datachain.cache import UniqueId
+from datachain.lib.file import File
 from datachain.storage import StorageURI
 from datachain.utils import TIME_ZERO, time_to_str
 
@@ -189,6 +190,18 @@ class Entry:
             return ""
         return split[0]
 
+    def to_file(self, source: str) -> File:
+        return File(
+            source=source,
+            path=self.path,
+            size=self.size,
+            version=self.version,
+            etag=self.etag,
+            is_latest=self.is_latest,
+            last_modified=self.last_modified,
+            location=self.location,
+        )
+
 
 def get_path(parent: str, name: str):
     return f"{parent}/{name}" if parent else name

datachain/query/dataset.py

@@ -1,7 +1,5 @@
 import contextlib
-import datetime
 import inspect
-import json
 import logging
 import os
 import random
@@ -38,11 +36,7 @@ from sqlalchemy.sql.selectable import Select
 from tqdm import tqdm
 
 from datachain.asyn import ASYNC_WORKERS, AsyncMapper, OrderedMapper
-from datachain.catalog import (
-    QUERY_SCRIPT_CANCELED_EXIT_CODE,
-    QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE,
-    get_catalog,
-)
+from datachain.catalog import QUERY_SCRIPT_CANCELED_EXIT_CODE, get_catalog
 from datachain.data_storage.schema import (
     PARTITION_COLUMN_ID,
     partition_col_names,
@@ -60,7 +54,6 @@ from datachain.utils import (
     get_datachain_executable,
 )
 
-from .metrics import metrics
 from .schema import C, UDFParamSpec, normalize_param
 from .session import Session
 from .udf import UDFBase, UDFClassWrapper, UDFFactory, UDFType
@@ -219,7 +212,7 @@ class IndexingStep(StartingStep):
                 recursive=self.recursive,
             )
 
-        storage = self.catalog.get_storage(uri)
+        storage = self.catalog.metastore.get_storage(uri)
 
         return step_result(q, dataset_rows.c, dependencies=[storage.uri])
 
@@ -1175,8 +1168,12 @@ class DatasetQuery:
         """
         return self.name is not None and self.version is not None
 
-    def c(self, name: Union[C, str]) -> "ColumnClause[Any]":
-        col = sqlalchemy.column(name) if isinstance(name, str) else name
+    def c(self, column: Union[C, str]) -> "ColumnClause[Any]":
+        col: sqlalchemy.ColumnClause = (
+            sqlalchemy.column(column)
+            if isinstance(column, str)
+            else sqlalchemy.column(column.name, column.type)
+        )
         col.table = self.table
         return col
 
@@ -1634,7 +1631,7 @@ class DatasetQuery:
                 )
             else:
                 # storage dependency - its name is a valid StorageURI
-                storage = self.catalog.get_storage(dependency)
+                storage = self.catalog.metastore.get_storage(dependency)
                 self.catalog.metastore.add_storage_dependency(
                     StorageURI(dataset.name),
                     version,
@@ -1712,113 +1709,23 @@ class DatasetQuery:
         return self.__class__(name=name, version=version, catalog=self.catalog)
 
 
-def _get_output_fd_for_write() -> Union[str, int]:
-    handle = os.getenv("DATACHAIN_OUTPUT_FD")
-    if not handle:
-        return os.devnull
-
-    if os.name != "nt":
-        return int(handle)
-
-    import msvcrt
-
-    return msvcrt.open_osfhandle(int(handle), os.O_WRONLY)  # type: ignore[attr-defined]
-
-
-@attrs.define
-class ExecutionResult:
-    preview: list[dict] = attrs.field(factory=list)
-    dataset: Optional[tuple[str, int]] = None
-    metrics: dict[str, Any] = attrs.field(factory=dict)
-
-
-def _send_result(dataset_query: DatasetQuery) -> None:
-    class JSONSerialize(json.JSONEncoder):
-        def default(self, obj):
-            if isinstance(obj, (datetime.datetime, datetime.date)):
-                return obj.isoformat()
-            if isinstance(obj, bytes):
-                return list(obj[:1024])
-            return super().default(obj)
-
-    try:
-        preview_args: dict[str, Any] = json.loads(
-            os.getenv("DATACHAIN_QUERY_PREVIEW_ARGS", "")
-        )
-    except ValueError:
-        preview_args = {}
-
-    columns = preview_args.get("columns") or []
-
-    if type(dataset_query) is DatasetQuery:
-        preview_query = dataset_query.select(*columns)
-    else:
-        preview_query = dataset_query.select(*columns, _sys=False)
-
-    preview_query = preview_query.limit(preview_args.get("limit", 10)).offset(
-        preview_args.get("offset", 0)
-    )
-
-    dataset: Optional[tuple[str, int]] = None
-    if dataset_query.attached:
-        assert dataset_query.name, "Dataset name should be provided"
-        assert dataset_query.version, "Dataset version should be provided"
-        dataset = dataset_query.name, dataset_query.version
-
-    preview = preview_query.to_db_records()
-    result = ExecutionResult(preview, dataset, metrics)
-    data = attrs.asdict(result)
-
-    with open(_get_output_fd_for_write(), mode="w") as f:
-        json.dump(data, f, cls=JSONSerialize)
-
-
-def query_wrapper(dataset_query: DatasetQuery) -> DatasetQuery:
+def query_wrapper(dataset_query: Any) -> Any:
     """
     Wrapper function that wraps the last statement of user query script.
     Last statement MUST be instance of DatasetQuery, otherwise script exits with
     error code 10
     """
     if not isinstance(dataset_query, DatasetQuery):
-        sys.exit(QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE)
+        return dataset_query
 
     catalog = dataset_query.catalog
     save = bool(os.getenv("DATACHAIN_QUERY_SAVE"))
-    save_as = os.getenv("DATACHAIN_QUERY_SAVE_AS")
 
     is_session_temp_dataset = dataset_query.name and dataset_query.name.startswith(
         dataset_query.session.get_temp_prefix()
     )
 
-    if save_as:
-        if dataset_query.attached:
-            dataset_name = dataset_query.name
-            version = dataset_query.version
-            assert dataset_name, "Dataset name should be provided in attached mode"
-            assert version, "Dataset version should be provided in attached mode"
-
-            dataset = catalog.get_dataset(dataset_name)
-
-            try:
-                target_dataset = catalog.get_dataset(save_as)
-            except DatasetNotFoundError:
-                target_dataset = None
-
-            if target_dataset:
-                dataset = catalog.register_dataset(dataset, version, target_dataset)
-            else:
-                dataset = catalog.register_new_dataset(dataset, version, save_as)
-
-            dataset_query = DatasetQuery(
-                name=dataset.name,
-                version=dataset.latest_version,
-                catalog=catalog,
-            )
-        else:
-            dataset_query = dataset_query.save(save_as)
-    elif save and (is_session_temp_dataset or not dataset_query.attached):
+    if save and (is_session_temp_dataset or not dataset_query.attached):
         name = catalog.generate_query_dataset_name()
         dataset_query = dataset_query.save(name)
-
-    _send_result(dataset_query)
     return dataset_query

datachain/query/metrics.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional, Union
 
 metrics: dict[str, Union[str, int, float, bool, None]] = {}
@@ -13,6 +14,13 @@ def set(key: str, value: Union[str, int, float, bool, None]) -> None:  # noqa: P
         raise TypeError("Value must be a string, int, float or bool")
     metrics[key] = value
 
+    if job_id := os.getenv("DATACHAIN_JOB_ID"):
+        from datachain.data_storage.job import JobStatus
+        from datachain.query.session import Session
+
+        metastore = Session.get().catalog.metastore
+        metastore.set_job_status(job_id, JobStatus.RUNNING, metrics=metrics)
+
 
 def get(key: str) -> Optional[Union[str, int, float, bool]]:
     """Get a metric value."""

datachain/utils.py

@@ -448,3 +448,8 @@ def get_datachain_executable() -> list[str]:
     if datachain_exec_path := os.getenv("DATACHAIN_EXEC_PATH"):
         return [datachain_exec_path]
     return [sys.executable, "-m", "datachain"]
+
+
+def uses_glob(path: str) -> bool:
+    """Checks if some URI path has glob syntax in it"""
+    return glob.has_magic(os.path.basename(os.path.normpath(path)))

{datachain-0.3.9.dist-info → datachain-0.3.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.9
+Version: 0.3.11
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -42,6 +42,7 @@ Requires-Dist: datamodel-code-generator >=0.25
 Requires-Dist: Pillow <11,>=10.0.0
 Requires-Dist: msgpack <2,>=1.0.4
 Requires-Dist: psutil
+Requires-Dist: huggingface-hub
 Requires-Dist: numpy <2,>=1 ; sys_platform == "win32"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests] ; extra == 'dev'
@@ -67,7 +68,7 @@ Requires-Dist: pdfplumber ==0.11.4 ; extra == 'examples'
 Requires-Dist: huggingface-hub[hf_transfer] ; extra == 'examples'
 Provides-Extra: hf
 Requires-Dist: numba >=0.60.0 ; extra == 'hf'
-Requires-Dist: datasets[audio,vision] ; extra == 'hf'
+Requires-Dist: datasets[audio,vision] >=2.21.0 ; extra == 'hf'
 Provides-Extra: remote
 Requires-Dist: lz4 ; extra == 'remote'
 Requires-Dist: requests >=2.22.0 ; extra == 'remote'
@@ -95,6 +96,10 @@ Requires-Dist: transformers >=4.36.0 ; extra == 'torch'
 Provides-Extra: vector
 Requires-Dist: usearch ; extra == 'vector'
 
+.. image:: docs/assets/datachain_logotype.svg
+   :height: 48
+   :alt: DataChain logo
+
 |PyPI| |Python Version| |Codecov| |Tests|
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/datachain.svg
@@ -110,7 +115,6 @@ Requires-Dist: usearch ; extra == 'vector'
    :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
    :alt: Tests
 
-AI 🔗 DataChain
 ----------------
 
 DataChain is a modern Pythonic data-frame library designed for artificial intelligence.

{datachain-0.3.9.dist-info → datachain-0.3.11.dist-info}/RECORD

@@ -2,56 +2,57 @@ datachain/__init__.py,sha256=GeyhE-5LgfJav2OKYGaieP2lBvf2Gm-ihj7thnK9zjI,800
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=biF8M8fQujtj5xs0VLi8S16eBtzG6kceWlO_NILbCsg,8197
 datachain/cache.py,sha256=wznC2pge6RhlPTaJfBVGjmBc6bxWCPThu4aTFMltvFU,4076
-datachain/cli.py,sha256=otR2eN0JL-JhZ9SOTPcPwt_-_TiT-vHifx2h4YzD6Tg,32052
+datachain/cli.py,sha256=ECf_z5X8ILDJdUn2Cpb_z-ZjSRIzn7skiuMGfM-y9i0,30999
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=PfC7W5yO6HFO6-iMB4YB-0RR88LPiGmD6sS_SfVbGso,1979
-datachain/dataset.py,sha256=MZezyuJWNj_3PEtzr0epPMNyWAOTrhTSPI5FmemV6L4,14470
+datachain/dataset.py,sha256=EcYjhHg1dxxPbDwSuIxc-mDRDo3v_pYf79fMy4re1oA,14740
 datachain/error.py,sha256=GY9KYTmb7GHXn2gGHV9X-PBhgwLj3i7VpK7tGHtAoGM,1279
-datachain/job.py,sha256=bk25bIqClhgRPzlXAhxpTtDeewibQe5l3S8Cf7db0gM,1229
+datachain/job.py,sha256=Jt4sNutMHJReaGsj3r3scueN5aESLGfhimAa8pUP7Is,1271
 datachain/listing.py,sha256=keLkvPfumDA3gijeIiinH5yGWe71qCxgF5HqqP5AeH4,8299
-datachain/node.py,sha256=ihrP5l9HKpXLR0fR1wyb7QIdb7NR26dX6bB09qGX5B4,6005
+datachain/node.py,sha256=dcm_7dVurFHpI0EHV2K6SjpJyh-gN4PVWAB-20quk04,6382
 datachain/nodes_fetcher.py,sha256=kca19yvu11JxoVY1t4_ydp1FmchiV88GnNicNBQ9NIA,831
 datachain/nodes_thread_pool.py,sha256=ZyzBvUImIPmi4WlKC2SW2msA0UhtembbTdcs2nx29A0,3191
 datachain/progress.py,sha256=7_8FtJs770ITK9sMq-Lt4k4k18QmYl4yIG_kCoWID3o,4559
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
-datachain/utils.py,sha256=ROVCLwb37VmFRzgTlSGUDw4eJNgYGiQ4yMX581HfUX8,12988
+datachain/utils.py,sha256=HKUdVqreBTzcCULAYRw1sC6z33OaomVD1WoMSoFcPHg,13148
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=kGpp9IEyr1YS7QFWjLYprRT1gp7freyt-WLaLNzqUZg,77859
+datachain/catalog/catalog.py,sha256=NgS7_SlmpJdUSp1v8KdCuLTjFklmYvT_jOLdzTyyK5I,72313
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
-datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
 datachain/client/__init__.py,sha256=T4wiYL9KIM0ZZ_UqIyzV8_ufzYlewmizlV4iymHNluE,86
 datachain/client/azure.py,sha256=LXSahE0Z6r4dXqpBkKnq3J5fg7N7ymC1lSn-1SoILGc,2687
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=G4QTm3KPhlaV74T3gLXJ86345_ak8CH38ezn2ET-oLc,13230
+datachain/client/fsspec.py,sha256=LQb5tr-pP9umCFYo3nGJR_dZxUyiSN7IDE8jhp1TXco,13333
 datachain/client/gcs.py,sha256=P_E3mhzhXR9mJ_wc3AYZuczzwOJ0-D3J5qhJXeSU-xk,4518
+datachain/client/hf.py,sha256=R-F6Ks6aVM9wSNkIXOkOnZFwsJlfdRwJjymRa78RLjM,1246
 datachain/client/local.py,sha256=H8TNY8pi2kA8y9_f_1XLUjJF66f229qC_b2y4xGkzdU,5300
 datachain/client/s3.py,sha256=aQxfMH8G8bUjmHF1-6P90MSkXsU5DgOPEVlKWLu459I,6568
 datachain/data_storage/__init__.py,sha256=cEOJpyu1JDZtfUupYucCDNFI6e5Wmp_Oyzq6rZv32Y8,398
 datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kTUCaru4,3406
 datachain/data_storage/id_generator.py,sha256=lCEoU0BM37Ai2aRpSbwo5oQT0GqZnSpYwwvizathRMQ,4292
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=nxcY6nwyEmQWMAo33sNGO-FgUFQs2amBGGnZz2ftEz0,55362
+datachain/data_storage/metastore.py,sha256=cHN0xmbUvChyayHHZm3Vqxr87jFqojPSlGBqhTPStlE,54519
 datachain/data_storage/schema.py,sha256=JKpSEz8igpwZ9zkpRPYVXZxEpiXuLKEs2WNhH0KqM6U,8552
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
-datachain/data_storage/sqlite.py,sha256=jLgkvikYkENQUO_ykoNFfsBc2ofZXwFHLMa1nyWP3aw,28316
-datachain/data_storage/warehouse.py,sha256=cvlfa-nyIxqrrpSRtCdeVjlTwhn7rcIoWjOq91HhItU,33668
+datachain/data_storage/sqlite.py,sha256=Z4B2KDL4C8Uio2aLMxaKv0t2MoOtCV3bSqWg4X9mTFg,28048
+datachain/data_storage/warehouse.py,sha256=f7ETnYIXx5KMcPfwg_4bh_00QJiMLIliwE_41vmRGUo,33037
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=17-jHLdYhsSdO5kfKWpBS5OAWbMjNi5r8ao0zGXUBoA,4941
+datachain/lib/arrow.py,sha256=dV17oGiknqEW55ogGK_9T0ycNFwd2z-EFOW0AQiR6TU,5840
 datachain/lib/clip.py,sha256=33RL11OIqfbwyhvBgiMGM8rDAnZx1IRmxk9dY89ls3Q,6130
 datachain/lib/data_model.py,sha256=gHIjlow84GMRDa78yLL1Ud-N18or21fnTyPEwsatpXY,2045
-datachain/lib/dataset_info.py,sha256=lONGr71ozo1DS4CQEhnpKORaU4qFb6Ketv8Xm8CVm2U,2188
-datachain/lib/dc.py,sha256=tY_ccOsv9njsXF23cwoZ7tSTCDKCfakyRvsIBLKE0SE,63976
-datachain/lib/file.py,sha256=ZHpdilDPYCob8uqtwUPtBvBNxVvQRq4AC_0IGg5m-G4,12003
-datachain/lib/hf.py,sha256=mYaHFPS4CW2-stRZHBMWW-NKN4dhrnhjZobBgRocnvo,5317
+datachain/lib/dataset_info.py,sha256=srPPhI2UHf6hFPBecyFEVw2SS5aPisIIMsvGgKqi7ss,2366
+datachain/lib/dc.py,sha256=s4E-bD6_T6JFJ7TEa5Y9RS705lIfcV9OUJwDD6RNCX0,68156
+datachain/lib/file.py,sha256=WOOYw3LcGROA6wshJ_aZkSgcTqfB4UxTbZDTx9KqAOg,11429
+datachain/lib/hf.py,sha256=ZiMvgy3DYiklGKZv-w7gevrHOgn3bGfpTlpDPOHCNqs,5336
 datachain/lib/image.py,sha256=WbcwSaFzuyqjg4x4hH5CUogeUQjkZFjQHqw_oDEV1nA,2655
-datachain/lib/listing.py,sha256=nXLmGae_oQke4hnurzzWiHTEjHjWiqqHdB41Wb-hMTk,3521
-datachain/lib/meta_formats.py,sha256=Hels85LJmNCz1aYVJvhymNdAt3qdJ2-qoxsIiUezrow,7198
+datachain/lib/listing.py,sha256=S9Xn_Saxu4xk3K_01VexkfMZW0INQiATlidt2bzgWKY,3938
+datachain/lib/listing_info.py,sha256=sr5KzCXlCxlPuRmy_pVadD4miLpp5y0btvyaIPcluwI,996
+datachain/lib/meta_formats.py,sha256=67uF9trQ2II6xFvN0u6eo5NNRf5xvCkpMHj7ThiG41Y,6777
 datachain/lib/model_store.py,sha256=c4USXsBBjrGH8VOh4seIgOiav-qHOwdoixtxfLgU63c,2409
 datachain/lib/pytorch.py,sha256=vK3GbWCy7kunN7ubul6w1hrWmJLja56uTCiMG_7XVQA,5623
 datachain/lib/settings.py,sha256=39thOpYJw-zPirzeNO6pmRC2vPrQvt4eBsw1xLWDFsw,2344
-datachain/lib/signal_schema.py,sha256=rW1R6nIzdtmqWzpXk7aNAfrQD58_gbvkvEGyNTQ4WNM,20099
+datachain/lib/signal_schema.py,sha256=hqQLwUmt3w8RLa96MtubK9N2CBXqqTPrUkSRXc0ktt4,20275
 datachain/lib/text.py,sha256=vqs1SQdsw1vCzfvOanIeT4xY2R2TmPonElBgYDVeZmY,1241
 datachain/lib/udf.py,sha256=nG7DDuPgZ5ZuijwvDoCq-OZMxlDM8vFNzyxMmik0Y1c,11716
 datachain/lib/udf_signature.py,sha256=gMStcEeYJka5M6cg50Z9orC6y6HzCAJ3MkFqqn1fjZg,7137
@@ -68,9 +69,9 @@ datachain/lib/convert/values_to_tuples.py,sha256=YOdbjzHq-uj6-cV2Qq43G72eN2avMND
 datachain/query/__init__.py,sha256=tv-spkjUCYamMN9ys_90scYrZ8kJ7C7d1MTYVmxGtk4,325
 datachain/query/batch.py,sha256=-vlpINJiertlnaoUVv1C95RatU0F6zuhpIYRufJRo1M,3660
 datachain/query/builtins.py,sha256=EmKPYsoQ46zwdyOn54MuCzvYFmfsBn5F8zyF7UBUfrc,2550
-datachain/query/dataset.py,sha256=mHqSyovJlCQ7pKVMQKKKCiTJs3bP1GDXLKpOSpzVxx8,61378
+datachain/query/dataset.py,sha256=B2EmGOL8gjrdU_WhU88Dj7FsxvxrNeKwe2STXnU9T9E,58369
 datachain/query/dispatch.py,sha256=GBh3EZHDp5AaXxrjOpfrpfsuy7Umnqxu-MAXcK9X3gc,12945
-datachain/query/metrics.py,sha256=vsECqbZfoSDBnvC3GQlziKXmISVYDLgHP1fMPEOtKyo,640
+datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
 datachain/query/schema.py,sha256=BvHipN79CnSTbVFcfIEwzo1npe7HmThnk0iY-CSLEkM,7899
@@ -95,9 +96,9 @@ datachain/sql/sqlite/base.py,sha256=WLPHBhZbXbiqPoRV1VgDrXJqku4UuvJpBhYeQ0k5rI8,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.9.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.9.dist-info/METADATA,sha256=r5uNlVdal7YrsX7nYE56c_Ak8YZIgXqCiSwNJF5KjlY,17015
-datachain-0.3.9.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-datachain-0.3.9.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.9.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.9.dist-info/RECORD,,
+datachain-0.3.11.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.3.11.dist-info/METADATA,sha256=iSdfjWpVT1Iqzlg82eN5QzJ-icaYxkG7TUKEpEOi5sk,17124
+datachain-0.3.11.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+datachain-0.3.11.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.3.11.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.3.11.dist-info/RECORD,,

{datachain-0.3.9.dist-info → datachain-0.3.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (74.1.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datachain/catalog/subclass.py

@@ -1,60 +0,0 @@
-import ast
-
-
-class SubclassFinder(ast.NodeVisitor):
-    """Finds subclasses of a target class in an AST."""
-
-    def __init__(self, target_classes: list[str]):
-        self.imports: list[ast.AST] = []
-        self.main_body: list[ast.AST] = []
-
-        self.target_classes: list[str] = target_classes
-        self.aliases: dict[str, str] = {}
-        self.feature_class: list[ast.AST] = []
-
-    def visit_ImportFrom(self, node):  # noqa: N802
-        module = node.module
-        for alias in node.names:
-            full_name = f"{module}.{alias.name}"
-            self.aliases[alias.asname or alias.name] = full_name
-        self.imports.append(node)
-
-    def visit_Import(self, node):  # noqa: N802
-        for alias in node.names:
-            self.aliases[alias.asname or alias.name] = alias.name
-        self.imports.append(node)
-
-    def visit_ClassDef(self, node):  # noqa: N802
-        base_names = [self.get_base_name(base) for base in node.bases]
-        if any(self.is_subclass(name) for name in base_names):
-            self.feature_class.append(node)
-        else:
-            self.main_body.append(node)
-
-    def visit(self, node):
-        if isinstance(
-            node,
-            (ast.Import, ast.ImportFrom, ast.ClassDef, ast.Module),
-        ):
-            return super().visit(node)
-        self.main_body.append(node)
-        return node
-
-    def get_base_name(self, node):
-        if isinstance(node, ast.Name):
-            return self.aliases.get(node.id, node.id)
-        if isinstance(node, ast.Attribute):
-            return self.get_full_attr_name(node)
-        if isinstance(node, ast.Subscript):
-            return self.get_base_name(node.value)
-        return None
-
-    def get_full_attr_name(self, node):
-        if isinstance(node.value, ast.Name):
-            return f"{node.value.id}.{node.attr}"
-        if isinstance(node.value, ast.Attribute):
-            return f"{self.get_full_attr_name(node.value)}.{node.attr}"
-        return node.attr
-
-    def is_subclass(self, base_name):
-        return base_name and base_name.split(".")[-1] in self.target_classes