datachain 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

datachain/data_storage/sqlite.py

@@ -29,12 +29,11 @@ from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse
 from datachain.data_storage.db_engine import DatabaseEngine
 from datachain.data_storage.id_generator import AbstractDBIDGenerator
 from datachain.data_storage.schema import DefaultSchema
-from datachain.dataset import DatasetRecord
+from datachain.dataset import DatasetRecord, StorageURI
 from datachain.error import DataChainError
 from datachain.sql.sqlite import create_user_defined_sql_functions, sqlite_dialect
 from datachain.sql.sqlite.base import load_usearch_extension
 from datachain.sql.types import SQLType
-from datachain.storage import StorageURI
 from datachain.utils import DataChainDir, batched_it
 
 if TYPE_CHECKING:
@@ -392,14 +391,14 @@ class SQLiteMetastore(AbstractDBMetastore):
     def __init__(
         self,
         id_generator: "SQLiteIDGenerator",
-        uri: StorageURI = StorageURI(""),
-        partial_id: Optional[int] = None,
+        uri: Optional[StorageURI] = None,
         db: Optional["SQLiteDatabaseEngine"] = None,
         db_file: Optional[str] = None,
         in_memory: bool = False,
     ):
+        uri = uri or StorageURI("")
         self.schema: DefaultSchema = DefaultSchema()
-        super().__init__(id_generator, uri, partial_id)
+        super().__init__(id_generator, uri)
 
         # needed for dropping tables in correct order for tests because of
         # foreign keys
@@ -417,21 +416,16 @@ class SQLiteMetastore(AbstractDBMetastore):
 
     def clone(
         self,
-        uri: StorageURI = StorageURI(""),
-        partial_id: Optional[int] = None,
+        uri: Optional[StorageURI] = None,
         use_new_connection: bool = False,
     ) -> "SQLiteMetastore":
-        if not uri:
-            if partial_id is not None:
-                raise ValueError("if partial_id is used, uri cannot be empty")
-            if self.uri:
-                uri = self.uri
-                if self.partial_id:
-                    partial_id = self.partial_id
+        uri = uri or StorageURI("")
+        if not uri and self.uri:
+            uri = self.uri
+
         return SQLiteMetastore(
             self.id_generator.clone(),
             uri=uri,
-            partial_id=partial_id,
             db=self.db.clone(),
         )
 
@@ -446,7 +440,6 @@ class SQLiteMetastore(AbstractDBMetastore):
             {
                 "id_generator_clone_params": self.id_generator.clone_params(),
                 "uri": self.uri,
-                "partial_id": self.partial_id,
                 "db_clone_params": self.db.clone_params(),
             },
         )
@@ -457,7 +450,6 @@ class SQLiteMetastore(AbstractDBMetastore):
         *,
         id_generator_clone_params: tuple[Callable, list, dict[str, Any]],
         uri: StorageURI,
-        partial_id: Optional[int],
         db_clone_params: tuple[Callable, list, dict[str, Any]],
     ) -> "SQLiteMetastore":
         (
@@ -469,14 +461,11 @@ class SQLiteMetastore(AbstractDBMetastore):
         return cls(
             id_generator=id_generator_class(*id_generator_args, **id_generator_kwargs),
             uri=uri,
-            partial_id=partial_id,
             db=db_class(*db_args, **db_kwargs),
         )
 
     def _init_tables(self) -> None:
         """Initialize tables."""
-        self.db.create_table(self._storages, if_not_exists=True)
-        self.default_table_names.append(self._storages.name)
         self.db.create_table(self._datasets, if_not_exists=True)
         self.default_table_names.append(self._datasets.name)
         self.db.create_table(self._datasets_versions, if_not_exists=True)
@@ -486,28 +475,11 @@ class SQLiteMetastore(AbstractDBMetastore):
         self.db.create_table(self._jobs, if_not_exists=True)
         self.default_table_names.append(self._jobs.name)
 
-    def init(self, uri: StorageURI) -> None:
-        if not uri:
-            raise ValueError("uri for init() cannot be empty")
-        partials_table = self._partials_table(uri)
-        self.db.create_table(partials_table, if_not_exists=True)
-
-    @classmethod
-    def _buckets_columns(cls) -> list["SchemaItem"]:
-        """Buckets (storages) table columns."""
-        return [*super()._buckets_columns(), UniqueConstraint("uri")]
-
     @classmethod
     def _datasets_columns(cls) -> list["SchemaItem"]:
         """Datasets table columns."""
         return [*super()._datasets_columns(), UniqueConstraint("name")]
 
-    def _storages_insert(self) -> "Insert":
-        return sqlite.insert(self._storages)
-
-    def _partials_insert(self) -> "Insert":
-        return sqlite.insert(self._partials)
-
     def _datasets_insert(self) -> "Insert":
         return sqlite.insert(self._datasets)
 
@@ -526,13 +498,9 @@ class SQLiteMetastore(AbstractDBMetastore):
             self._datasets_dependencies.c.id,
             self._datasets_dependencies.c.dataset_id,
             self._datasets_dependencies.c.dataset_version_id,
-            self._datasets_dependencies.c.bucket_id,
-            self._datasets_dependencies.c.bucket_version,
             self._datasets.c.name,
-            self._datasets.c.created_at,
             self._datasets_versions.c.version,
             self._datasets_versions.c.created_at,
-            self._storages.c.uri,
         ]
 
     #

datachain/data_storage/warehouse.py

@@ -19,11 +19,10 @@ from tqdm import tqdm
 from datachain.client import Client
 from datachain.data_storage.schema import convert_rows_custom_column_types
 from datachain.data_storage.serializer import Serializable
-from datachain.dataset import DatasetRecord
+from datachain.dataset import DatasetRecord, StorageURI
 from datachain.node import DirType, DirTypeGroup, Node, NodeWithPath, get_path
 from datachain.sql.functions import path as pathfunc
 from datachain.sql.types import Int, SQLType
-from datachain.storage import StorageURI
 from datachain.utils import sql_escape_like
 
 if TYPE_CHECKING:

datachain/dataset.py

@@ -3,21 +3,17 @@ import json
 from dataclasses import dataclass, fields
 from datetime import datetime
 from typing import (
-    TYPE_CHECKING,
     Any,
+    NewType,
     Optional,
     TypeVar,
     Union,
 )
 from urllib.parse import urlparse
 
-from datachain.client import Client
 from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
-if TYPE_CHECKING:
-    from datachain.storage import StorageURI
-
 T = TypeVar("T", bound="DatasetRecord")
 V = TypeVar("V", bound="DatasetVersion")
 DD = TypeVar("DD", bound="DatasetDependency")
@@ -27,6 +23,13 @@ QUERY_DATASET_PREFIX = "ds_query_"
 LISTING_PREFIX = "lst__"
 
 
+# StorageURI represents a normalised URI to a valid storage location (full bucket or
+# absolute local path).
+# Valid examples: s3://foo, file:///var/data
+# Invalid examples: s3://foo/, s3://foo/bar, file://~
+StorageURI = NewType("StorageURI", str)
+
+
 def parse_dataset_uri(uri: str) -> tuple[str, Optional[int]]:
     """
     Parse dataser uri to extract name and version out of it (if version is defined)
@@ -94,14 +97,11 @@ class DatasetDependency:
         id: int,
         dataset_id: Optional[int],
         dataset_version_id: Optional[int],
-        bucket_id: Optional[int],
-        bucket_version: Optional[str],
         dataset_name: Optional[str],
-        dataset_created_at: Optional[datetime],
         dataset_version: Optional[int],
         dataset_version_created_at: Optional[datetime],
-        bucket_uri: Optional["StorageURI"],
     ) -> Optional["DatasetDependency"]:
+        from datachain.client import Client
         from datachain.lib.listing import is_listing_dataset, listing_uri_from_name
 
         if not dataset_id:
@@ -124,7 +124,7 @@ class DatasetDependency:
                 if dataset_version
                 else None
             ),
-            dataset_version_created_at or dataset_created_at,  # type: ignore[arg-type]
+            dataset_version_created_at,  # type: ignore[arg-type]
             [],
         )
 
@@ -448,6 +448,8 @@ class DatasetRecord:
         For bucket listing we implicitly create underlying dataset to hold data. This
         method is checking if this is one of those datasets.
         """
+        from datachain.client import Client
+
         # TODO refactor and maybe remove method in
         # https://github.com/iterative/datachain/issues/318
         return Client.is_data_source_uri(self.name) or self.name.startswith(

datachain/error.py

@@ -18,10 +18,6 @@ class DatasetInvalidVersionError(Exception):
     pass
 
 
-class StorageNotFoundError(NotFoundError):
-    pass
-
-
 class PendingIndexingError(Exception):
     """An indexing operation is already in progress."""
 

datachain/lib/arrow.py

@@ -1,4 +1,3 @@
-import re
 from collections.abc import Sequence
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Optional
@@ -13,6 +12,7 @@ from datachain.lib.file import ArrowRow, File
 from datachain.lib.model_store import ModelStore
 from datachain.lib.signal_schema import SignalSchema
 from datachain.lib.udf import Generator
+from datachain.lib.utils import normalize_col_names
 
 if TYPE_CHECKING:
     from datasets.features.features import Features
@@ -128,7 +128,7 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
     signal_schema = _get_datachain_schema(schema)
     if signal_schema:
         return signal_schema.values
-    columns = _convert_col_names(col_names)  # type: ignore[arg-type]
+    columns = list(normalize_col_names(col_names).keys())  # type: ignore[arg-type]
     hf_schema = _get_hf_schema(schema)
     if hf_schema:
         return {
@@ -143,19 +143,6 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
     return output
 
 
-def _convert_col_names(col_names: Sequence[str]) -> list[str]:
-    default_column = 0
-    converted_col_names = []
-    for column in col_names:
-        column = column.lower()
-        column = re.sub("[^0-9a-z_]+", "", column)
-        if not column:
-            column = f"c{default_column}"
-            default_column += 1
-        converted_col_names.append(column)
-    return converted_col_names
-
-
 def arrow_type_mapper(col_type: pa.DataType, column: str = "") -> type:  # noqa: PLR0911
     """Convert pyarrow types to basic types."""
     from datetime import datetime

datachain/lib/data_model.py

@@ -2,9 +2,10 @@ from collections.abc import Sequence
 from datetime import datetime
 from typing import ClassVar, Union, get_args, get_origin
 
-from pydantic import BaseModel, create_model
+from pydantic import BaseModel, Field, create_model
 
 from datachain.lib.model_store import ModelStore
+from datachain.lib.utils import normalize_col_names
 
 StandardType = Union[
     type[int],
@@ -60,7 +61,14 @@ def is_chain_type(t: type) -> bool:
 
 
 def dict_to_data_model(name: str, data_dict: dict[str, DataType]) -> type[BaseModel]:
-    fields = {name: (anno, ...) for name, anno in data_dict.items()}
+    # Gets a map of a normalized_name -> original_name
+    columns = normalize_col_names(list(data_dict.keys()))
+    # We reverse if for convenience to original_name -> normalized_name
+    columns = {v: k for k, v in columns.items()}
+
+    fields = {
+        columns[name]: (anno, Field(alias=name)) for name, anno in data_dict.items()
+    }
     return create_model(
         name,
         __base__=(DataModel,),  # type: ignore[call-overload]

datachain/lib/utils.py

@@ -1,4 +1,6 @@
+import re
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 
 
 class AbstractUDF(ABC):
@@ -28,3 +30,31 @@ class DataChainParamsError(DataChainError):
 class DataChainColumnError(DataChainParamsError):
     def __init__(self, col_name, msg):
         super().__init__(f"Error for column {col_name}: {msg}")
+
+
+def normalize_col_names(col_names: Sequence[str]) -> dict[str, str]:
+    gen_col_counter = 0
+    new_col_names = {}
+    org_col_names = set(col_names)
+
+    for org_column in col_names:
+        new_column = org_column.lower()
+        new_column = re.sub("[^0-9a-z]+", "_", new_column)
+        new_column = new_column.strip("_")
+
+        generated_column = new_column
+
+        while (
+            not generated_column.isidentifier()
+            or generated_column in new_col_names
+            or (generated_column != org_column and generated_column in org_col_names)
+        ):
+            if new_column:
+                generated_column = f"c{gen_col_counter}_{new_column}"
+            else:
+                generated_column = f"c{gen_col_counter}"
+            gen_col_counter += 1
+
+        new_col_names[generated_column] = org_column
+
+    return new_col_names

datachain/node.py

@@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, Any, Optional
 
 import attrs
 
+from datachain.dataset import StorageURI
 from datachain.lib.file import File
-from datachain.storage import StorageURI
 from datachain.utils import TIME_ZERO, time_to_str
 
 if TYPE_CHECKING:

{datachain-0.6.2.dist-info → datachain-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.6.2
+Version: 0.6.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -47,7 +47,7 @@ Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client <1,>=0.21
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests] ; extra == 'dev'
-Requires-Dist: mypy ==1.12.1 ; extra == 'dev'
+Requires-Dist: mypy ==1.13.0 ; extra == 'dev'
 Requires-Dist: types-python-dateutil ; extra == 'dev'
 Requires-Dist: types-pytz ; extra == 'dev'
 Requires-Dist: types-PyYAML ; extra == 'dev'

{datachain-0.6.2.dist-info → datachain-0.6.4.dist-info}/RECORD

@@ -2,47 +2,46 @@ datachain/__init__.py,sha256=OGzc8xZWtwqxiiutjU4AxCRPY0lrX_csgERiTrq4G0o,908
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=Lg3Ck1PQLjQziMx9KU4atzbEnJXTE0924WMYkhgWtGU,8247
 datachain/cache.py,sha256=s0YHN7qurmQv-eC265TjeureK84TebWWAnL07cxchZQ,2997
-datachain/cli.py,sha256=EM6jlc9zunOJQi7-GwCyVtlumHmLM8NwN9Y6jqVGzyY,33769
+datachain/cli.py,sha256=Wl-xMpTRgrkg4drX5I_QxAB1IATyULHCXOdx_wfoLVg,33529
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=g8qbNV0vW2VEKpX-dGZ9pAn0DAz6G2ZFcr7SAV3PoSM,4272
-datachain/dataset.py,sha256=w7qqJP7xYrm9CmBSmSezSxUQHZDsHKkwviF8AYUob7o,14671
-datachain/error.py,sha256=vbIbamnFMIojh1UpmxWoA6Omup7WFAFNJnf8xAkGWwI,1146
+datachain/dataset.py,sha256=lLUbUbJP1TYL9Obkc0f2IDziGcDylZge9ORQjK-WtXs,14717
+datachain/error.py,sha256=bxAAL32lSeMgzsQDEHbGTGORj-mPzzpCRvWDPueJNN4,1092
 datachain/job.py,sha256=Jt4sNutMHJReaGsj3r3scueN5aESLGfhimAa8pUP7Is,1271
 datachain/listing.py,sha256=AV23WZq-k6e2zeeNBhVQP1-2PrwNCYidO0HBDKzpVaA,7152
-datachain/node.py,sha256=ThE6Ue4BqpaBvrkFFJW_ljLxchixUX2aWz3l_nbwY54,5195
+datachain/node.py,sha256=i7_jC8VcW6W5VYkDszAOu0H-rNBuqXB4UnLEh4wFzjc,5195
 datachain/nodes_fetcher.py,sha256=F-73-h19HHNGtHFBGKk7p3mc0ALm4a9zGnzhtuUjnp4,1107
 datachain/nodes_thread_pool.py,sha256=uPo-xl8zG5m9YgODjPFBpbcqqHjI-dcxH87yAbj_qco,3192
 datachain/progress.py,sha256=5KotcvvzAUL_RF0GEj4JY0IB1lyImnmHxe89YkT1XO4,4330
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/studio.py,sha256=d-jUsYpfI1LEv3g8KU-lLchVgb9L0TXvlHakieFud_E,3788
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=-mSFowjIidJ4_sMXInvNHLn4rK_QnHuIlLuH1_lMGmI,13897
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=PvJ-BRoSuI_FRCrXJ6tjMhYZD6L8Beq-ynrdPYRrwiw,58270
+datachain/catalog/catalog.py,sha256=qFlRrR01_9h1MjK6DEgVSgIwbtZEGV_SdG_E5qUsHmM,57352
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
 datachain/client/__init__.py,sha256=T4wiYL9KIM0ZZ_UqIyzV8_ufzYlewmizlV4iymHNluE,86
 datachain/client/azure.py,sha256=ffxs26zm6KLAL1aUWJm-vtzuZP3LSNha7UDGXynMBKo,2234
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=sB98CO7covhmFZg36hsnyv9UwUI8J94AD1QWgGdcBlY,12595
+datachain/client/fsspec.py,sha256=C6C5AO6ndkgcoUxCRN9_8fUzqX2cRWJWG6FL6oD9X_Q,12708
 datachain/client/gcs.py,sha256=cnTIr5GS6dbYOEYfqehhyQu3dr6XNjPHSg5U3FkivUk,4124
 datachain/client/hf.py,sha256=k24bpa6FEKNQn9zhoNC9kCigDwFSqobLsCnN_Nuzwh4,922
-datachain/client/local.py,sha256=Uaf_y_UGspOgprDysUTI9wDo334MLjGPUudqVtvef0c,4367
+datachain/client/local.py,sha256=vwbgCwZ7IqY2voj2l7tLJjgov7Dp--fEUvUwUBsMbls,4457
 datachain/client/s3.py,sha256=CVHBUZ1Ic2Q3370nl-Bbe69phuWjFlrVv9dTJKBpRT0,6019
 datachain/data_storage/__init__.py,sha256=cEOJpyu1JDZtfUupYucCDNFI6e5Wmp_Oyzq6rZv32Y8,398
 datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kTUCaru4,3406
 datachain/data_storage/id_generator.py,sha256=lCEoU0BM37Ai2aRpSbwo5oQT0GqZnSpYwwvizathRMQ,4292
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=HfCxk4lmDUg2Q4WsFNQGMWxllP0mToA00fxkFTwdNIE,52919
+datachain/data_storage/metastore.py,sha256=-TJCqG70VofSVOh2yEez4dwjHS3eQL8p7d9uO3WTVwM,35878
 datachain/data_storage/schema.py,sha256=CiRXrDYp5ZZopSyUgZ7MT2ml_6YvqSTYXdybatcbX9M,9849
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
-datachain/data_storage/sqlite.py,sha256=jopfVftng157TVcBKMB_QPlbkE6fTatiY4GYSSLNkig,28737
-datachain/data_storage/warehouse.py,sha256=iIjFOutYxhLev3CcUhUTwMJOkHeAEBwXZ2y3wmjrF1s,30756
+datachain/data_storage/sqlite.py,sha256=wb8xlMJYYyt59wft0psJj587d-AwpNThzIqspVcKnRI,27388
+datachain/data_storage/warehouse.py,sha256=xwMaR4jBpR13vjG3zrhphH4z2_CFLNj0KPF0LJCXCJ8,30727
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=0R2CYsN82nNa5_03iS6jVix9EKeeqNZNAMgpSQP2hfo,9482
+datachain/lib/arrow.py,sha256=M6SM4u2LeHgylzkPZBWckFeZt3CH3ehpBod3nGl6OYY,9138
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
-datachain/lib/data_model.py,sha256=ECTbvlnzM98hp2mZ4fo82Yi0-MuoqTIQasQKGIyd89I,2040
+datachain/lib/data_model.py,sha256=dau4AlZBhOFvF7pEKMeqCeRkcFFg5KFvTBWW_2CdH5g,2371
 datachain/lib/dataset_info.py,sha256=srPPhI2UHf6hFPBecyFEVw2SS5aPisIIMsvGgKqi7ss,2366
 datachain/lib/dc.py,sha256=pOyE8LqIwo86GrZTSpSMUJAYYwep7nCdIxebkSYlMGo,84484
 datachain/lib/file.py,sha256=LjTW_-PDAnoUhvyB4bJ8Y8n__XGqrxvmd9mDOF0Gir8,14875
@@ -59,7 +58,7 @@ datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
 datachain/lib/udf.py,sha256=4CqK51n3bntXCmkwoOQIrX34wMKOknkC23HtR4D_2vM,12705
 datachain/lib/udf_signature.py,sha256=GXw24A-Olna6DWCdgy2bC-gZh_gLGPQ-KvjuI6pUjC0,7281
-datachain/lib/utils.py,sha256=12elAX6eTFgMGKIf2UfZ4IW07kRwjK6wz8yGE41RtNM,618
+datachain/lib/utils.py,sha256=6NwgWLl5JrgtD4rsSFEe-yR2ntEwJMJEtAZ3FIxK3fg,1529
 datachain/lib/vfile.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/webdataset.py,sha256=o7SHk5HOUWsZ5Ln04xOM04eQqiBHiJNO7xLgyVBrwo8,6924
 datachain/lib/webdataset_laion.py,sha256=aGMWeFmeYNK75ewO9JTA11iB1i3QtTzUfenQA5jajfo,2535
@@ -101,9 +100,9 @@ datachain/sql/sqlite/base.py,sha256=aHSZVvh4XSVkvZ07h3jMoRlHI4sWD8y3SnmGs9xMG9Y,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.6.2.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.6.2.dist-info/METADATA,sha256=QJGHTrGZapho1am27dPKQCOKG_FiEMsvWNLloeU8qVQ,17188
-datachain-0.6.2.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-datachain-0.6.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.6.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.6.2.dist-info/RECORD,,
+datachain-0.6.4.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.6.4.dist-info/METADATA,sha256=zCHryMsrsacIST1qua0PHB6YRNgp1Qayuvsh57SqS9w,17188
+datachain-0.6.4.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+datachain-0.6.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.6.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.6.4.dist-info/RECORD,,

{datachain-0.6.2.dist-info → datachain-0.6.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datachain/storage.py

@@ -1,136 +0,0 @@
-import posixpath
-from abc import ABC, abstractmethod
-from datetime import datetime, timedelta, timezone
-from functools import cached_property
-from typing import NamedTuple, NewType, Optional, Union
-from urllib.parse import urlparse
-
-from datachain.utils import is_expired, time_to_local_str, time_to_str
-
-STALE_MINUTES_LIMIT = 15
-
-# StorageURI represents a normalised URI to a valid storage location (full bucket or
-# absolute local path).
-# Valid examples: s3://foo, file:///var/data
-# Invalid examples: s3://foo/, s3://foo/bar, file://~
-StorageURI = NewType("StorageURI", str)
-
-
-class StorageStatus:
-    CREATED = 1
-    PENDING = 2
-    FAILED = 3
-    COMPLETE = 4
-    PARTIAL = 5
-    STALE = 6
-    INDEXING_SCHEDULED = 7
-    DELETE_SCHEDULED = 8
-
-
-class AbstractStorage(ABC):
-    @property
-    @abstractmethod
-    def uri(self) -> StorageURI: ...
-
-    @property
-    @abstractmethod
-    def timestamp(self) -> Optional[Union[datetime, str]]: ...
-
-    @property
-    @abstractmethod
-    def expires(self) -> Optional[Union[datetime, str]]: ...
-
-    @property
-    @abstractmethod
-    def status(self) -> int: ...
-
-    @property
-    def type(self):
-        return self._parsed_uri.scheme
-
-    @property
-    def name(self):
-        return self._parsed_uri.netloc
-
-    @cached_property
-    def _parsed_uri(self):
-        return urlparse(self.uri)
-
-
-class StorageRecord(NamedTuple):
-    id: int
-    uri: StorageURI
-    timestamp: Optional[Union[datetime, str]] = None
-    expires: Optional[Union[datetime, str]] = None
-    started_inserting_at: Optional[Union[datetime, str]] = None
-    last_inserted_at: Optional[Union[datetime, str]] = None
-    status: int = StorageStatus.CREATED
-    error_message: str = ""
-    error_stack: str = ""
-
-
-class Storage(StorageRecord, AbstractStorage):
-    @property
-    def is_indexed(self) -> bool:
-        return self.status == StorageStatus.COMPLETE
-
-    @property
-    def is_expired(self) -> bool:
-        return is_expired(self.expires)
-
-    @property
-    def is_pending(self) -> bool:
-        return self.status == StorageStatus.PENDING
-
-    @property
-    def is_stale(self) -> bool:
-        limit = datetime.now(timezone.utc) - timedelta(minutes=STALE_MINUTES_LIMIT)
-        date_to_check = self.last_inserted_at or self.started_inserting_at
-
-        return self.is_pending and date_to_check < limit  # type: ignore [operator]
-
-    @property
-    def need_indexing(self) -> bool:
-        return self.is_expired or not self.is_indexed
-
-    @property
-    def timestamp_str(self) -> Optional[str]:
-        if not self.timestamp:
-            return None
-        return time_to_str(self.timestamp)
-
-    @property
-    def timestamp_to_local(self) -> Optional[str]:
-        if not self.timestamp:
-            return None
-        return time_to_local_str(self.timestamp)
-
-    @property
-    def expires_to_local(self) -> Optional[str]:
-        if not self.expires:
-            return None
-        return time_to_local_str(self.expires)
-
-    @staticmethod
-    def get_expiration_time(timestamp: datetime, ttl: int):
-        if ttl >= 0:
-            try:
-                return timestamp + timedelta(seconds=ttl)
-            except OverflowError:
-                return datetime.max
-        else:
-            return datetime.max
-
-    @staticmethod
-    def dataset_name(uri: str, partial_path: str) -> str:
-        return f"{uri}/{partial_path}"
-
-    def to_dict(self, file_path=""):
-        uri = self.uri
-        if file_path:
-            uri = posixpath.join(uri, *file_path.rstrip("/").split("/"))
-        return {
-            "uri": uri,
-            "timestamp": time_to_str(self.timestamp) if self.timestamp else None,
-            "expires": time_to_str(self.expires) if self.expires else None,
-        }