datachain 0.30.2__py3-none-any.whl → 0.30.4__py3-none-any.whl

datachain/__init__.py

@@ -6,6 +6,7 @@ from datachain.lib.dc import (
     Sys,
     datasets,
     delete_dataset,
+    is_studio,
     listings,
     move_dataset,
     read_csv,
@@ -74,6 +75,7 @@ __all__ = [
     "datasets",
     "delete_dataset",
     "is_chain_type",
+    "is_studio",
     "listings",
     "metrics",
     "move_dataset",

datachain/catalog/__init__.py

@@ -3,6 +3,7 @@ from .catalog import (
     QUERY_SCRIPT_CANCELED_EXIT_CODE,
     QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE,
     Catalog,
+    is_namespace_local,
 )
 from .loader import get_catalog
 
@@ -12,4 +13,5 @@ __all__ = [
     "QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE",
     "Catalog",
     "get_catalog",
+    "is_namespace_local",
 ]

datachain/catalog/catalog.py

@@ -113,6 +113,11 @@ else:
     SIGINT = signal.SIGINT
 
 
+def is_namespace_local(namespace_name) -> bool:
+    """Checks if namespace is from local environment, i.e. is `local`"""
+    return namespace_name == "local"
+
+
 def shutdown_process(
     proc: subprocess.Popen,
     interrupt_timeout: Optional[int] = None,
@@ -680,8 +685,9 @@ class Catalog:
                 ds_namespace, ds_project, ds_name = parse_dataset_name(ds_name)
                 assert ds_namespace
                 assert ds_project
-                project = self.metastore.get_project(ds_project, ds_namespace)
-                dataset = self.get_dataset(ds_name, project)
+                dataset = self.get_dataset(
+                    ds_name, namespace_name=ds_namespace, project_name=ds_project
+                )
                 if not ds_version:
                     ds_version = dataset.latest_version
                 dataset_sources = self.warehouse.get_dataset_sources(
@@ -807,7 +813,11 @@ class Catalog:
             )
         default_version = DEFAULT_DATASET_VERSION
         try:
-            dataset = self.get_dataset(name, project)
+            dataset = self.get_dataset(
+                name,
+                namespace_name=project.namespace.name if project else None,
+                project_name=project.name if project else None,
+            )
             default_version = dataset.next_version_patch
             if update_version == "major":
                 default_version = dataset.next_version_major
@@ -1016,7 +1026,11 @@ class Catalog:
             dc.save(name)
         except Exception as e:  # noqa: BLE001
             try:
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=project.namespace.name,
+                    project_name=project.name,
+                )
                 self.metastore.update_dataset_status(
                     ds,
                     DatasetStatus.FAILED,
@@ -1033,7 +1047,11 @@ class Catalog:
             except DatasetNotFoundError:
                 raise e from None
 
-        ds = self.get_dataset(name, project)
+        ds = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
         self.update_dataset_version_with_warehouse_info(
             ds,
@@ -1041,7 +1059,11 @@ class Catalog:
             sources="\n".join(sources),
         )
 
-        return self.get_dataset(name, project)
+        return self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
     def get_full_dataset_name(
         self,
@@ -1077,22 +1099,23 @@ class Catalog:
         return namespace_name, project_name, name
 
     def get_dataset(
-        self, name: str, project: Optional[Project] = None
+        self,
+        name: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
     ) -> DatasetRecord:
         from datachain.lib.listing import is_listing_dataset
 
-        project = project or self.metastore.default_project
+        namespace_name = namespace_name or self.metastore.default_namespace_name
+        project_name = project_name or self.metastore.default_project_name
 
         if is_listing_dataset(name):
-            project = self.metastore.listing_project
+            namespace_name = self.metastore.system_namespace_name
+            project_name = self.metastore.listing_project_name
 
-        try:
-            return self.metastore.get_dataset(name, project.id if project else None)
-        except DatasetNotFoundError:
-            raise DatasetNotFoundError(
-                f"Dataset {name} not found in namespace {project.namespace.name}"
-                f" and project {project.name}"
-            ) from None
+        return self.metastore.get_dataset(
+            name, namespace_name=namespace_name, project_name=project_name
+        )
 
     def get_dataset_with_remote_fallback(
         self,
@@ -1103,6 +1126,8 @@ class Catalog:
         pull_dataset: bool = False,
         update: bool = False,
     ) -> DatasetRecord:
+        from datachain.lib.dc.utils import is_studio
+
         # Intentionally ignore update flag is version is provided. Here only exact
         # version can be provided and update then doesn't make sense.
         # It corresponds to a query like this for example:
@@ -1111,16 +1136,24 @@ class Catalog:
         if version:
             update = False
 
-        if self.metastore.is_local_dataset(namespace_name) or not update:
+        # we don't do Studio fallback is script is already ran in Studio, or if we try
+        # to fetch dataset with local namespace as that one cannot
+        # exist in Studio in the first place
+        no_fallback = is_studio() or is_namespace_local(namespace_name)
+
+        if no_fallback or not update:
             try:
-                project = self.metastore.get_project(project_name, namespace_name)
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=namespace_name,
+                    project_name=project_name,
+                )
                 if not version or ds.has_version(version):
                     return ds
             except (NamespaceNotFoundError, ProjectNotFoundError, DatasetNotFoundError):
                 pass
 
-        if self.metastore.is_local_dataset(namespace_name):
+        if no_fallback:
             raise DatasetNotFoundError(
                 f"Dataset {name}"
                 + (f" version {version} " if version else " ")
@@ -1139,7 +1172,9 @@ class Catalog:
                 local_ds_version=version,
             )
             return self.get_dataset(
-                name, self.metastore.get_project(project_name, namespace_name)
+                name,
+                namespace_name=namespace_name,
+                project_name=project_name,
             )
 
         return self.get_remote_dataset(namespace_name, project_name, name)
@@ -1148,7 +1183,11 @@ class Catalog:
         """Returns dataset that contains version with specific uuid"""
         for dataset in self.ls_datasets():
             if dataset.has_version_with_uuid(uuid):
-                return self.get_dataset(dataset.name, dataset.project)
+                return self.get_dataset(
+                    dataset.name,
+                    namespace_name=dataset.project.namespace.name,
+                    project_name=dataset.project.name,
+                )
         raise DatasetNotFoundError(f"Dataset with version uuid {uuid} not found.")
 
     def get_remote_dataset(
@@ -1171,9 +1210,18 @@ class Catalog:
         return DatasetRecord.from_dict(dataset_info)
 
     def get_dataset_dependencies(
-        self, name: str, version: str, project: Optional[Project] = None, indirect=False
+        self,
+        name: str,
+        version: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        indirect=False,
     ) -> list[Optional[DatasetDependency]]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=namespace_name,
+            project_name=project_name,
+        )
 
         direct_dependencies = self.metastore.get_direct_dataset_dependencies(
             dataset, version
@@ -1187,10 +1235,13 @@ class Catalog:
                 # dependency has been removed
                 continue
             if d.is_dataset:
-                project = self.metastore.get_project(d.project, d.namespace)
                 # only datasets can have dependencies
                 d.dependencies = self.get_dataset_dependencies(
-                    d.name, d.version, project, indirect=indirect
+                    d.name,
+                    d.version,
+                    namespace_name=d.namespace,
+                    project_name=d.project,
+                    indirect=indirect,
                 )
 
         return direct_dependencies
@@ -1340,7 +1391,11 @@ class Catalog:
         project: Optional[Project] = None,
         client_config=None,
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
 
         return self.warehouse.export_dataset_table(
             bucket_uri, dataset, version, client_config
@@ -1349,7 +1404,11 @@ class Catalog:
     def dataset_table_export_file_names(
         self, name: str, version: str, project: Optional[Project] = None
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
     def remove_dataset(
@@ -1359,7 +1418,11 @@ class Catalog:
         version: Optional[str] = None,
         force: Optional[bool] = False,
     ):
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         if not version and not force:
             raise ValueError(f"Missing dataset version from input for dataset {name}")
         if version and not dataset.has_version(version):
@@ -1395,7 +1458,11 @@ class Catalog:
         if attrs is not None:
             update_data["attrs"] = attrs  # type: ignore[assignment]
 
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.update_dataset(dataset, **update_data)
 
     def ls(
@@ -1549,7 +1616,9 @@ class Catalog:
         )
 
         try:
-            local_dataset = self.get_dataset(local_ds_name, project=project)
+            local_dataset = self.get_dataset(
+                local_ds_name, namespace_name=namespace.name, project_name=project.name
+            )
             if local_dataset and local_dataset.has_version(local_ds_version):
                 raise DataChainError(
                     f"Local dataset {local_ds_uri} already exists with different uuid,"

datachain/catalog/loader.py

@@ -127,7 +127,8 @@ def get_udf_distributor_class() -> Optional[type["AbstractUDFDistributor"]]:
 
 
 def get_catalog(
-    client_config: Optional[dict[str, Any]] = None, in_memory: bool = False
+    client_config: Optional[dict[str, Any]] = None,
+    in_memory: bool = False,
 ) -> "Catalog":
     """
     Function that creates Catalog instance with appropriate metastore
@@ -142,8 +143,9 @@ def get_catalog(
     """
     from datachain.catalog import Catalog
 
+    metastore = get_metastore(in_memory=in_memory)
     return Catalog(
-        metastore=get_metastore(in_memory=in_memory),
+        metastore=metastore,
         warehouse=get_warehouse(in_memory=in_memory),
         client_config=client_config,
         in_memory=in_memory,

datachain/cli/__init__.py

@@ -6,6 +6,7 @@ from multiprocessing import freeze_support
 from typing import Optional
 
 from datachain.cli.utils import get_logging_level
+from datachain.error import DataChainError as DataChainError
 
 from .commands import (
     clear_cache,

datachain/cli/commands/datasets.py

@@ -6,6 +6,7 @@ from tabulate import tabulate
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
+from datachain.catalog import is_namespace_local
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
 from datachain.error import DataChainError, DatasetNotFoundError
@@ -107,8 +108,9 @@ def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
 def list_datasets_local_versions(catalog: "Catalog", name: str):
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
-    project = catalog.metastore.get_project(project_name, namespace_name)
-    ds = catalog.get_dataset(name, project)
+    ds = catalog.get_dataset(
+        name, namespace_name=namespace_name, project_name=project_name
+    )
     for v in ds.versions:
         yield (name, v.version)
 
@@ -137,15 +139,18 @@ def rm_dataset(
 ):
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
-    if not catalog.metastore.is_local_dataset(namespace_name) and studio:
+    if studio:
+        # removing Studio dataset from CLI
         from datachain.studio import remove_studio_dataset
 
-        token = Config().read().get("studio", {}).get("token")
-        if not token:
+        if Config().read().get("studio", {}).get("token"):
+            remove_studio_dataset(
+                team, name, namespace_name, project_name, version, force
+            )
+        else:
             raise DataChainError(
                 "Not logged in to Studio. Log in with 'datachain auth login'."
             )
-        remove_studio_dataset(team, name, namespace_name, project_name, version, force)
     else:
         try:
             project = catalog.metastore.get_project(project_name, namespace_name)
@@ -162,9 +167,11 @@ def edit_dataset(
     attrs: Optional[list[str]] = None,
     team: Optional[str] = None,
 ):
+    from datachain.lib.dc.utils import is_studio
+
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
-    if catalog.metastore.is_local_dataset(namespace_name):
+    if is_studio() or is_namespace_local(namespace_name):
         try:
             catalog.edit_dataset(
                 name, catalog.metastore.default_project, new_name, description, attrs
@@ -174,11 +181,11 @@ def edit_dataset(
     else:
         from datachain.studio import edit_studio_dataset
 
-        token = Config().read().get("studio", {}).get("token")
-        if not token:
+        if Config().read().get("studio", {}).get("token"):
+            edit_studio_dataset(
+                team, name, namespace_name, project_name, new_name, description, attrs
+            )
+        else:
             raise DataChainError(
                 "Not logged in to Studio. Log in with 'datachain auth login'."
             )
-        edit_studio_dataset(
-            team, name, namespace_name, project_name, new_name, description, attrs
-        )

datachain/data_storage/metastore.py

@@ -145,23 +145,6 @@ class AbstractMetastore(ABC, Serializable):
     def list_namespaces(self, conn=None) -> list[Namespace]:
         """Gets a list of all namespaces"""
 
-    @property
-    @abstractmethod
-    def is_studio(self) -> bool:
-        """Returns True if this code is ran in Studio"""
-
-    def is_local_dataset(self, dataset_namespace: str) -> bool:
-        """
-        Returns True if this is local dataset i.e. not pulled from Studio but
-        created locally. This is False if we ran code in CLI mode but using dataset
-        names that are present in Studio.
-        """
-        return self.is_studio or dataset_namespace == Namespace.default()
-
-    @property
-    def namespace_allowed_to_create(self):
-        return self.is_studio
-
     #
     # Projects
     #
@@ -215,10 +198,6 @@ class AbstractMetastore(ABC, Serializable):
     def list_projects(self, namespace_id: Optional[int], conn=None) -> list[Project]:
         """Gets list of projects in some namespace or in general (in all namespaces)"""
 
-    @property
-    def project_allowed_to_create(self):
-        return self.is_studio
-
     #
     # Datasets
     #
@@ -301,7 +280,13 @@ class AbstractMetastore(ABC, Serializable):
         """
 
     @abstractmethod
-    def get_dataset(self, name: str, project_id: Optional[int] = None) -> DatasetRecord:
+    def get_dataset(
+        self,
+        name: str,  # normal, not full dataset name
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        conn=None,
+    ) -> DatasetRecord:
         """Gets a single dataset by name."""
 
     @abstractmethod
@@ -912,11 +897,14 @@ class AbstractDBMetastore(AbstractMetastore):
         **kwargs,  # TODO registered = True / False
     ) -> DatasetRecord:
         """Creates new dataset."""
-        project_id = project_id or self.default_project.id
+        if not project_id:
+            project = self.default_project
+        else:
+            project = self.get_project_by_id(project_id)
 
         query = self._datasets_insert().values(
             name=name,
-            project_id=project_id,
+            project_id=project.id,
             status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=datetime.now(timezone.utc),
@@ -935,7 +923,9 @@ class AbstractDBMetastore(AbstractMetastore):
             query = query.on_conflict_do_nothing(index_elements=["project_id", "name"])
         self.db.execute(query)
 
-        return self.get_dataset(name, project_id)
+        return self.get_dataset(
+            name, namespace_name=project.namespace.name, project_name=project.name
+        )
 
     def create_dataset_version(  # noqa: PLR0913
         self,
@@ -992,7 +982,12 @@ class AbstractDBMetastore(AbstractMetastore):
             )
         self.db.execute(query, conn=conn)
 
-        return self.get_dataset(dataset.name, dataset.project.id, conn=conn)
+        return self.get_dataset(
+            dataset.name,
+            namespace_name=dataset.project.namespace.name,
+            project_name=dataset.project.name,
+            conn=conn,
+        )
 
     def remove_dataset(self, dataset: DatasetRecord) -> None:
         """Removes dataset."""
@@ -1216,21 +1211,30 @@ class AbstractDBMetastore(AbstractMetastore):
     def get_dataset(
         self,
         name: str,  # normal, not full dataset name
-        project_id: Optional[int] = None,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
         conn=None,
     ) -> DatasetRecord:
         """
         Gets a single dataset in project by dataset name.
         """
-        project_id = project_id or self.default_project.id
+        namespace_name = namespace_name or self.default_namespace_name
+        project_name = project_name or self.default_project_name
 
         d = self._datasets
+        n = self._namespaces
+        p = self._projects
         query = self._base_dataset_query()
-        query = query.where(d.c.name == name, d.c.project_id == project_id)  # type: ignore [attr-defined]
+        query = query.where(
+            d.c.name == name,
+            n.c.name == namespace_name,
+            p.c.name == project_name,
+        )  # type: ignore [attr-defined]
         ds = self._parse_dataset(self.db.execute(query, conn=conn))
         if not ds:
             raise DatasetNotFoundError(
-                f"Dataset {name} not found in project with id {project_id}"
+                f"Dataset {name} not found in namespace {namespace_name}"
+                f" and project {project_name}"
             )
 
         return ds

datachain/data_storage/sqlite.py

@@ -542,10 +542,6 @@ class SQLiteMetastore(AbstractDBMetastore):
     def _jobs_insert(self) -> "Insert":
         return sqlite.insert(self._jobs)
 
-    @property
-    def is_studio(self) -> bool:
-        return False
-
     #
     # Namespaces
     #

datachain/delta.py

@@ -77,7 +77,8 @@ def _get_delta_chain(
 
 def _get_retry_chain(
     name: str,
-    project: Project,
+    namespace_name: str,
+    project_name: str,
     latest_version: str,
     source_ds_name: str,
     source_ds_project: Project,
@@ -96,8 +97,8 @@ def _get_retry_chain(
     # Read the latest version of the result dataset for retry logic
     result_dataset = datachain.read_dataset(
         name,
-        namespace=project.namespace.name,
-        project=project.name,
+        namespace=namespace_name,
+        project=project_name,
         version=latest_version,
     )
     source_dc = datachain.read_dataset(
@@ -128,7 +129,8 @@ def _get_retry_chain(
 
 def _get_source_info(
     name: str,
-    project: Project,
+    namespace_name: str,
+    project_name: str,
     latest_version: str,
     catalog,
 ) -> tuple[
@@ -145,7 +147,11 @@ def _get_source_info(
         Returns (None, None, None, None) if source dataset was removed.
     """
     dependencies = catalog.get_dataset_dependencies(
-        name, latest_version, project=project, indirect=False
+        name,
+        latest_version,
+        namespace_name=namespace_name,
+        project_name=project_name,
+        indirect=False,
     )
 
     dep = dependencies[0]
@@ -157,7 +163,9 @@ def _get_source_info(
     source_ds_name = dep.name
     source_ds_version = dep.version
     source_ds_latest_version = catalog.get_dataset(
-        source_ds_name, project=source_ds_project
+        source_ds_name,
+        namespace_name=source_ds_project.namespace.name,
+        project_name=source_ds_project.name,
     ).latest_version
 
     return (
@@ -211,12 +219,14 @@ def delta_retry_update(
     """
 
     catalog = dc.session.catalog
-    project = catalog.metastore.get_project(project_name, namespace_name)
+    # project = catalog.metastore.get_project(project_name, namespace_name)
     dc._query.apply_listing_pre_step()
 
     # Check if dataset exists
     try:
-        dataset = catalog.get_dataset(name, project=project)
+        dataset = catalog.get_dataset(
+            name, namespace_name=namespace_name, project_name=project_name
+        )
         latest_version = dataset.latest_version
     except DatasetNotFoundError:
         # First creation of result dataset
@@ -234,7 +244,7 @@ def delta_retry_update(
         source_ds_version,
         source_ds_latest_version,
         dependencies,
-    ) = _get_source_info(name, project, latest_version, catalog)
+    ) = _get_source_info(name, namespace_name, project_name, latest_version, catalog)
 
     # If source_ds_name is None, starting dataset was removed
     if source_ds_name is None:
@@ -264,7 +274,8 @@ def delta_retry_update(
     if delta_retry:
         retry_chain = _get_retry_chain(
             name,
-            project,
+            namespace_name,
+            project_name,
             latest_version,
             source_ds_name,
             source_ds_project,
@@ -290,8 +301,8 @@ def delta_retry_update(
 
     latest_dataset = datachain.read_dataset(
         name,
-        namespace=project.namespace.name,
-        project=project.name,
+        namespace=namespace_name,
+        project=project_name,
         version=latest_version,
     )
     compared_chain = latest_dataset.diff(

datachain/func/string.py

@@ -6,6 +6,14 @@ from datachain.sql.functions import string
 
 from .func import ColT, Func
 
+__all__ = [
+    "byte_hamming_distance",
+    "length",
+    "regexp_replace",
+    "replace",
+    "split",
+]
+
 
 def length(col: ColT) -> Func:
     """

datachain/lib/dc/__init__.py

@@ -9,7 +9,7 @@ from .pandas import read_pandas
 from .parquet import read_parquet
 from .records import read_records
 from .storage import read_storage
-from .utils import DatasetMergeError, DatasetPrepareError, Sys
+from .utils import DatasetMergeError, DatasetPrepareError, Sys, is_studio
 from .values import read_values
 
 __all__ = [
@@ -21,6 +21,7 @@ __all__ = [
     "Sys",
     "datasets",
     "delete_dataset",
+    "is_studio",
     "listings",
     "move_dataset",
     "read_csv",