datachain 0.22.0__tar.gz → 0.23.0__tar.gz

{datachain-0.22.0 → datachain-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.22.0
+Version: 0.23.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.22.0 → datachain-0.23.0}/docs/guide/env.md

@@ -15,4 +15,8 @@ List of environment variables used to configure DataChain behavior.
 - `DATACHAIN_STUDIO_TOKEN` – Authentication token for Studio.
 - `DATACHAIN_STUDIO_TEAM` – Studio team name.
 
+### Namespaces and projects
+- `DATACHAIN_NAMESPACE` – Namespace name to use as default.
+- `DATACHAIN_PROJECT` – Project name or combination of namespace name and project name separated by `.` to use as default, example: `DATACHAIN_PROJECT=dev.analytics`
+
 Note: Some environment variables are used internally and may not be documented here. For the most up-to-date list, refer to the source code.

{datachain-0.22.0 → datachain-0.23.0}/docs/guide/namespaces.md

@@ -82,6 +82,49 @@ This is equivalent to saving to `dev.analytics.metrics`.
 
 In CLI, `.settings()` is only supported when both `namespace` and `project` are set to `"local"`.
 
+## Setting Namespace and Project via Environment Variables
+
+In addition to using `.settings()`, you can configure the namespace and project using environment variables:
+
+- `DATACHAIN_NAMESPACE` sets the namespace.
+- `DATACHAIN_PROJECT` sets the project name, or both the namespace and project using the format `namespace.project`.
+
+### Examples
+
+```
+# Set namespace only
+export DATACHAIN_NAMESPACE=dev
+
+# Set project only
+export DATACHAIN_PROJECT=analytics
+
+# Set both namespace and project
+export DATACHAIN_PROJECT=dev.analytics
+```
+
+##  How Namespace and Project Are Resolved
+
+When determining which namespace and project to use, Datachain applies the following precedence:
+
+1. **Fully qualified dataset name**
+   If the dataset name includes both the namespace and project, these values take highest precedence.
+   ```python
+   dc.read_dataset("dev.analytics.metrics")
+
+2. **Explicit settings in code**
+   Values provided via `.settings()` or passed directly to `read_dataset()` or similar methods.
+   ```python
+   dc.settings(namespace="dev", project="analytics")
+   dc.read_dataset("metrics", namespace="dev", project="analytics")
+   ```
+3. **Environment variables**
+   Namespace and project set using environment variables:
+   ```console
+   export DATACHAIN_PROJECT=dev.analytics
+   ```
+4. **Defaults**
+If none of the above are provided, Datachain falls back to the default namespace and project.
+
 ## Reading a Dataset from a Project
 
 To read a dataset from a specific namespace and project:
@@ -116,4 +159,3 @@ dc.read_values(scores=[0.8, 1.5, 2.1]).save("metrics")
 
 ds = dc.read_dataset("local.local.metrics")
 ds.show()
-```

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/catalog/catalog.py

@@ -1059,6 +1059,39 @@ class Catalog:
 
         return self.get_dataset(name, project)
 
+    def get_full_dataset_name(
+        self,
+        name: str,
+        project_name: Optional[str] = None,
+        namespace_name: Optional[str] = None,
+    ) -> tuple[str, str, str]:
+        """
+        Returns dataset name together with separated namespace and project name.
+        It takes into account all the ways namespace and project can be added.
+        """
+        parsed_namespace_name, parsed_project_name, name = parse_dataset_name(name)
+
+        namespace_env = os.environ.get("DATACHAIN_NAMESPACE")
+        project_env = os.environ.get("DATACHAIN_PROJECT")
+        if project_env and len(project_env.split(".")) == 2:
+            # we allow setting both namespace and project in DATACHAIN_PROJECT
+            namespace_env, project_env = project_env.split(".")
+
+        namespace_name = (
+            parsed_namespace_name
+            or namespace_name
+            or namespace_env
+            or self.metastore.default_namespace_name
+        )
+        project_name = (
+            parsed_project_name
+            or project_name
+            or project_env
+            or self.metastore.default_project_name
+        )
+
+        return namespace_name, project_name, name
+
     def get_dataset(
         self, name: str, project: Optional[Project] = None
     ) -> DatasetRecord:

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/cli/commands/datasets.py

@@ -8,7 +8,6 @@ if TYPE_CHECKING:
 
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
-from datachain.dataset import parse_dataset_name
 from datachain.error import DataChainError, DatasetNotFoundError
 from datachain.studio import list_datasets as list_datasets_studio
 
@@ -106,9 +105,8 @@ def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
 
 
 def list_datasets_local_versions(catalog: "Catalog", name: str):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
+    namespace_name, project_name, name = catalog.get_full_dataset_name(name)
+
     project = catalog.metastore.get_project(project_name, namespace_name)
     ds = catalog.get_dataset(name, project)
     for v in ds.versions:
@@ -137,9 +135,7 @@ def rm_dataset(
     studio: Optional[bool] = False,
     team: Optional[str] = None,
 ):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
+    namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     if not catalog.metastore.is_local_dataset(namespace_name) and studio:
         from datachain.studio import remove_studio_dataset
@@ -166,9 +162,7 @@ def edit_dataset(
     attrs: Optional[list[str]] = None,
     team: Optional[str] = None,
 ):
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = namespace_name or catalog.metastore.default_namespace_name
-    project_name = project_name or catalog.metastore.default_project_name
+    namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     if catalog.metastore.is_local_dataset(namespace_name):
         try:

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/data_storage/metastore.py

@@ -132,6 +132,7 @@ class AbstractMetastore(ABC, Serializable):
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
+        validate: bool = True,
         **kwargs,
     ) -> Namespace:
         """Creates new namespace"""
@@ -192,6 +193,7 @@ class AbstractMetastore(ABC, Serializable):
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
+        validate: bool = True,
         **kwargs,
     ) -> Project:
         """Creates new project in specific namespace"""
@@ -725,8 +727,11 @@ class AbstractDBMetastore(AbstractMetastore):
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
+        validate: bool = True,
         **kwargs,
     ) -> Namespace:
+        if validate:
+            Namespace.validate_name(name)
         query = self._namespaces_insert().values(
             name=name,
             uuid=uuid or str(uuid4()),
@@ -775,12 +780,15 @@ class AbstractDBMetastore(AbstractMetastore):
         description: Optional[str] = None,
         uuid: Optional[str] = None,
         ignore_if_exists: bool = True,
+        validate: bool = True,
         **kwargs,
     ) -> Project:
+        if validate:
+            Project.validate_name(name)
         try:
             namespace = self.get_namespace(namespace_name)
         except NamespaceNotFoundError:
-            namespace = self.create_namespace(namespace_name)
+            namespace = self.create_namespace(namespace_name, validate=validate)
 
         query = self._projects_insert().values(
             namespace_id=namespace.id,
@@ -817,11 +825,14 @@ class AbstractDBMetastore(AbstractMetastore):
         """Gets a single project inside some namespace by name"""
         n = self._namespaces
         p = self._projects
+        validate = True
+
         if self._is_listing_project(name, namespace_name) or self._is_default_project(
             name, namespace_name
         ):
             # we are always creating default and listing projects if they don't exist
             create = True
+            validate = False
 
         query = self._projects_select(
             *(getattr(n.c, f) for f in self._namespaces_fields),
@@ -834,7 +845,7 @@ class AbstractDBMetastore(AbstractMetastore):
         rows = list(self.db.execute(query, conn=conn))
         if not rows:
             if create:
-                return self.create_project(namespace_name, name)
+                return self.create_project(namespace_name, name, validate=validate)
             raise ProjectNotFoundError(
                 f"Project {name} in namespace {namespace_name} not found."
             )

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/data_storage/sqlite.py

@@ -468,8 +468,12 @@ class SQLiteMetastore(AbstractDBMetastore):
         be created implicitly though, to keep the same fully qualified name with
         Studio dataset.
         """
-        system_namespace = self.create_namespace(Namespace.system(), "System namespace")
-        self.create_project(system_namespace.name, Project.listing(), "Listing project")
+        system_namespace = self.create_namespace(
+            Namespace.system(), "System namespace", validate=False
+        )
+        self.create_project(
+            system_namespace.name, Project.listing(), "Listing project", validate=False
+        )
 
     def _check_schema_version(self) -> None:
         """

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/dataset.py

@@ -81,8 +81,10 @@ def create_dataset_uri(
 def parse_dataset_name(name: str) -> tuple[Optional[str], Optional[str], str]:
     """Parses dataset name and returns namespace, project and name"""
     if not name:
-        raise ValueError("Name must be defined to parse it")
+        raise InvalidDatasetNameError("Name must be defined to parse it")
     split = name.split(".")
+    if len(split) > 3:
+        raise InvalidDatasetNameError(f"Invalid dataset name {name}")
     name = split[-1]
     project_name = split[-2] if len(split) > 1 else None
     namespace_name = split[-3] if len(split) > 2 else None

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/lib/dc/datachain.py

@@ -24,7 +24,7 @@ from pydantic import BaseModel
 from tqdm import tqdm
 
 from datachain import semver
-from datachain.dataset import DatasetRecord, parse_dataset_name
+from datachain.dataset import DatasetRecord
 from datachain.delta import delta_disabled
 from datachain.error import ProjectCreateNotAllowedError, ProjectNotFoundError
 from datachain.func import literal
@@ -557,6 +557,7 @@ class DataChain:
             update_version: which part of the dataset version to automatically increase.
                 Available values: `major`, `minor` or `patch`. Default is `patch`.
         """
+        catalog = self.session.catalog
         if version is not None:
             semver.validate(version)
 
@@ -570,17 +571,10 @@ class DataChain:
                 " patch"
             )
 
-        namespace_name, project_name, name = parse_dataset_name(name)
-
-        namespace_name = (
-            namespace_name
-            or self._settings.namespace
-            or self.session.catalog.metastore.default_namespace_name
-        )
-        project_name = (
-            project_name
-            or self._settings.project
-            or self.session.catalog.metastore.default_project_name
+        namespace_name, project_name, name = catalog.get_full_dataset_name(
+            name,
+            namespace_name=self._settings.namespace,
+            project_name=self._settings.project,
         )
 
         try:

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/lib/dc/datasets.py

@@ -1,7 +1,6 @@
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Optional, Union, get_origin, get_type_hints
 
-from datachain.dataset import parse_dataset_name
 from datachain.error import (
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
@@ -125,11 +124,11 @@ def read_dataset(
     session = Session.get(session)
     catalog = session.catalog
 
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = (
-        namespace_name or namespace or catalog.metastore.default_namespace_name
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
     )
-    project_name = project_name or project or catalog.metastore.default_project_name
 
     if version is not None:
         try:
@@ -320,11 +319,11 @@ def delete_dataset(
     session = Session.get(session, in_memory=in_memory)
     catalog = session.catalog
 
-    namespace_name, project_name, name = parse_dataset_name(name)
-    namespace_name = (
-        namespace_name or namespace or catalog.metastore.default_namespace_name
+    namespace_name, project_name, name = catalog.get_full_dataset_name(
+        name,
+        project_name=project,
+        namespace_name=namespace,
     )
-    project_name = project_name or project or catalog.metastore.default_project_name
 
     if not catalog.metastore.is_local_dataset(namespace_name) and studio:
         return remove_studio_dataset(

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/lib/dc/records.py

@@ -97,4 +97,4 @@ def read_records(
     for chunk in batched(records, INSERT_BATCH_SIZE):
         warehouse.insert_rows(table, chunk)
     warehouse.insert_rows_done(table)
-    return read_dataset(name=dsr.name, session=session, settings=settings)
+    return read_dataset(name=dsr.full_name, session=session, settings=settings)

{datachain-0.22.0 → datachain-0.23.0}/src/datachain/lib/signal_schema.py

@@ -25,6 +25,7 @@ from pydantic import BaseModel, Field, create_model
 from sqlalchemy import ColumnElement
 from typing_extensions import Literal as LiteralEx
 
+from datachain.func import literal
 from datachain.func.func import Func
 from datachain.lib.convert.python_to_sql import python_to_sql
 from datachain.lib.convert.sql_to_python import sql_to_python
@@ -659,6 +660,7 @@ class SignalSchema:
 
     def mutate(self, args_map: dict) -> "SignalSchema":
         new_values = self.values.copy()
+        primitives = (bool, str, int, float)
 
         for name, value in args_map.items():
             if isinstance(value, Column) and value.name in self.values:
@@ -679,6 +681,12 @@ class SignalSchema:
                 # adding new signal with function
                 new_values[name] = value.get_result_type(self)
                 continue
+            if isinstance(value, primitives):
+                # For primitives, store the type, not the value
+                val = literal(value)
+                val.type = python_to_sql(type(value))()
+                new_values[name] = sql_to_python(val)
+                continue
             if isinstance(value, ColumnElement):
                 # adding new signal
                 new_values[name] = sql_to_python(value)

{datachain-0.22.0 → datachain-0.23.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.22.0
+Version: 0.23.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.22.0 → datachain-0.23.0}/tests/func/test_datachain.py

@@ -756,6 +756,52 @@ def test_mutate_existing_column(test_session):
     assert ds.order_by("ids").to_list() == [(2,), (3,), (4,)]
 
 
+def test_mutate_with_primitives_save_load(test_session):
+    """Test that mutate with primitive values properly persists schema
+    through save/load cycle."""
+    original_data = [1, 2, 3]
+
+    # Create dataset with multiple primitive columns added via mutate
+    ds = dc.read_values(data=original_data, session=test_session).mutate(
+        str_col="test_string",
+        int_col=42,
+        float_col=3.14,
+        bool_col=True,
+    )
+
+    # Verify schema before saving
+    schema = ds.signals_schema.values
+    assert schema.get("str_col") is str
+    assert schema.get("int_col") is int
+    assert schema.get("float_col") is float
+    assert schema.get("bool_col") is bool
+
+    ds.save("test_mutate_primitives")
+
+    # Load the dataset back
+    loaded_ds = dc.read_dataset("test_mutate_primitives", session=test_session)
+
+    # Verify schema after loading
+    loaded_schema = loaded_ds.signals_schema.values
+    assert loaded_schema.get("str_col") is str
+    assert loaded_schema.get("int_col") is int
+    assert loaded_schema.get("float_col") is float
+    assert loaded_schema.get("bool_col") is bool
+
+    # Verify data integrity
+    results = set(loaded_ds.to_list())
+    assert len(results) == 3
+
+    # Expected tuples: (data, str_col, int_col, float_col, bool_col)
+    expected_results = {
+        (1, "test_string", 42, 3.14, True),
+        (2, "test_string", 42, 3.14, True),
+        (3, "test_string", 42, 3.14, True),
+    }
+
+    assert results == expected_results
+
+
 @pytest.mark.parametrize("processes", [False, 2, True])
 @pytest.mark.xdist_group(name="tmpfile")
 def test_parallel(processes, test_session_tmpfile):

{datachain-0.22.0 → datachain-0.23.0}/tests/unit/lib/test_datachain.py

@@ -20,6 +20,9 @@ from datachain.error import (
     DatasetInvalidVersionError,
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
+    InvalidDatasetNameError,
+    InvalidNamespaceNameError,
+    InvalidProjectNameError,
     ProjectCreateNotAllowedError,
 )
 from datachain.lib.data_model import DataModel
@@ -3425,7 +3428,9 @@ def test_save_specify_only_non_default_project(
     default_namespace_name = catalog.metastore.default_namespace_name
 
     if project_created_upfront:
-        catalog.metastore.create_project(default_namespace_name, "numbers")
+        catalog.metastore.create_project(
+            default_namespace_name, "numbers", validate=False
+        )
 
     ds = dc.read_values(fib=[1, 1, 2, 3, 5, 8], session=test_session)
     if use_settings:
@@ -3445,6 +3450,111 @@ def test_save_specify_only_non_default_project(
         dc.read_dataset(name="fibonacci")
 
 
+@pytest.mark.parametrize(
+    (
+        "ds_name_namespace,ds_name_project,"
+        "settings_namespace,settings_project,"
+        "env_namespace,env_project,"
+        "result_ds_namespace,result_ds_project"
+    ),
+    [
+        ("n3", "p3", "n2", "p2", "n1", "p1", "n3", "p3"),
+        ("", "", "n2", "p2", "n1", "p1", "n2", "p2"),
+        ("", "", "", "", "n1", "p1", "n1", "p1"),
+        ("", "", "", "", "n5", "n1.p1", "n1", "p1"),
+        ("", "", "", "", "", "n1.p1", "n1", "p1"),
+        ("", "", "", "", "", "n5.p5", "n5", "p5"),
+        ("n3", "p3", "n2", "p2", "", "", "n3", "p3"),
+        ("n3", "p3", "", "", "", "", "n3", "p3"),
+        ("n3", "p3", "", "", "n1", "p1", "n3", "p3"),
+        ("", "", "", "", "", "", "", ""),
+    ],
+)
+def test_save_all_ways_to_set_project(
+    test_session,
+    monkeypatch,
+    ds_name_namespace,
+    ds_name_project,
+    settings_namespace,
+    settings_project,
+    env_namespace,
+    env_project,
+    result_ds_namespace,
+    result_ds_project,
+):
+    def _full_name(namespace, project, name) -> str:
+        if namespace and project:
+            return f"{namespace}.{project}.{name}"
+        return name
+
+    metastore = test_session.catalog.metastore
+    ds_name = "numbers"
+
+    monkeypatch.setenv("DATACHAIN_NAMESPACE", env_namespace)
+    monkeypatch.setenv("DATACHAIN_PROJECT", env_project)
+
+    if not result_ds_namespace and not result_ds_project:
+        # special case when nothing is defined - we set default ones
+        result_ds_namespace = metastore.default_namespace_name
+        result_ds_project = metastore.default_project_name
+
+    ds = (
+        dc.read_values(num=[1, 2, 3, 4], session=test_session)
+        .settings(namespace=settings_namespace, project=settings_project)
+        .save(_full_name(ds_name_namespace, ds_name_project, ds_name))
+    )
+
+    assert ds.dataset.project == metastore.get_project(
+        result_ds_project, result_ds_namespace
+    )
+    dc.read_dataset(_full_name(result_ds_namespace, result_ds_project, ds_name))
+
+
+@pytest.mark.parametrize(
+    (
+        "ds_name_namespace,ds_name_project,"
+        "settings_namespace,settings_project,"
+        "env_namespace,env_project,"
+        "error"
+    ),
+    [
+        ("n3.n3", "p3", "n2", "p2", "n1", "p1", InvalidDatasetNameError),
+        ("n3", "p3.p3", "n2", "p2", "n1", "p1", InvalidDatasetNameError),
+        ("", "", "n2.n2", "p2", "n1", "p1", InvalidNamespaceNameError),
+        ("", "", "n2", "p2.p2", "n1", "p1", InvalidProjectNameError),
+        ("", "", "", "", "n1.n1", "p1", InvalidNamespaceNameError),
+        ("", "", "", "", "n1", "p1.p1.p1", InvalidProjectNameError),
+    ],
+)
+def test_save_all_ways_to_set_project_invalid_name(
+    test_session,
+    monkeypatch,
+    ds_name_namespace,
+    ds_name_project,
+    settings_namespace,
+    settings_project,
+    env_namespace,
+    env_project,
+    error,
+):
+    def _full_name(namespace, project, name) -> str:
+        if namespace and project:
+            return f"{namespace}.{project}.{name}"
+        return name
+
+    ds_name = "numbers"
+
+    monkeypatch.setenv("DATACHAIN_NAMESPACE", env_namespace)
+    monkeypatch.setenv("DATACHAIN_PROJECT", env_project)
+
+    with pytest.raises(error):
+        (
+            dc.read_values(num=[1, 2, 3, 4], session=test_session)
+            .settings(namespace=settings_namespace, project=settings_project)
+            .save(_full_name(ds_name_namespace, ds_name_project, ds_name))
+        )
+
+
 @pytest.mark.parametrize("allow_create_project", [False])
 @skip_if_not_sqlite
 def test_save_create_project_not_allowed(test_session, allow_create_project):

{datachain-0.22.0 → datachain-0.23.0}/tests/unit/test_dataset.py

@@ -178,5 +178,5 @@ def test_parse_dataset_name(full_name, namespace, project, name):
 
 
 def test_parse_dataset_name_empty_name():
-    with pytest.raises(ValueError):
+    with pytest.raises(InvalidDatasetNameError):
         assert parse_dataset_name(None)