datachain 0.3.20__tar.gz → 0.5.0__tar.gz

{datachain-0.3.20 → datachain-0.5.0}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.6.5'
+    rev: 'v0.6.7'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.3.20/src/datachain.egg-info → datachain-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.20
+Version: 0.5.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.3.20 → datachain-0.5.0}/mkdocs.yml

@@ -15,6 +15,7 @@ validation:
 
 theme:
   name: material
+  custom_dir: overrides
   logo: assets/datachain-white.svg
   favicon: assets/datachain.svg
   icon:
@@ -71,6 +72,7 @@ nav:
       - references/udf.md
       - references/torch.md
       - references/sql.md
+  - DataChain Website: https://datachain.ai" target="_blank"
 
 markdown_extensions:
   - abbr

datachain-0.5.0/overrides/main.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block scripts %}
+
+{{ super() }}
+
+<script type="text/javascript">
+  !function () { var e, t, n; e = "14ffd92a6cbf5f2", t = function () { Reo.init({ clientID: "14ffd92a6cbf5f2" }) }, (n = document.createElement("script")).src = "https://static.reo.dev/" + e + "/reo.js", n.async = !0, n.onload = t, document.head.appendChild(n) }();
+</script>
+
+
+{% endblock %}

{datachain-0.3.20 → datachain-0.5.0}/src/datachain/catalog/catalog.py

@@ -979,7 +979,6 @@ class Catalog:
         script_output="",
         create_rows_table=True,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ) -> DatasetRecord:
         """
         Creates dataset version if it doesn't exist.
@@ -1001,7 +1000,6 @@ class Catalog:
             script_output=script_output,
             schema=schema,
             job_id=job_id,
-            is_job_result=is_job_result,
             ignore_if_exists=True,
         )
 
@@ -1211,7 +1209,6 @@ class Catalog:
             size=dataset_version.size,
             preview=dataset_version.preview,
             job_id=dataset_version.job_id,
-            is_job_result=dataset_version.is_job_result,
         )
         # to avoid re-creating rows table, we are just renaming it for a new version
         # of target dataset

{datachain-0.3.20 → datachain-0.5.0}/src/datachain/cli.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 
 import shtab
 
-from datachain import utils
+from datachain import Session, utils
 from datachain.cli_utils import BooleanOptionalAction, CommaSeparatedArgs, KeyValueArgs
 from datachain.lib.dc import DataChain
 from datachain.telemetry import telemetry
@@ -770,7 +770,8 @@ def show(
     show_records(records, collapse_columns=not no_collapse)
     if schema and dataset_version.feature_schema:
         print("\nSchema:")
-        dc = DataChain(name=name, version=version, catalog=catalog)
+        session = Session.get(catalog=catalog)
+        dc = DataChain.from_dataset(name=name, version=version, session=session)
         dc.print_schema()
 
 

{datachain-0.3.20 → datachain-0.5.0}/src/datachain/data_storage/metastore.py

@@ -15,7 +15,6 @@ from uuid import uuid4
 from sqlalchemy import (
     JSON,
     BigInteger,
-    Boolean,
     Column,
     DateTime,
     ForeignKey,
@@ -228,7 +227,7 @@ class AbstractMetastore(ABC, Serializable):
         self,
         dataset: DatasetRecord,
         version: int,
-        status: int = DatasetStatus.CREATED,
+        status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
         query_script: str = "",
@@ -243,7 +242,6 @@ class AbstractMetastore(ABC, Serializable):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ) -> DatasetRecord:
         """Creates new dataset version."""
 
@@ -449,7 +447,6 @@ class AbstractDBMetastore(AbstractMetastore):
             Column("name", Text, nullable=False),
             Column("description", Text),
             Column("labels", JSON, nullable=True),
-            Column("shadow", Boolean, nullable=False),
             Column("status", Integer, nullable=False),
             Column("feature_schema", JSON, nullable=True),
             Column("created_at", DateTime(timezone=True)),
@@ -482,8 +479,11 @@ class AbstractDBMetastore(AbstractMetastore):
                 nullable=False,
             ),
             Column("version", Integer, nullable=False),
-            # adding default for now until we fully remove shadow datasets
-            Column("status", Integer, nullable=False, default=DatasetStatus.COMPLETE),
+            Column(
+                "status",
+                Integer,
+                nullable=False,
+            ),
             Column("feature_schema", JSON, nullable=True),
             Column("created_at", DateTime(timezone=True)),
             Column("finished_at", DateTime(timezone=True)),
@@ -497,7 +497,6 @@ class AbstractDBMetastore(AbstractMetastore):
             Column("query_script", Text, nullable=False, default=""),
             Column("schema", JSON, nullable=True),
             Column("job_id", Text, nullable=True),
-            Column("is_job_result", Boolean, nullable=False, default=False),
             UniqueConstraint("dataset_id", "version"),
         ]
 
@@ -971,7 +970,6 @@ class AbstractDBMetastore(AbstractMetastore):
         # TODO abstract this method and add registered = True based on kwargs
         query = self._datasets_insert().values(
             name=name,
-            shadow=False,
             status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=datetime.now(timezone.utc),
@@ -994,7 +992,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self,
         dataset: DatasetRecord,
         version: int,
-        status: int = DatasetStatus.CREATED,
+        status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
         query_script: str = "",
@@ -1009,7 +1007,6 @@ class AbstractDBMetastore(AbstractMetastore):
         size: Optional[int] = None,
         preview: Optional[list[dict]] = None,
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
         conn=None,
     ) -> DatasetRecord:
         """Creates new dataset version."""
@@ -1021,7 +1018,7 @@ class AbstractDBMetastore(AbstractMetastore):
         query = self._datasets_versions_insert().values(
             dataset_id=dataset.id,
             version=version,
-            status=status,  # for now until we remove shadow datasets
+            status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=created_at or datetime.now(timezone.utc),
             finished_at=finished_at,
@@ -1035,7 +1032,6 @@ class AbstractDBMetastore(AbstractMetastore):
             size=size,
             preview=json.dumps(preview or []),
             job_id=job_id or os.getenv("DATACHAIN_JOB_ID"),
-            is_job_result=is_job_result,
         )
         if ignore_if_exists and hasattr(query, "on_conflict_do_nothing"):
             # SQLite and PostgreSQL both support 'on_conflict_do_nothing',

{datachain-0.3.20 → datachain-0.5.0}/src/datachain/data_storage/warehouse.py

@@ -919,9 +919,7 @@ class AbstractWarehouse(ABC, Serializable):
     def is_temp_table_name(self, name: str) -> bool:
         """Returns if the given table name refers to a temporary
         or no longer needed table."""
-        return name.startswith(
-            (self.TMP_TABLE_NAME_PREFIX, self.UDF_TABLE_NAME_PREFIX, "ds_shadow_")
-        ) or name.endswith("_shadow")
+        return name.startswith((self.TMP_TABLE_NAME_PREFIX, self.UDF_TABLE_NAME_PREFIX))
 
     def get_temp_table_names(self) -> list[str]:
         return [

{datachain-0.3.20 → datachain-0.5.0}/src/datachain/dataset.py

@@ -179,7 +179,6 @@ class DatasetVersion:
     sources: str = ""
     query_script: str = ""
     job_id: Optional[str] = None
-    is_job_result: bool = False
 
     @classmethod
     def parse(  # noqa: PLR0913
@@ -201,7 +200,6 @@ class DatasetVersion:
         sources: str = "",
         query_script: str = "",
         job_id: Optional[str] = None,
-        is_job_result: bool = False,
     ):
         return cls(
             id,
@@ -221,7 +219,6 @@ class DatasetVersion:
             sources,
             query_script,
             job_id,
-            is_job_result,
         )
 
     def __eq__(self, other):
@@ -270,7 +267,6 @@ class DatasetRecord:
     name: str
     description: Optional[str]
     labels: list[str]
-    shadow: bool
     schema: dict[str, Union[SQLType, type[SQLType]]]
     feature_schema: dict
     versions: list[DatasetVersion]
@@ -299,7 +295,6 @@ class DatasetRecord:
         name: str,
         description: Optional[str],
         labels: str,
-        shadow: int,
         status: int,
         feature_schema: Optional[str],
         created_at: datetime,
@@ -327,7 +322,6 @@ class DatasetRecord:
         version_query_script: Optional[str],
         version_schema: str,
         version_job_id: Optional[str] = None,
-        version_is_job_result: bool = False,
     ) -> "DatasetRecord":
         labels_lst: list[str] = json.loads(labels) if labels else []
         schema_dct: dict[str, Any] = json.loads(schema) if schema else {}
@@ -353,7 +347,6 @@ class DatasetRecord:
             version_sources,  # type: ignore[arg-type]
             version_query_script,  # type: ignore[arg-type]
             version_job_id,
-            version_is_job_result,
         )
 
         return cls(
@@ -361,7 +354,6 @@ class DatasetRecord:
             name,
             description,
             labels_lst,
-            bool(shadow),
             cls.parse_schema(schema_dct),  # type: ignore[arg-type]
             json.loads(feature_schema) if feature_schema else {},
             [dataset_version],