datachain 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

datachain/catalog/catalog.py

@@ -1627,8 +1627,17 @@ class Catalog:
         version = self.get_dataset(dataset_name).get_version(dataset_version)
 
         file_signals_values = {}
+        file_schemas = {}
+        # TODO: To remove after we properly fix deserialization
+        for signal, type_name in version.feature_schema.items():
+            from datachain.lib.model_store import ModelStore
 
-        schema = SignalSchema.deserialize(version.feature_schema)
+            type_name_parsed, v = ModelStore.parse_name_version(type_name)
+            fr = ModelStore.get(type_name_parsed, v)
+            if fr and issubclass(fr, File):
+                file_schemas[signal] = type_name
+
+        schema = SignalSchema.deserialize(file_schemas)
         for file_signals in schema.get_signals(File):
             prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
             file_signals_values[file_signals] = {

datachain/data_storage/schema.py

@@ -67,7 +67,11 @@ def convert_rows_custom_column_types(
     for row in rows:
         row_list = list(row)
         for idx, t in custom_columns_types:
-            row_list[idx] = t.on_read_convert(row_list[idx], dialect)
+            row_list[idx] = (
+                t.default_value(dialect)
+                if row_list[idx] is None
+                else t.on_read_convert(row_list[idx], dialect)
+            )
 
         yield tuple(row_list)
 
@@ -136,7 +140,15 @@ class DataTable:
         self.column_types: dict[str, SQLType] = column_types or {}
 
     @staticmethod
-    def copy_column(column: sa.Column):
+    def copy_column(
+        column: sa.Column,
+        primary_key: Optional[bool] = None,
+        index: Optional[bool] = None,
+        nullable: Optional[bool] = None,
+        default: Optional[Any] = None,
+        server_default: Optional[Any] = None,
+        unique: Optional[bool] = None,
+    ) -> sa.Column:
         """
         Copy a sqlalchemy Column object intended for use as a signal column.
 
@@ -150,12 +162,14 @@ class DataTable:
         return sa.Column(
             column.name,
             column.type,
-            primary_key=column.primary_key,
-            index=column.index,
-            nullable=column.nullable,
-            default=column.default,
-            server_default=column.server_default,
-            unique=column.unique,
+            primary_key=primary_key if primary_key is not None else column.primary_key,
+            index=index if index is not None else column.index,
+            nullable=nullable if nullable is not None else column.nullable,
+            default=default if default is not None else column.default,
+            server_default=(
+                server_default if server_default is not None else column.server_default
+            ),
+            unique=unique if unique is not None else column.unique,
         )
 
     @classmethod

datachain/data_storage/sqlite.py

@@ -122,6 +122,11 @@ class SQLiteDatabaseEngine(DatabaseEngine):
             engine = sqlalchemy.create_engine(
                 "sqlite+pysqlite:///", creator=lambda: db, future=True
             )
+            # ensure we run SA on_connect init (e.g it registers regexp function),
+            # also makes sure that it's consistent. Otherwise in some cases it
+            # seems we are getting different results if engine object is used in a
+            # different thread first and enine is not used in the Main thread.
+            engine.connect().close()
 
             db.isolation_level = None  # Use autocommit mode
             db.execute("PRAGMA foreign_keys = ON")

datachain/lib/dc.py

@@ -508,7 +508,7 @@ class DataChain(DatasetQuery):
 
     def print_json_schema(  # type: ignore[override]
         self, jmespath: Optional[str] = None, model_name: Optional[str] = None
-    ) -> "DataChain":
+    ) -> "Self":
         """Print JSON data model and save it. It returns the chain itself.
 
         Parameters:
@@ -533,7 +533,7 @@ class DataChain(DatasetQuery):
 
     def print_jsonl_schema(  # type: ignore[override]
         self, jmespath: Optional[str] = None, model_name: Optional[str] = None
-    ) -> "DataChain":
+    ) -> "Self":
         """Print JSON data model and save it. It returns the chain itself.
 
         Parameters:
@@ -549,7 +549,7 @@ class DataChain(DatasetQuery):
 
     def save(  # type: ignore[override]
         self, name: Optional[str] = None, version: Optional[int] = None
-    ) -> "DataChain":
+    ) -> "Self":
         """Save to a Dataset. It returns the chain itself.
 
         Parameters:
@@ -785,7 +785,7 @@ class DataChain(DatasetQuery):
             descending (bool): Whether to sort in descending order or not.
         """
         if descending:
-            args = tuple([sqlalchemy.desc(a) for a in args])
+            args = tuple(sqlalchemy.desc(a) for a in args)
 
         return super().order_by(*args)
 
@@ -1206,14 +1206,14 @@ class DataChain(DatasetQuery):
         """
         headers, max_length = self._effective_signals_schema.get_headers_with_length()
         if flatten or max_length < 2:
-            df = pd.DataFrame.from_records(self.to_records())
+            columns = []
             if headers:
-                df.columns = [".".join(filter(None, header)) for header in headers]
-            return df
+                columns = [".".join(filter(None, header)) for header in headers]
+            return pd.DataFrame.from_records(self.to_records(), columns=columns)
 
-        transposed_result = list(map(list, zip(*self.results())))
-        data = {tuple(n): val for n, val in zip(headers, transposed_result)}
-        return pd.DataFrame(data)
+        return pd.DataFrame(
+            self.results(), columns=pd.MultiIndex.from_tuples(map(tuple, headers))
+        )
 
     def show(
         self,
@@ -1232,6 +1232,12 @@ class DataChain(DatasetQuery):
         """
         dc = self.limit(limit) if limit > 0 else self
         df = dc.to_pandas(flatten)
+
+        if df.empty:
+            print("Empty result")
+            print(f"Columns: {list(df.columns)}")
+            return
+
         if transpose:
             df = df.T
 
@@ -1270,7 +1276,7 @@ class DataChain(DatasetQuery):
         source: bool = True,
         nrows: Optional[int] = None,
         **kwargs,
-    ) -> "DataChain":
+    ) -> "Self":
         """Generate chain from list of tabular files.
 
         Parameters:
@@ -1390,7 +1396,8 @@ class DataChain(DatasetQuery):
             dc = DataChain.from_csv("s3://mybucket/dir")
             ```
         """
-        from pyarrow.csv import ParseOptions, ReadOptions
+        from pandas.io.parsers.readers import STR_NA_VALUES
+        from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
         from pyarrow.dataset import CsvFileFormat
 
         chain = DataChain.from_storage(path, **kwargs)
@@ -1414,7 +1421,14 @@ class DataChain(DatasetQuery):
 
         parse_options = ParseOptions(delimiter=delimiter)
         read_options = ReadOptions(column_names=column_names)
-        format = CsvFileFormat(parse_options=parse_options, read_options=read_options)
+        convert_options = ConvertOptions(
+            strings_can_be_null=True, null_values=STR_NA_VALUES
+        )
+        format = CsvFileFormat(
+            parse_options=parse_options,
+            read_options=read_options,
+            convert_options=convert_options,
+        )
         return chain.parse_tabular(
             output=output,
             object_name=object_name,

datachain/lib/meta_formats.py

@@ -11,12 +11,16 @@ from collections.abc import Iterator
 from typing import Any, Callable
 
 import jmespath as jsp
-from pydantic import Field, ValidationError  # noqa: F401
+from pydantic import BaseModel, ConfigDict, Field, ValidationError  # noqa: F401
 
 from datachain.lib.data_model import DataModel  # noqa: F401
 from datachain.lib.file import File
 
 
+class UserModel(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+
 def generate_uuid():
     return uuid.uuid4()  # Generates a random UUID.
 
@@ -72,6 +76,8 @@ def read_schema(source_file, data_type="csv", expr=None, model_name=None):
         data_type,
         "--class-name",
         model_name,
+        "--base-class",
+        "datachain.lib.meta_formats.UserModel",
     ]
     try:
         result = subprocess.run(  # noqa: S603
@@ -87,7 +93,7 @@ def read_schema(source_file, data_type="csv", expr=None, model_name=None):
     except subprocess.CalledProcessError as e:
         model_output = f"An error occurred in datamodel-codegen: {e.stderr}"
     print(f"{model_output}")
-    print("\n" + "from datachain.lib.data_model import DataModel" + "\n")
+    print("from datachain.lib.data_model import DataModel")
     print("\n" + f"DataModel.register({model_name})" + "\n")
     print("\n" + f"spec={model_name}" + "\n")
     return model_output

datachain/node.py

@@ -47,7 +47,7 @@ class DirTypeGroup:
 @attrs.define
 class Node:
     sys__id: int = 0
-    sys__rand: int = -1
+    sys__rand: int = 0
     vtype: str = ""
     dir_type: Optional[int] = None
     path: str = ""

datachain/query/schema.py

@@ -45,6 +45,10 @@ class Column(sa.ColumnClause, metaclass=ColumnMeta):
         """Search for matches using glob pattern matching."""
         return self.op("GLOB")(glob_str)
 
+    def regexp(self, regexp_str):
+        """Search for matches using regexp pattern matching."""
+        return self.op("REGEXP")(regexp_str)
+
 
 class UDFParameter(ABC):
     @abstractmethod

datachain/sql/default/base.py

@@ -1,8 +1,10 @@
 from datachain.sql.types import (
+    DBDefaults,
     TypeConverter,
     TypeDefaults,
     TypeReadConverter,
     register_backend_types,
+    register_db_defaults,
     register_type_defaults,
     register_type_read_converters,
 )
@@ -18,5 +20,6 @@ def setup() -> None:
     register_backend_types("default", TypeConverter())
     register_type_read_converters("default", TypeReadConverter())
     register_type_defaults("default", TypeDefaults())
+    register_db_defaults("default", DBDefaults())
 
     setup_is_complete = True

datachain/sql/sqlite/base.py

@@ -22,8 +22,10 @@ from datachain.sql.sqlite.types import (
     register_type_converters,
 )
 from datachain.sql.types import (
+    DBDefaults,
     TypeDefaults,
     register_backend_types,
+    register_db_defaults,
     register_type_defaults,
     register_type_read_converters,
 )
@@ -66,6 +68,7 @@ def setup():
     register_backend_types("sqlite", SQLiteTypeConverter())
     register_type_read_converters("sqlite", SQLiteTypeReadConverter())
     register_type_defaults("sqlite", TypeDefaults())
+    register_db_defaults("sqlite", DBDefaults())
 
     compiles(sql_path.parent, "sqlite")(compile_path_parent)
     compiles(sql_path.name, "sqlite")(compile_path_name)

datachain/sql/types.py

@@ -17,6 +17,7 @@ from datetime import datetime
 from types import MappingProxyType
 from typing import Any, Union
 
+import sqlalchemy as sa
 from sqlalchemy import TypeDecorator, types
 
 _registry: dict[str, "TypeConverter"] = {}
@@ -28,6 +29,9 @@ read_converter_registry = MappingProxyType(_read_converter_registry)
 _type_defaults_registry: dict[str, "TypeDefaults"] = {}
 type_defaults_registry = MappingProxyType(_type_defaults_registry)
 
+_db_defaults_registry: dict[str, "DBDefaults"] = {}
+db_defaults_registry = MappingProxyType(_db_defaults_registry)
+
 NullType = types.NullType
 
 
@@ -43,6 +47,10 @@ def register_type_defaults(dialect_name: str, td: "TypeDefaults"):
     _type_defaults_registry[dialect_name] = td
 
 
+def register_db_defaults(dialect_name: str, dbd: "DBDefaults"):
+    _db_defaults_registry[dialect_name] = dbd
+
+
 def converter(dialect) -> "TypeConverter":
     name = dialect.name
     try:
@@ -71,6 +79,14 @@ def type_defaults(dialect) -> "TypeDefaults":
         raise ValueError(f"No type defaults registered for dialect: {name!r}") from None
 
 
+def db_defaults(dialect) -> "DBDefaults":
+    name = dialect.name
+    try:
+        return db_defaults_registry[name]
+    except KeyError:
+        raise ValueError(f"No DB defaults registered for dialect: {name!r}") from None
+
+
 class SQLType(TypeDecorator):
     impl: type[types.TypeEngine[Any]] = types.TypeEngine
     cache_ok = True
@@ -97,6 +113,10 @@ class String(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).string()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).string()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).string(value)
 
@@ -115,6 +135,10 @@ class Boolean(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).boolean()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).boolean()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).boolean(value)
 
@@ -133,6 +157,10 @@ class Int(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).int()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).int()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).int(value)
 
@@ -145,6 +173,10 @@ class Int32(Int):
     def default_value(dialect):
         return type_defaults(dialect).int32()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).int32()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).int32(value)
 
@@ -157,6 +189,10 @@ class Int64(Int):
     def default_value(dialect):
         return type_defaults(dialect).int64()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).int64()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).int64(value)
 
@@ -169,12 +205,16 @@ class UInt64(Int):
     def default_value(dialect):
         return type_defaults(dialect).uint64()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).uint64()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).uint64(value)
 
 
 class Float(SQLType):
-    impl = types.INTEGER
+    impl = types.FLOAT
 
     @property
     def python_type(self):
@@ -187,6 +227,10 @@ class Float(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).float()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).float()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).float(value)
 
@@ -199,6 +243,10 @@ class Float32(Float):
     def default_value(dialect):
         return type_defaults(dialect).float32()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).float32()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).float32(value)
 
@@ -211,6 +259,10 @@ class Float64(Float):
     def default_value(dialect):
         return type_defaults(dialect).float64()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).float64()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).float64(value)
 
@@ -247,6 +299,10 @@ class Array(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).array()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).array()
+
     def on_read_convert(self, value, dialect):
         r = read_converter(dialect).array(value, self.item_type, dialect)
         if isinstance(self.item_type, JSON):
@@ -268,6 +324,10 @@ class JSON(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).json()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).json()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).json(value)
 
@@ -286,6 +346,10 @@ class DateTime(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).datetime()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).datetime()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).datetime(value)
 
@@ -304,6 +368,10 @@ class Binary(SQLType):
     def default_value(dialect):
         return type_defaults(dialect).binary()
 
+    @staticmethod
+    def db_default_value(dialect):
+        return db_defaults(dialect).binary()
+
     def on_read_convert(self, value, dialect):
         return read_converter(dialect).binary(value)
 
@@ -328,13 +396,17 @@ class TypeReadConverter:
         return value
 
     def float(self, value):
+        if value is None:
+            return float("nan")
+        if isinstance(value, str) and value.lower() == "nan":
+            return float("nan")
         return value
 
     def float32(self, value):
-        return value
+        return self.float(value)
 
     def float64(self, value):
-        return value
+        return self.float(value)
 
     def array(self, value, item_type, dialect):
         if value is None or item_type is None:
@@ -347,10 +419,9 @@ class TypeReadConverter:
     def datetime(self, value):
         return value
 
-    def uuid(self, value):
-        return value
-
     def binary(self, value):
+        if isinstance(value, str):
+            return value.encode()
         return value
 
 
@@ -415,13 +486,13 @@ class TypeDefaults:
         return None
 
     def float(self):
-        return None
+        return float("nan")
 
     def float32(self):
-        return None
+        return self.float()
 
     def float64(self):
-        return None
+        return self.float()
 
     def array(self):
         return None
@@ -432,11 +503,49 @@ class TypeDefaults:
     def datetime(self):
         return None
 
-    def uuid(self):
+    def binary(self):
         return None
 
+
+class DBDefaults:
+    def string(self):
+        return sa.text("''")
+
+    def boolean(self):
+        return sa.text("False")
+
+    def int(self):
+        return sa.text("0")
+
+    def int32(self):
+        return self.int()
+
+    def int64(self):
+        return self.int()
+
+    def uint64(self):
+        return self.int()
+
+    def float(self):
+        return sa.text("NaN")
+
+    def float32(self):
+        return self.float()
+
+    def float64(self):
+        return self.float()
+
+    def array(self):
+        return sa.text("'[]'")
+
+    def json(self):
+        return sa.text("'{}'")
+
+    def datetime(self):
+        return sa.text("'1970-01-01 00:00:00'")
+
     def binary(self):
-        return None
+        return sa.text("''")
 
 
 TYPES = [

{datachain-0.3.1.dist-info → datachain-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.1
+Version: 0.3.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -55,6 +55,15 @@ Requires-Dist: mkdocs-material >=9.3.1 ; extra == 'docs'
 Requires-Dist: mkdocs-section-index >=0.3.6 ; extra == 'docs'
 Requires-Dist: mkdocstrings-python >=1.6.3 ; extra == 'docs'
 Requires-Dist: mkdocs-literate-nav >=0.6.1 ; extra == 'docs'
+Provides-Extra: examples
+Requires-Dist: datachain[tests] ; extra == 'examples'
+Requires-Dist: numpy <2,>=1 ; extra == 'examples'
+Requires-Dist: defusedxml ; extra == 'examples'
+Requires-Dist: accelerate ; extra == 'examples'
+Requires-Dist: unstructured[pdf] ; extra == 'examples'
+Requires-Dist: pdfplumber ==0.11.3 ; extra == 'examples'
+Requires-Dist: huggingface-hub[hf_transfer] ; extra == 'examples'
+Requires-Dist: nltk ==3.8.1 ; extra == 'examples'
 Provides-Extra: remote
 Requires-Dist: lz4 ; extra == 'remote'
 Requires-Dist: msgpack <2,>=1.0.4 ; extra == 'remote'
@@ -100,102 +109,78 @@ Requires-Dist: usearch ; extra == 'vector'
 AI 🔗 DataChain
 ----------------
 
-DataChain is a data-frame library designed for AI-specific scenarios. It helps ML and
-AI engineers build a metadata layer on top of unstructured files and analyze data using
-this layer.
+DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
+It is made to organize your unstructured data into datasets and wrangle it at scale on
+your local machine.
 
-📂 **Raw Files Processing**
-   Process raw files (images, video, text, PDFs) directly from storage (S3, GCP, Azure,
-   Local), version and update datasets.
+Key Features
+============
 
-🌟 **Metadata layer.**
-   Build a metadata layer on top of files using structured sources like CSV, Parquet,
-   and JSON files.
+📂 **Storage as a Source of Truth.**
+   - Process unstructured data without redundant copies: S3, GCP, Azure, and local
+     file systems.
+   - Multimodal data: images, video, text, PDFs, JSONs, CSVs, parquet.
+   - Join files and metadata together into persistent, versioned, columnar datasets.
 
-⭐ **Metadata enrichment.**
-   Enhance the metadata layer with outputs from local ML model inferences and LLM calls.
+🐍 **Python-friendly data pipelines.**
+   - Operate on Python objects and object fields.
+   - Built-in parallelization and out-of-memory compute without a need in SQL or
+     Spark jobs.
 
-🛠️ **Data Transformation.**
-   Transform metadata using traditional methods like filtering, grouping, joining, and
-   others.
+🧠 **Data Enrichment and Processing.**
+   - Generate metadata columns using local AI models and LLM APIs.
+   - Filter, join, and group by AI metadata. Vector similarity search.
+   - Pass datasets to Pytorch and Tensorflow, or export back into storage.
 
-🐍 **User-friendly interface.**
-   Operate efficiently with familiar Python objects and object fields, eliminating the
-   need for SQL.
+🚀 **Efficiency.**
+   - Parallelization, out-of-memory workloads and data caching.
+   - Vectorized operations on Python object fields: sum, count, avg, etc.
+   - Vector search on embeddings.
 
 
+Quick Start
+-----------
+
 .. code:: console
 
    $ pip install datachain
 
 
-Data Structures
-===============
-
-DataChain introduces expressive data structures tailored for AI-specific workload:
-
-- **Dataset:** Preserves the file-references and meta-information. Takes care of Python
-  object serialization, dataset versioning and difference. Operations on dataset:
-
-  - **Transformations:** traditional data-frame or SQL operations such as filtering,
-    grouping, joining.
-  - **Enrichments:** mapping, aggregating and generating using customer’s Python
-    code. This is needed to work with ML inference and LLM calls.
-
-- **Chain** is a sequence of operations on datasets. Chain executes operations in lazy
-  mode - only when needed.
-
-DataChain name comes from these major data structures: dataset and chaining.
-
+Selecting files using JSON metadata
+======================================
 
-What’s new in DataChain?
-========================
+A storage consists of images of cats and dogs (`dog.1048.jpg`, `cat.1009.jpg`),
+annotated with ground truth and model inferences in the 'json-pairs' format,
+where each image has a matching JSON file like `cat.1009.json`:
 
-The project combines multiple ideas from different areas in order to simplify AI
-use-cases and at the same time to fit it into traditional data infrastructure.
+.. code:: json
 
-- **Python-Native for AI.** Utilizes Python instead of SQL for data manipulation as the
-  native language for AI. It’s powered by `Pydantic`_ data models.
-- **Separation of CPU-GPU workloads.** Distinguishes CPU-heavy transformations (filter,
-  group_by, join) from GPU heavy enrichments (ML-inference or LLM calls). That’s mostly
-  needed for distributed computations.
-- **Resuming data processing** (in development). Introduces idempotent operations,
-  allowing data processing to resume from the last successful process file/record/batch
-  if it fails due to issues like failed LLM calls, ML inference or file download.
+    {
+        "class": "cat", "id": "1009", "num_annotators": 8,
+        "inference": {"class": "dog", "confidence": 0.68}
+    }
 
-Additional relatively new ideas:
+Example of downloading only high-confidence cat images using JSON metadata:
 
-- **Functional style data processing.** Using a functional/chaining approach to data
-  processing rather than declarative SQL, inspired by R-dplyr and some Python libraries.
-- **Data Versioning.** Treats raw files in cloud storage as the source of truth for data
-  and implements data versioning, extending ideas from DVC (developed by the same team).
 
+.. code:: py
 
-What DataChain is NOT?
-======================
-
-- **Not a database** (Postgres, MySQL). Instead, it uses databases under the hood:
-  `SQLite`_ in open-source and ClickHouse and other data warehouses for the commercial
-  version.
-- **Not a data processing tool / data warehouse** (Spark, Snowflake, Big Query) since
-  it delegates heavy data transformations to underlying data warehouses and focuses on
-  AI specific data enrichments and orchestrating all the pieces together.
-
+    from datachain import Column, DataChain
 
-Quick Start
------------
+    meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta")
+    images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg")
 
-Data curation with a local model
-=================================
+    images_id = images.map(id=lambda file: file.path.split('.')[-2])
+    annotated = images_id.merge(meta, on="id", right_on="meta.id")
 
-We will evaluate chatbot dialogs stored as text files in Google Cloud Storage
-- 50 files total in this example.
-These dialogs involve users chatting with a bot while looking for better wireless plans.
-Our goal is to identify the successful dialogs.
+    likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
+                                   & (Column("meta.inference.class_") == "cat"))
+    likely_cats.export_files("high-confidence-cats/", signal="file")
 
-The data used in the examples is `publicly available`_. The sample code is designed to run on a local machine.
 
-First, we'll show batch inference with a simple sentiment model using the `transformers` library:
+Data curation with a local AI model
+===================================
+Batch inference with a simple sentiment model using the `transformers` library:
 
 .. code:: shell
 
@@ -246,30 +231,30 @@ LLM judging chatbots
 =============================
 
 LLMs can work as efficient universal classifiers. In the example below,
-we employ a free API from Mistral to judge the chatbot performance. Please get a free
+we employ a free API from Mistral to judge the `publicly available`_ chatbot dialogs. Please get a free
 Mistral API key at https://console.mistral.ai
 
+
 .. code:: shell
 
-    $ pip install mistralai
+    $ pip install mistralai (Requires version >=1.0.0)
     $ export MISTRAL_API_KEY=_your_key_
 
 DataChain can parallelize API calls; the free Mistral tier supports up to 4 requests at the same time.
 
 .. code:: py
 
-    from mistralai.client import MistralClient
-    from mistralai.models.chat_completion import ChatMessage
+    from mistralai import Mistral
     from datachain import File, DataChain, Column
 
     PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
 
     def eval_dialogue(file: File) -> bool:
-         client = MistralClient()
-         response = client.chat(
+         client = Mistral()
+         response = client.chat.complete(
              model="open-mixtral-8x22b",
-             messages=[ChatMessage(role="system", content=PROMPT),
-                       ChatMessage(role="user", content=file.read())])
+             messages=[{"role": "system", "content": PROMPT},
+                       {"role": "user", "content": file.read()}])
          result = response.choices[0].message.content
          return result.lower().startswith("success")
 
@@ -309,8 +294,8 @@ Instead of extracting this information from the Mistral response data structure
 
 .. code:: py
 
-    from mistralai.client import MistralClient
-    from mistralai.models.chat_completion import ChatMessage, ChatCompletionResponse
+    from mistralai import Mistral
+    from mistralai.models import ChatCompletionResponse
     from datachain import File, DataChain, Column
 
     PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
@@ -319,8 +304,8 @@ Instead of extracting this information from the Mistral response data structure
          client = MistralClient()
          return client.chat(
              model="open-mixtral-8x22b",
-             messages=[ChatMessage(role="system", content=PROMPT),
-                       ChatMessage(role="user", content=file.read())])
+             messages=[{"role": "system", "content": PROMPT},
+                       {"role": "user", "content": file.read()}])
 
     chain = (
        DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file")
@@ -438,7 +423,10 @@ Tutorials
 ---------
 
 * `Getting Started`_
-* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
+* `Multimodal <https://github.com/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb>`__)
+* `LLM evaluations <https://github.com/iterative/datachain-examples/blob/main/llm/llm_chatbot_evaluation.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/llm/llm_chatbot_evaluation.ipynb>`__)
+* `Reading JSON metadata <https://github.com/iterative/datachain-examples/blob/main/formats/json-metadata-tutorial.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/formats/json-metadata-tutorial.ipynb>`__)
+
 
 Contributions
 -------------

{datachain-0.3.1.dist-info → datachain-0.3.2.dist-info}/RECORD

@@ -9,7 +9,7 @@ datachain/dataset.py,sha256=MZezyuJWNj_3PEtzr0epPMNyWAOTrhTSPI5FmemV6L4,14470
 datachain/error.py,sha256=GY9KYTmb7GHXn2gGHV9X-PBhgwLj3i7VpK7tGHtAoGM,1279
 datachain/job.py,sha256=bk25bIqClhgRPzlXAhxpTtDeewibQe5l3S8Cf7db0gM,1229
 datachain/listing.py,sha256=keLkvPfumDA3gijeIiinH5yGWe71qCxgF5HqqP5AeH4,8299
-datachain/node.py,sha256=frxZWoEvqUvk9pyXmVaeiNCs3W-xjC_sENmUD11V06Q,6006
+datachain/node.py,sha256=ihrP5l9HKpXLR0fR1wyb7QIdb7NR26dX6bB09qGX5B4,6005
 datachain/nodes_fetcher.py,sha256=kca19yvu11JxoVY1t4_ydp1FmchiV88GnNicNBQ9NIA,831
 datachain/nodes_thread_pool.py,sha256=ZyzBvUImIPmi4WlKC2SW2msA0UhtembbTdcs2nx29A0,3191
 datachain/progress.py,sha256=7_8FtJs770ITK9sMq-Lt4k4k18QmYl4yIG_kCoWID3o,4559
@@ -17,7 +17,7 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/utils.py,sha256=ROVCLwb37VmFRzgTlSGUDw4eJNgYGiQ4yMX581HfUX8,12988
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=9-7SnMjh5ruH9sdKDo8P5EklX9oC2EHH6bnku6ZqLko,80275
+datachain/catalog/catalog.py,sha256=9fxRJjiM8tK3ZePHFErYqY6LkJFA6bvlp-KHq-_kSYk,80703
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=GJ8zhEYkC7TuaPzCsjJQ4LtTdECu-wwYzC12MikPOMQ,7307
 datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
@@ -33,19 +33,19 @@ datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kT
 datachain/data_storage/id_generator.py,sha256=lCEoU0BM37Ai2aRpSbwo5oQT0GqZnSpYwwvizathRMQ,4292
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
 datachain/data_storage/metastore.py,sha256=nxcY6nwyEmQWMAo33sNGO-FgUFQs2amBGGnZz2ftEz0,55362
-datachain/data_storage/schema.py,sha256=Idi-29fckvZozzvkyz3nTR2FOIajPlSuPdIEO7SMvXM,7863
+datachain/data_storage/schema.py,sha256=GwJIHkjhrnBxJAV1WvCMM8jiJN5h79LXDyzMmUDtRw0,8523
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
-datachain/data_storage/sqlite.py,sha256=0r6L_a2hdGRoR_gl06v1qWhEFOS_Q31aldHyk07Yx-M,26857
+datachain/data_storage/sqlite.py,sha256=IKd4epEjVxAoQQHsE7WTY4kgOiFyUiWhvaGm-61rJfg,27218
 datachain/data_storage/warehouse.py,sha256=MXYkUG69UK2wbIFsZFvT7rKzXlnSitDMp3Vzj_IIsnA,33089
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/arrow.py,sha256=R8wDUDEa-5hYjI3HW9cqvOYYJpeeah5lbhFIL3gkmcE,4915
 datachain/lib/clip.py,sha256=16u4b_y2Y15nUS2UN_8ximMo6r_-_4IQpmct2ol-e-g,5730
 datachain/lib/data_model.py,sha256=qfTtQNncS5pt9SvXdMEa5kClniaT6XBGBfO7onEz2TI,1632
 datachain/lib/dataset_info.py,sha256=lONGr71ozo1DS4CQEhnpKORaU4qFb6Ketv8Xm8CVm2U,2188
-datachain/lib/dc.py,sha256=e24ecfIcypVkmVBqvr-p06zpwrw7GD20gy1gBJQPT-I,58012
+datachain/lib/dc.py,sha256=Q9HL7Axfo9i5hodlkD2GwklN4i0BVULm9_A11ckuj2A,58352
 datachain/lib/file.py,sha256=ZHpdilDPYCob8uqtwUPtBvBNxVvQRq4AC_0IGg5m-G4,12003
 datachain/lib/image.py,sha256=TgYhRhzd4nkytfFMeykQkPyzqb5Le_-tU81unVMPn4Q,2328
-datachain/lib/meta_formats.py,sha256=jlSYWRUeDMjun_YCsQ2JxyaDJpEpokzHDPmKUAoCXnU,7034
+datachain/lib/meta_formats.py,sha256=Hels85LJmNCz1aYVJvhymNdAt3qdJ2-qoxsIiUezrow,7198
 datachain/lib/model_store.py,sha256=c4USXsBBjrGH8VOh4seIgOiav-qHOwdoixtxfLgU63c,2409
 datachain/lib/pytorch.py,sha256=9PsypKseyKfIimTmTQOgb-pbNXgeeAHLdlWx0qRPULY,5660
 datachain/lib/settings.py,sha256=39thOpYJw-zPirzeNO6pmRC2vPrQvt4eBsw1xLWDFsw,2344
@@ -71,17 +71,17 @@ datachain/query/dispatch.py,sha256=GBh3EZHDp5AaXxrjOpfrpfsuy7Umnqxu-MAXcK9X3gc,1
 datachain/query/metrics.py,sha256=vsECqbZfoSDBnvC3GQlziKXmISVYDLgHP1fMPEOtKyo,640
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
-datachain/query/schema.py,sha256=O3mTM5DRjvRAJCI7O9mR8wOdFJbgI1jIjvtfl5YvjI4,7755
+datachain/query/schema.py,sha256=BvHipN79CnSTbVFcfIEwzo1npe7HmThnk0iY-CSLEkM,7899
 datachain/query/session.py,sha256=qTzkXgwMJdJhal3rVt3hdv3x1EXT1IHuXcwkC-Ex0As,4111
 datachain/query/udf.py,sha256=j3NhmKK5rYG5TclcM2Sr0LhS1tmYLMjzMugx9G9iFLM,8100
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/remote/studio.py,sha256=f5s6qSZ9uB4URGUoU_8_W1KZRRQQVSm6cgEBkBUEfuE,7226
 datachain/sql/__init__.py,sha256=A2djrbQwSMUZZEIKGnm-mnRA-NDSbiDJNpAmmwGNyIo,303
 datachain/sql/selectable.py,sha256=fBM-wS1TUA42kVEAAiwqGtibIevyZAEritwt8PZGyLQ,1589
-datachain/sql/types.py,sha256=SShudhdIpdfTKDxWDDqOajYRkTCkIgQbilA94g4i-4E,10389
+datachain/sql/types.py,sha256=1MFvECB_5A6QwQKKY3VPhvitgKDlc2aB7iBjY4hv1_s,13034
 datachain/sql/utils.py,sha256=rzlJw08etivdrcuQPqNVvVWhuVSyUPUQEEc6DOhu258,818
 datachain/sql/default/__init__.py,sha256=XQ2cEZpzWiABqjV-6yYHUBGI9vN_UHxbxZENESmVAWw,45
-datachain/sql/default/base.py,sha256=h44005q3qtMc9cjWmRufWwcBr5CfK_dnvG4IrcSQs_8,536
+datachain/sql/default/base.py,sha256=QD-31C6JnyOXzogyDx90sUhm7QvgXIYpeHEASH84igU,628
 datachain/sql/functions/__init__.py,sha256=Ioyy7nSetrTLVnHGcGcmZU99HxUFcx-5PFbrh2dPNH0,396
 datachain/sql/functions/array.py,sha256=EB7nJSncUc1PuxlHyzU2gVhF8DuXaxpGlxb5e8X2KFY,1297
 datachain/sql/functions/conditional.py,sha256=q7YUKfunXeEldXaxgT-p5pUTcOEVU_tcQ2BJlquTRPs,207
@@ -89,13 +89,13 @@ datachain/sql/functions/path.py,sha256=zixpERotTFP6LZ7I4TiGtyRA8kXOoZmH1yzH9oRW0
 datachain/sql/functions/random.py,sha256=vBwEEj98VH4LjWixUCygQ5Bz1mv1nohsCG0-ZTELlVg,271
 datachain/sql/functions/string.py,sha256=hIrF1fTvlPamDtm8UMnWDcnGfbbjCsHxZXS30U2Rzxo,651
 datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7d04,166
-datachain/sql/sqlite/base.py,sha256=LBYmXqXsVF30fbcnR55evCZHbPDCzMdGk_ogPLps63s,12236
+datachain/sql/sqlite/base.py,sha256=w6HbEkGdmNGDnDY3_75E-wDb6qNskVpq0qbHGADsERk,12327
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.1.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.1.dist-info/METADATA,sha256=qR3OMpGUkx0cKelnl51d9uksn5H-Wn4LvTJbUnTMDuQ,17268
-datachain-0.3.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-datachain-0.3.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.1.dist-info/RECORD,,
+datachain-0.3.2.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.3.2.dist-info/METADATA,sha256=i8evXYMe4FgBqxV7TYdWTRuh7MxRT6jfqmzL-tbk_JQ,16789
+datachain-0.3.2.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+datachain-0.3.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.3.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.3.2.dist-info/RECORD,,

{datachain-0.3.1.dist-info → datachain-0.3.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.1.0)
+Generator: setuptools (72.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any