datachain 0.8.9__tar.gz → 0.8.10__tar.gz

{datachain-0.8.9 → datachain-0.8.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.9
+Version: 0.8.10
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/__init__.py

@@ -48,7 +48,6 @@ def main(argv: Optional[list[str]] = None) -> int:
     logger.setLevel(logging_level)
 
     client_config = {
-        "aws_endpoint_url": args.aws_endpoint_url,
         "anon": args.anon,
     }
 
@@ -73,7 +72,7 @@ def main(argv: Optional[list[str]] = None) -> int:
 
 def handle_command(args, catalog, client_config) -> int:
     """Handle the different CLI commands."""
-    from datachain.studio import process_jobs_args, process_studio_cli_args
+    from datachain.studio import process_auth_cli_args, process_jobs_args
 
     command_handlers = {
         "cp": lambda: handle_cp_command(args, catalog),
@@ -89,7 +88,7 @@ def handle_command(args, catalog, client_config) -> int:
         "query": lambda: handle_query_command(args, catalog),
         "clear-cache": lambda: clear_cache(catalog),
         "gc": lambda: garbage_collect(catalog),
-        "studio": lambda: process_studio_cli_args(args),
+        "auth": lambda: process_auth_cli_args(args),
         "job": lambda: process_jobs_args(args),
     }
 

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/parser/__init__.py

@@ -7,7 +7,7 @@ import shtab
 from datachain.cli.utils import BooleanOptionalAction, KeyValueArgs
 
 from .job import add_jobs_parser
-from .studio import add_studio_parser
+from .studio import add_auth_parser
 from .utils import FIND_COLUMNS, add_show_args, add_sources_arg, find_columns_type
 
 
@@ -26,24 +26,25 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
 
     parent_parser = ArgumentParser(add_help=False)
     parent_parser.add_argument(
-        "--aws-endpoint-url",
-        type=str,
-        help="AWS endpoint URL",
-    )
-    parent_parser.add_argument(
-        "--anon",
-        action="store_true",
-        help="anon flag for remote storage (like awscli's --no-sign-request)",
+        "-v", "--verbose", action="count", default=0, help="Be verbose"
     )
     parent_parser.add_argument(
-        "-u", "--update", action="count", default=0, help="Update cache"
+        "-q", "--quiet", action="count", default=0, help="Be quiet"
     )
+
     parent_parser.add_argument(
-        "-v", "--verbose", action="count", default=0, help="Be verbose"
+        "--anon",
+        action="store_true",
+        help="Use anonymous access to storage",
     )
     parent_parser.add_argument(
-        "-q", "--quiet", action="count", default=0, help="Be quiet"
+        "-u",
+        "--update",
+        action="count",
+        default=0,
+        help="Update cached list of files for the sources",
     )
+
     parent_parser.add_argument(
         "--debug-sql",
         action="store_true",
@@ -67,7 +68,9 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "cp", parents=[parent_parser], description="Copy data files from the cloud."
     )
     add_sources_arg(parse_cp).complete = shtab.DIR  # type: ignore[attr-defined]
-    parse_cp.add_argument("output", type=str, help="Output")
+    parse_cp.add_argument(
+        "output", type=str, help="Path to a directory or file to put data to"
+    )
     parse_cp.add_argument(
         "-f",
         "--force",
@@ -94,7 +97,9 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "clone", parents=[parent_parser], description="Copy data files from the cloud."
     )
     add_sources_arg(parse_clone).complete = shtab.DIR  # type: ignore[attr-defined]
-    parse_clone.add_argument("output", type=str, help="Output")
+    parse_clone.add_argument(
+        "output", type=str, help="Path to a directory or file to put data to"
+    )
     parse_clone.add_argument(
         "-f",
         "--force",
@@ -123,7 +128,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="Do not copy files, just create a dataset",
     )
 
-    add_studio_parser(subp, parent_parser)
+    add_auth_parser(subp, parent_parser)
     add_jobs_parser(subp, parent_parser)
 
     datasets_parser = subp.add_parser(

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/parser/job.py

@@ -6,7 +6,7 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     )
     jobs_subparser = jobs_parser.add_subparsers(
         dest="cmd",
-        help="Use `datachain studio CMD --help` to display command-specific help",
+        help="Use `datachain auth CMD --help` to display command-specific help",
     )
 
     studio_run_help = "Run a job in Studio"

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/parser/studio.py

@@ -1,31 +1,31 @@
-def add_studio_parser(subparsers, parent_parser) -> None:
-    studio_help = "Manage Studio authentication"
-    studio_description = (
+def add_auth_parser(subparsers, parent_parser) -> None:
+    auth_help = "Manage Studio authentication"
+    auth_description = (
         "Manage authentication and settings for Studio. "
         "Configure tokens for sharing datasets and using Studio features."
     )
 
-    studio_parser = subparsers.add_parser(
-        "studio",
+    auth_parser = subparsers.add_parser(
+        "auth",
         parents=[parent_parser],
-        description=studio_description,
-        help=studio_help,
+        description=auth_description,
+        help=auth_help,
     )
-    studio_subparser = studio_parser.add_subparsers(
+    auth_subparser = auth_parser.add_subparsers(
         dest="cmd",
-        help="Use `datachain studio CMD --help` to display command-specific help",
+        help="Use `datachain auth CMD --help` to display command-specific help",
     )
 
-    studio_login_help = "Authenticate with Studio"
-    studio_login_description = (
+    auth_login_help = "Authenticate with Studio"
+    auth_login_description = (
         "Authenticate with Studio using default scopes. "
         "A random name will be assigned as the token name if not specified."
     )
-    login_parser = studio_subparser.add_parser(
+    login_parser = auth_subparser.add_parser(
         "login",
         parents=[parent_parser],
-        description=studio_login_description,
-        help=studio_login_help,
+        description=auth_login_description,
+        help=auth_login_help,
     )
 
     login_parser.add_argument(
@@ -58,26 +58,26 @@ def add_studio_parser(subparsers, parent_parser) -> None:
         help="Use code-based authentication without browser",
     )
 
-    studio_logout_help = "Log out from Studio"
-    studio_logout_description = (
+    auth_logout_help = "Log out from Studio"
+    auth_logout_description = (
         "Remove the Studio authentication token from global config."
     )
 
-    studio_subparser.add_parser(
+    auth_subparser.add_parser(
         "logout",
         parents=[parent_parser],
-        description=studio_logout_description,
-        help=studio_logout_help,
+        description=auth_logout_description,
+        help=auth_logout_help,
     )
 
-    studio_team_help = "Set default team for Studio operations"
-    studio_team_description = "Set the default team for Studio operations."
+    auth_team_help = "Set default team for Studio operations"
+    auth_team_description = "Set the default team for Studio operations."
 
-    team_parser = studio_subparser.add_parser(
+    team_parser = auth_subparser.add_parser(
         "team",
         parents=[parent_parser],
-        description=studio_team_description,
-        help=studio_team_help,
+        description=auth_team_description,
+        help=auth_team_help,
     )
     team_parser.add_argument(
         "team_name",
@@ -91,12 +91,12 @@ def add_studio_parser(subparsers, parent_parser) -> None:
         help="Set team globally for all projects",
     )
 
-    studio_token_help = "View Studio authentication token"  # noqa: S105
-    studio_token_description = "Display the current authentication token for Studio."  # noqa: S105
+    auth_token_help = "View Studio authentication token"  # noqa: S105
+    auth_token_description = "Display the current authentication token for Studio."  # noqa: S105
 
-    studio_subparser.add_parser(
+    auth_subparser.add_parser(
         "token",
         parents=[parent_parser],
-        description=studio_token_description,
-        help=studio_token_help,
+        description=auth_token_description,
+        help=auth_token_help,
     )

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/parser/utils.py

@@ -30,7 +30,7 @@ def add_sources_arg(parser: ArgumentParser, nargs: Union[str, int] = "+") -> Act
         "sources",
         type=str,
         nargs=nargs,
-        help="Data sources - paths to cloud storage directories",
+        help="Data sources - paths to source storage directories or files",
     )
 
 

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/cli/utils.py

@@ -87,7 +87,7 @@ def get_logging_level(args: Namespace) -> int:
 def determine_flavors(studio: bool, local: bool, all: bool, token: Optional[str]):
     if studio and not token:
         raise DataChainError(
-            "Not logged in to Studio. Log in with 'datachain studio login'."
+            "Not logged in to Studio. Log in with 'datachain auth login'."
         )
 
     if local or studio:

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/data_storage/sqlite.py

@@ -19,6 +19,7 @@ from sqlalchemy import MetaData, Table, UniqueConstraint, exists, select
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.schema import CreateIndex, CreateTable, DropTable
 from sqlalchemy.sql import func
+from sqlalchemy.sql.elements import BinaryExpression, BooleanClauseList
 from sqlalchemy.sql.expression import bindparam, cast
 from sqlalchemy.sql.selectable import Select
 from tqdm.auto import tqdm
@@ -40,7 +41,6 @@ if TYPE_CHECKING:
     from sqlalchemy.schema import SchemaItem
     from sqlalchemy.sql._typing import _FromClauseArgument, _OnClauseArgument
     from sqlalchemy.sql.elements import ColumnElement
-    from sqlalchemy.sql.selectable import Join
     from sqlalchemy.types import TypeEngine
 
     from datachain.lib.file import File
@@ -654,16 +654,47 @@ class SQLiteWarehouse(AbstractWarehouse):
         right: "_FromClauseArgument",
         onclause: "_OnClauseArgument",
         inner: bool = True,
-    ) -> "Join":
+        full: bool = False,
+        columns=None,
+    ) -> "Select":
         """
         Join two tables together.
         """
-        return sqlalchemy.join(
-            left,
-            right,
-            onclause,
-            isouter=not inner,
+        if not full:
+            join_query = sqlalchemy.join(
+                left,
+                right,
+                onclause,
+                isouter=not inner,
+            )
+            return sqlalchemy.select(*columns).select_from(join_query)
+
+        left_right_join = sqlalchemy.select(*columns).select_from(
+            sqlalchemy.join(left, right, onclause, isouter=True)
         )
+        right_left_join = sqlalchemy.select(*columns).select_from(
+            sqlalchemy.join(right, left, onclause, isouter=True)
+        )
+
+        def add_left_rows_filter(exp: BinaryExpression):
+            """
+            Adds filter to right_left_join to remove unmatched left table rows by
+            getting column names that need to be NULL from BinaryExpressions in onclause
+            """
+            return right_left_join.where(
+                getattr(left.c, exp.left.name) == None  # type: ignore[union-attr] # noqa: E711
+            )
+
+        if isinstance(onclause, BinaryExpression):
+            right_left_join = add_left_rows_filter(onclause)
+
+        if isinstance(onclause, BooleanClauseList):
+            for c in onclause.get_children():
+                if isinstance(c, BinaryExpression):
+                    right_left_join = add_left_rows_filter(c)
+
+        union = sqlalchemy.union(left_right_join, right_left_join).subquery()
+        return sqlalchemy.select(*union.c).select_from(union)
 
     def create_pre_udf_table(self, query: "Select") -> "Table":
         """

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/data_storage/warehouse.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
         _FromClauseArgument,
         _OnClauseArgument,
     )
-    from sqlalchemy.sql.selectable import Join, Select
+    from sqlalchemy.sql.selectable import Select
     from sqlalchemy.types import TypeEngine
 
     from datachain.data_storage import schema
@@ -873,7 +873,7 @@ class AbstractWarehouse(ABC, Serializable):
         right: "_FromClauseArgument",
         onclause: "_OnClauseArgument",
         inner: bool = True,
-    ) -> "Join":
+    ) -> "Select":
         """
         Join two tables together.
         """

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/lib/convert/python_to_sql.py

@@ -52,15 +52,15 @@ def python_to_sql(typ):  # noqa: PLR0911
 
     args = get_args(typ)
     if inspect.isclass(orig) and (issubclass(list, orig) or issubclass(tuple, orig)):
-        if args is None or len(args) != 1:
+        if args is None:
             raise TypeError(f"Cannot resolve type '{typ}' for flattening features")
 
         args0 = args[0]
         if ModelStore.is_pydantic(args0):
             return Array(JSON())
 
-        next_type = python_to_sql(args0)
-        return Array(next_type)
+        list_type = list_of_args_to_type(args)
+        return Array(list_type)
 
     if orig is Annotated:
         # Ignoring annotations
@@ -82,6 +82,18 @@ def python_to_sql(typ):  # noqa: PLR0911
     raise TypeError(f"Cannot recognize type {typ}")
 
 
+def list_of_args_to_type(args) -> SQLType:
+    first_type = python_to_sql(args[0])
+    for next_arg in args[1:]:
+        try:
+            next_type = python_to_sql(next_arg)
+            if next_type != first_type:
+                return JSON()
+        except TypeError:
+            return JSON()
+    return first_type
+
+
 def _is_json_inside_union(orig, args) -> bool:
     if orig == Union and len(args) >= 2:
         # List in JSON: Union[dict, list[dict]]

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/lib/dc.py

@@ -1332,6 +1332,7 @@ class DataChain:
         on: Union[MergeColType, Sequence[MergeColType]],
         right_on: Optional[Union[MergeColType, Sequence[MergeColType]]] = None,
         inner=False,
+        full=False,
         rname="right_",
     ) -> "Self":
         """Merge two chains based on the specified criteria.
@@ -1345,6 +1346,7 @@ class DataChain:
             right_on: Optional predicate or list of Predicates for the `right_ds`
                 to join.
             inner (bool): Whether to run inner join or outer join.
+            full (bool): Whether to run full outer join.
             rname (str): Name prefix for conflicting signal names.
 
         Examples:
@@ -1419,7 +1421,7 @@ class DataChain:
             )
 
         query = self._query.join(
-            right_ds._query, sqlalchemy.and_(*ops), inner, rname + "{name}"
+            right_ds._query, sqlalchemy.and_(*ops), inner, full, rname + "{name}"
         )
         query.feature_schema = None
         ds = self._evolve(query=query)

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/query/dataset.py

@@ -875,6 +875,7 @@ class SQLJoin(Step):
     query2: "DatasetQuery"
     predicates: Union[JoinPredicateType, tuple[JoinPredicateType, ...]]
     inner: bool
+    full: bool
     rname: str
 
     def get_query(self, dq: "DatasetQuery", temp_tables: list[str]) -> sa.Subquery:
@@ -977,14 +978,14 @@ class SQLJoin(Step):
         self.validate_expression(join_expression, q1, q2)
 
         def q(*columns):
-            join_query = self.catalog.warehouse.join(
+            return self.catalog.warehouse.join(
                 q1,
                 q2,
                 join_expression,
                 inner=self.inner,
+                full=self.full,
+                columns=columns,
             )
-            return sqlalchemy.select(*columns).select_from(join_query)
-            # return sqlalchemy.select(*subquery.c).select_from(subquery)
 
         return step_result(
             q,
@@ -1489,6 +1490,7 @@ class DatasetQuery:
         dataset_query: "DatasetQuery",
         predicates: Union[JoinPredicateType, Sequence[JoinPredicateType]],
         inner=False,
+        full=False,
         rname="{name}_right",
     ) -> "Self":
         left = self.clone(new_table=False)
@@ -1504,7 +1506,9 @@ class DatasetQuery:
             if isinstance(predicates, (str, ColumnClause, ColumnElement))
             else tuple(predicates)
         )
-        new_query.steps = [SQLJoin(self.catalog, left, right, predicates, inner, rname)]
+        new_query.steps = [
+            SQLJoin(self.catalog, left, right, predicates, inner, full, rname)
+        ]
         return new_query
 
     @detach

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/remote/studio.py

@@ -75,7 +75,7 @@ class StudioClient:
 
         if not token:
             raise DataChainError(
-                "Studio token is not set. Use `datachain studio login` "
+                "Studio token is not set. Use `datachain auth login` "
                 "or environment variable `DVC_STUDIO_TOKEN` to set it."
             )
 
@@ -105,7 +105,7 @@ class StudioClient:
         if not team:
             raise DataChainError(
                 "Studio team is not set. "
-                "Use `datachain studio team <team_name>` "
+                "Use `datachain auth team <team_name>` "
                 "or environment variable `DVC_STUDIO_TEAM` to set it."
                 "You can also set it in the config file as team under studio."
             )

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/sql/sqlite/base.py

@@ -4,6 +4,7 @@ import sqlite3
 import warnings
 from collections.abc import Iterable
 from datetime import MAXYEAR, MINYEAR, datetime, timezone
+from functools import cache
 from types import MappingProxyType
 from typing import Callable, Optional
 
@@ -526,24 +527,44 @@ def compile_collect(element, compiler, **kwargs):
     return compiler.process(func.json_group_array(*element.clauses.clauses), **kwargs)
 
 
-def load_usearch_extension(conn: sqlite3.Connection) -> bool:
+@cache
+def usearch_sqlite_path() -> Optional[str]:
     try:
-        # usearch is part of the vector optional dependencies
-        # we use the extension's cosine and euclidean distance functions
-        from usearch import sqlite_path
+        import usearch
+    except ImportError:
+        return None
 
-        conn.enable_load_extension(True)
+    with warnings.catch_warnings():
+        # usearch binary is not available for Windows, see: https://github.com/unum-cloud/usearch/issues/427.
+        # and, sometimes fail to download the binary in other platforms
+        # triggering UserWarning.
 
-        with warnings.catch_warnings():
-            # usearch binary is not available for Windows, see: https://github.com/unum-cloud/usearch/issues/427.
-            # and, sometimes fail to download the binary in other platforms
-            # triggering UserWarning.
+        warnings.filterwarnings("ignore", category=UserWarning, module="usearch")
 
-            warnings.filterwarnings("ignore", category=UserWarning, module="usearch")
-            conn.load_extension(sqlite_path())
+        try:
+            return usearch.sqlite_path()
+        except FileNotFoundError:
+            return None
 
-        conn.enable_load_extension(False)
-        return True
 
-    except Exception:  # noqa: BLE001
+def load_usearch_extension(conn: sqlite3.Connection) -> bool:
+    # usearch is part of the vector optional dependencies
+    # we use the extension's cosine and euclidean distance functions
+    ext_path = usearch_sqlite_path()
+    if ext_path is None:
+        return False
+
+    try:
+        conn.enable_load_extension(True)
+    except AttributeError:
+        # sqlite3 module is not built with loadable extension support by default.
+        return False
+
+    try:
+        conn.load_extension(ext_path)
+    except sqlite3.OperationalError:
         return False
+    else:
+        return True
+    finally:
+        conn.enable_load_extension(False)

{datachain-0.8.9 → datachain-0.8.10}/src/datachain/studio.py

@@ -47,7 +47,7 @@ def process_jobs_args(args: "Namespace"):
     raise DataChainError(f"Unknown command '{args.cmd}'.")
 
 
-def process_studio_cli_args(args: "Namespace"):
+def process_auth_cli_args(args: "Namespace"):
     if args.cmd is None:
         print(
             f"Use 'datachain {args.command} --help' to see available options",
@@ -95,7 +95,7 @@ def login(args: "Namespace"):
         raise DataChainError(
             "Token already exists. "
             "To login with a different token, "
-            "logout using `datachain studio logout`."
+            "logout using `datachain auth logout`."
         )
 
     open_browser = not args.no_open
@@ -121,12 +121,12 @@ def logout():
         token = conf.get("studio", {}).get("token")
         if not token:
             raise DataChainError(
-                "Not logged in to Studio. Log in with 'datachain studio login'."
+                "Not logged in to Studio. Log in with 'datachain auth login'."
             )
 
         del conf["studio"]["token"]
 
-    print("Logged out from Studio. (you can log back in with 'datachain studio login')")
+    print("Logged out from Studio. (you can log back in with 'datachain auth login')")
 
 
 def token():
@@ -134,7 +134,7 @@ def token():
     token = config.get("token")
     if not token:
         raise DataChainError(
-            "Not logged in to Studio. Log in with 'datachain studio login'."
+            "Not logged in to Studio. Log in with 'datachain auth login'."
         )
 
     print(token)
@@ -299,7 +299,7 @@ def cancel_job(job_id: str, team_name: Optional[str]):
     token = Config().read().get("studio", {}).get("token")
     if not token:
         raise DataChainError(
-            "Not logged in to Studio. Log in with 'datachain studio login'."
+            "Not logged in to Studio. Log in with 'datachain auth login'."
         )
 
     client = StudioClient(team=team_name)
@@ -314,7 +314,7 @@ def show_job_logs(job_id: str, team_name: Optional[str]):
     token = Config().read().get("studio", {}).get("token")
     if not token:
         raise DataChainError(
-            "Not logged in to Studio. Log in with 'datachain studio login'."
+            "Not logged in to Studio. Log in with 'datachain auth login'."
         )
 
     client = StudioClient(team=team_name)

{datachain-0.8.9 → datachain-0.8.10}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.9
+Version: 0.8.10
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.8.9 → datachain-0.8.10}/src/datachain.egg-info/SOURCES.txt

@@ -284,6 +284,7 @@ tests/unit/lib/test_hf.py
 tests/unit/lib/test_image.py
 tests/unit/lib/test_listing_info.py
 tests/unit/lib/test_models.py
+tests/unit/lib/test_python_to_sql.py
 tests/unit/lib/test_schema.py
 tests/unit/lib/test_signal_schema.py
 tests/unit/lib/test_sql_to_python.py