datachain 0.8.8__tar.gz → 0.8.9__tar.gz

{datachain-0.8.8 → datachain-0.8.9}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.8.6'
+    rev: 'v0.9.1'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.8.8 → datachain-0.8.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.8
+Version: 0.8.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -99,7 +99,7 @@ Requires-Dist: unstructured[pdf]<0.16.12; extra == "examples"
 Requires-Dist: pdfplumber==0.11.5; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.58; extra == "examples"
+Requires-Dist: ultralytics==8.3.61; extra == "examples"
 
 ================
 |logo| DataChain
@@ -189,13 +189,14 @@ Python code:
 
 .. code:: py
 
+    import os
     from mistralai import Mistral
     from datachain import File, DataChain, Column
 
     PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
 
     def eval_dialogue(file: File) -> bool:
-         client = Mistral()
+         client = Mistral(api_key = os.environ["MISTRAL_API_KEY"])
          response = client.chat.complete(
              model="open-mixtral-8x22b",
              messages=[{"role": "system", "content": PROMPT},

{datachain-0.8.8 → datachain-0.8.9}/README.rst

@@ -86,13 +86,14 @@ Python code:
 
 .. code:: py
 
+    import os
     from mistralai import Mistral
     from datachain import File, DataChain, Column
 
     PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
 
     def eval_dialogue(file: File) -> bool:
-         client = Mistral()
+         client = Mistral(api_key = os.environ["MISTRAL_API_KEY"])
          response = client.chat.complete(
              model="open-mixtral-8x22b",
              messages=[{"role": "system", "content": PROMPT},

{datachain-0.8.8 → datachain-0.8.9}/pyproject.toml

@@ -112,7 +112,7 @@ examples = [
   "pdfplumber==0.11.5",
   "huggingface_hub[hf_transfer]",
   "onnx==1.16.1",
-  "ultralytics==8.3.58"
+  "ultralytics==8.3.61"
 ]
 
 [project.urls]

{datachain-0.8.8 → datachain-0.8.9}/src/datachain/cli/__init__.py

@@ -39,6 +39,10 @@ def main(argv: Optional[list[str]] = None) -> int:
     if args.command in ("internal-run-udf", "internal-run-udf-worker"):
         return handle_udf(args.command)
 
+    if args.command is None:
+        datachain_parser.print_help(sys.stderr)
+        return 1
+
     logger.addHandler(logging.StreamHandler())
     logging_level = get_logging_level(args)
     logger.setLevel(logging_level)
@@ -120,12 +124,17 @@ def handle_clone_command(args, catalog):
         recursive=bool(args.recursive),
         no_glob=args.no_glob,
         no_cp=args.no_cp,
-        edatachain=args.edatachain,
-        edatachain_file=args.edatachain_file,
     )
 
 
 def handle_dataset_command(args, catalog):
+    if args.datasets_cmd is None:
+        print(
+            f"Use 'datachain {args.command} --help' to see available options",
+            file=sys.stderr,
+        )
+        return 1
+
     dataset_commands = {
         "pull": lambda: catalog.pull_dataset(
             args.dataset,
@@ -134,8 +143,6 @@ def handle_dataset_command(args, catalog):
             local_ds_version=args.local_version,
             cp=args.cp,
             force=bool(args.force),
-            edatachain=args.edatachain,
-            edatachain_file=args.edatachain_file,
         ),
         "edit": lambda: edit_dataset(
             catalog,
@@ -187,6 +194,7 @@ def handle_dataset_command(args, catalog):
     handler = dataset_commands.get(args.datasets_cmd)
     if handler:
         return handler()
+
     raise Exception(f"Unexpected command {args.datasets_cmd}")
 
 

{datachain-0.8.8 → datachain-0.8.9}/src/datachain/cli/commands/datasets.py

@@ -11,6 +11,7 @@ if TYPE_CHECKING:
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
 from datachain.error import DatasetNotFoundError
+from datachain.studio import list_datasets as list_datasets_studio
 
 
 def list_datasets(
@@ -20,14 +21,12 @@ def list_datasets(
     all: bool = True,
     team: Optional[str] = None,
 ):
-    from datachain.studio import list_datasets
-
     token = Config().read().get("studio", {}).get("token")
     all, local, studio = determine_flavors(studio, local, all, token)
 
     local_datasets = set(list_datasets_local(catalog)) if all or local else set()
     studio_datasets = (
-        set(list_datasets(team=team)) if (all or studio) and token else set()
+        set(list_datasets_studio(team=team)) if (all or studio) and token else set()
     )
 
     rows = [

{datachain-0.8.8 → datachain-0.8.9}/src/datachain/cli/parser/__init__.py

@@ -1,3 +1,4 @@
+import argparse
 from argparse import ArgumentParser
 from importlib.metadata import PackageNotFoundError, version
 
@@ -18,7 +19,8 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         __version__ = "unknown"
 
     parser = ArgumentParser(
-        description="DataChain: Wrangle unstructured AI data at scale", prog="datachain"
+        description="DataChain: Wrangle unstructured AI data at scale.",
+        prog="datachain",
     )
     parser.add_argument("-V", "--version", action="version", version=__version__)
 
@@ -31,13 +33,13 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     parent_parser.add_argument(
         "--anon",
         action="store_true",
-        help="AWS anon (aka awscli's --no-sign-request)",
+        help="anon flag for remote storage (like awscli's --no-sign-request)",
     )
     parent_parser.add_argument(
         "-u", "--update", action="count", default=0, help="Update cache"
     )
     parent_parser.add_argument(
-        "-v", "--verbose", action="count", default=0, help="Verbose"
+        "-v", "--verbose", action="count", default=0, help="Be verbose"
     )
     parent_parser.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet"
@@ -46,24 +48,23 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--debug-sql",
         action="store_true",
         default=False,
-        help="Show All SQL Queries (very verbose output, for debugging only)",
+        help=argparse.SUPPRESS,
     )
     parent_parser.add_argument(
         "--pdb",
         action="store_true",
         default=False,
-        help="Drop into the pdb debugger on fatal exception",
+        help=argparse.SUPPRESS,
     )
 
     subp = parser.add_subparsers(
         title="Available Commands",
         metavar="command",
         dest="command",
-        help=f"Use `{parser.prog} command --help` for command-specific help.",
-        required=True,
+        help=f"Use `{parser.prog} command --help` for command-specific help",
     )
     parse_cp = subp.add_parser(
-        "cp", parents=[parent_parser], description="Copy data files from the cloud"
+        "cp", parents=[parent_parser], description="Copy data files from the cloud."
     )
     add_sources_arg(parse_cp).complete = shtab.DIR  # type: ignore[attr-defined]
     parse_cp.add_argument("output", type=str, help="Output")
@@ -72,7 +73,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--force",
         default=False,
         action="store_true",
-        help="Force creating outputs",
+        help="Force creating files even if they already exist",
     )
     parse_cp.add_argument(
         "-r",
@@ -90,7 +91,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     parse_clone = subp.add_parser(
-        "clone", parents=[parent_parser], description="Copy data files from the cloud"
+        "clone", parents=[parent_parser], description="Copy data files from the cloud."
     )
     add_sources_arg(parse_clone).complete = shtab.DIR  # type: ignore[attr-defined]
     parse_clone.add_argument("output", type=str, help="Output")
@@ -121,16 +122,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         action="store_true",
         help="Do not copy files, just create a dataset",
     )
-    parse_clone.add_argument(
-        "--edatachain",
-        default=False,
-        action="store_true",
-        help="Create a .edatachain file",
-    )
-    parse_clone.add_argument(
-        "--edatachain-file",
-        help="Use a different filename for the resulting .edatachain file",
-    )
 
     add_studio_parser(subp, parent_parser)
     add_jobs_parser(subp, parent_parser)
@@ -139,22 +130,22 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "dataset",
         aliases=["ds"],
         parents=[parent_parser],
-        description="Commands for managing datasers",
+        description="Commands for managing datasets.",
     )
     datasets_subparser = datasets_parser.add_subparsers(
         dest="datasets_cmd",
-        help="Use `datachain datasets CMD --help` to display command specific help",
+        help="Use `datachain dataset CMD --help` to display command-specific help",
     )
 
     parse_pull = datasets_subparser.add_parser(
         "pull",
         parents=[parent_parser],
-        description="Pull specific dataset version from SaaS",
+        description="Pull specific dataset version from Studio.",
     )
     parse_pull.add_argument(
         "dataset",
         type=str,
-        help="Name and version of remote dataset created in SaaS",
+        help="Name and version of remote dataset created in Studio",
     )
     parse_pull.add_argument("-o", "--output", type=str, help="Output")
     parse_pull.add_argument(
@@ -178,16 +169,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         action="store_true",
         help="Copy actual files after pulling remote dataset into local DB",
     )
-    parse_pull.add_argument(
-        "--edatachain",
-        default=False,
-        action="store_true",
-        help="Create .edatachain file",
-    )
-    parse_pull.add_argument(
-        "--edatachain-file",
-        help="Use a different filename for the resulting .edatachain file",
-    )
+
     parse_pull.add_argument(
         "--local-name",
         action="store",
@@ -202,7 +184,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     parse_edit_dataset = datasets_subparser.add_parser(
-        "edit", parents=[parent_parser], description="Edit dataset metadata"
+        "edit", parents=[parent_parser], description="Edit dataset metadata."
     )
     parse_edit_dataset.add_argument("name", type=str, help="Dataset name")
     parse_edit_dataset.add_argument(
@@ -244,41 +226,41 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--team",
         action="store",
         default=None,
-        help="The team to edit a dataset. By default, it will use team from config.",
+        help="The team to edit a dataset. By default, it will use team from config",
     )
 
-    datasets_parser = datasets_subparser.add_parser(
-        "ls", parents=[parent_parser], description="List datasets"
+    datasets_ls_parser = datasets_subparser.add_parser(
+        "ls", parents=[parent_parser], description="List datasets."
     )
-    datasets_parser.add_argument(
+    datasets_ls_parser.add_argument(
         "--studio",
         action="store_true",
         default=False,
         help="List the files in the Studio",
     )
-    datasets_parser.add_argument(
+    datasets_ls_parser.add_argument(
         "-L",
         "--local",
         action="store_true",
         default=False,
         help="List local files only",
     )
-    datasets_parser.add_argument(
+    datasets_ls_parser.add_argument(
         "-a",
         "--all",
         action="store_true",
         default=True,
         help="List all files including hidden files",
     )
-    datasets_parser.add_argument(
+    datasets_ls_parser.add_argument(
         "--team",
         action="store",
         default=None,
-        help="The team to list datasets for. By default, it will use team from config.",
+        help="The team to list datasets for. By default, it will use team from config",
     )
 
     rm_dataset_parser = datasets_subparser.add_parser(
-        "rm", parents=[parent_parser], description="Removes dataset", aliases=["remove"]
+        "rm", parents=[parent_parser], description="Remove dataset.", aliases=["remove"]
     )
     rm_dataset_parser.add_argument("name", type=str, help="Dataset name")
     rm_dataset_parser.add_argument(
@@ -292,7 +274,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--force",
         default=False,
         action=BooleanOptionalAction,
-        help="Force delete registered dataset with all of it's versions",
+        help="Force delete registered dataset with all of its versions",
     )
     rm_dataset_parser.add_argument(
         "--studio",
@@ -318,13 +300,11 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--team",
         action="store",
         default=None,
-        help="The team to delete a dataset. By default, it will use team from config.",
+        help="The team to delete a dataset. By default, it will use team from config",
     )
 
     dataset_stats_parser = datasets_subparser.add_parser(
-        "stats",
-        parents=[parent_parser],
-        description="Shows basic dataset stats",
+        "stats", parents=[parent_parser], description="Show basic dataset statistics."
     )
     dataset_stats_parser.add_argument("name", type=str, help="Dataset name")
     dataset_stats_parser.add_argument(
@@ -349,7 +329,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     parse_ls = subp.add_parser(
-        "ls", parents=[parent_parser], description="List storage contents"
+        "ls", parents=[parent_parser], description="List storage contents."
     )
     add_sources_arg(parse_ls, nargs="*")
     parse_ls.add_argument(
@@ -357,7 +337,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--long",
         action="count",
         default=0,
-        help="List files in the long format",
+        help="List files in long format",
     )
     parse_ls.add_argument(
         "--studio",
@@ -383,11 +363,11 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--team",
         action="store",
         default=None,
-        help="The team to list datasets for. By default, it will use team from config.",
+        help="The team to list datasets for. By default, it will use team from config",
     )
 
     parse_du = subp.add_parser(
-        "du", parents=[parent_parser], description="Display space usage"
+        "du", parents=[parent_parser], description="Display space usage."
     )
     add_sources_arg(parse_du)
     parse_du.add_argument(
@@ -405,8 +385,8 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         type=int,
         metavar="N",
         help=(
-            "Display sizes for N directory depths below the given directory, "
-            "the default is 0 (summarize provided directory only)."
+            "Display sizes up to N directory levels deep "
+            "(default: 0, summarize provided directory only)"
         ),
     )
     parse_du.add_argument(
@@ -417,32 +397,32 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     parse_find = subp.add_parser(
-        "find", parents=[parent_parser], description="Search in a directory hierarchy"
+        "find", parents=[parent_parser], description="Search in a directory hierarchy."
     )
     add_sources_arg(parse_find)
     parse_find.add_argument(
         "--name",
         type=str,
         action="append",
-        help="Filename to match pattern.",
+        help="Match filename pattern",
     )
     parse_find.add_argument(
         "--iname",
         type=str,
         action="append",
-        help="Like -name but case insensitive.",
+        help="Match filename pattern (case insensitive)",
     )
     parse_find.add_argument(
         "--path",
         type=str,
         action="append",
-        help="Path to match pattern.",
+        help="Path to match pattern",
     )
     parse_find.add_argument(
         "--ipath",
         type=str,
         action="append",
-        help="Like -path but case insensitive.",
+        help="Like -path but case insensitive",
     )
     parse_find.add_argument(
         "--size",
@@ -450,7 +430,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help=(
             "Filter by size (+ is greater or equal, - is less or equal). "
             "Specified size is in bytes, or use a suffix like K, M, G for "
-            "kilobytes, megabytes, gigabytes, etc."
+            "kilobytes, megabytes, gigabytes, etc"
         ),
     )
     parse_find.add_argument(
@@ -470,14 +450,14 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     parse_index = subp.add_parser(
-        "index", parents=[parent_parser], description="Index storage location"
+        "index", parents=[parent_parser], description="Index storage location."
     )
     add_sources_arg(parse_index)
 
     show_parser = subp.add_parser(
         "show",
         parents=[parent_parser],
-        description="Create a new dataset with a query script",
+        description="Create a new dataset with a query script.",
     )
     show_parser.add_argument("name", type=str, help="Dataset name")
     show_parser.add_argument(
@@ -493,7 +473,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     query_parser = subp.add_parser(
         "query",
         parents=[parent_parser],
-        description="Create a new dataset with a query script",
+        description="Create a new dataset with a query script.",
     )
     query_parser.add_argument(
         "script", metavar="<script.py>", type=str, help="Filepath for script"
@@ -507,7 +487,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         metavar="N",
         help=(
             "Use multiprocessing to run any query script UDFs with N worker processes. "
-            "N defaults to the CPU count."
+            "N defaults to the CPU count"
         ),
     )
     query_parser.add_argument(
@@ -520,10 +500,12 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
     )
 
     subp.add_parser(
-        "clear-cache", parents=[parent_parser], description="Clear the local file cache"
+        "clear-cache",
+        parents=[parent_parser],
+        description="Clear the local file cache.",
     )
     subp.add_parser(
-        "gc", parents=[parent_parser], description="Garbage collect temporary tables"
+        "gc", parents=[parent_parser], description="Garbage collect temporary tables."
     )
 
     subp.add_parser("internal-run-udf", parents=[parent_parser])
@@ -536,12 +518,12 @@ def add_completion_parser(subparsers, parents):
     parser = subparsers.add_parser(
         "completion",
         parents=parents,
-        description="Output shell completion script",
+        description="Output shell completion script.",
     )
     parser.add_argument(
         "-s",
         "--shell",
-        help="Shell syntax for completions.",
+        help="Shell syntax for completions",
         default="bash",
         choices=shtab.SUPPORTED_SHELLS,
     )

{datachain-0.8.8 → datachain-0.8.9}/src/datachain/cli/parser/job.py

@@ -1,19 +1,16 @@
 def add_jobs_parser(subparsers, parent_parser) -> None:
-    jobs_help = "Commands to handle the Job running with Iterative Studio"
-    jobs_description = (
-        "This will help us to run, cancel and view the status of the job in Studio. "
-    )
+    jobs_help = "Manage jobs in Studio"
+    jobs_description = "Commands to manage job execution in Studio."
     jobs_parser = subparsers.add_parser(
         "job", parents=[parent_parser], description=jobs_description, help=jobs_help
     )
     jobs_subparser = jobs_parser.add_subparsers(
         dest="cmd",
-        help="Use `DataChain studio CMD --help` to display command-specific help.",
-        required=True,
+        help="Use `datachain studio CMD --help` to display command-specific help",
     )
 
     studio_run_help = "Run a job in Studio"
-    studio_run_description = "This command runs a job in Studio."
+    studio_run_description = "Run a job in Studio."
 
     studio_run_parser = jobs_subparser.add_parser(
         "run",
@@ -25,56 +22,56 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     studio_run_parser.add_argument(
         "query_file",
         action="store",
-        help="The query file to run.",
+        help="Query file to run",
     )
 
     studio_run_parser.add_argument(
         "--team",
         action="store",
         default=None,
-        help="The team to run a job for. By default, it will use team from config.",
+        help="Team to run job for (default: from config)",
     )
     studio_run_parser.add_argument(
         "--env-file",
         action="store",
-        help="File containing environment variables to set for the job.",
+        help="File with environment variables for the job",
     )
 
     studio_run_parser.add_argument(
         "--env",
         nargs="+",
-        help="Environment variable. Can be specified multiple times. Format: KEY=VALUE",
+        help="Environment variables in KEY=VALUE format",
     )
 
     studio_run_parser.add_argument(
         "--workers",
         type=int,
-        help="Number of workers to use for the job.",
+        help="Number of workers for the job",
     )
     studio_run_parser.add_argument(
         "--files",
         nargs="+",
-        help="Files to include in the job.",
+        help="Additional files to include in the job",
     )
     studio_run_parser.add_argument(
         "--python-version",
         action="store",
-        help="Python version to use for the job (e.g. '3.9', '3.10', '3.11').",
+        help="Python version for the job (e.g., 3.9, 3.10, 3.11)",
     )
     studio_run_parser.add_argument(
         "--req-file",
         action="store",
-        help="File containing Python package requirements.",
+        help="Python requirements file",
     )
 
     studio_run_parser.add_argument(
         "--req",
         nargs="+",
-        help="Python package requirement. Can be specified multiple times.",
+        help="Python package requirements",
     )
 
     studio_cancel_help = "Cancel a job in Studio"
-    studio_cancel_description = "This command cancels a job in Studio."
+    studio_cancel_description = "Cancel a running job in Studio."
 
     studio_cancel_parser = jobs_subparser.add_parser(
         "cancel",
@@ -86,19 +83,17 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     studio_cancel_parser.add_argument(
         "job_id",
         action="store",
-        help="The job ID to cancel.",
+        help="Job ID to cancel",
     )
     studio_cancel_parser.add_argument(
         "--team",
         action="store",
         default=None,
-        help="The team to cancel a job for. By default, it will use team from config.",
+        help="Team to cancel job for (default: from config)",
     )
 
-    studio_log_help = "Show the logs and latest status of Jobs in Studio"
-    studio_log_description = (
-        "This will display the logs and latest status of jobs in Studio"
-    )
+    studio_log_help = "Show job logs and status in Studio"
+    studio_log_description = "Display logs and current status of jobs in Studio."
 
     studio_log_parser = jobs_subparser.add_parser(
         "logs",
@@ -110,11 +105,11 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     studio_log_parser.add_argument(
         "job_id",
         action="store",
-        help="The job ID to show the logs.",
+        help="Job ID to show logs for",
     )
     studio_log_parser.add_argument(
         "--team",
         action="store",
         default=None,
-        help="The team to check the logs. By default, it will use team from config.",
+        help="Team to check logs for (default: from config)",
     )