datachain 0.34.6__py3-none-any.whl → 0.35.0__py3-none-any.whl

datachain/remote/studio.py

@@ -4,13 +4,7 @@ import os
 from collections.abc import AsyncIterator, Iterable, Iterator
 from datetime import datetime, timedelta, timezone
 from struct import unpack
-from typing import (
-    Any,
-    BinaryIO,
-    Generic,
-    Optional,
-    TypeVar,
-)
+from typing import Any, BinaryIO, Generic, TypeVar
 from urllib.parse import urlparse, urlunparse
 
 import websockets
@@ -22,14 +16,14 @@ from datachain.error import DataChainError
 from datachain.utils import STUDIO_URL, retry_with_backoff
 
 T = TypeVar("T")
-LsData = Optional[list[dict[str, Any]]]
-DatasetInfoData = Optional[dict[str, Any]]
-DatasetRowsData = Optional[Iterable[dict[str, Any]]]
-DatasetJobVersionsData = Optional[dict[str, Any]]
-DatasetExportStatus = Optional[dict[str, Any]]
-DatasetExportSignedUrls = Optional[list[str]]
-FileUploadData = Optional[dict[str, Any]]
-JobData = Optional[dict[str, Any]]
+LsData = list[dict[str, Any]] | None
+DatasetInfoData = dict[str, Any] | None
+DatasetRowsData = Iterable[dict[str, Any]] | None
+DatasetJobVersionsData = dict[str, Any] | None
+DatasetExportStatus = dict[str, Any] | None
+DatasetExportSignedUrls = list[str] | None
+FileUploadData = dict[str, Any] | None
+JobData = dict[str, Any] | None
 JobListData = list[dict[str, Any]]
 ClusterListData = list[dict[str, Any]]
 
@@ -93,7 +87,7 @@ class Response(Generic[T]):
 
 
 class StudioClient:
-    def __init__(self, timeout: float = 3600.0, team: Optional[str] = None) -> None:
+    def __init__(self, timeout: float = 3600.0, team: str | None = None) -> None:
         self._check_dependencies()
         self.timeout = timeout
         self._config = None
@@ -154,7 +148,7 @@ class StudioClient:
             ) from None
 
     def _send_request_msgpack(
-        self, route: str, data: dict[str, Any], method: Optional[str] = "POST"
+        self, route: str, data: dict[str, Any], method: str | None = "POST"
     ) -> Response[Any]:
         import msgpack
         import requests
@@ -192,7 +186,7 @@ class StudioClient:
 
     @retry_with_backoff(retries=3, errors=(HTTPError, Timeout))
     def _send_request(
-        self, route: str, data: dict[str, Any], method: Optional[str] = "POST"
+        self, route: str, data: dict[str, Any], method: str | None = "POST"
     ) -> Response[Any]:
         """
         Function that communicate Studio API.
@@ -241,7 +235,7 @@ class StudioClient:
         return Response(data, ok, message, response.status_code)
 
     def _send_multipart_request(
-        self, route: str, files: dict[str, Any], params: Optional[dict[str, Any]] = None
+        self, route: str, files: dict[str, Any], params: dict[str, Any] | None = None
     ) -> Response[Any]:
         """
         Function that communicates with Studio API using multipart/form-data.
@@ -345,7 +339,7 @@ class StudioClient:
             response = self._send_request_msgpack("datachain/ls", {"source": path})
             yield path, response
 
-    def ls_datasets(self, prefix: Optional[str] = None) -> Response[LsData]:
+    def ls_datasets(self, prefix: str | None = None) -> Response[LsData]:
         return self._send_request(
             "datachain/datasets", {"prefix": prefix}, method="GET"
         )
@@ -355,9 +349,9 @@ class StudioClient:
         name: str,
         namespace: str,
         project: str,
-        new_name: Optional[str] = None,
-        description: Optional[str] = None,
-        attrs: Optional[list[str]] = None,
+        new_name: str | None = None,
+        description: str | None = None,
+        attrs: list[str] | None = None,
     ) -> Response[DatasetInfoData]:
         body = {
             "new_name": new_name,
@@ -378,8 +372,8 @@ class StudioClient:
         name: str,
         namespace: str,
         project: str,
-        version: Optional[str] = None,
-        force: Optional[bool] = False,
+        version: str | None = None,
+        force: bool | None = False,
     ) -> Response[DatasetInfoData]:
         return self._send_request(
             "datachain/datasets",
@@ -461,18 +455,18 @@ class StudioClient:
         self,
         query: str,
         query_type: str,
-        environment: Optional[str] = None,
-        workers: Optional[int] = None,
-        query_name: Optional[str] = None,
-        files: Optional[list[str]] = None,
-        python_version: Optional[str] = None,
-        requirements: Optional[str] = None,
-        repository: Optional[str] = None,
-        priority: Optional[int] = None,
-        cluster: Optional[str] = None,
-        start_time: Optional[str] = None,
-        cron: Optional[str] = None,
-        credentials_name: Optional[str] = None,
+        environment: str | None = None,
+        workers: int | None = None,
+        query_name: str | None = None,
+        files: list[str] | None = None,
+        python_version: str | None = None,
+        requirements: str | None = None,
+        repository: str | None = None,
+        priority: int | None = None,
+        cluster: str | None = None,
+        start_time: str | None = None,
+        cron: str | None = None,
+        credentials_name: str | None = None,
     ) -> Response[JobData]:
         data = {
             "query": query,
@@ -494,9 +488,9 @@ class StudioClient:
 
     def get_jobs(
         self,
-        status: Optional[str] = None,
+        status: str | None = None,
         limit: int = 20,
-        job_id: Optional[str] = None,
+        job_id: str | None = None,
     ) -> Response[JobListData]:
         params: dict[str, Any] = {"limit": limit}
         if status is not None:

datachain/script_meta.py

@@ -1,6 +1,6 @@
 import re
 from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Any
 
 try:
     import tomllib
@@ -59,23 +59,23 @@ class ScriptConfig:
 
     """
 
-    python_version: Optional[str]
+    python_version: str | None
     dependencies: list[str]
     attachments: dict[str, str]
     params: dict[str, Any]
     inputs: dict[str, Any]
     outputs: dict[str, Any]
-    num_workers: Optional[int] = None
+    num_workers: int | None = None
 
     def __init__(
         self,
-        python_version: Optional[str] = None,
-        dependencies: Optional[list[str]] = None,
-        attachments: Optional[dict[str, str]] = None,
-        params: Optional[dict[str, Any]] = None,
-        inputs: Optional[dict[str, Any]] = None,
-        outputs: Optional[dict[str, Any]] = None,
-        num_workers: Optional[int] = None,
+        python_version: str | None = None,
+        dependencies: list[str] | None = None,
+        attachments: dict[str, str] | None = None,
+        params: dict[str, Any] | None = None,
+        inputs: dict[str, Any] | None = None,
+        outputs: dict[str, Any] | None = None,
+        num_workers: int | None = None,
     ):
         self.python_version = python_version
         self.dependencies = dependencies or []
@@ -98,7 +98,7 @@ class ScriptConfig:
         return self.attachments.get(name, default)
 
     @staticmethod
-    def read(script: str) -> Optional[dict]:
+    def read(script: str) -> dict | None:
         """Converts inline script metadata to dict with all found data"""
         regex = (
             r"(?m)^# \/\/\/ (?P<type>[a-zA-Z0-9-]+)[ \t]*$[\r\n|\r|\n]"
@@ -119,7 +119,7 @@ class ScriptConfig:
         return None
 
     @staticmethod
-    def parse(script: str) -> Optional["ScriptConfig"]:
+    def parse(script: str) -> "ScriptConfig | None":
         """
         Method that is parsing inline script metadata from datachain script and
         instantiating ScriptConfig class with found data. If no inline metadata is

datachain/sql/sqlite/base.py

@@ -2,11 +2,10 @@ import logging
 import re
 import sqlite3
 import warnings
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from datetime import MAXYEAR, MINYEAR, datetime, timezone
 from functools import cache
 from types import MappingProxyType
-from typing import Callable, Optional
 
 import sqlalchemy as sa
 import ujson as json
@@ -132,7 +131,7 @@ def run_compiler_hook(name):
 
 
 def functions_exist(
-    names: Iterable[str], connection: Optional[sqlite3.Connection] = None
+    names: Iterable[str], connection: sqlite3.Connection | None = None
 ) -> bool:
     """
     Returns True if all function names are defined for the given connection.
@@ -201,9 +200,7 @@ def sqlite_int_hash_64(x: int) -> int:
 def sqlite_bit_hamming_distance(a: int, b: int) -> int:
     """Calculate the Hamming distance between two integers."""
     diff = (a & MAX_INT64) ^ (b & MAX_INT64)
-    if hasattr(diff, "bit_count"):
-        return diff.bit_count()
-    return bin(diff).count("1")
+    return diff.bit_count()
 
 
 def sqlite_byte_hamming_distance(a: str, b: str) -> int:
@@ -215,7 +212,7 @@ def sqlite_byte_hamming_distance(a: str, b: str) -> int:
     elif len(b) < len(a):
         diff = len(a) - len(b)
         a = a[: len(b)]
-    return diff + sum(c1 != c2 for c1, c2 in zip(a, b))
+    return diff + sum(c1 != c2 for c1, c2 in zip(a, b, strict=False))
 
 
 def register_user_defined_sql_functions() -> None:
@@ -470,7 +467,7 @@ def py_json_array_get_element(val, idx):
         return None
 
 
-def py_json_array_slice(val, offset: int, length: Optional[int] = None):
+def py_json_array_slice(val, offset: int, length: int | None = None):
     arr = json.loads(val)
     try:
         return json.dumps(
@@ -605,7 +602,7 @@ def compile_collect(element, compiler, **kwargs):
 
 
 @cache
-def usearch_sqlite_path() -> Optional[str]:
+def usearch_sqlite_path() -> str | None:
     try:
         import usearch
     except ImportError:

datachain/studio.py

@@ -2,7 +2,7 @@ import asyncio
 import os
 import sys
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 import dateparser
 import tabulate
@@ -175,7 +175,7 @@ def token():
     print(token)
 
 
-def list_datasets(team: Optional[str] = None, name: Optional[str] = None):
+def list_datasets(team: str | None = None, name: str | None = None):
     def ds_full_name(ds: dict) -> str:
         return (
             f"{ds['project']['namespace']['name']}.{ds['project']['name']}.{ds['name']}"
@@ -206,7 +206,7 @@ def list_datasets(team: Optional[str] = None, name: Optional[str] = None):
             yield (full_name, version)
 
 
-def list_dataset_versions(team: Optional[str] = None, name: str = ""):
+def list_dataset_versions(team: str | None = None, name: str = ""):
     client = StudioClient(team=team)
 
     namespace_name, project_name, name = parse_dataset_name(name)
@@ -226,13 +226,13 @@ def list_dataset_versions(team: Optional[str] = None, name: str = ""):
 
 
 def edit_studio_dataset(
-    team_name: Optional[str],
+    team_name: str | None,
     name: str,
     namespace: str,
     project: str,
-    new_name: Optional[str] = None,
-    description: Optional[str] = None,
-    attrs: Optional[list[str]] = None,
+    new_name: str | None = None,
+    description: str | None = None,
+    attrs: list[str] | None = None,
 ):
     client = StudioClient(team=team_name)
     response = client.edit_dataset(
@@ -245,12 +245,12 @@ def edit_studio_dataset(
 
 
 def remove_studio_dataset(
-    team_name: Optional[str],
+    team_name: str | None,
     name: str,
     namespace: str,
     project: str,
-    version: Optional[str] = None,
-    force: Optional[bool] = False,
+    version: str | None = None,
+    force: bool | None = False,
 ):
     client = StudioClient(team=team_name)
     response = client.rm_dataset(name, namespace, project, version, force)
@@ -271,7 +271,7 @@ def save_config(hostname, token, level=ConfigLevel.GLOBAL):
     return config.config_file()
 
 
-def parse_start_time(start_time_str: Optional[str]) -> Optional[str]:
+def parse_start_time(start_time_str: str | None) -> str | None:
     if not start_time_str:
         return None
 
@@ -343,21 +343,21 @@ def show_logs_from_client(client, job_id):
 
 def create_job(
     query_file: str,
-    team_name: Optional[str],
-    env_file: Optional[str] = None,
-    env: Optional[list[str]] = None,
-    workers: Optional[int] = None,
-    files: Optional[list[str]] = None,
-    python_version: Optional[str] = None,
-    repository: Optional[str] = None,
-    req: Optional[list[str]] = None,
-    req_file: Optional[str] = None,
-    priority: Optional[int] = None,
-    cluster: Optional[str] = None,
-    start_time: Optional[str] = None,
-    cron: Optional[str] = None,
-    no_wait: Optional[bool] = False,
-    credentials_name: Optional[str] = None,
+    team_name: str | None,
+    env_file: str | None = None,
+    env: list[str] | None = None,
+    workers: int | None = None,
+    files: list[str] | None = None,
+    python_version: str | None = None,
+    repository: str | None = None,
+    req: list[str] | None = None,
+    req_file: str | None = None,
+    priority: int | None = None,
+    cluster: str | None = None,
+    start_time: str | None = None,
+    cron: str | None = None,
+    no_wait: bool | None = False,
+    credentials_name: str | None = None,
 ):
     query_type = "PYTHON" if query_file.endswith(".py") else "SHELL"
     with open(query_file) as f:
@@ -433,7 +433,7 @@ def upload_files(client: StudioClient, files: list[str]) -> list[str]:
     return file_ids
 
 
-def cancel_job(job_id: str, team_name: Optional[str]):
+def cancel_job(job_id: str, team_name: str | None):
     token = Config().read().get("studio", {}).get("token")
     if not token:
         raise DataChainError(
@@ -448,7 +448,7 @@ def cancel_job(job_id: str, team_name: Optional[str]):
     print(f"Job {job_id} canceled")
 
 
-def list_jobs(status: Optional[str], team_name: Optional[str], limit: int):
+def list_jobs(status: str | None, team_name: str | None, limit: int):
     client = StudioClient(team=team_name)
     response = client.get_jobs(status, limit)
     if not response.ok:
@@ -473,7 +473,7 @@ def list_jobs(status: Optional[str], team_name: Optional[str], limit: int):
     print(tabulate.tabulate(rows, headers="keys", tablefmt="grid"))
 
 
-def show_job_logs(job_id: str, team_name: Optional[str]):
+def show_job_logs(job_id: str, team_name: str | None):
     token = Config().read().get("studio", {}).get("token")
     if not token:
         raise DataChainError(
@@ -484,7 +484,7 @@ def show_job_logs(job_id: str, team_name: Optional[str]):
     return show_logs_from_client(client, job_id)
 
 
-def list_clusters(team_name: Optional[str]):
+def list_clusters(team_name: str | None):
     client = StudioClient(team=team_name)
     response = client.get_clusters()
     if not response.ok:

datachain/toolkit/split.py

@@ -1,5 +1,4 @@
 import random
-from typing import Optional
 
 from datachain import C, DataChain
 
@@ -9,7 +8,7 @@ RESOLUTION = 2**31 - 1  # Maximum positive value for a 32-bit signed integer.
 def train_test_split(
     dc: DataChain,
     weights: list[float],
-    seed: Optional[int] = None,
+    seed: int | None = None,
 ) -> list[DataChain]:
     """
     Splits a DataChain into multiple subsets based on the provided weights.

datachain/utils.py

@@ -11,7 +11,7 @@ import time
 from collections.abc import Iterable, Iterator, Sequence
 from contextlib import contextmanager
 from datetime import date, datetime, timezone
-from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeVar
 from uuid import UUID
 
 import cloudpickle
@@ -53,11 +53,11 @@ class DataChainDir:
 
     def __init__(
         self,
-        root: Optional[str] = None,
-        cache: Optional[str] = None,
-        tmp: Optional[str] = None,
-        db: Optional[str] = None,
-        config: Optional[str] = None,
+        root: str | None = None,
+        cache: str | None = None,
+        tmp: str | None = None,
+        db: str | None = None,
+        config: str | None = None,
     ) -> None:
         self.root = osp.abspath(root) if root is not None else self.default_root()
         self.cache = (
@@ -122,7 +122,7 @@ def global_config_dir():
     )
 
 
-def human_time_to_int(time: str) -> Optional[int]:
+def human_time_to_int(time: str) -> int | None:
     if not time:
         return None
 
@@ -146,7 +146,7 @@ def time_to_str(dt):
     return dt.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def time_to_local(dt: Union[datetime, str]) -> datetime:
+def time_to_local(dt: datetime | str) -> datetime:
     # TODO check usage
     if isinstance(dt, str):
         dt = isoparse(dt)
@@ -156,11 +156,11 @@ def time_to_local(dt: Union[datetime, str]) -> datetime:
         return dt
 
 
-def time_to_local_str(dt: Union[datetime, str]) -> str:
+def time_to_local_str(dt: datetime | str) -> str:
     return time_to_str(time_to_local(dt))
 
 
-def is_expired(expires: Optional[Union[datetime, str]]):
+def is_expired(expires: datetime | str | None):
     if expires:
         return time_to_local(expires) < time_to_local(datetime.now())  # noqa: DTZ005
 
@@ -301,9 +301,9 @@ def retry_with_backoff(retries=5, backoff_sec=1, errors=(Exception,)):
 
 
 def determine_workers(
-    workers: Union[bool, int],
-    rows_total: Optional[int] = None,
-) -> Union[bool, int]:
+    workers: bool | int,
+    rows_total: int | None = None,
+) -> bool | int:
     """Determine the number of workers to use for distributed processing."""
     if rows_total is not None and rows_total <= 1:
         # Disable distributed processing if there is no rows or only one row.
@@ -322,9 +322,9 @@ def determine_workers(
 
 
 def determine_processes(
-    parallel: Optional[Union[bool, int]] = None,
-    rows_total: Optional[int] = None,
-) -> Union[bool, int]:
+    parallel: bool | int | None = None,
+    rows_total: int | None = None,
+) -> bool | int:
     """Determine the number of processes to use for parallel processing."""
     if rows_total is not None and rows_total <= 1:
         # Disable parallel processing if there is no rows or only one row.
@@ -344,8 +344,8 @@ def determine_processes(
 
 
 def get_env_list(
-    key: str, default: Optional[Sequence] = None, sep: str = ","
-) -> Optional[Sequence[str]]:
+    key: str, default: Sequence | None = None, sep: str = ","
+) -> Sequence[str] | None:
     try:
         str_val = os.environ[key]
     except KeyError:
@@ -386,10 +386,10 @@ def show_df(
 
 
 def show_records(
-    records: Optional[list[dict]],
+    records: list[dict] | None,
     collapse_columns: bool = False,
     system_columns: bool = False,
-    hidden_fields: Optional[list[str]] = None,
+    hidden_fields: list[str] | None = None,
 ) -> None:
     import pandas as pd
 
@@ -518,7 +518,7 @@ def row_to_nested_dict(
 ) -> dict[str, Any]:
     """Converts a row to a nested dict based on the provided headers."""
     result: dict[str, Any] = {}
-    for h, v in zip(headers, row):
+    for h, v in zip(headers, row, strict=False):
         nested_dict_path_set(result, h, v)
     return result
 

{datachain-0.34.6.dist-info → datachain-0.35.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.6
+Version: 0.35.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -8,13 +8,12 @@ Project-URL: Documentation, https://datachain.dvc.ai
 Project-URL: Issues, https://github.com/iterative/datachain/issues
 Project-URL: Source, https://github.com/iterative/datachain
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Development Status :: 2 - Pre-Alpha
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: pyyaml