datachain 0.3.2__tar.gz → 0.3.3__tar.gz

{datachain-0.3.2 → datachain-0.3.3}/.github/workflows/benchmarks.yml

@@ -5,23 +5,24 @@ on:
     - cron: '0 0 * * *'
   pull_request:
     types: [opened, reopened, labeled, synchronize]
-  workflow_dispatch: {}
+  workflow_dispatch:
 
 env:
   FORCE_COLOR: "1"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   run:
-    if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }}
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v4
-
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip'
 
       - name: Upgrade nox and uv
@@ -30,5 +31,7 @@ jobs:
           nox --version
           uv --version
 
+      - run: uv pip install dvc[gs] --system
+      - run: dvc --cd tests/benchmarks/datasets pull
       - name: Run benchmarks
         run: nox -s bench

datachain-0.3.3/.github/workflows/tests-studio.yml

@@ -0,0 +1,103 @@
+name: Studio Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+env:
+  FORCE_COLOR: "1"
+  BRANCH: ${{ github.head_ref || github.ref_name }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  studio:
+    if: '!github.event.pull_request.head.repo.fork'
+    runs-on: ubuntu-latest-16-cores
+    strategy:
+      matrix:
+        pyv: ['3.12']
+        group: [1, 2, 3, 4, 5, 6]
+    services:
+      postgres:
+        image: postgres:16.3
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: test
+          POSTGRES_DB: database
+          POSTGRES_HOST_AUTH_METHOD: trust
+      clickhouse:
+        image: clickhouse/clickhouse-server:24
+        ports:
+          - 8123:8123
+          - 9010:9000
+        env:
+          CLICKHOUSE_DB: studio_local_db
+          CLICKHOUSE_USER: studio_local
+          CLICKHOUSE_PASSWORD: ch123456789!
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+      redis:
+        image: redis:7.2.5
+        ports:
+          - 6379:6379
+    steps:
+      - name: Studio branch name
+        env:
+          BRANCH: ${{ env.BRANCH }}
+          STUDIO_READ_ACCESS_TOKEN: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
+        run: |
+          echo "DataChain branch: $BRANCH"
+          if [[ "$BRANCH" == "main" ]]
+          then
+              STUDIO_BRANCH=develop
+          elif git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/iterative/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null
+          then
+              STUDIO_BRANCH="$BRANCH"
+          else
+              STUDIO_BRANCH=develop
+          fi
+          echo "STUDIO_BRANCH=$STUDIO_BRANCH" >> $GITHUB_ENV
+          echo "Studio branch: $STUDIO_BRANCH"
+
+      - name: Check out Studio
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          repository: iterative/studio
+          ref: ${{ env.STUDIO_BRANCH }}
+          token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          path: './backend/datachain'
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.pyv }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pyv }}
+          cache: 'pip'
+
+      - name: Install uv
+        run: |
+          python -m pip install --upgrade uv
+          uv --version
+
+      - name: Install dependencies
+        run: uv pip install --system ./backend/datachain_server[tests] ./backend/datachain[tests]
+
+      - name: Run tests
+        # Generate `.test_durations` file with `pytest --store-durations --durations-path ../.github/.test_durations ...`
+        run: >
+          pytest
+          --config-file=pyproject.toml -rs
+          --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
+          -m 'not benchmark'
+          tests ../datachain/tests
+        working-directory: backend/datachain_server

{datachain-0.3.2 → datachain-0.3.3}/.github/workflows/tests.yml

@@ -8,7 +8,6 @@ on:
 
 env:
   FORCE_COLOR: "1"
-  BRANCH: ${{ github.head_ref || github.ref_name }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -18,7 +17,6 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-
       - name: Check out the repository
         uses: actions/checkout@v4
         with:
@@ -112,95 +110,6 @@ jobs:
       - name: Build docs
         run: nox -s docs
 
-
-  studio:
-    if: '!github.event.pull_request.head.repo.fork'
-    runs-on: ubuntu-latest-16-cores
-    strategy:
-      matrix:
-        pyv: ['3.12']
-        group: [1, 2, 3, 4, 5, 6]
-    services:
-      postgres:
-        image: postgres:16.3
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_USER: test
-          POSTGRES_DB: database
-          POSTGRES_HOST_AUTH_METHOD: trust
-      clickhouse:
-        image: clickhouse/clickhouse-server:24
-        ports:
-          - 8123:8123
-          - 9010:9000
-        env:
-          CLICKHOUSE_DB: studio_local_db
-          CLICKHOUSE_USER: studio_local
-          CLICKHOUSE_PASSWORD: ch123456789!
-          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
-      redis:
-        image: redis:7.2.5
-        ports:
-          - 6379:6379
-    steps:
-
-      - name: Studio branch name
-        env:
-          BRANCH: ${{ env.BRANCH }}
-          STUDIO_READ_ACCESS_TOKEN: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
-        run: |
-          echo "DataChain branch: $BRANCH"
-          if [[ "$BRANCH" == "main" ]]
-          then
-              STUDIO_BRANCH=develop
-          elif git ls-remote --heads https://"$STUDIO_READ_ACCESS_TOKEN"@github.com/iterative/studio.git "$BRANCH" | grep -F "$BRANCH" 2>&1>/dev/null
-          then
-              STUDIO_BRANCH="$BRANCH"
-          else
-              STUDIO_BRANCH=develop
-          fi
-          echo "STUDIO_BRANCH=$STUDIO_BRANCH" >> $GITHUB_ENV
-          echo "Studio branch: $STUDIO_BRANCH"
-
-      - name: Check out Studio
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          repository: iterative/studio
-          ref: ${{ env.STUDIO_BRANCH }}
-          token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
-
-      - name: Check out repository
-        uses: actions/checkout@v4
-        with:
-          path: './backend/datachain'
-          fetch-depth: 0
-
-      - name: Set up Python ${{ matrix.pyv }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.pyv }}
-          cache: 'pip'
-
-      - name: Install uv
-        run: |
-          python -m pip install --upgrade uv
-          uv --version
-
-      - name: Install dependencies
-        run: uv pip install --system ./backend/datachain_server[tests] ./backend/datachain[tests]
-
-      - name: Run tests
-        # Generate `.test_durations` file with `pytest --store-durations --durations-path ../.github/.test_durations ...`
-        run: >
-          pytest
-          --config-file=pyproject.toml -rs
-          --splits=6 --group=${{ matrix.group }} --durations-path=../../.github/.test_durations
-          tests ../datachain/tests
-        working-directory: backend/datachain_server
-
-
   examples:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
@@ -211,7 +120,6 @@ jobs:
         pyv: ['3.9', '3.12']
         group: ['get_started', 'llm_and_nlp or computer_vision', 'multimodal']
     steps:
-
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.pyv }}

{datachain-0.3.2/src/datachain.egg-info → datachain-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.2
+Version: 0.3.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -76,7 +76,7 @@ Requires-Dist: aiotools>=1.7.0; extra == "tests"
 Requires-Dist: requests-mock; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.10.1; extra == "dev"
+Requires-Dist: mypy==1.11.1; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"

{datachain-0.3.2 → datachain-0.3.3}/examples/computer_vision/openimage-detect.py

@@ -54,7 +54,7 @@ source = "gs://datachain-demo/openimages-v6-test-jsonpairs/"
     .filter(C("file.path").glob("*.jpg") | C("file.path").glob("*.json"))
     .agg(
         openimage_detect,
-        partition_by=path.file_stem(path.name(C("file.path"))),
+        partition_by=path.file_stem(C("file.path")),
         params=["file"],
         output={"file": File, "bbox": BBox},
     )

{datachain-0.3.2 → datachain-0.3.3}/examples/get_started/common_sql_functions.py

@@ -26,8 +26,8 @@ dc.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).select(
 
 (
     dc.mutate(
-        stem=path.file_stem(path.name(C("file.path"))),
-        ext=path.file_ext(path.name(C("file.path"))),
+        stem=path.file_stem(C("file.path")),
+        ext=path.file_ext(C("file.path")),
     )
     .select("file.path", "stem", "ext")
     .show(5)

{datachain-0.3.2 → datachain-0.3.3}/pyproject.toml

@@ -87,7 +87,7 @@ tests = [
 ]
 dev = [
   "datachain[docs,tests]",
-  "mypy==1.10.1",
+  "mypy==1.11.1",
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/asyn.py

@@ -224,3 +224,23 @@ class OrderedMapper(AsyncMapper[InputT, ResultT]):
     async def _break_iteration(self) -> None:
         self.heap = []
         self._push_result(self._next_yield, None)
+
+
+def iter_over_async(ait, loop):
+    """Wrap an asynchronous iterator into a synchronous one"""
+    ait = ait.__aiter__()
+
+    # helper async fn that just gets the next element from the async iterator
+    async def get_next():
+        try:
+            obj = await ait.__anext__()
+            return False, obj
+        except StopAsyncIteration:
+            return True, None
+
+    # actual sync iterator
+    while True:
+        done, obj = asyncio.run_coroutine_threadsafe(get_next(), loop).result()
+        if done:
+            break
+        yield obj

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/catalog/catalog.py

@@ -577,6 +577,7 @@ class Catalog:
         warehouse_ready_callback: Optional[
             Callable[["AbstractWarehouse"], None]
         ] = None,
+        in_memory: bool = False,
     ):
         datachain_dir = DataChainDir(cache=cache_dir, tmp=tmp_dir)
         datachain_dir.init()
@@ -590,6 +591,7 @@ class Catalog:
             "tmp_dir": tmp_dir,
         }
         self._warehouse_ready_callback = warehouse_ready_callback
+        self.in_memory = in_memory
 
     @cached_property
     def warehouse(self) -> "AbstractWarehouse":

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/catalog/loader.py

@@ -28,8 +28,10 @@ WAREHOUSE_ARG_PREFIX = "DATACHAIN_WAREHOUSE_ARG_"
 DISTRIBUTED_IMPORT_PATH = "DATACHAIN_DISTRIBUTED"
 DISTRIBUTED_ARG_PREFIX = "DATACHAIN_DISTRIBUTED_ARG_"
 
+IN_MEMORY_ERROR_MESSAGE = "In-memory is only supported on SQLite"
 
-def get_id_generator() -> "AbstractIDGenerator":
+
+def get_id_generator(in_memory: bool = False) -> "AbstractIDGenerator":
     id_generator_serialized = os.environ.get(ID_GENERATOR_SERIALIZED)
     if id_generator_serialized:
         id_generator_obj = deserialize(id_generator_serialized)
@@ -43,25 +45,31 @@ def get_id_generator() -> "AbstractIDGenerator":
     id_generator_import_path = os.environ.get(ID_GENERATOR_IMPORT_PATH)
     id_generator_arg_envs = get_envs_by_prefix(ID_GENERATOR_ARG_PREFIX)
     # Convert env variable names to keyword argument names by lowercasing them
-    id_generator_args = {k.lower(): v for k, v in id_generator_arg_envs.items()}
-
-    if id_generator_import_path:
-        # ID generator paths are specified as (for example):
-        # datachain.data_storage.SQLiteIDGenerator
-        if "." not in id_generator_import_path:
-            raise RuntimeError(
-                f"Invalid {ID_GENERATOR_IMPORT_PATH} import path:"
-                f"{id_generator_import_path}"
-            )
-        module_name, _, class_name = id_generator_import_path.rpartition(".")
-        id_generator = import_module(module_name)
-        id_generator_class = getattr(id_generator, class_name)
-    else:
-        id_generator_class = SQLiteIDGenerator
+    id_generator_args: dict[str, Any] = {
+        k.lower(): v for k, v in id_generator_arg_envs.items()
+    }
+
+    if not id_generator_import_path:
+        id_generator_args["in_memory"] = in_memory
+        return SQLiteIDGenerator(**id_generator_args)
+    if in_memory:
+        raise RuntimeError(IN_MEMORY_ERROR_MESSAGE)
+    # ID generator paths are specified as (for example):
+    # datachain.data_storage.SQLiteIDGenerator
+    if "." not in id_generator_import_path:
+        raise RuntimeError(
+            f"Invalid {ID_GENERATOR_IMPORT_PATH} import path:"
+            f"{id_generator_import_path}"
+        )
+    module_name, _, class_name = id_generator_import_path.rpartition(".")
+    id_generator = import_module(module_name)
+    id_generator_class = getattr(id_generator, class_name)
     return id_generator_class(**id_generator_args)
 
 
-def get_metastore(id_generator: Optional["AbstractIDGenerator"]) -> "AbstractMetastore":
+def get_metastore(
+    id_generator: Optional["AbstractIDGenerator"], in_memory: bool = False
+) -> "AbstractMetastore":
     metastore_serialized = os.environ.get(METASTORE_SERIALIZED)
     if metastore_serialized:
         metastore_obj = deserialize(metastore_serialized)
@@ -78,24 +86,32 @@ def get_metastore(id_generator: Optional["AbstractIDGenerator"]) -> "AbstractMet
     metastore_import_path = os.environ.get(METASTORE_IMPORT_PATH)
     metastore_arg_envs = get_envs_by_prefix(METASTORE_ARG_PREFIX)
     # Convert env variable names to keyword argument names by lowercasing them
-    metastore_args = {k.lower(): v for k, v in metastore_arg_envs.items()}
-
-    if metastore_import_path:
-        # Metastore paths are specified as (for example):
-        # datachain.data_storage.SQLiteMetastore
-        if "." not in metastore_import_path:
-            raise RuntimeError(
-                f"Invalid {METASTORE_IMPORT_PATH} import path: {metastore_import_path}"
-            )
-        module_name, _, class_name = metastore_import_path.rpartition(".")
-        metastore = import_module(module_name)
-        metastore_class = getattr(metastore, class_name)
-    else:
-        metastore_class = SQLiteMetastore
+    metastore_args: dict[str, Any] = {
+        k.lower(): v for k, v in metastore_arg_envs.items()
+    }
+
+    if not metastore_import_path:
+        if not isinstance(id_generator, SQLiteIDGenerator):
+            raise ValueError("SQLiteMetastore can only be used with SQLiteIDGenerator")
+        metastore_args["in_memory"] = in_memory
+        return SQLiteMetastore(id_generator, **metastore_args)
+    if in_memory:
+        raise RuntimeError(IN_MEMORY_ERROR_MESSAGE)
+    # Metastore paths are specified as (for example):
+    # datachain.data_storage.SQLiteMetastore
+    if "." not in metastore_import_path:
+        raise RuntimeError(
+            f"Invalid {METASTORE_IMPORT_PATH} import path: {metastore_import_path}"
+        )
+    module_name, _, class_name = metastore_import_path.rpartition(".")
+    metastore = import_module(module_name)
+    metastore_class = getattr(metastore, class_name)
     return metastore_class(id_generator, **metastore_args)
 
 
-def get_warehouse(id_generator: Optional["AbstractIDGenerator"]) -> "AbstractWarehouse":
+def get_warehouse(
+    id_generator: Optional["AbstractIDGenerator"], in_memory: bool = False
+) -> "AbstractWarehouse":
     warehouse_serialized = os.environ.get(WAREHOUSE_SERIALIZED)
     if warehouse_serialized:
         warehouse_obj = deserialize(warehouse_serialized)
@@ -112,20 +128,26 @@ def get_warehouse(id_generator: Optional["AbstractIDGenerator"]) -> "AbstractWar
     warehouse_import_path = os.environ.get(WAREHOUSE_IMPORT_PATH)
     warehouse_arg_envs = get_envs_by_prefix(WAREHOUSE_ARG_PREFIX)
     # Convert env variable names to keyword argument names by lowercasing them
-    warehouse_args = {k.lower(): v for k, v in warehouse_arg_envs.items()}
-
-    if warehouse_import_path:
-        # Warehouse paths are specified as (for example):
-        # datachain.data_storage.SQLiteWarehouse
-        if "." not in warehouse_import_path:
-            raise RuntimeError(
-                f"Invalid {WAREHOUSE_IMPORT_PATH} import path: {warehouse_import_path}"
-            )
-        module_name, _, class_name = warehouse_import_path.rpartition(".")
-        warehouse = import_module(module_name)
-        warehouse_class = getattr(warehouse, class_name)
-    else:
-        warehouse_class = SQLiteWarehouse
+    warehouse_args: dict[str, Any] = {
+        k.lower(): v for k, v in warehouse_arg_envs.items()
+    }
+
+    if not warehouse_import_path:
+        if not isinstance(id_generator, SQLiteIDGenerator):
+            raise ValueError("SQLiteWarehouse can only be used with SQLiteIDGenerator")
+        warehouse_args["in_memory"] = in_memory
+        return SQLiteWarehouse(id_generator, **warehouse_args)
+    if in_memory:
+        raise RuntimeError(IN_MEMORY_ERROR_MESSAGE)
+    # Warehouse paths are specified as (for example):
+    # datachain.data_storage.SQLiteWarehouse
+    if "." not in warehouse_import_path:
+        raise RuntimeError(
+            f"Invalid {WAREHOUSE_IMPORT_PATH} import path: {warehouse_import_path}"
+        )
+    module_name, _, class_name = warehouse_import_path.rpartition(".")
+    warehouse = import_module(module_name)
+    warehouse_class = getattr(warehouse, class_name)
     return warehouse_class(id_generator, **warehouse_args)
 
 
@@ -152,7 +174,9 @@ def get_distributed_class(**kwargs):
     return distributed_class(**distributed_args | kwargs)
 
 
-def get_catalog(client_config: Optional[dict[str, Any]] = None) -> Catalog:
+def get_catalog(
+    client_config: Optional[dict[str, Any]] = None, in_memory: bool = False
+) -> Catalog:
     """
     Function that creates Catalog instance with appropriate metastore
     and warehouse classes. Metastore class can be provided with env variable
@@ -164,10 +188,11 @@ def get_catalog(client_config: Optional[dict[str, Any]] = None) -> Catalog:
     and name of variable after, e.g. if it accepts team_id as kwargs
     we can provide DATACHAIN_METASTORE_ARG_TEAM_ID=12345 env variable.
     """
-    id_generator = get_id_generator()
+    id_generator = get_id_generator(in_memory=in_memory)
     return Catalog(
         id_generator=id_generator,
-        metastore=get_metastore(id_generator),
-        warehouse=get_warehouse(id_generator),
+        metastore=get_metastore(id_generator, in_memory=in_memory),
+        warehouse=get_warehouse(id_generator, in_memory=in_memory),
         client_config=client_config,
+        in_memory=in_memory,
     )

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/client/azure.py

@@ -3,6 +3,7 @@ from typing import Any
 from adlfs import AzureBlobFileSystem
 from tqdm import tqdm
 
+from datachain.lib.file import File
 from datachain.node import Entry
 
 from .fsspec import DELIMITER, Client, ResultQueue
@@ -24,6 +25,18 @@ class AzureClient(Client):
             size=v.get("size", ""),
         )
 
+    def info_to_file(self, v: dict[str, Any], path: str) -> File:
+        version_id = v.get("version_id")
+        return File(
+            source=self.uri,
+            path=path,
+            etag=v.get("etag", "").strip('"'),
+            version=version_id or "",
+            is_latest=version_id is None or bool(v.get("is_current_version")),
+            last_modified=v["last_modified"],
+            size=v.get("size", ""),
+        )
+
     async def _fetch_flat(self, start_prefix: str, result_queue: ResultQueue) -> None:
         prefix = start_prefix
         if prefix:

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/client/gcs.py

@@ -9,6 +9,7 @@ from dateutil.parser import isoparse
 from gcsfs import GCSFileSystem
 from tqdm import tqdm
 
+from datachain.lib.file import File
 from datachain.node import Entry
 
 from .fsspec import DELIMITER, Client, ResultQueue
@@ -120,3 +121,14 @@ class GCSClient(Client):
             last_modified=self.parse_timestamp(v["updated"]),
             size=v.get("size", ""),
         )
+
+    def info_to_file(self, v: dict[str, Any], path: str) -> File:
+        return File(
+            source=self.uri,
+            path=path,
+            etag=v.get("etag", ""),
+            version=v.get("generation", ""),
+            is_latest=not v.get("timeDeleted"),
+            last_modified=self.parse_timestamp(v["updated"]),
+            size=v.get("size", ""),
+        )

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/client/local.py

@@ -7,6 +7,7 @@ from urllib.parse import urlparse
 
 from fsspec.implementations.local import LocalFileSystem
 
+from datachain.lib.file import File
 from datachain.node import Entry
 from datachain.storage import StorageURI
 
@@ -144,6 +145,16 @@ class FileClient(Client):
             size=v.get("size", ""),
         )
 
+    def info_to_file(self, v: dict[str, Any], path: str) -> File:
+        return File(
+            source=self.uri,
+            path=path,
+            size=v.get("size", ""),
+            etag=v["mtime"].hex(),
+            is_latest=True,
+            last_modified=datetime.fromtimestamp(v["mtime"], timezone.utc),
+        )
+
     def fetch_nodes(
         self,
         nodes,

{datachain-0.3.2 → datachain-0.3.3}/src/datachain/client/s3.py

@@ -5,6 +5,7 @@ from botocore.exceptions import NoCredentialsError
 from s3fs import S3FileSystem
 from tqdm import tqdm
 
+from datachain.lib.file import File
 from datachain.node import Entry
 
 from .fsspec import DELIMITER, Client, ResultQueue
@@ -167,3 +168,14 @@ class ClientS3(Client):
             owner_name=v.get("Owner", {}).get("DisplayName", ""),
             owner_id=v.get("Owner", {}).get("ID", ""),
         )
+
+    def info_to_file(self, v: dict[str, Any], path: str) -> File:
+        return File(
+            source=self.uri,
+            path=path,
+            size=v["size"],
+            version=ClientS3.clean_s3_version(v.get("VersionId", "")),
+            etag=v.get("ETag", "").strip('"'),
+            is_latest=v.get("IsLatest", True),
+            last_modified=v.get("LastModified", ""),
+        )