cognite-toolkit 0.5.97__py3-none-any.whl → 0.7.34__py3-none-any.whl

cognite_toolkit/_cdf_tk/client/data_classes/streamlit_.py

@@ -17,13 +17,13 @@ class _StreamlitCore(WriteableCogniteResource["StreamlitWrite"], ABC):
         external_id: str,
         name: str,
         creator: str,
-        entrypoint: str,
+        entrypoint: str | None = None,
         description: str | None = None,
         published: bool = False,
         theme: Literal["Light", "Dark"] = "Light",
         thumbnail: str | None = None,
         data_set_id: int | None = None,
-        cognite_toolkit_app_hash: str = "MISSING",
+        cognite_toolkit_app_hash: str | None = None,
     ) -> None:
         self.external_id = external_id
         self.name = name
@@ -43,11 +43,13 @@ class _StreamlitCore(WriteableCogniteResource["StreamlitWrite"], ABC):
             "name": self.name,
             "published": self.published,
             "theme": self.theme,
-            "entrypoint": self.entrypoint,
-            "cdf-toolkit-app-hash": self.cognite_toolkit_app_hash,
         }
         if self.thumbnail:
             metadata["thumbnail"] = self.thumbnail
+        if self.cognite_toolkit_app_hash:
+            metadata["cdf-toolkit-app-hash"] = self.cognite_toolkit_app_hash
+        if self.entrypoint:
+            metadata["entrypoint"] = self.entrypoint
         return {
             "external_id": self.external_id,
             "name": f"{self.name}-source.json",
@@ -64,7 +66,7 @@ class StreamlitWrite(_StreamlitCore):
             external_id=resource["externalId"],
             name=resource["name"],
             creator=resource["creator"],
-            entrypoint=resource["entrypoint"],
+            entrypoint=resource.get("entrypoint"),
             description=resource.get("description"),
             thumbnail=resource.get("thumbnail"),
             data_set_id=resource.get("dataSetId"),
@@ -90,15 +92,15 @@ class Streamlit(_StreamlitCore):
         external_id: str,
         name: str,
         creator: str,
-        entrypoint: str,
         created_time: int,
         last_updated_time: int,
+        entrypoint: str | None = None,
         description: str | None = None,
         published: bool = False,
         theme: Literal["Light", "Dark"] = "Light",
         thumbnail: str | None = None,
         data_set_id: int | None = None,
-        cognite_toolkit_app_hash: str = "MISSING",
+        cognite_toolkit_app_hash: str | None = None,
     ) -> None:
         super().__init__(
             external_id,
@@ -121,7 +123,7 @@ class Streamlit(_StreamlitCore):
             external_id=resource["externalId"],
             name=resource["name"],
             creator=resource["creator"],
-            entrypoint=resource["entrypoint"],
+            entrypoint=resource.get("entrypoint"),
             created_time=resource["createdTime"],
             last_updated_time=resource["lastUpdatedTime"],
             description=resource.get("description"),
@@ -129,7 +131,7 @@ class Streamlit(_StreamlitCore):
             data_set_id=resource.get("dataSetId"),
         )
         # Trick to avoid specifying defaults twice
-        for key in ["theme", "app_hash"]:
+        for key in ["theme"]:
             if key in resource:
                 args[key] = resource[key]
         if "cogniteToolkitAppHash" in resource:
@@ -148,8 +150,6 @@ class Streamlit(_StreamlitCore):
             dumped.update(dumped.pop("metadata"))
         if "cdf-toolkit-app-hash" in dumped:
             dumped["cogniteToolkitAppHash"] = dumped.pop("cdf-toolkit-app-hash")
-        if "entrypoint" not in dumped:
-            dumped["entrypoint"] = "MISSING"
         return cls._load(dumped)
 
     def as_write(self) -> StreamlitWrite:

cognite_toolkit/_cdf_tk/client/data_classes/streams.py

@@ -0,0 +1,90 @@
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.constants import StreamTemplateName
+from cognite_toolkit._cdf_tk.protocols import (
+    ResourceRequestListProtocol,
+    ResourceResponseListProtocol,
+)
+
+from .base import BaseModelObject, BaseResourceList, RequestResource, ResponseResource
+
+
+class StreamRequest(RequestResource):
+    """Stream request resource class."""
+
+    external_id: str
+    settings: dict[Literal["template"], dict[Literal["name"], StreamTemplateName]]
+
+    def as_id(self) -> str:
+        return self.external_id
+
+
+class StreamRequestList(BaseResourceList[StreamRequest], ResourceRequestListProtocol):
+    """List of Stream request resources."""
+
+    _RESOURCE = StreamRequest
+
+
+class LifecycleObject(BaseModelObject):
+    """Lifecycle object."""
+
+    hot_phase_duration: str | None = None
+    data_deleted_after: str | None = None
+    retained_after_soft_delete: str
+
+
+class ResourceUsage(BaseModelObject):
+    """Resource quota with provisioned and consumed values."""
+
+    provisioned: int
+    consumed: int | None = None
+
+
+class LimitsObject(BaseModelObject):
+    """Limits object."""
+
+    max_records_total: ResourceUsage
+    max_giga_bytes_total: ResourceUsage
+    max_filtering_interval: str | None = None
+
+
+class StreamSettings(BaseModelObject):
+    """Stream settings object."""
+
+    lifecycle: LifecycleObject
+    limits: LimitsObject
+
+
+class StreamResponse(ResponseResource["StreamRequest"]):
+    """Stream response resource class."""
+
+    external_id: str
+    created_time: int
+    created_from_template: StreamTemplateName
+    type: Literal["Mutable", "Immutable"]
+    settings: StreamSettings | None = None
+
+    def as_request_resource(self) -> StreamRequest:
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
+        )
+
+    def as_write(self) -> StreamRequest:
+        return StreamRequest.model_validate(
+            {
+                "externalId": self.external_id,
+                "settings": {"template": {"name": self.created_from_template}},
+            }
+        )
+
+
+class StreamResponseList(BaseResourceList[StreamResponse], ResourceResponseListProtocol):
+    """List of Stream response resources."""
+
+    _RESOURCE = StreamResponse
+
+    def as_write(self) -> StreamRequestList:
+        return StreamRequestList([item.as_write() for item in self.data])

cognite_toolkit/_cdf_tk/client/data_classes/three_d.py

@@ -0,0 +1,53 @@
+from typing import Literal
+
+from .base import BaseModelObject, RequestResource, ResponseResource
+
+
+class NodeReference(BaseModelObject):
+    space: str
+    external_id: str
+
+
+class RevisionStatus(BaseModelObject):
+    status: Literal["Queued", "Processing", "Done", "Failed"] | None = None
+    revision_id: int | None = None
+    created_time: int | None = None
+    revision_count: int | None = None
+    types: list[str] | None = None
+
+
+class ThreeDModelRequest(RequestResource):
+    name: str
+
+    def as_id(self) -> str:
+        return self.name
+
+
+class ThreeDModelClassicRequest(ThreeDModelRequest):
+    data_set_id: int | None = None
+    metadata: dict[str, str] | None = None
+
+    def as_id(self) -> str:
+        return self.name
+
+
+class ThreeDModelDMSRequest(ThreeDModelRequest):
+    space: str
+    type: Literal["CAD", "PointCloud", "Image360"]
+    thumbnail_reference: NodeReference | None = None
+
+
+class ThreeDModelResponse(ResponseResource[ThreeDModelRequest]):
+    name: str
+    id: int
+    created_time: int
+    data_set_id: int | None = None
+    metadata: dict[str, str] | None = None
+    space: str | None = None
+    last_revision_info: RevisionStatus | None = None
+
+    def as_request_resource(self) -> ThreeDModelRequest:
+        if self.space is None:
+            return ThreeDModelClassicRequest._load(self.dump())
+        else:
+            return ThreeDModelDMSRequest._load(self.dump())

cognite_toolkit/_cdf_tk/client/testing.py

@@ -5,26 +5,46 @@ from unittest.mock import MagicMock
 
 from cognite.client._api.datapoints import DatapointsAPI
 from cognite.client._api.datapoints_subscriptions import DatapointsSubscriptionAPI
+from cognite.client._api.functions import FunctionCallsAPI, FunctionSchedulesAPI
+from cognite.client._api.raw import RawDatabasesAPI, RawRowsAPI, RawTablesAPI
 from cognite.client._api.synthetic_time_series import SyntheticDatapointsAPI
 from cognite.client.testing import CogniteClientMock
+from rich.console import Console
 
 from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
 
+from ._toolkit_client import ToolAPI
 from .api.canvas import CanvasAPI, IndustrialCanvasAPI
+from .api.charts import ChartsAPI
 from .api.dml import DMLAPI
 from .api.extended_data_modeling import ExtendedInstancesAPI
+from .api.extended_files import ExtendedFileMetadataAPI
+from .api.extended_functions import ExtendedFunctionsAPI
+from .api.extended_raw import ExtendedRawAPI
 from .api.extended_timeseries import ExtendedTimeSeriesAPI
+from .api.infield import InfieldAPI, InFieldCDMConfigAPI, InfieldConfigAPI
 from .api.location_filters import LocationFiltersAPI
 from .api.lookup import (
     AssetLookUpAPI,
     DataSetLookUpAPI,
+    EventLookUpAPI,
     ExtractionPipelineLookUpAPI,
+    FileMetadataLookUpAPI,
+    FunctionLookUpAPI,
     LocationFiltersLookUpAPI,
     LookUpGroup,
     SecurityCategoriesLookUpAPI,
     TimeSeriesLookUpAPI,
 )
-from .api.migration import InstanceSourceAPI, MigrationAPI
+from .api.migration import (
+    CreatedSourceSystemAPI,
+    InstanceSourceAPI,
+    LookupAPI,
+    MigrationAPI,
+    MigrationLookupAPI,
+    ResourceViewMappingAPI,
+)
+from .api.project import ProjectAPI
 from .api.robotics import RoboticsAPI
 from .api.robotics.capabilities import CapabilitiesAPI
 from .api.robotics.data_postprocessing import DataPostProcessingAPI
@@ -33,6 +53,8 @@ from .api.robotics.locations import LocationsAPI as RoboticsLocationsAPI
 from .api.robotics.maps import MapsAPI
 from .api.search import SearchAPI
 from .api.search_config import SearchConfigurationsAPI
+from .api.streams import StreamsAPI
+from .api.three_d import ThreeDAPI, ThreeDModelAPI
 from .api.token import TokenAPI
 from .api.verify import VerifyAPI
 
@@ -48,6 +70,7 @@ class ToolkitClientMock(CogniteClientMock):
             super().__init__(*args, **kwargs)
             return None
         super().__init__(*args, **kwargs)
+        self.console = Console()
         # Developer note:
         # - Please add your mocked APIs in chronological order
         # - For nested APIs:
@@ -56,6 +79,17 @@ class ToolkitClientMock(CogniteClientMock):
         #   - Use `spec_set=MyNestedAPI` for all nested APIs
         self.canvas = MagicMock(spec=CanvasAPI)
         self.canvas.industrial = MagicMock(spec_set=IndustrialCanvasAPI)
+        self.charts = MagicMock(spec_set=ChartsAPI)
+        self.files = MagicMock(spec_set=ExtendedFileMetadataAPI)
+        self.functions = MagicMock(spec=ExtendedFunctionsAPI)
+        self.functions.calls = MagicMock(spec_set=FunctionCallsAPI)
+        self.functions.schedules = MagicMock(spec_set=FunctionSchedulesAPI)
+        self.infield = MagicMock(spec=InfieldAPI)
+        self.infield.config = MagicMock(spec_set=InfieldConfigAPI)
+        self.infield.cdm_config = MagicMock(spec_set=InFieldCDMConfigAPI)
+
+        self.project = MagicMock(spec_set=ProjectAPI)
+
         self.search = MagicMock(spec=SearchAPI)
         self.search.locations = MagicMock(spec_set=LocationFiltersAPI)
         self.search.configurations = MagicMock(spec_set=SearchConfigurationsAPI)
@@ -64,11 +98,25 @@ class ToolkitClientMock(CogniteClientMock):
         self.lookup.data_sets = MagicMock(spec_set=DataSetLookUpAPI)
         self.lookup.assets = MagicMock(spec_set=AssetLookUpAPI)
         self.lookup.time_series = MagicMock(spec_set=TimeSeriesLookUpAPI)
+        self.lookup.files = MagicMock(spec_set=FileMetadataLookUpAPI)
+        self.lookup.events = MagicMock(spec_set=EventLookUpAPI)
         self.lookup.security_categories = MagicMock(spec_set=SecurityCategoriesLookUpAPI)
         self.lookup.location_filters = MagicMock(spec_set=LocationFiltersLookUpAPI)
         self.lookup.extraction_pipelines = MagicMock(spec_set=ExtractionPipelineLookUpAPI)
+        self.lookup.functions = MagicMock(spec_set=FunctionLookUpAPI)
         self.migration = MagicMock(spec=MigrationAPI)
         self.migration.instance_source = MagicMock(spec_set=InstanceSourceAPI)
+        self.migration.lookup = MagicMock(spec=MigrationLookupAPI)
+        self.migration.lookup.assets = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.events = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.files = MagicMock(spec_set=LookupAPI)
+        self.migration.lookup.time_series = MagicMock(spec_set=LookupAPI)
+        self.migration.resource_view_mapping = MagicMock(spec_set=ResourceViewMappingAPI)
+        self.migration.created_source_system = MagicMock(spec_set=CreatedSourceSystemAPI)
+        self.raw = MagicMock(spec=ExtendedRawAPI)
+        self.raw.databases = MagicMock(spec_set=RawDatabasesAPI)
+        self.raw.rows = MagicMock(spec_set=RawRowsAPI)
+        self.raw.tables = MagicMock(spec_set=RawTablesAPI)
 
         self.robotics = MagicMock()
         self.robotics.robots = MagicMock(spec=RoboticsAPI)
@@ -85,6 +133,12 @@ class ToolkitClientMock(CogniteClientMock):
         self.time_series.data.synthetic = MagicMock(spec_set=SyntheticDatapointsAPI)
         self.time_series.subscriptions = MagicMock(spec_set=DatapointsSubscriptionAPI)
 
+        self.tool = MagicMock(spec=ToolAPI)
+        self.tool.three_d = MagicMock(spec=ThreeDAPI)
+        self.tool.three_d.models = MagicMock(spec_set=ThreeDModelAPI)
+
+        self.streams = MagicMock(spec=StreamsAPI)
+
         # This is a helper API, not a real API.
         self.token = TokenAPI(self)
         self.verify = MagicMock(spec_set=VerifyAPI)
@@ -93,6 +147,8 @@ class ToolkitClientMock(CogniteClientMock):
 @contextmanager
 def monkeypatch_toolkit_client() -> Iterator[ToolkitClientMock]:
     toolkit_client_mock = ToolkitClientMock()
-    ToolkitClient.__new__ = lambda *args, **kwargs: toolkit_client_mock  # type: ignore[method-assign]
-    yield toolkit_client_mock
-    ToolkitClient.__new__ = lambda cls, *args, **kwargs: object.__new__(cls)  # type: ignore[method-assign]
+    try:
+        ToolkitClient.__new__ = lambda *args, **kwargs: toolkit_client_mock  # type: ignore[method-assign]
+        yield toolkit_client_mock
+    finally:
+        ToolkitClient.__new__ = lambda cls, *args, **kwargs: object.__new__(cls)  # type: ignore[method-assign]

cognite_toolkit/_cdf_tk/commands/__init__.py

@@ -1,44 +1,39 @@
+from ._download import DownloadCommand
 from ._migrate import (
-    MigrateAssetsCommand,
-    MigrateFilesCommand,
-    MigrateTimeseriesCommand,
-    MigrationCanvasCommand,
+    MigrationCommand,
     MigrationPrepareCommand,
 )
-from ._populate import PopulateCommand
 from ._profile import ProfileAssetCentricCommand, ProfileAssetCommand, ProfileRawCommand, ProfileTransformationCommand
 from ._purge import PurgeCommand
+from ._upload import UploadCommand
+from .about import AboutCommand
 from .auth import AuthCommand
 from .build_cmd import BuildCommand
 from .clean import CleanCommand
 from .collect import CollectCommand
 from .deploy import DeployCommand
-from .dump_data import DumpDataCommand
 from .dump_resource import DumpResourceCommand
-from .featureflag import FeatureFlagCommand
 from .init import InitCommand
 from .modules import ModulesCommand
 from .pull import PullCommand
 from .repo import RepoCommand
+from .resources import ResourcesCommand
 from .run import RunFunctionCommand, RunTransformationCommand, RunWorkflowCommand
 
 __all__ = [
+    "AboutCommand",
     "AuthCommand",
     "BuildCommand",
+    "BuildCommandV2",
     "CleanCommand",
     "CollectCommand",
     "DeployCommand",
-    "DumpDataCommand",
+    "DownloadCommand",
     "DumpResourceCommand",
-    "FeatureFlagCommand",
     "InitCommand",
-    "MigrateAssetsCommand",
-    "MigrateFilesCommand",
-    "MigrateTimeseriesCommand",
-    "MigrationCanvasCommand",
+    "MigrationCommand",
     "MigrationPrepareCommand",
     "ModulesCommand",
-    "PopulateCommand",
     "ProfileAssetCentricCommand",
     "ProfileAssetCommand",
     "ProfileRawCommand",
@@ -46,7 +41,9 @@ __all__ = [
     "PullCommand",
     "PurgeCommand",
     "RepoCommand",
+    "ResourcesCommand",
     "RunFunctionCommand",
     "RunTransformationCommand",
     "RunWorkflowCommand",
+    "UploadCommand",
 ]

cognite_toolkit/_cdf_tk/commands/_base.py

@@ -5,6 +5,7 @@ from typing import Any
 from rich import print
 from rich.console import Console
 
+from cognite_toolkit._cdf_tk.data_classes import CommandTrackingInfo
 from cognite_toolkit._cdf_tk.tk_warnings import (
     ToolkitWarning,
     WarningList,
@@ -20,13 +21,19 @@ class ToolkitCommand:
         self.silent = silent
         self.warning_list = WarningList[ToolkitWarning]()
         self.tracker = Tracker(skip_tracking)
+        self._additional_tracking_info = CommandTrackingInfo()
 
     @property
     def print_warning(self) -> bool:
         return self._print_warning and not self.silent
 
     def _track_command(self, result: str | Exception) -> None:
-        self.tracker.track_cli_command(self.warning_list, result, type(self).__name__.removesuffix("Command"))
+        self.tracker.track_cli_command(
+            self.warning_list,
+            result,
+            type(self).__name__.removesuffix("Command"),
+            self._additional_tracking_info,
+        )
 
     def run(self, execute: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
         global _HAS_PRINTED_COLLECT_MESSAGE

cognite_toolkit/_cdf_tk/commands/_changes.py

@@ -11,9 +11,8 @@ from packaging.version import Version
 from packaging.version import parse as parse_version
 from rich import print
 
-from cognite_toolkit._cdf_tk.builders import get_loader
+from cognite_toolkit._cdf_tk.cdf_toml import CDFToml
 from cognite_toolkit._cdf_tk.constants import DOCKER_IMAGE_NAME
-from cognite_toolkit._cdf_tk.data_classes import ModuleDirectories
 from cognite_toolkit._cdf_tk.utils import iterate_modules, read_yaml_file, safe_read, safe_write
 from cognite_toolkit._version import __version__
 
@@ -51,40 +50,6 @@ class ManualChange(Change):
         return ""
 
 
-class SetKindOnFile(AutomaticChange):
-    """Adds the kind to the filename of all resource files.
-
-Before `your_file.yaml`:
-After `your_file.FileMetadata.yaml`:
-    """
-
-    deprecated_from = Version("0.4.0")
-    has_file_changes = True
-
-    def do(self) -> set[Path]:
-        module_directories = ModuleDirectories.load(self._organization_dir)
-        changed: set[Path] = set()
-        for module in module_directories:
-            for resource_folder, source_files in module.source_paths_by_resource_folder.items():
-                for source_file in source_files:
-                    if source_file.suffix not in {".yaml", ".yml"}:
-                        continue
-                    loader, warning = get_loader(source_file, resource_folder, force_pattern=True)
-                    if loader is None:
-                        print(f"Could not find loader for {source_file}")
-                        continue
-                    if source_file.stem.casefold().endswith(loader.kind.casefold()):
-                        continue
-                    new_name = source_file.with_name(f"{source_file.stem}.{loader.kind}{source_file.suffix}")
-                    source_file.rename(new_name)
-                    changed.add(source_file)
-                    for suffix in [".sql", ".csv", ".parquet"]:
-                        if (adjacent_file := source_file.with_suffix(suffix)).exists():
-                            adjacent_file.rename(new_name.with_suffix(suffix))
-                            changed.add(adjacent_file)
-
-        return changed
-
 
 class FixViewBasedLocationFilter(AutomaticChange):
     """The created view-based location filter has been fixed to be compatible with the CDF API.
@@ -362,7 +327,6 @@ After:
 
     def do(self) -> set[Path]:
         # Avoid circular import
-        from .modules import ModulesCommand
 
         system_yaml = self._organization_dir / "_system.yaml"
         if not system_yaml.exists():
@@ -372,11 +336,8 @@ After:
         content = read_yaml_file(system_yaml)
         current_version = content.get("cdf_toolkit_version", __version__)
 
-        cdf_toml_content = ModulesCommand(skip_tracking=True).create_cdf_toml(self._organization_dir)
-        cdf_toml_content = cdf_toml_content.replace(f'version = "{__version__}"', f'version = "{current_version}"')
-
-        cdf_toml_path = Path.cwd() / "cdf.toml"
-        cdf_toml_path.write_text(cdf_toml_content, encoding="utf-8")
+        CDFToml.write(self._organization_dir, version=current_version)
+        cdf_toml_path = Path.cwd() / CDFToml.file_name
         system_yaml.unlink()
         return {cdf_toml_path, system_yaml}
 

cognite_toolkit/_cdf_tk/commands/_download.py

@@ -0,0 +1,154 @@
+from collections.abc import Callable, Iterable
+from functools import partial
+from pathlib import Path
+
+from cognite_toolkit._cdf_tk.constants import DATA_MANIFEST_STEM, DATA_RESOURCE_DIR
+from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError
+from cognite_toolkit._cdf_tk.protocols import T_ResourceResponse
+from cognite_toolkit._cdf_tk.storageio import (
+    ConfigurableStorageIO,
+    Page,
+    StorageIO,
+    T_Selector,
+    TableStorageIO,
+)
+from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning
+from cognite_toolkit._cdf_tk.utils.file import safe_write, sanitize_filename, yaml_safe_dump
+from cognite_toolkit._cdf_tk.utils.fileio import (
+    TABLE_WRITE_CLS_BY_FORMAT,
+    Compression,
+    FileWriter,
+    SchemaColumn,
+)
+from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
+from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
+
+from ._base import ToolkitCommand
+
+
+class DownloadCommand(ToolkitCommand):
+    def download(
+        self,
+        selectors: Iterable[T_Selector],
+        io: StorageIO[T_Selector, T_ResourceResponse],
+        output_dir: Path,
+        verbose: bool,
+        file_format: str,
+        compression: str,
+        limit: int | None = 100_000,
+    ) -> None:
+        """Downloads data from CDF to the specified output directory.
+
+        Args:
+            selectors: The selectors of the resources to download.
+            io: The StorageIO instance that defines how to download and process the data.
+            output_dir: The directory where the downloaded files will be saved.
+            verbose: If True, prints detailed information about the download process.
+            file_format: The format of the files to be written (e.g., ".ndjson").
+            compression: The compression method to use for the downloaded files (e.g., "none", "gzip").
+            limit: The maximum number of items to download for each selected set. If None, all items will be downloaded.
+        """
+        compression_cls = Compression.from_name(compression)
+
+        console = io.client.console
+        for selector in selectors:
+            target_dir = output_dir / sanitize_filename(selector.group)
+            if verbose:
+                console.print(f"Downloading {selector.display_name} '{selector!s}' to {target_dir.as_posix()!r}")
+
+            iteration_count = self._get_iteration_count(io, selector, limit)
+            filestem = sanitize_filename(str(selector))
+            if self._already_downloaded(target_dir, filestem):
+                warning = LowSeverityWarning(
+                    f"Data for {selector!s} already exists in {target_dir.as_posix()!r}. Skipping download."
+                )
+                self.warn(warning, console=console)
+                continue
+
+            selector.dump_to_file(target_dir)
+            columns: list[SchemaColumn] | None = None
+            is_table = file_format in TABLE_WRITE_CLS_BY_FORMAT
+            if is_table and isinstance(io, TableStorageIO):
+                columns = io.get_schema(selector)
+            elif is_table:
+                raise ToolkitValueError(
+                    f"Cannot download {selector.kind} in {file_format!r} format. The {selector.kind!r} storage type does not support table schemas."
+                )
+
+            with FileWriter.create_from_format(
+                file_format, target_dir, selector.kind, compression_cls, columns=columns
+            ) as writer:
+                executor = ProducerWorkerExecutor[Page[T_ResourceResponse], list[dict[str, JsonVal]]](
+                    download_iterable=io.stream_data(selector, limit),
+                    process=self.create_data_process(io=io, selector=selector, is_table=is_table),
+                    write=partial(writer.write_chunks, filestem=filestem),
+                    iteration_count=iteration_count,
+                    # Limit queue size to avoid filling up memory before the workers can write to disk.
+                    max_queue_size=8 * 10,  # 8 workers, 10 items per worker
+                    download_description=f"Downloading {selector!s}",
+                    process_description="Processing",
+                    write_description=f"Writing to {target_dir.as_posix()!r} in files with stem {filestem!r}",
+                    console=console,
+                )
+                executor.run()
+                executor.raise_on_error()
+                file_count = writer.file_count
+
+            if isinstance(io, ConfigurableStorageIO):
+                for config in io.configurations(selector):
+                    filename = config.filename or filestem
+                    config_file = target_dir / DATA_RESOURCE_DIR / config.folder_name / f"{filename}.{config.kind}.yaml"
+                    config_file.parent.mkdir(parents=True, exist_ok=True)
+                    safe_write(config_file, yaml_safe_dump(config.value))
+
+            console.print(f"Downloaded {selector!s} to {file_count} file(s) in {target_dir.as_posix()!r}.")
+
+    @staticmethod
+    def _get_iteration_count(
+        io: StorageIO[T_Selector, T_ResourceResponse],
+        selector: T_Selector,
+        limit: int | None,
+    ) -> int | None:
+        total = io.count(selector)
+        if total is not None and limit is not None and total > limit:
+            total = limit
+        iteration_count: int | None = None
+        if total is not None:
+            iteration_count = total // io.CHUNK_SIZE + (1 if total % io.CHUNK_SIZE > 0 else 0)
+        return iteration_count
+
+    @staticmethod
+    def _already_downloaded(output_dir: Path, filestem: str) -> bool:
+        if not output_dir.exists():
+            return False
+
+        # Check for multi-part files (e.g. ndjson, csv, parquet)
+        if any(output_dir.glob(f"{filestem}-part-*")):
+            return True
+
+        # Check for single files (e.g. yaml) and exclude the metadata file.
+        manifest_file_name = f"{filestem}.{DATA_MANIFEST_STEM}.yaml"
+        for f in output_dir.glob(f"{filestem}.*"):
+            if f.name != manifest_file_name:
+                return True
+
+        return False
+
+    @staticmethod
+    def create_data_process(
+        io: StorageIO[T_Selector, T_ResourceResponse],
+        selector: T_Selector,
+        is_table: bool,
+    ) -> Callable[[Page[T_ResourceResponse]], list[dict[str, JsonVal]]]:
+        """Creates a data processing function based on the IO type and whether the output is a table."""
+        if is_table and isinstance(io, TableStorageIO):
+
+            def row_data_process(chunk: Page[T_ResourceResponse]) -> list[dict[str, JsonVal]]:
+                return io.data_to_row(chunk.items, selector)
+
+            return row_data_process
+
+        def chunk_data_process(data_page: Page[T_ResourceResponse]) -> list[dict[str, JsonVal]]:
+            return io.data_to_json_chunk(data_page.items, selector)
+
+        return chunk_data_process