cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.30__py3-none-any.whl

cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py

@@ -1,201 +0,0 @@
-from uuid import uuid4
-
-from cognite.client.data_classes.capabilities import (
-    Capability,
-    DataModelInstancesAcl,
-    DataModelsAcl,
-    SpaceIDScope,
-)
-from cognite.client.exceptions import CogniteException
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.client.data_classes.canvas import (
-    CANVAS_INSTANCE_SPACE,
-    Canvas,
-    ContainerReferenceApply,
-    FdmInstanceContainerReferenceApply,
-)
-from cognite_toolkit._cdf_tk.client.data_classes.migration import InstanceSource
-from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
-from cognite_toolkit._cdf_tk.commands._migrate.data_model import (
-    INSTANCE_SOURCE_VIEW_ID,
-    MODEL_ID,
-    RESOURCE_VIEW_MAPPING_VIEW_ID,
-)
-from cognite_toolkit._cdf_tk.exceptions import AuthenticationError, ToolkitMigrationError
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning, MediumSeverityWarning
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.interactive_select import InteractiveCanvasSelect
-
-
-class MigrationCanvasCommand(ToolkitCommand):
-    canvas_schema_space = Canvas.get_source().space
-    # Note sequences are not supported in Canvas, so we do not include them here.
-    asset_centric_resource_types = frozenset({"asset", "event", "file", "timeseries"})
-
-    def migrate_canvas(
-        self,
-        client: ToolkitClient,
-        external_ids: list[str] | None = None,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        self.validate_access(
-            client,
-            instance_spaces=[CANVAS_INSTANCE_SPACE],
-            schema_spaces=[self.canvas_schema_space, INSTANCE_SOURCE_VIEW_ID.space],
-        )
-        self.validate_migration_model_available(client)
-        external_ids = external_ids or InteractiveCanvasSelect(client).select_external_ids()
-        if external_ids is None or not external_ids:
-            self.console("No canvases selected for migration.")
-            return
-        action = "Would migrate" if dry_run else "Migrating"
-        self.console(f"{action} {len(external_ids)} canvases.")
-        for external_id in external_ids:
-            self._migrate_single_canvas(client, external_id, dry_run=dry_run, verbose=verbose)
-
-    def _migrate_single_canvas(
-        self,
-        client: ToolkitClient,
-        external_id: str,
-        dry_run: bool = False,
-        verbose: bool = False,
-    ) -> None:
-        canvas = client.canvas.industrial.retrieve(external_id=external_id)
-        if canvas is None:
-            self.warn(MediumSeverityWarning(f"Canvas with external ID '{external_id}' not found. Skipping.. "))
-            return
-        update = canvas.as_write()
-        to_migrate = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type in self.asset_centric_resource_types
-        ]
-        if not to_migrate:
-            self.warn(
-                LowSeverityWarning(
-                    f"Canvas with name '{canvas.canvas.name}' does not have any asset-centric references. Skipping.. "
-                )
-            )
-        if verbose:
-            self.console(f"Found canvas: {canvas.canvas.name}")
-        reference_ids = [ref.as_asset_centric_id() for ref in to_migrate]
-        instance_sources = client.migration.instance_source.retrieve(reference_ids)
-        source_by_reference_id = {source.as_asset_centric_id(): source for source in instance_sources}
-        missing = set(reference_ids) - set(source_by_reference_id.keys())
-        if missing:
-            self.warn(
-                HighSeverityWarning(
-                    f"Canvas '{canvas.canvas.name}' has references to resources that are not been migrated: {humanize_collection(missing)}. Skipping.. "
-                )
-            )
-            return
-        if dry_run:
-            self.console(
-                f"Canvas '{canvas.canvas.name}' is ready for migration all {len(instance_sources)} references asset-centric resources found."
-            )
-            return
-        if verbose:
-            self.console(
-                f"Migrating canvas '{canvas.canvas.name}' with {len(instance_sources)} references to asset-centric resources."
-            )
-        backup = canvas.as_write().create_backup()
-
-        update.container_references = [
-            ref
-            for ref in update.container_references
-            if ref.container_reference_type not in self.asset_centric_resource_types
-        ]
-        for ref in to_migrate:
-            source = source_by_reference_id[ref.as_asset_centric_id()]
-            fdm_ref = self.migrate_container_reference(ref, source, canvas.canvas.external_id)
-            update.fdm_instance_container_references.append(fdm_ref)
-
-        try:
-            client.canvas.industrial.create(backup)
-        except CogniteException as e:
-            raise ToolkitMigrationError(f"Failed to create backup for canvas '{canvas.canvas.name}': {e!s}. ") from e
-        try:
-            client.canvas.industrial.update(update)
-        except CogniteException as e:
-            raise ToolkitMigrationError(
-                f"Failed to migrate canvas '{canvas.canvas.name}': {e!s}. A backup was created with external ID '{backup.canvas.external_id}'."
-            ) from e
-        else:
-            self.console(
-                f'Canvas "{canvas.canvas.name}" migrated successfully with {len(to_migrate)} references to data model instances.'
-            )
-
-    @classmethod
-    def migrate_container_reference(
-        cls, reference: ContainerReferenceApply, source: InstanceSource, canvas_external_id: str
-    ) -> FdmInstanceContainerReferenceApply:
-        """Migrate a single container reference by replacing the asset-centric ID with the data model instance ID."""
-        consumer_view = source.consumer_view()
-        new_id = str(uuid4())
-        new_external_id = f"{canvas_external_id}_{new_id}"
-        return FdmInstanceContainerReferenceApply(
-            external_id=new_external_id,
-            id_=new_id,
-            container_reference_type="fdmInstance",
-            instance_space=source.space,
-            instance_external_id=source.external_id,
-            view_space=consumer_view.space,
-            view_external_id=consumer_view.external_id,
-            view_version=consumer_view.version,
-            label=reference.label,
-            properties_=reference.properties_,
-            x=reference.x,
-            y=reference.y,
-            width=reference.width,
-            height=reference.height,
-            max_width=reference.max_width,
-            max_height=reference.max_height,
-        )
-
-    @staticmethod
-    def validate_access(
-        client: ToolkitClient,
-        instance_spaces: list[str] | None = None,
-        schema_spaces: list[str] | None = None,
-    ) -> None:
-        required_capabilities: list[Capability] = []
-        if instance_spaces is not None:
-            required_capabilities.append(
-                DataModelInstancesAcl(
-                    actions=[
-                        DataModelInstancesAcl.Action.Read,
-                        DataModelInstancesAcl.Action.Write,
-                        DataModelInstancesAcl.Action.Write_Properties,
-                    ],
-                    scope=SpaceIDScope(instance_spaces),
-                )
-            )
-        if schema_spaces is not None:
-            required_capabilities.append(
-                DataModelsAcl(actions=[DataModelsAcl.Action.Read], scope=SpaceIDScope(schema_spaces)),
-            )
-        if missing := client.iam.verify_capabilities(required_capabilities):
-            raise AuthenticationError(f"Missing required capabilities: {humanize_collection(missing)}.", missing)
-
-    @staticmethod
-    def validate_migration_model_available(client: ToolkitClient) -> None:
-        models = client.data_modeling.data_models.retrieve([MODEL_ID], inline_views=False)
-        if not models:
-            raise ToolkitMigrationError(
-                f"The migration data model {MODEL_ID!r} does not exist. "
-                "Please run the `cdf migrate prepare` command to deploy the migration data model."
-            )
-        elif len(models) > 1:
-            raise ToolkitMigrationError(
-                f"Multiple migration models {MODEL_ID!r}. "
-                "Please delete the duplicate models before proceeding with the migration."
-            )
-        model = models[0]
-        missing_views = {INSTANCE_SOURCE_VIEW_ID, RESOURCE_VIEW_MAPPING_VIEW_ID} - set(model.views or [])
-        if missing_views:
-            raise ToolkitMigrationError(
-                f"Invalid migration model. Missing views {humanize_collection(missing_views)}. "
-                f"Please run the `cdf migrate prepare` command to deploy the migration data model."
-            )

cognite_toolkit/_cdf_tk/commands/dump_data.py

@@ -1,489 +0,0 @@
-import time
-from abc import ABC, abstractmethod
-from collections.abc import Callable, Iterable, Iterator
-from functools import lru_cache
-from itertools import groupby
-from pathlib import Path
-from typing import Any, ClassVar, Generic, Literal
-
-from cognite.client.data_classes import (
-    Asset,
-    AssetFilter,
-    DataSetList,
-    Event,
-    EventFilter,
-    FileMetadata,
-    FileMetadataFilter,
-    LabelDefinitionList,
-    TimeSeries,
-    TimeSeriesFilter,
-)
-from cognite.client.data_classes._base import T_CogniteResource
-from rich.console import Console
-from rich.progress import track
-
-from cognite_toolkit._cdf_tk.client import ToolkitClient
-from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand
-from cognite_toolkit._cdf_tk.cruds import (
-    AssetCRUD,
-    DataSetsCRUD,
-    EventCRUD,
-    FileMetadataCRUD,
-    LabelCRUD,
-    ResourceCRUD,
-    TimeSeriesCRUD,
-)
-from cognite_toolkit._cdf_tk.exceptions import (
-    ToolkitFileExistsError,
-    ToolkitIsADirectoryError,
-    ToolkitValueError,
-)
-from cognite_toolkit._cdf_tk.utils import humanize_collection
-from cognite_toolkit._cdf_tk.utils.cdf import metadata_key_counts
-from cognite_toolkit._cdf_tk.utils.file import safe_rmtree
-from cognite_toolkit._cdf_tk.utils.producer_worker import ProducerWorkerExecutor
-from cognite_toolkit._cdf_tk.utils.table_writers import (
-    FileFormat,
-    Schema,
-    SchemaColumn,
-    SchemaColumnList,
-    TableFileWriter,
-)
-
-
-class DataFinder:
-    supported_formats: ClassVar[frozenset[FileFormat]] = frozenset()
-    # This is the standard maximum items that can be returns by most CDF endpoints.
-    chunk_size: ClassVar[int] = 1000
-
-    def validate_format(self, format_: str) -> Literal[FileFormat]:
-        if format_ in self.supported_formats:
-            return format_  # type: ignore[return-value]
-        raise ToolkitValueError(
-            f"Unsupported format {format_}. Supported formats are {humanize_collection(self.supported_formats)}."
-        )
-
-    @abstractmethod
-    def create_iterators(
-        self, format_: FileFormat, limit: int | None
-    ) -> Iterator[tuple[Schema, int, Iterable, Callable]]:
-        """Create an iterator for the specified format."""
-        raise NotImplementedError("This method should be implemented in subclasses.")
-
-
-class AssetCentricFinder(DataFinder, ABC, Generic[T_CogniteResource]):
-    def __init__(self, client: ToolkitClient, hierarchies: list[str], data_sets: list[str]):
-        self.client = client
-        self.hierarchies = hierarchies
-        self.data_sets = data_sets
-        self.loader = self._create_loader(client)
-        self._hierarchy_set = set(self.hierarchies)
-        self._data_set_set = set(self.data_sets)
-        self._used_labels: set[str] = set()
-        self._used_data_sets: set[str] = set()
-
-    @abstractmethod
-    def _create_loader(self, client: ToolkitClient) -> ResourceCRUD:
-        """Create the appropriate loader for the finder."""
-        raise NotImplementedError()
-
-    @lru_cache
-    def aggregate_count(self, hierarchies: tuple[str, ...], data_sets: tuple[str, ...]) -> int:
-        return self._aggregate_count(list(hierarchies), list(data_sets))
-
-    @abstractmethod
-    def _aggregate_count(self, hierarchies: list[str], data_sets: list[str]) -> int:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def _get_resource_columns(self) -> SchemaColumnList:
-        """Get the columns for the schema."""
-        raise NotImplementedError()
-
-    @abstractmethod
-    def create_resource_iterator(self, limit: int | None) -> Iterable:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def _resource_processor(self, items: Iterable[T_CogniteResource]) -> list[tuple[str, list[dict[str, Any]]]]:
-        """Process the resources and return them in a format suitable for writing."""
-        raise NotImplementedError()
-
-    def _to_write(self, items: Iterable[T_CogniteResource]) -> list[dict[str, Any]]:
-        write_items: list[dict[str, Any]] = []
-        for item in items:
-            dumped = self.loader.dump_resource(item)
-            if "metadata" in dumped:
-                metadata = dumped.pop("metadata")
-                for key, value in metadata.items():
-                    dumped[f"metadata.{key}"] = value
-            if isinstance(dumped.get("labels"), list):
-                dumped["labels"] = [label["externalId"] for label in dumped["labels"]]
-                self._used_labels.update(dumped["labels"])
-            if "dataSetExternalId" in dumped:
-                self._used_data_sets.add(dumped["dataSetExternalId"])
-            write_items.append(dumped)
-        return write_items
-
-    def create_iterators(
-        self, format_: FileFormat, limit: int | None
-    ) -> Iterator[tuple[Schema, int, Iterable, Callable]]:
-        total = self.aggregate_count(tuple(self.hierarchies), tuple(self.data_sets))
-        columns = self._get_resource_columns()
-
-        iteration_count = total // self.chunk_size + (1 if total % self.chunk_size > 0 else 0)
-        if iteration_count == 0:
-            return
-
-        yield (
-            Schema(
-                display_name=self.loader.display_name,
-                format_=format_,
-                columns=columns,
-                folder_name=self.loader.folder_name,
-                kind=self.loader.kind,
-            ),
-            iteration_count,
-            self.create_resource_iterator(limit),
-            self._resource_processor,
-        )
-        if self._used_data_sets:
-            yield self._data_sets()
-        if self._used_labels:
-            yield self._labels()
-
-    def _data_sets(self) -> tuple[Schema, int, Iterable, Callable]:
-        data_sets = self.client.data_sets.retrieve_multiple(
-            external_ids=list(self._used_data_sets), ignore_unknown_ids=True
-        )
-        loader = DataSetsCRUD.create_loader(self.client)
-
-        def process_data_sets(items: DataSetList) -> list[tuple[str, list[dict[str, Any]]]]:
-            # All data sets are written to a single group, thus the empty string as the group key.
-            # (Group keys are for example used in CSV files to create separate files for each
-            # data set an asset belongs to.)
-            return [("", [loader.dump_resource(item) for item in items])]
-
-        return (
-            # YAML format does not need columns.
-            Schema(
-                display_name=loader.display_name,
-                format_="yaml",
-                columns=SchemaColumnList(),
-                folder_name=loader.folder_name,
-                kind=loader.kind,
-            ),
-            1,
-            [data_sets],
-            process_data_sets,
-        )
-
-    def _labels(self) -> tuple[Schema, int, Iterable, Callable]:
-        labels = self.client.labels.retrieve(external_id=list(self._used_labels))
-        loader = LabelCRUD.create_loader(self.client)
-
-        def process_labels(items: LabelDefinitionList) -> list[tuple[str, list[dict[str, Any]]]]:
-            # All labels are written to a single group, thus the empty string as the group key.
-            # (Group keys are for example used in CSV files to create separate files for each
-            # label an asset belongs to.)
-            return [("", [loader.dump_resource(item) for item in items])]
-
-        return (
-            # YAML format does not need columns.
-            Schema(
-                display_name=loader.display_name,
-                format_="yaml",
-                columns=SchemaColumnList(),
-                folder_name=loader.folder_name,
-                kind=loader.kind,
-            ),
-            1,
-            [labels],
-            process_labels,
-        )
-
-
-class AssetFinder(AssetCentricFinder[Asset]):
-    supported_formats = frozenset({"csv", "parquet", "yaml"})
-
-    def _create_loader(self, client: ToolkitClient) -> ResourceCRUD:
-        return AssetCRUD.create_loader(client)
-
-    def _aggregate_count(self, hierarchies: list[str], data_sets: list[str]) -> int:
-        return self.client.assets.aggregate_count(
-            filter=AssetFilter(
-                data_set_ids=[{"externalId": item} for item in data_sets] or None,
-                asset_subtree_ids=[{"externalId": item} for item in hierarchies] or None,
-            )
-        )
-
-    def create_resource_iterator(self, limit: int | None) -> Iterator:
-        return self.client.assets(
-            chunk_size=self.chunk_size,
-            asset_subtree_external_ids=self.hierarchies or None,
-            data_set_external_ids=self.data_sets or None,
-            limit=limit,
-        )
-
-    def _resource_processor(self, assets: Iterable[Asset]) -> list[tuple[str, list[dict[str, Any]]]]:
-        grouped_assets: list[tuple[str, list[dict[str, object]]]] = []
-        for group, asset_group in groupby(
-            sorted([(self._group(asset), asset) for asset in assets], key=lambda x: x[0]), key=lambda x: x[0]
-        ):
-            grouped_assets.append((group, self._to_write([asset for _, asset in asset_group])))
-        return grouped_assets
-
-    def _group(self, item: Asset) -> str:
-        if self.hierarchies and self.data_sets:
-            asset_external_id = self.client.lookup.assets.external_id(item.root_id or 0)
-            data_set_external_id = self.client.lookup.data_sets.external_id(item.data_set_id or 0)
-            if asset_external_id and data_set_external_id:
-                return f"{asset_external_id}.{data_set_external_id}"
-            elif asset_external_id:
-                return asset_external_id
-            elif data_set_external_id:
-                return data_set_external_id
-            return ""
-        elif self.hierarchies:
-            return self.client.lookup.assets.external_id(item.root_id or 0) or ""
-        elif self.data_sets:
-            return self.client.lookup.data_sets.external_id(item.data_set_id or 0) or ""
-        return ""
-
-    def _get_resource_columns(self) -> SchemaColumnList:
-        columns = SchemaColumnList(
-            [
-                SchemaColumn(name="externalId", type="string"),
-                SchemaColumn(name="name", type="string"),
-                SchemaColumn(name="parentExternalId", type="string"),
-                SchemaColumn(name="description", type="string"),
-                SchemaColumn(name="dataSetExternalId", type="string"),
-                SchemaColumn(name="source", type="string"),
-                SchemaColumn(name="labels", type="string", is_array=True),
-                SchemaColumn(name="geoLocation", type="json"),
-            ]
-        )
-        data_set_ids = self.client.lookup.data_sets.id(self.data_sets) if self.data_sets else []
-        root_ids = self.client.lookup.assets.id(self.hierarchies) if self.hierarchies else []
-        metadata_keys = metadata_key_counts(self.client, "assets", data_set_ids or None, root_ids or None)
-        sorted_keys = sorted([key for key, count in metadata_keys if count > 0])
-        columns.extend([SchemaColumn(name=f"metadata.{key}", type="string") for key in sorted_keys])
-        return columns
-
-
-class FileMetadataFinder(AssetCentricFinder[FileMetadata]):
-    supported_formats = frozenset({"csv", "parquet"})
-
-    def _create_loader(self, client: ToolkitClient) -> ResourceCRUD:
-        return FileMetadataCRUD.create_loader(client)
-
-    def _aggregate_count(self, hierarchies: list[str], data_sets: list[str]) -> int:
-        result = self.client.files.aggregate(
-            filter=FileMetadataFilter(
-                data_set_ids=[{"externalId": item} for item in data_sets] or None,
-                asset_subtree_ids=[{"externalId": item} for item in hierarchies] or None,
-            )
-        )
-        return result[0].count if result else 0
-
-    def _get_resource_columns(self) -> SchemaColumnList:
-        columns = SchemaColumnList(
-            [
-                SchemaColumn(name="externalId", type="string"),
-                SchemaColumn(name="name", type="string"),
-                SchemaColumn(name="directory", type="string"),
-                SchemaColumn(name="source", type="string"),
-                SchemaColumn(name="mimeType", type="string"),
-                SchemaColumn(name="assetExternalIds", type="string", is_array=True),
-                SchemaColumn(name="dataSetExternalId", type="string"),
-                SchemaColumn(name="sourceCreatedTime", type="integer"),
-                SchemaColumn(name="sourceModifiedTime", type="integer"),
-                SchemaColumn(name="securityCategories", type="string", is_array=True),
-                SchemaColumn(name="labels", type="string", is_array=True),
-                SchemaColumn(name="geoLocation", type="json"),
-            ]
-        )
-        data_set_ids = self.client.lookup.data_sets.id(self.data_sets) if self.data_sets else []
-        root_ids = self.client.lookup.assets.id(self.hierarchies) if self.hierarchies else []
-        metadata_keys = metadata_key_counts(self.client, "files", data_set_ids or None, root_ids or None)
-        sorted_keys = sorted([key for key, count in metadata_keys if count > 0])
-        columns.extend([SchemaColumn(name=f"metadata.{key}", type="string") for key in sorted_keys])
-        return columns
-
-    def create_resource_iterator(self, limit: int | None) -> Iterable:
-        return self.client.files(
-            chunk_size=self.chunk_size,
-            asset_subtree_external_ids=self.hierarchies or None,
-            data_set_external_ids=self.data_sets or None,
-            limit=limit,
-        )
-
-    def _resource_processor(self, items: Iterable[FileMetadata]) -> list[tuple[str, list[dict[str, Any]]]]:
-        return [("", self._to_write(items))]
-
-
-class TimeSeriesFinder(AssetCentricFinder[TimeSeries]):
-    supported_formats = frozenset({"csv", "parquet", "yaml"})
-
-    def _create_loader(self, client: ToolkitClient) -> TimeSeriesCRUD:
-        return TimeSeriesCRUD.create_loader(client)
-
-    def _aggregate_count(self, hierarchies: list[str], data_sets: list[str]) -> int:
-        return self.client.time_series.aggregate_count(
-            filter=TimeSeriesFilter(
-                data_set_ids=[{"externalId": item} for item in data_sets] or None,
-                asset_subtree_ids=[{"externalId": item} for item in hierarchies] or None,
-            )
-        )
-
-    def create_resource_iterator(self, limit: int | None) -> Iterator:
-        return self.client.time_series(
-            chunk_size=self.chunk_size,
-            asset_subtree_external_ids=self.hierarchies or None,
-            data_set_external_ids=self.data_sets or None,
-            limit=limit,
-        )
-
-    def _resource_processor(self, time_series: Iterable[TimeSeries]) -> list[tuple[str, list[dict[str, Any]]]]:
-        return [("", self._to_write(time_series))]
-
-    def _get_resource_columns(self) -> SchemaColumnList:
-        columns = SchemaColumnList(
-            [
-                SchemaColumn(name="externalId", type="string"),
-                SchemaColumn(name="name", type="string"),
-                SchemaColumn(name="isString", type="boolean"),
-                SchemaColumn(name="unit", type="string"),
-                SchemaColumn(name="unitExternalId", type="string"),
-                SchemaColumn(name="assetExternalId", type="string"),
-                SchemaColumn(name="isStep", type="boolean"),
-                SchemaColumn(name="description", type="string"),
-                SchemaColumn(name="dataSetExternalId", type="string"),
-                SchemaColumn(name="securityCategories", type="string", is_array=True),
-            ]
-        )
-        data_set_ids = self.client.lookup.data_sets.id(self.data_sets) if self.data_sets else []
-        root_ids = self.client.lookup.assets.id(self.hierarchies) if self.hierarchies else []
-        metadata_keys = metadata_key_counts(self.client, "timeseries", data_set_ids or None, root_ids or None)
-        sorted_keys = sorted([key for key, count in metadata_keys if count > 0])
-        columns.extend([SchemaColumn(name=f"metadata.{key}", type="string") for key in sorted_keys])
-        return columns
-
-
-class EventFinder(AssetCentricFinder[Event]):
-    supported_formats = frozenset({"csv", "parquet"})
-
-    def _create_loader(self, client: ToolkitClient) -> ResourceCRUD:
-        return EventCRUD.create_loader(client)
-
-    def _aggregate_count(self, hierarchies: list[str], data_sets: list[str]) -> int:
-        return self.client.events.aggregate_count(
-            filter=EventFilter(
-                data_set_ids=[{"externalId": item} for item in data_sets] or None,
-                asset_subtree_ids=[{"externalId": item} for item in hierarchies] or None,
-            )
-        )
-
-    def _get_resource_columns(self) -> SchemaColumnList:
-        columns = SchemaColumnList(
-            [
-                SchemaColumn(name="externalId", type="string"),
-                SchemaColumn(name="dataSetExternalId", type="string"),
-                SchemaColumn(name="startTime", type="integer"),
-                SchemaColumn(name="endTime", type="integer"),
-                SchemaColumn(name="type", type="string"),
-                SchemaColumn(name="subtype", type="string"),
-                SchemaColumn(name="description", type="string"),
-                SchemaColumn(name="assetExternalIds", type="string", is_array=True),
-                SchemaColumn(name="source", type="string"),
-            ]
-        )
-        data_set_ids = self.client.lookup.data_sets.id(self.data_sets) if self.data_sets else []
-        root_ids = self.client.lookup.assets.id(self.hierarchies) if self.hierarchies else []
-        metadata_keys = metadata_key_counts(self.client, "events", data_set_ids or None, root_ids or None)
-        sorted_keys = sorted([key for key, count in metadata_keys if count > 0])
-        columns.extend([SchemaColumn(name=f"metadata.{key}", type="string") for key in sorted_keys])
-        return columns
-
-    def create_resource_iterator(self, limit: int | None) -> Iterable:
-        return self.client.events(
-            chunk_size=self.chunk_size,
-            asset_subtree_external_ids=self.hierarchies or None,
-            data_set_external_ids=self.data_sets or None,
-            limit=limit,
-        )
-
-    def _resource_processor(self, items: Iterable[Event]) -> list[tuple[str, list[dict[str, Any]]]]:
-        return [("", self._to_write(items))]
-
-
-class DumpDataCommand(ToolkitCommand):
-    def dump_table(
-        self,
-        finder: DataFinder,
-        output_dir: Path,
-        clean: bool,
-        limit: int | None = None,
-        format_: str = "csv",
-        verbose: bool = False,
-        parallel_threshold: int = 10,
-        max_queue_size: int = 10,
-    ) -> None:
-        """Dumps data from CDF to a file
-
-        Args:
-            finder (DataFinder): The finder object to use for fetching data.
-            output_dir (Path): The directory to write the output files to.
-            clean (bool): Whether to clean the output directory before writing files.
-            limit (int | None, optional): The maximum number of rows to write. Defaults to None.
-            format_ (Literal["yaml", "csv", "parquet"], optional): The format of the output file. Defaults to "csv".
-            verbose (bool, optional): Whether to print detailed progress information. Defaults to False.
-            parallel_threshold (int, optional): The iteration threshold for parallel processing. Defaults to 10.
-            max_queue_size (int, optional): If using parallel processing, the maximum size of the queue. Defaults to 10.
-
-        """
-        valid_format = finder.validate_format(format_)
-        self.validate_directory(output_dir, clean)
-
-        console = Console()
-        # The ignore is used as MyPy does not understand that is_supported_format
-        # above guarantees that the format is valid.
-        for schema, iteration_count, resource_iterator, resource_processor in finder.create_iterators(
-            valid_format, limit
-        ):
-            writer_cls = TableFileWriter.get_write_cls(schema.format_)
-            row_counts = 0
-            t0 = time.perf_counter()
-            with writer_cls(schema, output_dir) as writer:
-                if iteration_count > parallel_threshold:
-                    executor = ProducerWorkerExecutor(
-                        download_iterable=resource_iterator,
-                        process=resource_processor,
-                        write=writer.write_rows,
-                        iteration_count=iteration_count,
-                        max_queue_size=max_queue_size,
-                        download_description=f"Downloading {schema.display_name}",
-                        process_description=f"Processing {schema.display_name}",
-                        write_description=f"Writing {schema.display_name} to file",
-                    )
-                    executor.run()
-                    executor.raise_on_error()
-                    row_counts = executor.total_items
-                else:
-                    for resources in track(
-                        resource_iterator, total=iteration_count, description=f"Dumping {schema.display_name}"
-                    ):
-                        row_counts += len(resources)
-                        processed = resource_processor(resources)
-                        writer.write_rows(processed)
-            elapsed = time.perf_counter() - t0
-            console.print(f"Dumped {row_counts:,} rows to {output_dir} in {elapsed:,.2f} seconds.")
-
-    @staticmethod
-    def validate_directory(output_dir: Path, clean: bool) -> None:
-        if output_dir.exists() and clean:
-            safe_rmtree(output_dir)
-        elif output_dir.exists():
-            raise ToolkitFileExistsError(f"Output directory {output_dir!s} already exists. Use --clean to remove it.")
-        elif output_dir.suffix:
-            raise ToolkitIsADirectoryError(f"Output directory {output_dir!s} is not a directory.")