cognite-toolkit 0.7.12__py3-none-any.whl → 0.7.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,15 +10,19 @@ from rich import print
10
10
  from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
11
11
  from cognite_toolkit._cdf_tk.commands import DownloadCommand
12
12
  from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR
13
+ from cognite_toolkit._cdf_tk.feature_flags import Flags
13
14
  from cognite_toolkit._cdf_tk.storageio import (
14
15
  AssetIO,
15
16
  CanvasIO,
16
17
  ChartIO,
18
+ DataSelector,
17
19
  EventIO,
20
+ FileContentIO,
18
21
  FileMetadataIO,
19
22
  HierarchyIO,
20
23
  InstanceIO,
21
24
  RawIO,
25
+ StorageIO,
22
26
  TimeSeriesIO,
23
27
  )
24
28
  from cognite_toolkit._cdf_tk.storageio.selectors import (
@@ -28,11 +32,14 @@ from cognite_toolkit._cdf_tk.storageio.selectors import (
28
32
  ChartExternalIdSelector,
29
33
  ChartSelector,
30
34
  DataSetSelector,
35
+ FileIdentifierSelector,
31
36
  InstanceSpaceSelector,
32
37
  RawTableSelector,
33
38
  SelectedTable,
34
39
  SelectedView,
35
40
  )
41
+ from cognite_toolkit._cdf_tk.storageio.selectors._file_content import FileInternalID
42
+ from cognite_toolkit._cdf_tk.utils import sanitize_filename
36
43
  from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
37
44
  from cognite_toolkit._cdf_tk.utils.interactive_select import (
38
45
  AssetCentricInteractiveSelect,
@@ -45,7 +52,6 @@ from cognite_toolkit._cdf_tk.utils.interactive_select import (
45
52
  RawTableInteractiveSelect,
46
53
  TimeSeriesInteractiveSelect,
47
54
  )
48
- from cognite_toolkit._cdf_tk.utils.useful_types import AssetCentricKind
49
55
 
50
56
 
51
57
  class RawFormats(str, Enum):
@@ -59,6 +65,10 @@ class AssetCentricFormats(str, Enum):
59
65
  ndjson = "ndjson"
60
66
 
61
67
 
68
+ class FileContentFormats(str, Enum):
69
+ ndjson = "ndjson"
70
+
71
+
62
72
  class HierarchyFormats(str, Enum):
63
73
  ndjson = "ndjson"
64
74
 
@@ -261,7 +271,7 @@ class DownloadApp(typer.Typer):
261
271
  compression,
262
272
  output_dir,
263
273
  limit,
264
- "Assets",
274
+ "assets",
265
275
  )
266
276
 
267
277
  selectors = [DataSetSelector(kind="Assets", data_set_external_id=data_set) for data_set in data_sets]
@@ -286,13 +296,14 @@ class DownloadApp(typer.Typer):
286
296
  compression: CompressionFormat,
287
297
  output_dir: Path,
288
298
  limit: int,
289
- kind: AssetCentricKind,
299
+ display_name: str,
300
+ max_limit: int | None = None,
301
+ available_formats: type[Enum] = AssetCentricFormats,
290
302
  ) -> tuple[list[str], AssetCentricFormats, CompressionFormat, Path, int]:
291
303
  data_sets = selector.select_data_sets()
292
- display_name = kind.casefold() + "s"
293
304
  file_format = questionary.select(
294
305
  f"Select format to download the {display_name} in:",
295
- choices=[Choice(title=format_.value, value=format_) for format_ in AssetCentricFormats],
306
+ choices=[Choice(title=format_.value, value=format_) for format_ in available_formats],
296
307
  default=file_format,
297
308
  ).ask()
298
309
  compression = questionary.select(
@@ -302,7 +313,7 @@ class DownloadApp(typer.Typer):
302
313
  ).ask()
303
314
  output_dir = Path(
304
315
  questionary.path(
305
- "Where to download the assets:",
316
+ f"Where to download the {display_name}:",
306
317
  default=str(output_dir),
307
318
  only_directories=True,
308
319
  ).ask()
@@ -316,9 +327,15 @@ class DownloadApp(typer.Typer):
316
327
  raise typer.Abort()
317
328
  try:
318
329
  limit = int(limit_str)
319
- break
320
330
  except ValueError:
321
331
  print("[red]Please enter a valid integer for the limit.[/]")
332
+ else:
333
+ if max_limit is not None and limit > max_limit:
334
+ print(
335
+ f"[red]The maximum limit for downloading {display_name} is {max_limit}. Please enter a lower value.[/]"
336
+ )
337
+ else:
338
+ break
322
339
  return data_sets, file_format, compression, output_dir, limit
323
340
 
324
341
  def download_timeseries_cmd(
@@ -383,7 +400,7 @@ class DownloadApp(typer.Typer):
383
400
  compression,
384
401
  output_dir,
385
402
  limit,
386
- "TimeSeries",
403
+ "time series",
387
404
  )
388
405
 
389
406
  selectors = [DataSetSelector(kind="TimeSeries", data_set_external_id=data_set) for data_set in data_sets]
@@ -462,7 +479,7 @@ class DownloadApp(typer.Typer):
462
479
  compression,
463
480
  output_dir,
464
481
  limit,
465
- "Events",
482
+ "events",
466
483
  )
467
484
 
468
485
  selectors = [DataSetSelector(kind="Events", data_set_external_id=data_set) for data_set in data_sets]
@@ -491,6 +508,16 @@ class DownloadApp(typer.Typer):
491
508
  help="List of data sets to download file metadata from. If this is not provided, an interactive selection will be made.",
492
509
  ),
493
510
  ] = None,
511
+ include_file_contents: Annotated[
512
+ bool,
513
+ typer.Option(
514
+ "--include-file-contents",
515
+ "-c",
516
+ help="Whether to include file contents when downloading assets. Note if you enable this option, you can"
517
+ "only download 1000 files at a time.",
518
+ hidden=not Flags.EXTEND_DOWNLOAD.is_enabled(),
519
+ ),
520
+ ] = False,
494
521
  file_format: Annotated[
495
522
  AssetCentricFormats,
496
523
  typer.Option(
@@ -536,21 +563,58 @@ class DownloadApp(typer.Typer):
536
563
  """This command will download file metadata from CDF into a temporary directory."""
537
564
  client = EnvironmentVariables.create_from_environment().get_client()
538
565
  if data_sets is None:
566
+ if Flags.EXTEND_DOWNLOAD.is_enabled():
567
+ include_file_contents = questionary.select(
568
+ "Do you want to include file contents when downloading file metadata?",
569
+ choices=[
570
+ Choice(title="Yes", value=True),
571
+ Choice(title="No", value=False),
572
+ ],
573
+ ).ask()
574
+ else:
575
+ include_file_contents = False
576
+
577
+ available_formats = FileContentFormats if include_file_contents else AssetCentricFormats
578
+ file_format = FileContentFormats.ndjson if include_file_contents else file_format # type: ignore[assignment]
539
579
  data_sets, file_format, compression, output_dir, limit = self._asset_centric_interactive(
540
580
  FileMetadataInteractiveSelect(client, "download"),
541
581
  file_format,
542
582
  compression,
543
583
  output_dir,
544
- limit,
545
- "FileMetadata",
584
+ limit if not include_file_contents else 1000,
585
+ "file metadata",
586
+ max_limit=1000 if include_file_contents else None,
587
+ available_formats=available_formats,
546
588
  )
547
589
 
548
- selectors = [DataSetSelector(kind="FileMetadata", data_set_external_id=data_set) for data_set in data_sets]
590
+ io: StorageIO
591
+ selectors: list[DataSelector]
592
+ if include_file_contents:
593
+ if limit == -1 or limit > 1000:
594
+ limit = 1000
595
+ print(
596
+ "[yellow]When including file contents, the maximum number of files that can be downloaded at a time is 1000. "
597
+ )
598
+ if file_format == AssetCentricFormats.csv or file_format == AssetCentricFormats.parquet:
599
+ print(
600
+ "[red]When including file contents, the only supported format is ndjson. Overriding the format to ndjson.[/]"
601
+ )
602
+ file_format = AssetCentricFormats.ndjson
603
+ files = client.files.list(data_set_external_ids=data_sets, limit=limit)
604
+ selector = FileIdentifierSelector(
605
+ identifiers=tuple([FileInternalID(internal_id=file.id) for file in files]) # type: ignore[call-arg]
606
+ )
607
+ selectors = [selector]
608
+ io = FileContentIO(client, output_dir / sanitize_filename(selector.group))
609
+ else:
610
+ selectors = [DataSetSelector(kind="FileMetadata", data_set_external_id=data_set) for data_set in data_sets]
611
+ io = FileMetadataIO(client)
612
+
549
613
  cmd = DownloadCommand()
550
614
  cmd.run(
551
615
  lambda: cmd.download(
552
- selectors=selectors,
553
- io=FileMetadataIO(client),
616
+ selectors=selectors, # type: ignore[misc]
617
+ io=io,
554
618
  output_dir=output_dir,
555
619
  file_format=f".{file_format.value}",
556
620
  compression=compression.value,
@@ -38,6 +38,7 @@ class ToolkitClient(CogniteClient):
38
38
  def __init__(self, config: ToolkitClientConfig | None = None, enable_set_pending_ids: bool = False) -> None:
39
39
  super().__init__(config=config)
40
40
  http_client = HTTPClient(self.config)
41
+ self.http_client = http_client
41
42
  toolkit_config = ToolkitClientConfig.from_client_config(self.config)
42
43
  self.console = Console()
43
44
  self.tool = ToolAPI(http_client, self.console)
@@ -26,6 +26,7 @@ import questionary
26
26
  from cognite.client.data_classes.capabilities import (
27
27
  AssetsAcl,
28
28
  Capability,
29
+ ExtractionConfigsAcl,
29
30
  FunctionsAcl,
30
31
  GroupsAcl,
31
32
  ProjectsAcl,
@@ -46,6 +47,7 @@ from cognite_toolkit._cdf_tk.constants import (
46
47
  TOOLKIT_DEMO_GROUP_NAME,
47
48
  TOOLKIT_SERVICE_PRINCIPAL_GROUP_NAME,
48
49
  )
50
+ from cognite_toolkit._cdf_tk.cruds import ExtractionPipelineConfigCRUD
49
51
  from cognite_toolkit._cdf_tk.exceptions import (
50
52
  AuthenticationError,
51
53
  AuthorizationError,
@@ -434,8 +436,23 @@ class AuthCommand(ToolkitCommand):
434
436
  crud = crud_cls.create_loader(client)
435
437
  if crud.prerequisite_warning() is not None:
436
438
  continue
437
- capability = crud_cls.get_required_capability(None, read_only=False)
438
- capabilities = capability if isinstance(capability, list) else [capability]
439
+ if isinstance(crud, ExtractionPipelineConfigCRUD):
440
+ # The Extraction Pipeline Config CRUD requires special handling.
441
+ # The .get_required_capability is used in the DeployCommand as well. Since, there is no way to no
442
+ # the extraction pipeline ID or dataSetId from an ExtractionPipelineConfigWrite object, we do not
443
+ # check those there. If we returned the full capability, it would always have to be all scoped.
444
+ # That is too restrictive in the deploy command, so we return an empty list, essentially not checking
445
+ # anything there. Here, we want to add the all scoped capability, so that the Toolkit group gets the
446
+ # correct capability.
447
+ capabilities: list[Capability] = [
448
+ ExtractionConfigsAcl(
449
+ [ExtractionConfigsAcl.Action.Read, ExtractionConfigsAcl.Action.Write],
450
+ ExtractionConfigsAcl.Scope.All(),
451
+ )
452
+ ]
453
+ else:
454
+ capability = crud_cls.get_required_capability(None, read_only=False)
455
+ capabilities = capability if isinstance(capability, list) else [capability]
439
456
  for cap in capabilities:
440
457
  if project_type == "DATA_MODELING_ONLY" and isinstance(cap, AssetsAcl | RelationshipsAcl):
441
458
  continue
@@ -53,6 +53,10 @@ class Flags(Enum):
53
53
  visible=True,
54
54
  description="Enables the support for the resources create command under dev plugin",
55
55
  )
56
+ EXTEND_DOWNLOAD = FlagMetadata(
57
+ visible=True,
58
+ description="Enables extended download to support downloading file content",
59
+ )
56
60
 
57
61
  def is_enabled(self) -> bool:
58
62
  return FeatureFlag.is_enabled(self)
@@ -5,6 +5,7 @@ from dataclasses import dataclass
5
5
  from pathlib import Path
6
6
  from typing import cast
7
7
 
8
+ import httpx
8
9
  from cognite.client.data_classes import FileMetadata, FileMetadataWrite
9
10
  from cognite.client.data_classes.data_modeling import NodeId, ViewId
10
11
 
@@ -26,8 +27,16 @@ from cognite_toolkit._cdf_tk.utils.http_client import (
26
27
  from cognite_toolkit._cdf_tk.utils.useful_types import JsonVal
27
28
 
28
29
  from ._base import Page, UploadableStorageIO, UploadItem
29
- from .selectors import FileContentSelector, FileMetadataTemplateSelector
30
- from .selectors._file_content import FILEPATH, FileDataModelingTemplateSelector
30
+ from .selectors import FileContentSelector, FileIdentifierSelector, FileMetadataTemplateSelector
31
+ from .selectors._file_content import (
32
+ FILEPATH,
33
+ FileDataModelingTemplateSelector,
34
+ FileExternalID,
35
+ FileIdentifier,
36
+ FileInstanceID,
37
+ FileInternalID,
38
+ )
39
+ from .selectors._file_content import NodeId as SelectorNodeId
31
40
 
32
41
  COGNITE_FILE_VIEW = ViewId("cdf_cdm", "CogniteFile", "v1")
33
42
 
@@ -47,23 +56,150 @@ class FileContentIO(UploadableStorageIO[FileContentSelector, FileMetadata, FileM
47
56
  SUPPORTED_READ_FORMATS = frozenset({".ndjson"})
48
57
  UPLOAD_ENDPOINT = "/files"
49
58
 
50
- def __init__(self, client: ToolkitClient) -> None:
59
+ def __init__(self, client: ToolkitClient, target_dir: Path = Path.cwd()) -> None:
51
60
  super().__init__(client)
52
61
  self._crud = FileMetadataCRUD(client, None, None)
62
+ self._target_dir = target_dir
53
63
 
54
64
  def as_id(self, item: FileMetadata) -> str:
55
65
  return item.external_id or str(item.id)
56
66
 
57
- def stream_data(self, selector: FileContentSelector, limit: int | None = None) -> Iterable[Page]:
58
- raise NotImplementedError("Download of FileContent is not yet supported")
67
+ def stream_data(self, selector: FileContentSelector, limit: int | None = None) -> Iterable[Page[FileMetadata]]:
68
+ if not isinstance(selector, FileIdentifierSelector):
69
+ raise ToolkitNotImplementedError(
70
+ f"Download with the manifest, {type(selector).__name__}, is not supported for FileContentIO"
71
+ )
72
+ selected_identifiers = selector.identifiers
73
+ if limit is not None and limit < len(selected_identifiers):
74
+ selected_identifiers = selected_identifiers[:limit]
75
+ for identifiers in chunker_sequence(selected_identifiers, self.CHUNK_SIZE):
76
+ metadata = self._retrieve_metadata(identifiers)
77
+ if metadata is None:
78
+ continue
79
+ identifiers_map = self._as_metadata_map(metadata)
80
+ downloaded_files: list[FileMetadata] = []
81
+ for identifier in identifiers:
82
+ if identifier not in identifiers_map:
83
+ continue
84
+
85
+ meta = identifiers_map[identifier]
86
+ filepath = self._create_filepath(meta, selector)
87
+ download_url = self._retrieve_download_url(identifier)
88
+ if download_url is None:
89
+ continue
90
+ filepath.parent.mkdir(parents=True, exist_ok=True)
91
+ with httpx.stream("GET", download_url) as response:
92
+ if response.status_code != 200:
93
+ continue
94
+ with filepath.open(mode="wb") as file_stream:
95
+ for chunk in response.iter_bytes(chunk_size=8192):
96
+ file_stream.write(chunk)
97
+ downloaded_files.append(meta)
98
+ yield Page(items=downloaded_files, worker_id="Main")
99
+
100
+ def _retrieve_metadata(self, identifiers: Sequence[FileIdentifier]) -> Sequence[FileMetadata] | None:
101
+ config = self.client.config
102
+ responses = self.client.http_client.request_with_retries(
103
+ message=SimpleBodyRequest(
104
+ endpoint_url=config.create_api_url("/files/byids"),
105
+ method="POST",
106
+ body_content={
107
+ "items": [
108
+ identifier.model_dump(mode="json", by_alias=True, exclude={"id_type"})
109
+ for identifier in identifiers
110
+ ],
111
+ "ignoreUnknownIds": True,
112
+ },
113
+ )
114
+ )
115
+ if responses.has_failed:
116
+ return None
117
+ body = responses.get_first_body()
118
+ items_data = body.get("items", [])
119
+ if not isinstance(items_data, list):
120
+ return None
121
+ # MyPy does not understand that JsonVal is valid dict[Any, Any]
122
+ return [FileMetadata._load(item) for item in items_data] # type: ignore[arg-type]
123
+
124
+ @staticmethod
125
+ def _as_metadata_map(metadata: Sequence[FileMetadata]) -> dict[FileIdentifier, FileMetadata]:
126
+ identifiers_map: dict[FileIdentifier, FileMetadata] = {}
127
+ for item in metadata:
128
+ if item.id is not None:
129
+ # MyPy does cooperate well with Pydantic.
130
+ identifiers_map[FileInternalID(internal_id=item.id)] = item # type: ignore[call-arg]
131
+ if item.external_id is not None:
132
+ identifiers_map[FileExternalID(external_id=item.external_id)] = item
133
+ if item.instance_id is not None:
134
+ identifiers_map[
135
+ FileInstanceID(
136
+ instance_id=SelectorNodeId(
137
+ space=item.instance_id.space, external_id=item.instance_id.external_id
138
+ )
139
+ )
140
+ ] = item
141
+ return identifiers_map
142
+
143
+ def _create_filepath(self, meta: FileMetadata, selector: FileIdentifierSelector) -> Path:
144
+ # We now that metadata always have name set
145
+ filename = Path(cast(str, meta.name))
146
+ if len(filename.suffix) == 0 and meta.mime_type:
147
+ if mime_ext := mimetypes.guess_extension(meta.mime_type):
148
+ filename = filename.with_suffix(mime_ext)
149
+ directory = selector.file_directory
150
+ if isinstance(meta.directory, str) and meta.directory != "":
151
+ directory = Path(meta.directory.removeprefix("/"))
152
+
153
+ counter = 1
154
+ filepath = self._target_dir / directory / filename
155
+ while filepath.exists():
156
+ filepath = self._target_dir / directory / f"{filename} ({counter})"
157
+ counter += 1
158
+
159
+ return filepath
160
+
161
+ def _retrieve_download_url(self, identifier: FileIdentifier) -> str | None:
162
+ config = self.client.config
163
+ responses = self.client.http_client.request_with_retries(
164
+ message=SimpleBodyRequest(
165
+ endpoint_url=config.create_api_url("/files/downloadlink"),
166
+ method="POST",
167
+ body_content={"items": [identifier.model_dump(mode="json", by_alias=True, exclude={"id_type"})]},
168
+ )
169
+ )
170
+ try:
171
+ body = responses.get_first_body()
172
+ except ValueError:
173
+ return None
174
+ if "items" in body and isinstance(body["items"], list) and len(body["items"]) > 0:
175
+ # The API responses is not following the API docs, this is a workaround
176
+ body = body["items"][0] # type: ignore[assignment]
177
+ try:
178
+ return cast(str, body["downloadUrl"])
179
+ except (KeyError, IndexError):
180
+ return None
59
181
 
60
182
  def count(self, selector: FileContentSelector) -> int | None:
183
+ if isinstance(selector, FileIdentifierSelector):
184
+ return len(selector.identifiers)
61
185
  return None
62
186
 
63
187
  def data_to_json_chunk(
64
188
  self, data_chunk: Sequence[FileMetadata], selector: FileContentSelector | None = None
65
189
  ) -> list[dict[str, JsonVal]]:
66
- raise NotImplementedError("Download of FileContent is not yet supported")
190
+ """Convert a writable Cognite resource list to a JSON-compatible chunk of data.
191
+
192
+ Args:
193
+ data_chunk: A writable Cognite resource list representing the data.
194
+ selector: The selector used for the data. (Not used in this implementation)
195
+ Returns:
196
+ A list of dictionaries, each representing the data in a JSON-compatible format.
197
+ """
198
+ result: list[dict[str, JsonVal]] = []
199
+ for item in data_chunk:
200
+ item_json = self._crud.dump_resource(item)
201
+ result.append(item_json)
202
+ return result
67
203
 
68
204
  def json_chunk_to_data(self, data_chunk: list[tuple[str, dict[str, JsonVal]]]) -> Sequence[UploadFileContentItem]:
69
205
  """Convert a JSON-compatible chunk of data back to a writable Cognite resource list.
@@ -17,6 +17,7 @@ from ._file_content import (
17
17
  FileContentSelector,
18
18
  FileDataModelingTemplate,
19
19
  FileDataModelingTemplateSelector,
20
+ FileIdentifierSelector,
20
21
  FileMetadataTemplate,
21
22
  FileMetadataTemplateSelector,
22
23
  )
@@ -43,7 +44,8 @@ Selector = Annotated[
43
44
  | ChartExternalIdSelector
44
45
  | CanvasExternalIdSelector
45
46
  | FileMetadataTemplateSelector
46
- | FileDataModelingTemplateSelector,
47
+ | FileDataModelingTemplateSelector
48
+ | FileIdentifierSelector,
47
49
  Field(discriminator="type"),
48
50
  ]
49
51
 
@@ -67,6 +69,7 @@ __all__ = [
67
69
  "FileContentSelector",
68
70
  "FileDataModelingTemplate",
69
71
  "FileDataModelingTemplateSelector",
72
+ "FileIdentifierSelector",
70
73
  "FileMetadataTemplate",
71
74
  "FileMetadataTemplateSelector",
72
75
  "InstanceColumn",
@@ -1,9 +1,12 @@
1
+ import hashlib
1
2
  import json
2
3
  from abc import ABC, abstractmethod
3
4
  from pathlib import Path
4
- from typing import Any, Literal
5
+ from typing import Annotated, Any, Literal
5
6
 
6
- from pydantic import ConfigDict, field_validator, model_validator
7
+ from pydantic import ConfigDict, Field, field_validator, model_validator
8
+
9
+ from cognite_toolkit._cdf_tk.exceptions import ToolkitNotImplementedError
7
10
 
8
11
  from ._base import DataSelector, SelectorObject
9
12
  from ._instances import SelectedView
@@ -107,3 +110,57 @@ class FileDataModelingTemplateSelector(FileContentSelector):
107
110
 
108
111
  def create_instance(self, filepath: Path) -> dict[str, Any]:
109
112
  return self.template.create_instance(filepath.name)
113
+
114
+
115
+ class FileIdentifierDefinition(SelectorObject):
116
+ id_type: str
117
+
118
+
119
+ class FileInternalID(FileIdentifierDefinition):
120
+ id_type: Literal["internalId"] = "internalId"
121
+ internal_id: int = Field(alias="id")
122
+
123
+ def __str__(self) -> str:
124
+ return f"internalId_{self.internal_id}"
125
+
126
+
127
+ class FileExternalID(FileIdentifierDefinition):
128
+ id_type: Literal["externalId"] = "externalId"
129
+ external_id: str
130
+
131
+ def __str__(self) -> str:
132
+ return f"externalId_{self.external_id}"
133
+
134
+
135
+ class NodeId(SelectorObject):
136
+ space: str
137
+ external_id: str
138
+
139
+
140
+ class FileInstanceID(FileIdentifierDefinition):
141
+ id_type: Literal["instanceId"] = "instanceId"
142
+ instance_id: NodeId
143
+
144
+ def __str__(self) -> str:
145
+ return f"instanceId_{self.instance_id.space}_{self.instance_id.external_id}"
146
+
147
+
148
+ FileIdentifier = Annotated[FileInstanceID | FileExternalID | FileInternalID, Field(discriminator="id_type")]
149
+
150
+
151
+ class FileIdentifierSelector(FileContentSelector):
152
+ type: Literal["fileIdentifier"] = "fileIdentifier"
153
+ file_directory: Path = Path("file_content")
154
+ use_metadata_directory: bool = True
155
+ identifiers: tuple[FileIdentifier, ...]
156
+
157
+ @property
158
+ def group(self) -> str:
159
+ return "Files"
160
+
161
+ def __str__(self) -> str:
162
+ hash_ = hashlib.md5(",".join(sorted(str(self.identifiers))).encode()).hexdigest()[:8]
163
+ return f"file_{len(self.identifiers)}_identifiers_{hash_}"
164
+
165
+ def create_instance(self, filepath: Path) -> dict[str, Any]:
166
+ raise ToolkitNotImplementedError("FileIdentifierSelector does not support creating instances from file paths.")
@@ -338,6 +338,18 @@ class ResponseList(UserList[ResponseMessage | FailedRequestMessage]):
338
338
  error_messages += "; ".join(f"Request error: {err.error}" for err in failed_requests)
339
339
  raise ToolkitAPIError(f"One or more requests failed: {error_messages}")
340
340
 
341
+ @property
342
+ def has_failed(self) -> bool:
343
+ """Indicates whether any response in the list indicates a failure.
344
+
345
+ Returns:
346
+ bool: True if there are any failed responses or requests, False otherwise.
347
+ """
348
+ for resp in self.data:
349
+ if isinstance(resp, FailedResponse | FailedRequestMessage):
350
+ return True
351
+ return False
352
+
341
353
  def get_first_body(self) -> dict[str, JsonVal]:
342
354
  """Returns the body of the first successful response in the list.
343
355
 
@@ -12,7 +12,7 @@ jobs:
12
12
  environment: dev
13
13
  name: Deploy
14
14
  container:
15
- image: cognite/toolkit:0.7.12
15
+ image: cognite/toolkit:0.7.14
16
16
  env:
17
17
  CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
18
18
  CDF_PROJECT: ${{ vars.CDF_PROJECT }}
@@ -10,7 +10,7 @@ jobs:
10
10
  environment: dev
11
11
  name: Deploy Dry Run
12
12
  container:
13
- image: cognite/toolkit:0.7.12
13
+ image: cognite/toolkit:0.7.14
14
14
  env:
15
15
  CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
16
16
  CDF_PROJECT: ${{ vars.CDF_PROJECT }}
@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
4
4
  [modules]
5
5
  # This is the version of the modules. It should not be changed manually.
6
6
  # It will be updated by the 'cdf modules upgrade' command.
7
- version = "0.7.12"
7
+ version = "0.7.14"
8
8
 
9
9
 
10
10
  [plugins]
@@ -1 +1 @@
1
- __version__ = "0.7.12"
1
+ __version__ = "0.7.14"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cognite_toolkit
3
- Version: 0.7.12
3
+ Version: 0.7.14
4
4
  Summary: Official Cognite Data Fusion tool for project templates and configuration deployment
5
5
  Project-URL: Homepage, https://docs.cognite.com/cdf/deploy/cdf_toolkit/
6
6
  Project-URL: Changelog, https://github.com/cognitedata/toolkit/releases
@@ -1,11 +1,11 @@
1
1
  cognite_toolkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
2
  cognite_toolkit/_cdf.py,sha256=YoiPOMgf1Mufy7wNYWWSF3bNBTGqKr26Ey5kj38fCXU,6008
3
- cognite_toolkit/_version.py,sha256=PjQNisQKLFjBRbjgoDmIiFnMrUFWFPsbOBfj_imtP4Q,23
3
+ cognite_toolkit/_version.py,sha256=7BOggTpADa3sz2Lo7K_EuX10-Jk_mW3PS3k2rLh8AF8,23
4
4
  cognite_toolkit/_cdf_tk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
5
  cognite_toolkit/_cdf_tk/cdf_toml.py,sha256=VSWV9h44HusWIaKpWgjrOMrc3hDoPTTXBXlp6-NOrIM,9079
6
6
  cognite_toolkit/_cdf_tk/constants.py,sha256=TplKm2J9pGRHq7nAnLI0caTMHetS04OIz3hfq-jvGzo,7236
7
7
  cognite_toolkit/_cdf_tk/exceptions.py,sha256=xG0jMwi5A20nvPvyo6sCyz_cyKycynPyIzpYiGR4gcU,6064
8
- cognite_toolkit/_cdf_tk/feature_flags.py,sha256=5JnJc5aRHAZ4-D6ueRNlemNKDW644akRwev0h-6hNXw,1907
8
+ cognite_toolkit/_cdf_tk/feature_flags.py,sha256=nCI71ldaZyoXEu0zI53XMFzcAMQW08KaM6CmuBIOJyw,2056
9
9
  cognite_toolkit/_cdf_tk/hints.py,sha256=UI1ymi2T5wCcYOpEbKbVaDnlyFReFy8TDtMVt-5E1h8,6493
10
10
  cognite_toolkit/_cdf_tk/plugins.py,sha256=0V14rceAWLZQF8iWdyL5QmK7xB796YaDEtb9RIj5AOc,836
11
11
  cognite_toolkit/_cdf_tk/protocols.py,sha256=Lc8XnBfmDZN6dwmSopmK7cFE9a9jZ2zdUryEeCXn27I,3052
@@ -16,7 +16,7 @@ cognite_toolkit/_cdf_tk/apps/_auth_app.py,sha256=ER7uYb3ViwsHMXiQEZpyhwU6TIjKaB9
16
16
  cognite_toolkit/_cdf_tk/apps/_core_app.py,sha256=YK0MOK7Tv3cDSe5_6o9GtM5n_6sE7I0Wm-Se4eJnyNM,13744
17
17
  cognite_toolkit/_cdf_tk/apps/_data_app.py,sha256=LeplXlxXtyIymRPgbatQrRFodU4VZBFxI0bqDutLSbg,806
18
18
  cognite_toolkit/_cdf_tk/apps/_dev_app.py,sha256=FaY67PFdKwdiMKgJbTcjHT1X2Xfbog2PKL6T-kcawyc,2818
19
- cognite_toolkit/_cdf_tk/apps/_download_app.py,sha256=g-VA51KI91wziVuO3w305rmr33xIb0ghYTtW06LhNz8,31994
19
+ cognite_toolkit/_cdf_tk/apps/_download_app.py,sha256=gcmcYZmB_ppWTiqDxc_6aVS9zoIACoV_cVU1sZVYwXY,34985
20
20
  cognite_toolkit/_cdf_tk/apps/_dump_app.py,sha256=EPq6fWSaScj9ncKfRY253rRJ37er47PIM60IFgkQK_k,37127
21
21
  cognite_toolkit/_cdf_tk/apps/_landing_app.py,sha256=YR9z83OY7PhhgBVC5gmRLgo9iTXoGoZfRhOU3gd_r2o,888
22
22
  cognite_toolkit/_cdf_tk/apps/_migrate_app.py,sha256=g4S_53kbIgk57ziPLdRMuR6xUe434gkMqa69VmVm5Vg,39619
@@ -37,7 +37,7 @@ cognite_toolkit/_cdf_tk/builders/_streamlit.py,sha256=8Pu_zgyKZjbAsPWywjzB2KWD7h
37
37
  cognite_toolkit/_cdf_tk/builders/_transformation.py,sha256=STB42zhzOW5M_-b8cKOQ_cegnr7FtMoMxZ87gPLXft4,4723
38
38
  cognite_toolkit/_cdf_tk/client/__init__.py,sha256=a6rQXDGfW2g7K5WwrOW5oakh1TdFlBjUVjf9wusOox8,135
39
39
  cognite_toolkit/_cdf_tk/client/_constants.py,sha256=COUGcea37mDF2sf6MGqJXWmecTY_6aCImslxXrYW1I0,73
40
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py,sha256=ucy_vXXatbUdB8gvNITmVsH1ghirZqmez-TR2EGrY2w,3260
40
+ cognite_toolkit/_cdf_tk/client/_toolkit_client.py,sha256=1syPhlnCWJZzLuF9e1cjID06C-eOw5VLkOYz97cg3lA,3299
41
41
  cognite_toolkit/_cdf_tk/client/api_client.py,sha256=CQdD_gfDqQkz5OYHrTnKvBvEvzHPdHDB1BkZPWRoahg,440
42
42
  cognite_toolkit/_cdf_tk/client/config.py,sha256=weMR43z-gqHMn-Jqvfmh_nJ0HbgEdyeCGtISuEf3OuY,4269
43
43
  cognite_toolkit/_cdf_tk/client/testing.py,sha256=axKaW6rdzq_TI7DD4ijnLeHC4a74WKWDAxQ3QyZ7Kvg,6720
@@ -114,7 +114,7 @@ cognite_toolkit/_cdf_tk/commands/_upload.py,sha256=jFt4NG2zEcUu8WUCgRQEoYDIs5C70
114
114
  cognite_toolkit/_cdf_tk/commands/_utils.py,sha256=UxMJW5QYKts4om5n6x2Tq2ihvfO9gWjhQKeqZNFTlKg,402
115
115
  cognite_toolkit/_cdf_tk/commands/_virtual_env.py,sha256=GFAid4hplixmj9_HkcXqU5yCLj-fTXm4cloGD6U2swY,2180
116
116
  cognite_toolkit/_cdf_tk/commands/about.py,sha256=pEXNdCeJYONOalH8x-7QRsKLgj-9gdIqN16pPxA3bhg,9395
117
- cognite_toolkit/_cdf_tk/commands/auth.py,sha256=TD6X3CKFNhDHoIv2b2uI2nKBHgQljLntp9vZffdA-jc,31384
117
+ cognite_toolkit/_cdf_tk/commands/auth.py,sha256=TX_8YuVCjMVIQjEdfBE66bSDrojJhCnxf_jcT3-9wM8,32550
118
118
  cognite_toolkit/_cdf_tk/commands/build_cmd.py,sha256=6m-lK0vccje1gaQ_fd68UvA4CbhuBszDapnHwu4VU_U,30897
119
119
  cognite_toolkit/_cdf_tk/commands/clean.py,sha256=JNPIjNSiOJaP-cUhFzT8H3s30luA9lI39yH18QxHaYM,16603
120
120
  cognite_toolkit/_cdf_tk/commands/collect.py,sha256=zBMKhhvjOpuASMnwP0eeHRI02tANcvFEZgv0CQO1ECc,627
@@ -247,16 +247,16 @@ cognite_toolkit/_cdf_tk/storageio/_asset_centric.py,sha256=GZZSQ8NLCP8tSQKOc8BUb
247
247
  cognite_toolkit/_cdf_tk/storageio/_base.py,sha256=ElvqhIEBnhcz0yY1Ds164wVN0_7CFNK-uT0-z7LcR9U,13067
248
248
  cognite_toolkit/_cdf_tk/storageio/_data_classes.py,sha256=s3TH04BJ1q7rXndRhEbVMEnoOXjxrGg4n-w9Z5uUL-o,3480
249
249
  cognite_toolkit/_cdf_tk/storageio/_datapoints.py,sha256=AGTQm9CBRbu1oXbBh0X7UGzFrHnlWZExqNvAohT0hM0,8641
250
- cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=ufuQ-ZTxQrLVaKtJhe7WGMD3CTgJRRA2tQZ-UkbRLnw,11000
250
+ cognite_toolkit/_cdf_tk/storageio/_file_content.py,sha256=qcJDk7YGUZ7YmVTYUDAi8eOK2sozEoxmehpmWrA45Ak,17127
251
251
  cognite_toolkit/_cdf_tk/storageio/_instances.py,sha256=t9fNpHnT6kCk8LDoPj3qZXmHpyDbPF5BZ6pI8ziTyFw,10810
252
252
  cognite_toolkit/_cdf_tk/storageio/_raw.py,sha256=pgZN5MbqDwMZl9Ow1KouDJUO2Ngga8_b6hwv7H31SVQ,5161
253
- cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=ELUCirQmhDR52PVIhLRd_2M1DWYURwL44U6WiN9_hEA,2225
253
+ cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py,sha256=eJkVXJpXt9iWRppO8kMyZB2Er6prCiVc0-48tZnjKbY,2312
254
254
  cognite_toolkit/_cdf_tk/storageio/selectors/_asset_centric.py,sha256=7Iv_ccVX6Vzt3ZLFZ0Er3hN92iEsFTm9wgF-yermOWE,1467
255
255
  cognite_toolkit/_cdf_tk/storageio/selectors/_base.py,sha256=hjFkbmNGsK3QIW-jnJV_8YNmvVROERxzG82qIZhU7SM,3065
256
256
  cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py,sha256=E9S-wr-JUqRosI_2cSCfR0tF8MdIFTrMxDItuWRcuO4,597
257
257
  cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py,sha256=lQHuNtF3i6SEIMPAlziMm0QlqRcvZJ7MKIug6HMTDrs,1012
258
258
  cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py,sha256=tMdSa-xj_Mf94SwGE8E1I4l_UpioK9OvE8vA8SbBeqY,1719
259
- cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=J_lP04QsvqCOIiw_u2rGMqIa_7obIxCaFkC70ukUjpI,3628
259
+ cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py,sha256=A2ikNImKTC-WuG5c-WLEJ6LfaB7FytXS57-D2ORuA1k,5326
260
260
  cognite_toolkit/_cdf_tk/storageio/selectors/_instances.py,sha256=NCFSJrAw52bNX6UTfOali8PvNjlqHnvxzL0hYBr7ZmA,4934
261
261
  cognite_toolkit/_cdf_tk/storageio/selectors/_raw.py,sha256=sZq9C4G9DMe3S46_usKet0FphQ6ow7cWM_PfXrEAakk,503
262
262
  cognite_toolkit/_cdf_tk/tk_warnings/__init__.py,sha256=U9bT-G2xKrX6mmtZ7nZ1FfQeCjNKfKP_p7pev90dwOE,2316
@@ -295,7 +295,7 @@ cognite_toolkit/_cdf_tk/utils/fileio/_readers.py,sha256=IjOSHyW0GiID_lKdgAwQZao9
295
295
  cognite_toolkit/_cdf_tk/utils/fileio/_writers.py,sha256=mc23m0kJgl57FUDvwLmS7yR3xVZWQguPJa_63-qQ_L0,17731
296
296
  cognite_toolkit/_cdf_tk/utils/http_client/__init__.py,sha256=G8b7Bg4yIet5R4Igh3dS2SntWzE6I0iTGBeNlNsSxkQ,857
297
297
  cognite_toolkit/_cdf_tk/utils/http_client/_client.py,sha256=NTRfloXkCiS_rl5Vl1D_hsyTTowMKWDsiIR4oGwTADI,11208
298
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=gNEJLb-tCoRh-OQA0BcJpESWl416ctC_6xKhWdwI4BU,13920
298
+ cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py,sha256=wbSDEtd4tXPwYp4VOKaasbz6WD3DlVpmUHroD1TjmUc,14313
299
299
  cognite_toolkit/_cdf_tk/utils/http_client/_exception.py,sha256=fC9oW6BN0HbUe2AkYABMP7Kj0-9dNYXVFBY5RQztq2c,126
300
300
  cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py,sha256=EBBnd-JZ7nc_jYNFJokCHN2UZ9sx0McFLZvlceUYYic,1215
301
301
  cognite_toolkit/_repo_files/.env.tmpl,sha256=UmgKZVvIp-OzD8oOcYuwb_6c7vSJsqkLhuFaiVgK7RI,972
@@ -303,13 +303,13 @@ cognite_toolkit/_repo_files/.gitignore,sha256=ip9kf9tcC5OguF4YF4JFEApnKYw0nG0vPi
303
303
  cognite_toolkit/_repo_files/AzureDevOps/.devops/README.md,sha256=OLA0D7yCX2tACpzvkA0IfkgQ4_swSd-OlJ1tYcTBpsA,240
304
304
  cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml,sha256=brULcs8joAeBC_w_aoWjDDUHs3JheLMIR9ajPUK96nc,693
305
305
  cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml,sha256=OBFDhFWK1mlT4Dc6mDUE2Es834l8sAlYG50-5RxRtHk,723
306
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=nL-mCBkPP8pUL2SgvmWouawboQps1xkiaK4QVTzpWAM,667
307
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=HFP5UDDyeDmZWHZUj1Mti3qFrVxvsqyMvQkpvZtvkJE,2430
308
- cognite_toolkit/_resources/cdf.toml,sha256=sMvPlsrR0qaY0RVUdLdw3e14_zxbNE_zKB6ronK_baU,475
306
+ cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml,sha256=v6a0VebiQ65SHtahhnUq_SMWcVbnZY4QJs4Fxzgh2JU,667
307
+ cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml,sha256=2Hoq3FVDM9BgWnJ0vhHaSCSD2UjTcbWL1Q-GEUCL-xk,2430
308
+ cognite_toolkit/_resources/cdf.toml,sha256=rlGz8YpUF6Ed6nSNd43gWkGdMnUnLS0z4VF3WPxKOdA,475
309
309
  cognite_toolkit/demo/__init__.py,sha256=-m1JoUiwRhNCL18eJ6t7fZOL7RPfowhCuqhYFtLgrss,72
310
310
  cognite_toolkit/demo/_base.py,sha256=6xKBUQpXZXGQ3fJ5f7nj7oT0s2n7OTAGIa17ZlKHZ5U,8052
311
- cognite_toolkit-0.7.12.dist-info/METADATA,sha256=6wCTZJ1s95fQEV-Za7qiiZYviLS_MUyCsp2fiX5-enQ,4501
312
- cognite_toolkit-0.7.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
313
- cognite_toolkit-0.7.12.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
314
- cognite_toolkit-0.7.12.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
315
- cognite_toolkit-0.7.12.dist-info/RECORD,,
311
+ cognite_toolkit-0.7.14.dist-info/METADATA,sha256=6EL0Rpt2IHDCRcvMdPKnGV966WeDY271wNIpHJ9CMOM,4501
312
+ cognite_toolkit-0.7.14.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
313
+ cognite_toolkit-0.7.14.dist-info/entry_points.txt,sha256=JlR7MH1_UMogC3QOyN4-1l36VbrCX9xUdQoHGkuJ6-4,83
314
+ cognite_toolkit-0.7.14.dist-info/licenses/LICENSE,sha256=CW0DRcx5tL-pCxLEN7ts2S9g2sLRAsWgHVEX4SN9_Mc,752
315
+ cognite_toolkit-0.7.14.dist-info/RECORD,,