cognite-toolkit 0.6.97__py3-none-any.whl → 0.7.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognite_toolkit/_cdf.py +21 -23
- cognite_toolkit/_cdf_tk/apps/__init__.py +4 -0
- cognite_toolkit/_cdf_tk/apps/_core_app.py +19 -5
- cognite_toolkit/_cdf_tk/apps/_data_app.py +1 -1
- cognite_toolkit/_cdf_tk/apps/_dev_app.py +86 -0
- cognite_toolkit/_cdf_tk/apps/_download_app.py +693 -25
- cognite_toolkit/_cdf_tk/apps/_dump_app.py +44 -102
- cognite_toolkit/_cdf_tk/apps/_import_app.py +41 -0
- cognite_toolkit/_cdf_tk/apps/_landing_app.py +18 -4
- cognite_toolkit/_cdf_tk/apps/_migrate_app.py +424 -9
- cognite_toolkit/_cdf_tk/apps/_modules_app.py +0 -3
- cognite_toolkit/_cdf_tk/apps/_purge.py +15 -43
- cognite_toolkit/_cdf_tk/apps/_run.py +11 -0
- cognite_toolkit/_cdf_tk/apps/_upload_app.py +45 -6
- cognite_toolkit/_cdf_tk/builders/__init__.py +2 -2
- cognite_toolkit/_cdf_tk/builders/_base.py +28 -42
- cognite_toolkit/_cdf_tk/builders/_raw.py +1 -1
- cognite_toolkit/_cdf_tk/cdf_toml.py +20 -1
- cognite_toolkit/_cdf_tk/client/_toolkit_client.py +32 -12
- cognite_toolkit/_cdf_tk/client/api/infield.py +114 -17
- cognite_toolkit/_cdf_tk/client/api/{canvas.py → legacy/canvas.py} +15 -7
- cognite_toolkit/_cdf_tk/client/api/{charts.py → legacy/charts.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_data_modeling.py → legacy/extended_data_modeling.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_files.py → legacy/extended_files.py} +2 -2
- cognite_toolkit/_cdf_tk/client/api/{extended_functions.py → legacy/extended_functions.py} +15 -18
- cognite_toolkit/_cdf_tk/client/api/{extended_raw.py → legacy/extended_raw.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/{extended_timeseries.py → legacy/extended_timeseries.py} +5 -2
- cognite_toolkit/_cdf_tk/client/api/{location_filters.py → legacy/location_filters.py} +1 -1
- cognite_toolkit/_cdf_tk/client/api/legacy/robotics/__init__.py +8 -0
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/capabilities.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/data_postprocessing.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/frames.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/locations.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/maps.py +1 -1
- cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/robots.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/{search_config.py → legacy/search_config.py} +5 -1
- cognite_toolkit/_cdf_tk/client/api/migration.py +177 -4
- cognite_toolkit/_cdf_tk/client/api/project.py +9 -8
- cognite_toolkit/_cdf_tk/client/api/search.py +2 -2
- cognite_toolkit/_cdf_tk/client/api/streams.py +88 -0
- cognite_toolkit/_cdf_tk/client/api/three_d.py +384 -0
- cognite_toolkit/_cdf_tk/client/data_classes/api_classes.py +13 -0
- cognite_toolkit/_cdf_tk/client/data_classes/base.py +37 -33
- cognite_toolkit/_cdf_tk/client/data_classes/charts_data.py +95 -213
- cognite_toolkit/_cdf_tk/client/data_classes/infield.py +32 -18
- cognite_toolkit/_cdf_tk/client/data_classes/instance_api.py +18 -13
- cognite_toolkit/_cdf_tk/client/data_classes/legacy/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/client/data_classes/{canvas.py → legacy/canvas.py} +47 -4
- cognite_toolkit/_cdf_tk/client/data_classes/{charts.py → legacy/charts.py} +3 -3
- cognite_toolkit/_cdf_tk/client/data_classes/{migration.py → legacy/migration.py} +10 -2
- cognite_toolkit/_cdf_tk/client/data_classes/streams.py +90 -0
- cognite_toolkit/_cdf_tk/client/data_classes/three_d.py +112 -0
- cognite_toolkit/_cdf_tk/client/testing.py +42 -18
- cognite_toolkit/_cdf_tk/commands/__init__.py +7 -6
- cognite_toolkit/_cdf_tk/commands/_changes.py +3 -42
- cognite_toolkit/_cdf_tk/commands/_download.py +21 -11
- cognite_toolkit/_cdf_tk/commands/_migrate/__init__.py +0 -2
- cognite_toolkit/_cdf_tk/commands/_migrate/command.py +22 -20
- cognite_toolkit/_cdf_tk/commands/_migrate/conversion.py +140 -92
- cognite_toolkit/_cdf_tk/commands/_migrate/creators.py +1 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/data_classes.py +108 -26
- cognite_toolkit/_cdf_tk/commands/_migrate/data_mapper.py +448 -45
- cognite_toolkit/_cdf_tk/commands/_migrate/data_model.py +1 -0
- cognite_toolkit/_cdf_tk/commands/_migrate/default_mappings.py +6 -6
- cognite_toolkit/_cdf_tk/commands/_migrate/issues.py +52 -1
- cognite_toolkit/_cdf_tk/commands/_migrate/migration_io.py +377 -11
- cognite_toolkit/_cdf_tk/commands/_migrate/selectors.py +9 -4
- cognite_toolkit/_cdf_tk/commands/_profile.py +1 -1
- cognite_toolkit/_cdf_tk/commands/_purge.py +36 -39
- cognite_toolkit/_cdf_tk/commands/_questionary_style.py +16 -0
- cognite_toolkit/_cdf_tk/commands/_upload.py +109 -86
- cognite_toolkit/_cdf_tk/commands/about.py +221 -0
- cognite_toolkit/_cdf_tk/commands/auth.py +19 -12
- cognite_toolkit/_cdf_tk/commands/build_cmd.py +16 -62
- cognite_toolkit/_cdf_tk/commands/build_v2/__init__.py +0 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_cmd.py +241 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_input.py +85 -0
- cognite_toolkit/_cdf_tk/commands/build_v2/build_issues.py +27 -0
- cognite_toolkit/_cdf_tk/commands/clean.py +63 -16
- cognite_toolkit/_cdf_tk/commands/deploy.py +20 -17
- cognite_toolkit/_cdf_tk/commands/dump_resource.py +10 -8
- cognite_toolkit/_cdf_tk/commands/init.py +225 -3
- cognite_toolkit/_cdf_tk/commands/modules.py +20 -44
- cognite_toolkit/_cdf_tk/commands/pull.py +6 -19
- cognite_toolkit/_cdf_tk/commands/resources.py +179 -0
- cognite_toolkit/_cdf_tk/commands/run.py +1 -1
- cognite_toolkit/_cdf_tk/constants.py +20 -1
- cognite_toolkit/_cdf_tk/cruds/__init__.py +19 -5
- cognite_toolkit/_cdf_tk/cruds/_base_cruds.py +14 -70
- cognite_toolkit/_cdf_tk/cruds/_data_cruds.py +10 -19
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/__init__.py +4 -1
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/agent.py +11 -9
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/auth.py +5 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/classic.py +45 -44
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/configuration.py +5 -12
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/data_organization.py +4 -13
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py +206 -67
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/extraction_pipeline.py +6 -18
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/fieldops.py +126 -35
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py +7 -28
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/function.py +23 -30
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/hosted_extractors.py +12 -30
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/industrial_tool.py +4 -8
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/location.py +4 -16
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/migration.py +5 -13
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/raw.py +5 -11
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/relationship.py +3 -8
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/robotics.py +16 -45
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/streams.py +94 -0
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/three_d_model.py +3 -7
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/timeseries.py +5 -15
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/transformation.py +75 -32
- cognite_toolkit/_cdf_tk/cruds/_resource_cruds/workflow.py +20 -40
- cognite_toolkit/_cdf_tk/cruds/_worker.py +24 -36
- cognite_toolkit/_cdf_tk/data_classes/_module_toml.py +1 -0
- cognite_toolkit/_cdf_tk/feature_flags.py +16 -36
- cognite_toolkit/_cdf_tk/plugins.py +2 -1
- cognite_toolkit/_cdf_tk/resource_classes/__init__.py +4 -0
- cognite_toolkit/_cdf_tk/resource_classes/capabilities.py +12 -0
- cognite_toolkit/_cdf_tk/resource_classes/functions.py +3 -1
- cognite_toolkit/_cdf_tk/resource_classes/infield_cdm_location_config.py +109 -0
- cognite_toolkit/_cdf_tk/resource_classes/migration.py +8 -17
- cognite_toolkit/_cdf_tk/resource_classes/search_config.py +1 -1
- cognite_toolkit/_cdf_tk/resource_classes/streams.py +29 -0
- cognite_toolkit/_cdf_tk/resource_classes/workflow_version.py +164 -5
- cognite_toolkit/_cdf_tk/storageio/__init__.py +9 -21
- cognite_toolkit/_cdf_tk/storageio/_annotations.py +19 -16
- cognite_toolkit/_cdf_tk/storageio/_applications.py +340 -28
- cognite_toolkit/_cdf_tk/storageio/_asset_centric.py +67 -104
- cognite_toolkit/_cdf_tk/storageio/_base.py +61 -29
- cognite_toolkit/_cdf_tk/storageio/_datapoints.py +276 -20
- cognite_toolkit/_cdf_tk/storageio/_file_content.py +435 -0
- cognite_toolkit/_cdf_tk/storageio/_instances.py +35 -3
- cognite_toolkit/_cdf_tk/storageio/_raw.py +26 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/__init__.py +71 -4
- cognite_toolkit/_cdf_tk/storageio/selectors/_base.py +14 -2
- cognite_toolkit/_cdf_tk/storageio/selectors/_canvas.py +14 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/_charts.py +14 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/_datapoints.py +23 -3
- cognite_toolkit/_cdf_tk/storageio/selectors/_file_content.py +164 -0
- cognite_toolkit/_cdf_tk/storageio/selectors/_three_d.py +34 -0
- cognite_toolkit/_cdf_tk/tk_warnings/other.py +4 -0
- cognite_toolkit/_cdf_tk/tracker.py +2 -2
- cognite_toolkit/_cdf_tk/utils/cdf.py +1 -1
- cognite_toolkit/_cdf_tk/utils/dtype_conversion.py +9 -3
- cognite_toolkit/_cdf_tk/utils/fileio/__init__.py +2 -0
- cognite_toolkit/_cdf_tk/utils/fileio/_base.py +5 -1
- cognite_toolkit/_cdf_tk/utils/fileio/_readers.py +112 -20
- cognite_toolkit/_cdf_tk/utils/fileio/_writers.py +15 -15
- cognite_toolkit/_cdf_tk/utils/http_client/__init__.py +28 -0
- cognite_toolkit/_cdf_tk/utils/http_client/_client.py +285 -18
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes.py +56 -4
- cognite_toolkit/_cdf_tk/utils/http_client/_data_classes2.py +247 -0
- cognite_toolkit/_cdf_tk/utils/http_client/_tracker.py +5 -2
- cognite_toolkit/_cdf_tk/utils/interactive_select.py +60 -18
- cognite_toolkit/_cdf_tk/utils/sql_parser.py +2 -3
- cognite_toolkit/_cdf_tk/utils/useful_types.py +6 -2
- cognite_toolkit/_cdf_tk/validation.py +83 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml +1 -1
- cognite_toolkit/_repo_files/GitHub/.github/workflows/dry-run.yaml +1 -1
- cognite_toolkit/_resources/cdf.toml +5 -4
- cognite_toolkit/_version.py +1 -1
- cognite_toolkit/config.dev.yaml +13 -0
- {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.39.dist-info}/METADATA +24 -24
- cognite_toolkit-0.7.39.dist-info/RECORD +322 -0
- cognite_toolkit-0.7.39.dist-info/WHEEL +4 -0
- {cognite_toolkit-0.6.97.dist-info → cognite_toolkit-0.7.39.dist-info}/entry_points.txt +1 -0
- cognite_toolkit/_cdf_tk/client/api/robotics/__init__.py +0 -3
- cognite_toolkit/_cdf_tk/commands/_migrate/canvas.py +0 -201
- cognite_toolkit/_cdf_tk/commands/dump_data.py +0 -489
- cognite_toolkit/_cdf_tk/commands/featureflag.py +0 -27
- cognite_toolkit/_cdf_tk/prototypes/import_app.py +0 -41
- cognite_toolkit/_cdf_tk/utils/table_writers.py +0 -434
- cognite_toolkit-0.6.97.dist-info/RECORD +0 -306
- cognite_toolkit-0.6.97.dist-info/WHEEL +0 -4
- cognite_toolkit-0.6.97.dist-info/licenses/LICENSE +0 -18
- /cognite_toolkit/_cdf_tk/{prototypes/commands → client/api/legacy}/__init__.py +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{dml.py → legacy/dml.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{fixed_transformations.py → legacy/fixed_transformations.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/api.py +0 -0
- /cognite_toolkit/_cdf_tk/client/api/{robotics → legacy/robotics}/utlis.py +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{apm_config_v1.py → legacy/apm_config_v1.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{extendable_cognite_file.py → legacy/extendable_cognite_file.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{extended_filemetadata.py → legacy/extended_filemetadata.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{extended_filemetdata.py → legacy/extended_filemetdata.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{extended_timeseries.py → legacy/extended_timeseries.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{functions.py → legacy/functions.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{graphql_data_models.py → legacy/graphql_data_models.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{instances.py → legacy/instances.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{location_filters.py → legacy/location_filters.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{pending_instances_ids.py → legacy/pending_instances_ids.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{project.py → legacy/project.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{raw.py → legacy/raw.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{robotics.py → legacy/robotics.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{search_config.py → legacy/search_config.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{sequences.py → legacy/sequences.py} +0 -0
- /cognite_toolkit/_cdf_tk/client/data_classes/{streamlit_.py → legacy/streamlit_.py} +0 -0
- /cognite_toolkit/_cdf_tk/{prototypes/commands/import_.py → commands/_import_cmd.py} +0 -0
|
@@ -2,32 +2,57 @@ from enum import Enum
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import Annotated, Any
|
|
4
4
|
|
|
5
|
+
import questionary
|
|
5
6
|
import typer
|
|
7
|
+
from questionary import Choice
|
|
6
8
|
from rich import print
|
|
7
9
|
|
|
8
|
-
from cognite_toolkit._cdf_tk.client.data_classes.raw import RawTable
|
|
10
|
+
from cognite_toolkit._cdf_tk.client.data_classes.legacy.raw import RawTable
|
|
9
11
|
from cognite_toolkit._cdf_tk.commands import DownloadCommand
|
|
10
12
|
from cognite_toolkit._cdf_tk.constants import DATA_DEFAULT_DIR
|
|
13
|
+
from cognite_toolkit._cdf_tk.feature_flags import Flags
|
|
11
14
|
from cognite_toolkit._cdf_tk.storageio import (
|
|
12
15
|
AssetIO,
|
|
16
|
+
CanvasIO,
|
|
17
|
+
ChartIO,
|
|
18
|
+
DatapointsIO,
|
|
19
|
+
DataSelector,
|
|
20
|
+
EventIO,
|
|
21
|
+
FileContentIO,
|
|
22
|
+
FileMetadataIO,
|
|
13
23
|
HierarchyIO,
|
|
14
24
|
InstanceIO,
|
|
15
25
|
RawIO,
|
|
26
|
+
StorageIO,
|
|
27
|
+
TimeSeriesIO,
|
|
16
28
|
)
|
|
17
29
|
from cognite_toolkit._cdf_tk.storageio.selectors import (
|
|
18
|
-
AssetCentricSelector,
|
|
19
30
|
AssetSubtreeSelector,
|
|
31
|
+
CanvasExternalIdSelector,
|
|
32
|
+
CanvasSelector,
|
|
33
|
+
ChartExternalIdSelector,
|
|
34
|
+
ChartSelector,
|
|
35
|
+
DataPointsDataSetSelector,
|
|
20
36
|
DataSetSelector,
|
|
37
|
+
FileIdentifierSelector,
|
|
21
38
|
InstanceSpaceSelector,
|
|
22
39
|
RawTableSelector,
|
|
23
40
|
SelectedTable,
|
|
24
41
|
SelectedView,
|
|
25
42
|
)
|
|
43
|
+
from cognite_toolkit._cdf_tk.storageio.selectors._file_content import FileInternalID
|
|
44
|
+
from cognite_toolkit._cdf_tk.utils import sanitize_filename
|
|
26
45
|
from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables
|
|
27
46
|
from cognite_toolkit._cdf_tk.utils.interactive_select import (
|
|
47
|
+
AssetCentricInteractiveSelect,
|
|
28
48
|
AssetInteractiveSelect,
|
|
29
49
|
DataModelingSelect,
|
|
50
|
+
EventInteractiveSelect,
|
|
51
|
+
FileMetadataInteractiveSelect,
|
|
52
|
+
InteractiveCanvasSelect,
|
|
53
|
+
InteractiveChartSelect,
|
|
30
54
|
RawTableInteractiveSelect,
|
|
55
|
+
TimeSeriesInteractiveSelect,
|
|
31
56
|
)
|
|
32
57
|
|
|
33
58
|
|
|
@@ -42,14 +67,36 @@ class AssetCentricFormats(str, Enum):
|
|
|
42
67
|
ndjson = "ndjson"
|
|
43
68
|
|
|
44
69
|
|
|
70
|
+
class FileContentFormats(str, Enum):
|
|
71
|
+
ndjson = "ndjson"
|
|
72
|
+
|
|
73
|
+
|
|
45
74
|
class HierarchyFormats(str, Enum):
|
|
46
75
|
ndjson = "ndjson"
|
|
47
76
|
|
|
48
77
|
|
|
78
|
+
class DatapointFormats(str, Enum):
|
|
79
|
+
csv = "csv"
|
|
80
|
+
parquet = "parquet"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class DatapointsDataTypes(str, Enum):
|
|
84
|
+
numeric = "numeric"
|
|
85
|
+
string = "string"
|
|
86
|
+
|
|
87
|
+
|
|
49
88
|
class InstanceFormats(str, Enum):
|
|
50
89
|
ndjson = "ndjson"
|
|
51
90
|
|
|
52
91
|
|
|
92
|
+
class ChartFormats(str, Enum):
|
|
93
|
+
ndjson = "ndjson"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class CanvasFormats(str, Enum):
|
|
97
|
+
ndjson = "ndjson"
|
|
98
|
+
|
|
99
|
+
|
|
53
100
|
class InstanceTypes(str, Enum):
|
|
54
101
|
node = "node"
|
|
55
102
|
edge = "edge"
|
|
@@ -69,8 +116,15 @@ class DownloadApp(typer.Typer):
|
|
|
69
116
|
self.callback(invoke_without_command=True)(self.download_main)
|
|
70
117
|
self.command("raw")(self.download_raw_cmd)
|
|
71
118
|
self.command("assets")(self.download_assets_cmd)
|
|
119
|
+
self.command("timeseries")(self.download_timeseries_cmd)
|
|
120
|
+
self.command("events")(self.download_events_cmd)
|
|
121
|
+
self.command("files")(self.download_files_cmd)
|
|
72
122
|
self.command("hierarchy")(self.download_hierarchy_cmd)
|
|
123
|
+
if Flags.EXTEND_DOWNLOAD.is_enabled():
|
|
124
|
+
self.command("datapoints")(self.download_datapoints_cmd)
|
|
73
125
|
self.command("instances")(self.download_instances_cmd)
|
|
126
|
+
self.command("charts")(self.download_charts_cmd)
|
|
127
|
+
self.command("canvas")(self.download_canvas_cmd)
|
|
74
128
|
|
|
75
129
|
@staticmethod
|
|
76
130
|
def download_main(ctx: typer.Context) -> None:
|
|
@@ -169,23 +223,15 @@ class DownloadApp(typer.Typer):
|
|
|
169
223
|
)
|
|
170
224
|
)
|
|
171
225
|
|
|
172
|
-
@staticmethod
|
|
173
226
|
def download_assets_cmd(
|
|
227
|
+
self,
|
|
174
228
|
ctx: typer.Context,
|
|
175
229
|
data_sets: Annotated[
|
|
176
230
|
list[str] | None,
|
|
177
231
|
typer.Option(
|
|
178
232
|
"--data-set",
|
|
179
233
|
"-d",
|
|
180
|
-
help="List of data sets to download assets from. If this
|
|
181
|
-
),
|
|
182
|
-
] = None,
|
|
183
|
-
hierarchy: Annotated[
|
|
184
|
-
list[str] | None,
|
|
185
|
-
typer.Option(
|
|
186
|
-
"--hierarchy",
|
|
187
|
-
"-r",
|
|
188
|
-
help="List of asset hierarchies to download assets from. If this and data sets are not provided, an interactive selection will be made.",
|
|
234
|
+
help="List of data sets to download assets from. If this is not provided, an interactive selection will be made.",
|
|
189
235
|
),
|
|
190
236
|
] = None,
|
|
191
237
|
file_format: Annotated[
|
|
@@ -232,20 +278,17 @@ class DownloadApp(typer.Typer):
|
|
|
232
278
|
) -> None:
|
|
233
279
|
"""This command will download assets from CDF into a temporary directory."""
|
|
234
280
|
client = EnvironmentVariables.create_from_environment().get_client()
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
281
|
+
if data_sets is None:
|
|
282
|
+
data_sets, file_format, compression, output_dir, limit = self._asset_centric_interactive(
|
|
283
|
+
AssetInteractiveSelect(client, "download"),
|
|
284
|
+
file_format,
|
|
285
|
+
compression,
|
|
286
|
+
output_dir,
|
|
287
|
+
limit,
|
|
288
|
+
"assets",
|
|
289
|
+
)
|
|
243
290
|
|
|
244
|
-
selectors
|
|
245
|
-
if data_sets:
|
|
246
|
-
selectors.extend([DataSetSelector(data_set_external_id=ds, kind="Assets") for ds in data_sets])
|
|
247
|
-
if hierarchy:
|
|
248
|
-
selectors.extend([AssetSubtreeSelector(hierarchy=h, kind="Assets") for h in hierarchy])
|
|
291
|
+
selectors = [DataSetSelector(kind="Assets", data_set_external_id=data_set) for data_set in data_sets]
|
|
249
292
|
cmd = DownloadCommand()
|
|
250
293
|
cmd.run(
|
|
251
294
|
lambda: cmd.download(
|
|
@@ -259,6 +302,341 @@ class DownloadApp(typer.Typer):
|
|
|
259
302
|
)
|
|
260
303
|
)
|
|
261
304
|
|
|
305
|
+
@classmethod
|
|
306
|
+
def _asset_centric_interactive(
|
|
307
|
+
cls,
|
|
308
|
+
selector: AssetCentricInteractiveSelect,
|
|
309
|
+
file_format: AssetCentricFormats,
|
|
310
|
+
compression: CompressionFormat,
|
|
311
|
+
output_dir: Path,
|
|
312
|
+
limit: int,
|
|
313
|
+
display_name: str,
|
|
314
|
+
max_limit: int | None = None,
|
|
315
|
+
available_formats: type[Enum] = AssetCentricFormats,
|
|
316
|
+
) -> tuple[list[str], AssetCentricFormats, CompressionFormat, Path, int]:
|
|
317
|
+
data_sets = selector.select_data_sets()
|
|
318
|
+
file_format = questionary.select(
|
|
319
|
+
f"Select format to download the {display_name} in:",
|
|
320
|
+
choices=[Choice(title=format_.value, value=format_) for format_ in available_formats],
|
|
321
|
+
default=file_format,
|
|
322
|
+
).ask()
|
|
323
|
+
compression = questionary.select(
|
|
324
|
+
f"Select compression format to use when downloading the {display_name}:",
|
|
325
|
+
choices=[Choice(title=comp.value, value=comp) for comp in CompressionFormat],
|
|
326
|
+
default=compression,
|
|
327
|
+
).ask()
|
|
328
|
+
output_dir = Path(
|
|
329
|
+
questionary.path(
|
|
330
|
+
f"Where to download the {display_name}:",
|
|
331
|
+
default=str(output_dir),
|
|
332
|
+
only_directories=True,
|
|
333
|
+
).ask()
|
|
334
|
+
)
|
|
335
|
+
while True:
|
|
336
|
+
limit_str = questionary.text(
|
|
337
|
+
f"The maximum number of {display_name} to download from each dataset. Use -1 to download all {display_name}.",
|
|
338
|
+
default=str(limit),
|
|
339
|
+
).ask()
|
|
340
|
+
if limit_str is None:
|
|
341
|
+
raise typer.Abort()
|
|
342
|
+
try:
|
|
343
|
+
limit = int(limit_str)
|
|
344
|
+
except ValueError:
|
|
345
|
+
print("[red]Please enter a valid integer for the limit.[/]")
|
|
346
|
+
else:
|
|
347
|
+
if max_limit is not None and limit > max_limit:
|
|
348
|
+
print(
|
|
349
|
+
f"[red]The maximum limit for downloading {display_name} is {max_limit}. Please enter a lower value.[/]"
|
|
350
|
+
)
|
|
351
|
+
else:
|
|
352
|
+
break
|
|
353
|
+
return data_sets, file_format, compression, output_dir, limit
|
|
354
|
+
|
|
355
|
+
def download_timeseries_cmd(
|
|
356
|
+
self,
|
|
357
|
+
ctx: typer.Context,
|
|
358
|
+
data_sets: Annotated[
|
|
359
|
+
list[str] | None,
|
|
360
|
+
typer.Option(
|
|
361
|
+
"--data-set",
|
|
362
|
+
"-d",
|
|
363
|
+
help="List of data sets to download time series from. If this is not provided, an interactive selection will be made.",
|
|
364
|
+
),
|
|
365
|
+
] = None,
|
|
366
|
+
file_format: Annotated[
|
|
367
|
+
AssetCentricFormats,
|
|
368
|
+
typer.Option(
|
|
369
|
+
"--format",
|
|
370
|
+
"-f",
|
|
371
|
+
help="Format to download the time series in.",
|
|
372
|
+
),
|
|
373
|
+
] = AssetCentricFormats.csv,
|
|
374
|
+
compression: Annotated[
|
|
375
|
+
CompressionFormat,
|
|
376
|
+
typer.Option(
|
|
377
|
+
"--compression",
|
|
378
|
+
"-z",
|
|
379
|
+
help="Compression format to use when downloading the time series.",
|
|
380
|
+
),
|
|
381
|
+
] = CompressionFormat.none,
|
|
382
|
+
output_dir: Annotated[
|
|
383
|
+
Path,
|
|
384
|
+
typer.Option(
|
|
385
|
+
"--output-dir",
|
|
386
|
+
"-o",
|
|
387
|
+
help="Where to download the time series.",
|
|
388
|
+
allow_dash=True,
|
|
389
|
+
),
|
|
390
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
391
|
+
limit: Annotated[
|
|
392
|
+
int,
|
|
393
|
+
typer.Option(
|
|
394
|
+
"--limit",
|
|
395
|
+
"-l",
|
|
396
|
+
help="The maximum number of time series to download from each dataset. Use -1 to download all time series.",
|
|
397
|
+
),
|
|
398
|
+
] = 100_000,
|
|
399
|
+
verbose: Annotated[
|
|
400
|
+
bool,
|
|
401
|
+
typer.Option(
|
|
402
|
+
"--verbose",
|
|
403
|
+
"-v",
|
|
404
|
+
help="Turn on to get more verbose output when running the command",
|
|
405
|
+
),
|
|
406
|
+
] = False,
|
|
407
|
+
) -> None:
|
|
408
|
+
"""This command will download time series from CDF into a temporary directory."""
|
|
409
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
410
|
+
if data_sets is None:
|
|
411
|
+
data_sets, file_format, compression, output_dir, limit = self._asset_centric_interactive(
|
|
412
|
+
TimeSeriesInteractiveSelect(client, "download"),
|
|
413
|
+
file_format,
|
|
414
|
+
compression,
|
|
415
|
+
output_dir,
|
|
416
|
+
limit,
|
|
417
|
+
"time series",
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
selectors = [DataSetSelector(kind="TimeSeries", data_set_external_id=data_set) for data_set in data_sets]
|
|
421
|
+
cmd = DownloadCommand()
|
|
422
|
+
cmd.run(
|
|
423
|
+
lambda: cmd.download(
|
|
424
|
+
selectors=selectors,
|
|
425
|
+
io=TimeSeriesIO(client),
|
|
426
|
+
output_dir=output_dir,
|
|
427
|
+
file_format=f".{file_format.value}",
|
|
428
|
+
compression=compression.value,
|
|
429
|
+
limit=limit if limit != -1 else None,
|
|
430
|
+
verbose=verbose,
|
|
431
|
+
)
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
def download_events_cmd(
|
|
435
|
+
self,
|
|
436
|
+
ctx: typer.Context,
|
|
437
|
+
data_sets: Annotated[
|
|
438
|
+
list[str] | None,
|
|
439
|
+
typer.Option(
|
|
440
|
+
"--data-set",
|
|
441
|
+
"-d",
|
|
442
|
+
help="List of data sets to download events from. If this is not provided, an interactive selection will be made.",
|
|
443
|
+
),
|
|
444
|
+
] = None,
|
|
445
|
+
file_format: Annotated[
|
|
446
|
+
AssetCentricFormats,
|
|
447
|
+
typer.Option(
|
|
448
|
+
"--format",
|
|
449
|
+
"-f",
|
|
450
|
+
help="Format to download the events in.",
|
|
451
|
+
),
|
|
452
|
+
] = AssetCentricFormats.csv,
|
|
453
|
+
compression: Annotated[
|
|
454
|
+
CompressionFormat,
|
|
455
|
+
typer.Option(
|
|
456
|
+
"--compression",
|
|
457
|
+
"-z",
|
|
458
|
+
help="Compression format to use when downloading the events.",
|
|
459
|
+
),
|
|
460
|
+
] = CompressionFormat.none,
|
|
461
|
+
output_dir: Annotated[
|
|
462
|
+
Path,
|
|
463
|
+
typer.Option(
|
|
464
|
+
"--output-dir",
|
|
465
|
+
"-o",
|
|
466
|
+
help="Where to download the events.",
|
|
467
|
+
allow_dash=True,
|
|
468
|
+
),
|
|
469
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
470
|
+
limit: Annotated[
|
|
471
|
+
int,
|
|
472
|
+
typer.Option(
|
|
473
|
+
"--limit",
|
|
474
|
+
"-l",
|
|
475
|
+
help="The maximum number of events to download from each dataset. Use -1 to download all events.",
|
|
476
|
+
),
|
|
477
|
+
] = 100_000,
|
|
478
|
+
verbose: Annotated[
|
|
479
|
+
bool,
|
|
480
|
+
typer.Option(
|
|
481
|
+
"--verbose",
|
|
482
|
+
"-v",
|
|
483
|
+
help="Turn on to get more verbose output when running the command",
|
|
484
|
+
),
|
|
485
|
+
] = False,
|
|
486
|
+
) -> None:
|
|
487
|
+
"""This command will download events from CDF into a temporary directory."""
|
|
488
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
489
|
+
if data_sets is None:
|
|
490
|
+
data_sets, file_format, compression, output_dir, limit = self._asset_centric_interactive(
|
|
491
|
+
EventInteractiveSelect(client, "download"),
|
|
492
|
+
file_format,
|
|
493
|
+
compression,
|
|
494
|
+
output_dir,
|
|
495
|
+
limit,
|
|
496
|
+
"events",
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
selectors = [DataSetSelector(kind="Events", data_set_external_id=data_set) for data_set in data_sets]
|
|
500
|
+
cmd = DownloadCommand()
|
|
501
|
+
|
|
502
|
+
cmd.run(
|
|
503
|
+
lambda: cmd.download(
|
|
504
|
+
selectors=selectors,
|
|
505
|
+
io=EventIO(client),
|
|
506
|
+
output_dir=output_dir,
|
|
507
|
+
file_format=f".{file_format.value}",
|
|
508
|
+
compression=compression.value,
|
|
509
|
+
limit=limit if limit != -1 else None,
|
|
510
|
+
verbose=verbose,
|
|
511
|
+
)
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
def download_files_cmd(
|
|
515
|
+
self,
|
|
516
|
+
ctx: typer.Context,
|
|
517
|
+
data_sets: Annotated[
|
|
518
|
+
list[str] | None,
|
|
519
|
+
typer.Option(
|
|
520
|
+
"--data-set",
|
|
521
|
+
"-d",
|
|
522
|
+
help="List of data sets to download file metadata from. If this is not provided, an interactive selection will be made.",
|
|
523
|
+
),
|
|
524
|
+
] = None,
|
|
525
|
+
include_file_contents: Annotated[
|
|
526
|
+
bool,
|
|
527
|
+
typer.Option(
|
|
528
|
+
"--include-file-contents",
|
|
529
|
+
"-c",
|
|
530
|
+
help="Whether to include file contents when downloading assets. Note if you enable this option, you can"
|
|
531
|
+
"only download 1000 files at a time.",
|
|
532
|
+
hidden=not Flags.EXTEND_DOWNLOAD.is_enabled(),
|
|
533
|
+
),
|
|
534
|
+
] = False,
|
|
535
|
+
file_format: Annotated[
|
|
536
|
+
AssetCentricFormats,
|
|
537
|
+
typer.Option(
|
|
538
|
+
"--format",
|
|
539
|
+
"-f",
|
|
540
|
+
help="Format to download the file metadata in.",
|
|
541
|
+
),
|
|
542
|
+
] = AssetCentricFormats.csv,
|
|
543
|
+
compression: Annotated[
|
|
544
|
+
CompressionFormat,
|
|
545
|
+
typer.Option(
|
|
546
|
+
"--compression",
|
|
547
|
+
"-z",
|
|
548
|
+
help="Compression format to use when downloading the file metadata.",
|
|
549
|
+
),
|
|
550
|
+
] = CompressionFormat.none,
|
|
551
|
+
output_dir: Annotated[
|
|
552
|
+
Path,
|
|
553
|
+
typer.Option(
|
|
554
|
+
"--output-dir",
|
|
555
|
+
"-o",
|
|
556
|
+
help="Where to download the file metadata.",
|
|
557
|
+
allow_dash=True,
|
|
558
|
+
),
|
|
559
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
560
|
+
limit: Annotated[
|
|
561
|
+
int,
|
|
562
|
+
typer.Option(
|
|
563
|
+
"--limit",
|
|
564
|
+
"-l",
|
|
565
|
+
help="The maximum number of file metadata to download from each dataset. Use -1 to download all file metadata.",
|
|
566
|
+
),
|
|
567
|
+
] = 100_000,
|
|
568
|
+
verbose: Annotated[
|
|
569
|
+
bool,
|
|
570
|
+
typer.Option(
|
|
571
|
+
"--verbose",
|
|
572
|
+
"-v",
|
|
573
|
+
help="Turn on to get more verbose output when running the command",
|
|
574
|
+
),
|
|
575
|
+
] = False,
|
|
576
|
+
) -> None:
|
|
577
|
+
"""This command will download file metadata from CDF into a temporary directory."""
|
|
578
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
579
|
+
if data_sets is None:
|
|
580
|
+
if Flags.EXTEND_DOWNLOAD.is_enabled():
|
|
581
|
+
include_file_contents = questionary.select(
|
|
582
|
+
"Do you want to include file contents when downloading file metadata?",
|
|
583
|
+
choices=[
|
|
584
|
+
Choice(title="Yes", value=True),
|
|
585
|
+
Choice(title="No", value=False),
|
|
586
|
+
],
|
|
587
|
+
).ask()
|
|
588
|
+
else:
|
|
589
|
+
include_file_contents = False
|
|
590
|
+
|
|
591
|
+
available_formats = FileContentFormats if include_file_contents else AssetCentricFormats
|
|
592
|
+
file_format = FileContentFormats.ndjson if include_file_contents else file_format # type: ignore[assignment]
|
|
593
|
+
data_sets, file_format, compression, output_dir, limit = self._asset_centric_interactive(
|
|
594
|
+
FileMetadataInteractiveSelect(client, "download"),
|
|
595
|
+
file_format,
|
|
596
|
+
compression,
|
|
597
|
+
output_dir,
|
|
598
|
+
limit if not include_file_contents else 1000,
|
|
599
|
+
"file metadata",
|
|
600
|
+
max_limit=1000 if include_file_contents else None,
|
|
601
|
+
available_formats=available_formats,
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
io: StorageIO
|
|
605
|
+
selectors: list[DataSelector]
|
|
606
|
+
if include_file_contents:
|
|
607
|
+
if limit == -1 or limit > 1000:
|
|
608
|
+
limit = 1000
|
|
609
|
+
print(
|
|
610
|
+
"[yellow]When including file contents, the maximum number of files that can be downloaded at a time is 1000. "
|
|
611
|
+
)
|
|
612
|
+
if file_format == AssetCentricFormats.csv or file_format == AssetCentricFormats.parquet:
|
|
613
|
+
print(
|
|
614
|
+
"[red]When including file contents, the only supported format is ndjson. Overriding the format to ndjson.[/]"
|
|
615
|
+
)
|
|
616
|
+
file_format = AssetCentricFormats.ndjson
|
|
617
|
+
files = client.files.list(data_set_external_ids=data_sets, limit=limit)
|
|
618
|
+
selector = FileIdentifierSelector(
|
|
619
|
+
identifiers=tuple([FileInternalID(internal_id=file.id) for file in files])
|
|
620
|
+
)
|
|
621
|
+
selectors = [selector]
|
|
622
|
+
io = FileContentIO(client, output_dir / sanitize_filename(selector.group))
|
|
623
|
+
else:
|
|
624
|
+
selectors = [DataSetSelector(kind="FileMetadata", data_set_external_id=data_set) for data_set in data_sets]
|
|
625
|
+
io = FileMetadataIO(client)
|
|
626
|
+
|
|
627
|
+
cmd = DownloadCommand()
|
|
628
|
+
cmd.run(
|
|
629
|
+
lambda: cmd.download(
|
|
630
|
+
selectors=selectors, # type: ignore[misc]
|
|
631
|
+
io=io,
|
|
632
|
+
output_dir=output_dir,
|
|
633
|
+
file_format=f".{file_format.value}",
|
|
634
|
+
compression=compression.value,
|
|
635
|
+
limit=limit if limit != -1 else None,
|
|
636
|
+
verbose=verbose,
|
|
637
|
+
)
|
|
638
|
+
)
|
|
639
|
+
|
|
262
640
|
@staticmethod
|
|
263
641
|
def download_hierarchy_cmd(
|
|
264
642
|
ctx: typer.Context,
|
|
@@ -476,3 +854,293 @@ class DownloadApp(typer.Typer):
|
|
|
476
854
|
verbose=verbose,
|
|
477
855
|
)
|
|
478
856
|
)
|
|
857
|
+
|
|
858
|
+
@staticmethod
|
|
859
|
+
def download_datapoints_cmd(
|
|
860
|
+
dataset: Annotated[
|
|
861
|
+
str | None,
|
|
862
|
+
typer.Argument(
|
|
863
|
+
help="The dataset to download timeseries from. If not provided, an interactive selection will be made.",
|
|
864
|
+
),
|
|
865
|
+
] = None,
|
|
866
|
+
start_time: Annotated[
|
|
867
|
+
str | None,
|
|
868
|
+
typer.Option(
|
|
869
|
+
"--start-time",
|
|
870
|
+
"-s",
|
|
871
|
+
help="The start time for the datapoints to download. Can be in RFC3339 format or as a relative time (e.g., '1d-ago'). If not provided, all datapoints from the beginning will be downloaded.",
|
|
872
|
+
),
|
|
873
|
+
] = None,
|
|
874
|
+
end_time: Annotated[
|
|
875
|
+
str | None,
|
|
876
|
+
typer.Option(
|
|
877
|
+
"--end-time",
|
|
878
|
+
"-e",
|
|
879
|
+
help="The end time for the datapoints to download. Can be in RFC3339 format or as a relative time (e.g., '1d-ago'). If not provided, all datapoints up to the latest will be downloaded.",
|
|
880
|
+
),
|
|
881
|
+
] = None,
|
|
882
|
+
datapoint_type: Annotated[
|
|
883
|
+
DatapointsDataTypes,
|
|
884
|
+
typer.Option(
|
|
885
|
+
"--data-type",
|
|
886
|
+
"-d",
|
|
887
|
+
help="The type of datapoints to download.",
|
|
888
|
+
),
|
|
889
|
+
] = DatapointsDataTypes.numeric,
|
|
890
|
+
file_format: Annotated[
|
|
891
|
+
DatapointFormats,
|
|
892
|
+
typer.Option(
|
|
893
|
+
"--format",
|
|
894
|
+
"-f",
|
|
895
|
+
help="Format for downloading the datapoints.",
|
|
896
|
+
),
|
|
897
|
+
] = DatapointFormats.csv,
|
|
898
|
+
output_dir: Annotated[
|
|
899
|
+
Path,
|
|
900
|
+
typer.Option(
|
|
901
|
+
"--output-dir",
|
|
902
|
+
"-o",
|
|
903
|
+
help="Where to download the datapoints.",
|
|
904
|
+
allow_dash=True,
|
|
905
|
+
),
|
|
906
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
907
|
+
limit: Annotated[
|
|
908
|
+
int,
|
|
909
|
+
typer.Option(
|
|
910
|
+
"--limit",
|
|
911
|
+
"-l",
|
|
912
|
+
help="The maximum number of timeseries to download datapoints from. Use -1 to download all timeseries."
|
|
913
|
+
"The maximum number of datapoints in total is 10 million and 100 000 per timeseries.",
|
|
914
|
+
max=10_000_000,
|
|
915
|
+
),
|
|
916
|
+
] = 1000,
|
|
917
|
+
verbose: Annotated[
|
|
918
|
+
bool,
|
|
919
|
+
typer.Option(
|
|
920
|
+
"--verbose",
|
|
921
|
+
"-v",
|
|
922
|
+
help="Turn on to get more verbose output when running the command",
|
|
923
|
+
),
|
|
924
|
+
] = False,
|
|
925
|
+
) -> None:
|
|
926
|
+
"""This command will download Datapoints from CDF into a temporary ."""
|
|
927
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
928
|
+
if dataset is None:
|
|
929
|
+
interactive = TimeSeriesInteractiveSelect(client, "download datapoints")
|
|
930
|
+
dataset = interactive.select_data_set(allow_empty=False)
|
|
931
|
+
|
|
932
|
+
datapoint_type = questionary.select(
|
|
933
|
+
"Select the type of datapoints to download:",
|
|
934
|
+
choices=[Choice(title=dt.value, value=dt) for dt in DatapointsDataTypes],
|
|
935
|
+
default=datapoint_type,
|
|
936
|
+
).ask()
|
|
937
|
+
|
|
938
|
+
start_time = (
|
|
939
|
+
questionary.text(
|
|
940
|
+
"Enter the start time for the datapoints to download (RFC3339 format or relative time, e.g., '1d-ago'). Leave empty to download from the beginning.",
|
|
941
|
+
default=start_time or "",
|
|
942
|
+
).ask()
|
|
943
|
+
or None
|
|
944
|
+
)
|
|
945
|
+
end_time = (
|
|
946
|
+
questionary.text(
|
|
947
|
+
"Enter the end time for the datapoints to download (RFC3339 format or relative time, e.g., '1d-ago'). Leave empty to download up to the latest.",
|
|
948
|
+
default=end_time or "",
|
|
949
|
+
).ask()
|
|
950
|
+
or None
|
|
951
|
+
)
|
|
952
|
+
file_format = questionary.select(
|
|
953
|
+
"Select format to download the datapoints in:",
|
|
954
|
+
choices=[Choice(title=format_.value, value=format_) for format_ in DatapointFormats],
|
|
955
|
+
default=file_format,
|
|
956
|
+
).ask()
|
|
957
|
+
output_dir = Path(
|
|
958
|
+
questionary.path(
|
|
959
|
+
"Where to download the datapoints:", default=str(output_dir), only_directories=True
|
|
960
|
+
).ask()
|
|
961
|
+
)
|
|
962
|
+
while True:
|
|
963
|
+
limit_str = questionary.text(
|
|
964
|
+
"The maximum number of timeseries to download datapoints from. Use -1 to download all timeseries."
|
|
965
|
+
"The maximum number of datapoints in total is 10 million and 100 000 per timeseries.",
|
|
966
|
+
default=str(limit),
|
|
967
|
+
).ask()
|
|
968
|
+
if limit_str is None:
|
|
969
|
+
raise typer.Abort()
|
|
970
|
+
try:
|
|
971
|
+
limit = int(limit_str)
|
|
972
|
+
except ValueError:
|
|
973
|
+
print("[red]Please enter a valid integer for the limit.[/]")
|
|
974
|
+
else:
|
|
975
|
+
if limit != -1 and limit < 1:
|
|
976
|
+
print("[red]Please enter a valid integer greater than 0 or -1 for unlimited.[/]")
|
|
977
|
+
else:
|
|
978
|
+
break
|
|
979
|
+
verbose = questionary.confirm(
|
|
980
|
+
"Turn on to get more verbose output when running the command?", default=verbose
|
|
981
|
+
).ask()
|
|
982
|
+
|
|
983
|
+
cmd = DownloadCommand()
|
|
984
|
+
selector = DataPointsDataSetSelector(
|
|
985
|
+
data_set_external_id=dataset,
|
|
986
|
+
start=start_time,
|
|
987
|
+
end=end_time,
|
|
988
|
+
data_type=datapoint_type.value,
|
|
989
|
+
)
|
|
990
|
+
cmd.run(
|
|
991
|
+
lambda: cmd.download(
|
|
992
|
+
selectors=[selector],
|
|
993
|
+
io=DatapointsIO(client),
|
|
994
|
+
output_dir=output_dir,
|
|
995
|
+
file_format=f".{file_format.value}",
|
|
996
|
+
compression="none",
|
|
997
|
+
limit=limit if limit != -1 else None,
|
|
998
|
+
verbose=verbose,
|
|
999
|
+
)
|
|
1000
|
+
)
|
|
1001
|
+
|
|
1002
|
+
@staticmethod
|
|
1003
|
+
def download_charts_cmd(
|
|
1004
|
+
ctx: typer.Context,
|
|
1005
|
+
external_ids: Annotated[
|
|
1006
|
+
list[str] | None,
|
|
1007
|
+
typer.Argument(
|
|
1008
|
+
help="List of chart external IDs to download. If not provided, an interactive selection will be made.",
|
|
1009
|
+
),
|
|
1010
|
+
] = None,
|
|
1011
|
+
file_format: Annotated[
|
|
1012
|
+
ChartFormats,
|
|
1013
|
+
typer.Option(
|
|
1014
|
+
"--format",
|
|
1015
|
+
"-f",
|
|
1016
|
+
help="Format for downloading the charts.",
|
|
1017
|
+
),
|
|
1018
|
+
] = ChartFormats.ndjson,
|
|
1019
|
+
compression: Annotated[
|
|
1020
|
+
CompressionFormat,
|
|
1021
|
+
typer.Option(
|
|
1022
|
+
"--compression",
|
|
1023
|
+
"-z",
|
|
1024
|
+
help="Compression format to use when downloading the instances.",
|
|
1025
|
+
),
|
|
1026
|
+
] = CompressionFormat.none,
|
|
1027
|
+
output_dir: Annotated[
|
|
1028
|
+
Path,
|
|
1029
|
+
typer.Option(
|
|
1030
|
+
"--output-dir",
|
|
1031
|
+
"-o",
|
|
1032
|
+
help="Where to download the charts.",
|
|
1033
|
+
allow_dash=True,
|
|
1034
|
+
),
|
|
1035
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
1036
|
+
limit: Annotated[
|
|
1037
|
+
int,
|
|
1038
|
+
typer.Option(
|
|
1039
|
+
"--limit",
|
|
1040
|
+
"-l",
|
|
1041
|
+
help="The maximum number of charts to download. Use -1 to download all charts.",
|
|
1042
|
+
),
|
|
1043
|
+
] = 1000,
|
|
1044
|
+
verbose: Annotated[
|
|
1045
|
+
bool,
|
|
1046
|
+
typer.Option(
|
|
1047
|
+
"--verbose",
|
|
1048
|
+
"-v",
|
|
1049
|
+
help="Turn on to get more verbose output when running the command",
|
|
1050
|
+
),
|
|
1051
|
+
] = False,
|
|
1052
|
+
) -> None:
|
|
1053
|
+
"""This command will download Charts from CDF into a temporary directory."""
|
|
1054
|
+
cmd = DownloadCommand()
|
|
1055
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
1056
|
+
selector: ChartSelector
|
|
1057
|
+
if external_ids is None:
|
|
1058
|
+
selected_external_ids = InteractiveChartSelect(client).select_external_ids()
|
|
1059
|
+
selector = ChartExternalIdSelector(external_ids=tuple(selected_external_ids))
|
|
1060
|
+
else:
|
|
1061
|
+
selector = ChartExternalIdSelector(external_ids=tuple(external_ids))
|
|
1062
|
+
|
|
1063
|
+
cmd.run(
|
|
1064
|
+
lambda: cmd.download(
|
|
1065
|
+
selectors=[selector],
|
|
1066
|
+
io=ChartIO(client),
|
|
1067
|
+
output_dir=output_dir,
|
|
1068
|
+
file_format=f".{file_format.value}",
|
|
1069
|
+
compression=compression.value,
|
|
1070
|
+
limit=limit if limit != -1 else None,
|
|
1071
|
+
verbose=verbose,
|
|
1072
|
+
)
|
|
1073
|
+
)
|
|
1074
|
+
|
|
1075
|
+
@staticmethod
|
|
1076
|
+
def download_canvas_cmd(
|
|
1077
|
+
ctx: typer.Context,
|
|
1078
|
+
external_ids: Annotated[
|
|
1079
|
+
list[str] | None,
|
|
1080
|
+
typer.Argument(
|
|
1081
|
+
help="List of canvas external IDs to download. If not provided, an interactive selection will be made.",
|
|
1082
|
+
),
|
|
1083
|
+
] = None,
|
|
1084
|
+
file_format: Annotated[
|
|
1085
|
+
CanvasFormats,
|
|
1086
|
+
typer.Option(
|
|
1087
|
+
"--format",
|
|
1088
|
+
"-f",
|
|
1089
|
+
help="Format for downloading the canvas.",
|
|
1090
|
+
),
|
|
1091
|
+
] = CanvasFormats.ndjson,
|
|
1092
|
+
compression: Annotated[
|
|
1093
|
+
CompressionFormat,
|
|
1094
|
+
typer.Option(
|
|
1095
|
+
"--compression",
|
|
1096
|
+
"-z",
|
|
1097
|
+
help="Compression format to use when downloading the canvas.",
|
|
1098
|
+
),
|
|
1099
|
+
] = CompressionFormat.none,
|
|
1100
|
+
output_dir: Annotated[
|
|
1101
|
+
Path,
|
|
1102
|
+
typer.Option(
|
|
1103
|
+
"--output-dir",
|
|
1104
|
+
"-o",
|
|
1105
|
+
help="Where to download the canvas.",
|
|
1106
|
+
allow_dash=True,
|
|
1107
|
+
),
|
|
1108
|
+
] = DEFAULT_DOWNLOAD_DIR,
|
|
1109
|
+
limit: Annotated[
|
|
1110
|
+
int,
|
|
1111
|
+
typer.Option(
|
|
1112
|
+
"--limit",
|
|
1113
|
+
"-l",
|
|
1114
|
+
help="The maximum number of canvas to download. Use -1 to download all canvas.",
|
|
1115
|
+
),
|
|
1116
|
+
] = 1000,
|
|
1117
|
+
verbose: Annotated[
|
|
1118
|
+
bool,
|
|
1119
|
+
typer.Option(
|
|
1120
|
+
"--verbose",
|
|
1121
|
+
"-v",
|
|
1122
|
+
help="Turn on to get more verbose output when running the command",
|
|
1123
|
+
),
|
|
1124
|
+
] = False,
|
|
1125
|
+
) -> None:
|
|
1126
|
+
"""This command will download Canvas from CDF into a temporary directory."""
|
|
1127
|
+
cmd = DownloadCommand()
|
|
1128
|
+
client = EnvironmentVariables.create_from_environment().get_client()
|
|
1129
|
+
selector: CanvasSelector
|
|
1130
|
+
if external_ids is None:
|
|
1131
|
+
selected_external_ids = InteractiveCanvasSelect(client).select_external_ids()
|
|
1132
|
+
selector = CanvasExternalIdSelector(external_ids=tuple(selected_external_ids))
|
|
1133
|
+
else:
|
|
1134
|
+
selector = CanvasExternalIdSelector(external_ids=tuple(external_ids))
|
|
1135
|
+
|
|
1136
|
+
cmd.run(
|
|
1137
|
+
lambda: cmd.download(
|
|
1138
|
+
selectors=[selector],
|
|
1139
|
+
io=CanvasIO(client),
|
|
1140
|
+
output_dir=output_dir,
|
|
1141
|
+
file_format=f".{file_format.value}",
|
|
1142
|
+
compression=compression.value,
|
|
1143
|
+
limit=limit if limit != -1 else None,
|
|
1144
|
+
verbose=verbose,
|
|
1145
|
+
)
|
|
1146
|
+
)
|