fprime-gds 3.4.1__py3-none-any.whl → 3.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fprime_gds/common/encoders/seq_writer.py +2 -2
- fprime_gds/common/files/uplinker.py +4 -4
- fprime_gds/common/loaders/xml_loader.py +9 -0
- fprime_gds/common/pipeline/files.py +5 -4
- fprime_gds/common/pipeline/standard.py +23 -5
- fprime_gds/common/testing_fw/api.py +4 -3
- fprime_gds/executables/cli.py +19 -6
- fprime_gds/executables/comm.py +1 -1
- fprime_gds/executables/run_deployment.py +1 -1
- fprime_gds/executables/utils.py +11 -8
- fprime_gds/flask/app.py +4 -9
- fprime_gds/flask/default_settings.py +1 -12
- fprime_gds/flask/static/addons/commanding/command-history.js +23 -2
- fprime_gds/flask/updown.py +57 -6
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/METADATA +1 -1
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/RECORD +21 -22
- fprime_gds/flask/flask_uploads.py +0 -542
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/LICENSE.txt +0 -0
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/NOTICE.txt +0 -0
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/WHEEL +0 -0
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/entry_points.txt +0 -0
- {fprime_gds-3.4.1.dist-info → fprime_gds-3.4.2.dist-info}/top_level.txt +0 -0
@@ -137,9 +137,9 @@ class SeqBinaryWriter:
|
|
137
137
|
for cmd in seq_cmds_list:
|
138
138
|
sequence += self.__binaryCmdRecord(cmd)
|
139
139
|
size = len(sequence)
|
140
|
-
tb_txt =
|
140
|
+
tb_txt = "ANY" if self.__timebase == 0xFFFF else hex(self.__timebase)
|
141
141
|
|
142
|
-
print("Sequence is
|
142
|
+
print(f"Sequence is {size} bytes with timebase {tb_txt}")
|
143
143
|
|
144
144
|
header = b""
|
145
145
|
header += U32Type(
|
@@ -47,7 +47,9 @@ class UplinkQueue:
|
|
47
47
|
self.queue = queue.Queue()
|
48
48
|
self.__file_store = []
|
49
49
|
self.__exit = threading.Event()
|
50
|
-
self.__thread = threading.Thread(
|
50
|
+
self.__thread = threading.Thread(
|
51
|
+
target=self.run, name="UplinkerThread", args=()
|
52
|
+
)
|
51
53
|
self.__thread.start()
|
52
54
|
|
53
55
|
def enqueue(self, filepath, destination):
|
@@ -228,9 +230,7 @@ class FileUplinker(fprime_gds.common.handlers.DataHandler):
|
|
228
230
|
# Prevent multiple uplinks at once
|
229
231
|
if self.state != FileStates.IDLE:
|
230
232
|
msg = f"Currently uplinking file '{self.active.source}' cannot start uplinking '{file_obj.source}'"
|
231
|
-
raise FileUplinkerBusyException(
|
232
|
-
msg
|
233
|
-
)
|
233
|
+
raise FileUplinkerBusyException(msg)
|
234
234
|
self.state = FileStates.RUNNING
|
235
235
|
self.active = file_obj
|
236
236
|
self.active.open(TransmitFileState.READ)
|
@@ -61,6 +61,7 @@ class XmlLoader(dict_loader.DictLoader):
|
|
61
61
|
SER_MEMB_FMT_STR_TAG = "format_specifier"
|
62
62
|
SER_MEMB_DESC_TAG = "description"
|
63
63
|
SER_MEMB_TYPE_TAG = "type"
|
64
|
+
SER_MEMB_SIZE_TAG = "size"
|
64
65
|
|
65
66
|
# Xml section names and tags for array types
|
66
67
|
ARR_SECT = "arrays"
|
@@ -256,7 +257,15 @@ class XmlLoader(dict_loader.DictLoader):
|
|
256
257
|
fmt_str = memb.get(self.SER_MEMB_FMT_STR_TAG)
|
257
258
|
desc = memb.get(self.SER_MEMB_DESC_TAG)
|
258
259
|
memb_type_name = memb.get(self.SER_MEMB_TYPE_TAG)
|
260
|
+
memb_size = memb.get(self.SER_MEMB_SIZE_TAG)
|
259
261
|
type_obj = self.parse_type(memb_type_name, memb, xml_obj)
|
262
|
+
# memb_size is not None for member array
|
263
|
+
if(memb_size is not None):
|
264
|
+
type_obj = ArrayType.construct_type(
|
265
|
+
f"Array_{type_obj.__name__}_{memb_size}",
|
266
|
+
type_obj,
|
267
|
+
int(memb_size),
|
268
|
+
fmt_str)
|
260
269
|
|
261
270
|
members.append((name, type_obj, fmt_str, desc))
|
262
271
|
|
@@ -7,7 +7,7 @@ communications layer.
|
|
7
7
|
|
8
8
|
@author mstarch
|
9
9
|
"""
|
10
|
-
import
|
10
|
+
from pathlib import Path
|
11
11
|
import fprime_gds.common.files.downlinker
|
12
12
|
import fprime_gds.common.files.uplinker
|
13
13
|
|
@@ -43,10 +43,11 @@ class Filing:
|
|
43
43
|
)
|
44
44
|
file_decoder.register(self.__downlinker)
|
45
45
|
distributor.register("FW_PACKET_HAND", self.__uplinker)
|
46
|
-
|
46
|
+
try:
|
47
|
+
Path(down_store).mkdir(parents=True, exist_ok=True)
|
48
|
+
except PermissionError:
|
47
49
|
raise PermissionError(
|
48
|
-
f"{down_store} is not writable.
|
49
|
-
"Fix permissions or change storage directory with --file-storage-directory."
|
50
|
+
f"{down_store} is not writable. Fix permissions or change storage directory with --file-storage-directory."
|
50
51
|
)
|
51
52
|
|
52
53
|
@property
|
@@ -44,6 +44,8 @@ class StandardPipeline:
|
|
44
44
|
self.client_socket = None
|
45
45
|
self.logger = None
|
46
46
|
self.dictionary_path = None
|
47
|
+
self.up_store = None
|
48
|
+
self.down_store = None
|
47
49
|
|
48
50
|
self.__dictionaries = dictionaries.Dictionaries()
|
49
51
|
self.__coders = encoding.EncodingDecoding()
|
@@ -52,7 +54,7 @@ class StandardPipeline:
|
|
52
54
|
self.__transport_type = ThreadedTCPSocketClient
|
53
55
|
|
54
56
|
def setup(
|
55
|
-
self, config, dictionary,
|
57
|
+
self, config, dictionary, file_store, logging_prefix=None, packet_spec=None
|
56
58
|
):
|
57
59
|
"""
|
58
60
|
Setup the standard pipeline for moving data from the middleware layer through the GDS layers using the standard
|
@@ -60,11 +62,23 @@ class StandardPipeline:
|
|
60
62
|
|
61
63
|
:param config: config object used when constructing the pipeline.
|
62
64
|
:param dictionary: dictionary path. Used to setup loading of dictionaries.
|
63
|
-
:param
|
65
|
+
:param file_store: uplink/downlink storage directory
|
64
66
|
:param logging_prefix: logging prefix. Defaults to not logging at all.
|
65
67
|
:param packet_spec: location of packetized telemetry XML specification.
|
66
68
|
"""
|
67
|
-
assert
|
69
|
+
assert (
|
70
|
+
dictionary is not None and Path(dictionary).is_file()
|
71
|
+
), f"Dictionary {dictionary} does not exist"
|
72
|
+
# File storage configuration for uplink and downlink
|
73
|
+
self.up_store = Path(file_store) / "fprime-uplink"
|
74
|
+
self.down_store = Path(file_store) / "fprime-downlink"
|
75
|
+
try:
|
76
|
+
self.down_store.mkdir(parents=True, exist_ok=True)
|
77
|
+
self.up_store.mkdir(parents=True, exist_ok=True)
|
78
|
+
except PermissionError:
|
79
|
+
raise PermissionError(
|
80
|
+
f"{file_store} is not writable. Fix permissions or change storage directory with --file-storage-directory."
|
81
|
+
)
|
68
82
|
self.dictionary_path = Path(dictionary)
|
69
83
|
# Loads the distributor and client socket
|
70
84
|
self.distributor = fprime_gds.common.distributor.distributor.Distributor(config)
|
@@ -76,7 +90,7 @@ class StandardPipeline:
|
|
76
90
|
)
|
77
91
|
self.histories.setup_histories(self.coders)
|
78
92
|
self.files.setup_file_handling(
|
79
|
-
down_store,
|
93
|
+
self.down_store,
|
80
94
|
self.coders.file_encoder,
|
81
95
|
self.coders.file_decoder,
|
82
96
|
self.distributor,
|
@@ -152,7 +166,11 @@ class StandardPipeline:
|
|
152
166
|
outgoing_tag: this pipeline will produce data for supplied tag (FSW, GUI). Default: FSW
|
153
167
|
"""
|
154
168
|
# Backwards compatibility with the old method .connect(host, port)
|
155
|
-
if
|
169
|
+
if (
|
170
|
+
isinstance(incoming_tag, int)
|
171
|
+
and ":" not in connection_uri
|
172
|
+
and outgoing_tag == RoutingTag.FSW
|
173
|
+
):
|
156
174
|
connection_uri = f"{connection_uri}:{incoming_tag}"
|
157
175
|
incoming_tag = RoutingTag.GUI
|
158
176
|
self.client_socket.connect(connection_uri, incoming_tag, outgoing_tag)
|
@@ -402,7 +402,7 @@ class IntegrationTestAPI(DataHandler):
|
|
402
402
|
return self.await_event_sequence(events, start=start, timeout=timeout)
|
403
403
|
return self.await_event(events, start=start, timeout=timeout)
|
404
404
|
|
405
|
-
def send_and_assert_command(self, command, args=[], max_delay=None, timeout=5, events=None):
|
405
|
+
def send_and_assert_command(self, command, args=[], max_delay=None, timeout=5, events=None, commander="cmdDisp"):
|
406
406
|
"""
|
407
407
|
This helper will send a command and verify that the command was dispatched and completed
|
408
408
|
within the F' deployment. This helper can retroactively check that the delay between
|
@@ -414,12 +414,13 @@ class IntegrationTestAPI(DataHandler):
|
|
414
414
|
max_delay: the maximum allowable delay between dispatch and completion (int/float)
|
415
415
|
timeout: the number of seconds to wait before terminating the search (int)
|
416
416
|
events: extra event predicates to check between dispatch and complete
|
417
|
+
commander: the command dispatching component. Defaults to cmdDisp
|
417
418
|
Return:
|
418
419
|
returns a list of the EventData objects found by the search
|
419
420
|
"""
|
420
421
|
cmd_id = self.translate_command_name(command)
|
421
|
-
dispatch = [self.get_event_pred("
|
422
|
-
complete = [self.get_event_pred("
|
422
|
+
dispatch = [self.get_event_pred(f"{commander}.OpCodeDispatched", [cmd_id, None])]
|
423
|
+
complete = [self.get_event_pred(f"{commander}.OpCodeCompleted", [cmd_id])]
|
423
424
|
events = dispatch + (events if events else []) + complete
|
424
425
|
results = self.send_and_assert_event(command, args, events, timeout=timeout)
|
425
426
|
if max_delay is not None:
|
fprime_gds/executables/cli.py
CHANGED
@@ -547,18 +547,31 @@ class FileHandlingParser(ParserBase):
|
|
547
547
|
|
548
548
|
return {
|
549
549
|
("--file-storage-directory",): {
|
550
|
-
"dest": "
|
550
|
+
"dest": "files_storage_directory",
|
551
551
|
"action": "store",
|
552
|
-
"default": "/tmp/" + username
|
552
|
+
"default": "/tmp/" + username,
|
553
553
|
"required": False,
|
554
554
|
"type": str,
|
555
|
-
"help": "
|
556
|
-
}
|
555
|
+
"help": "Directory to store uplink and downlink files. Default: %(default)s",
|
556
|
+
},
|
557
|
+
("--remote-sequence-directory",): {
|
558
|
+
"dest": "remote_sequence_directory",
|
559
|
+
"action": "store",
|
560
|
+
"default": "/seq",
|
561
|
+
"required": False,
|
562
|
+
"type": str,
|
563
|
+
"help": "Directory to save command sequence binaries, on the remote FSW. Default: %(default)s",
|
564
|
+
},
|
557
565
|
}
|
558
566
|
|
559
567
|
def handle_arguments(self, args, **kwargs):
|
560
568
|
"""Handle arguments as parsed"""
|
561
|
-
|
569
|
+
try:
|
570
|
+
Path(args.files_storage_directory).mkdir(parents=True, exist_ok=True)
|
571
|
+
except PermissionError:
|
572
|
+
raise PermissionError(
|
573
|
+
f"{args.files_storage_directory} is not writable. Fix permissions or change storage directory with --file-storage-directory."
|
574
|
+
)
|
562
575
|
return args
|
563
576
|
|
564
577
|
|
@@ -584,7 +597,7 @@ class StandardPipelineParser(CompositeParser):
|
|
584
597
|
pipeline_arguments = {
|
585
598
|
"config": ConfigManager(),
|
586
599
|
"dictionary": args_ns.dictionary,
|
587
|
-
"
|
600
|
+
"file_store": args_ns.files_storage_directory,
|
588
601
|
"packet_spec": args_ns.packet_spec,
|
589
602
|
"logging_prefix": args_ns.logs,
|
590
603
|
}
|
fprime_gds/executables/comm.py
CHANGED
@@ -64,7 +64,7 @@ def main():
|
|
64
64
|
)
|
65
65
|
fprime_gds.common.communication.checksum = args.checksum_type
|
66
66
|
if args.comm_adapter == "none":
|
67
|
-
print("[ERROR] Comm adapter set to 'none'. Nothing to do but exit.")
|
67
|
+
print("[ERROR] Comm adapter set to 'none'. Nothing to do but exit.", file=sys.stderr)
|
68
68
|
sys.exit(-1)
|
69
69
|
|
70
70
|
# Create the handling components for either side of this script, adapter for hardware, and ground for the GDS side
|
@@ -168,7 +168,7 @@ def main():
|
|
168
168
|
if parsed_args.adapter == "ip":
|
169
169
|
launchers.append(launch_app)
|
170
170
|
else:
|
171
|
-
print("[WARNING] App cannot be auto-launched without IP adapter")
|
171
|
+
print("[WARNING] App cannot be auto-launched without IP adapter", file=sys.stderr)
|
172
172
|
|
173
173
|
# Launch the desired GUI package
|
174
174
|
if parsed_args.gui == "html":
|
fprime_gds/executables/utils.py
CHANGED
@@ -146,11 +146,12 @@ def get_artifacts_root() -> Path:
|
|
146
146
|
ini_settings = IniSettings.load(ini_file)
|
147
147
|
except FprimeLocationUnknownException:
|
148
148
|
print(
|
149
|
-
"[ERROR] Not in fprime project and no deployment path provided, unable to find dictionary and/or app"
|
149
|
+
"[ERROR] Not in fprime project and no deployment path provided, unable to find dictionary and/or app",
|
150
|
+
file=sys.stderr
|
150
151
|
)
|
151
152
|
sys.exit(-1)
|
152
153
|
except FprimeSettingsException as e:
|
153
|
-
print("[ERROR]", e)
|
154
|
+
print("[ERROR]", e, file=sys.stderr)
|
154
155
|
sys.exit(-1)
|
155
156
|
assert (
|
156
157
|
"install_destination" in ini_settings
|
@@ -165,17 +166,18 @@ def find_app(root: Path) -> Path:
|
|
165
166
|
bin_dir = root / "bin"
|
166
167
|
|
167
168
|
if not bin_dir.exists():
|
168
|
-
print(f"[ERROR] binary location {bin_dir} does not exist")
|
169
|
+
print(f"[ERROR] binary location {bin_dir} does not exist", file=sys.stderr)
|
169
170
|
sys.exit(-1)
|
170
171
|
|
171
172
|
files = [child for child in bin_dir.iterdir() if child.is_file()]
|
172
173
|
if not files:
|
173
|
-
print(f"[ERROR] App not found in {bin_dir}")
|
174
|
+
print(f"[ERROR] App not found in {bin_dir}", file=sys.stderr)
|
174
175
|
sys.exit(-1)
|
175
176
|
|
176
177
|
if len(files) > 1:
|
177
178
|
print(
|
178
|
-
f"[ERROR] Multiple app candidates in binary location {bin_dir}. Specify app manually with --app."
|
179
|
+
f"[ERROR] Multiple app candidates in binary location {bin_dir}. Specify app manually with --app.",
|
180
|
+
file=sys.stderr
|
179
181
|
)
|
180
182
|
sys.exit(-1)
|
181
183
|
|
@@ -186,7 +188,7 @@ def find_dict(root: Path) -> Path:
|
|
186
188
|
dict_dir = root / "dict"
|
187
189
|
|
188
190
|
if not dict_dir.exists():
|
189
|
-
print(f"[ERROR] dictionary location {dict_dir} does not exist")
|
191
|
+
print(f"[ERROR] dictionary location {dict_dir} does not exist", file=sys.stderr)
|
190
192
|
sys.exit(-1)
|
191
193
|
|
192
194
|
files = [
|
@@ -196,12 +198,13 @@ def find_dict(root: Path) -> Path:
|
|
196
198
|
]
|
197
199
|
|
198
200
|
if not files:
|
199
|
-
print(f"[ERROR] No xml dictionary found in dictionary location {dict_dir}")
|
201
|
+
print(f"[ERROR] No xml dictionary found in dictionary location {dict_dir}", file=sys.stderr)
|
200
202
|
sys.exit(-1)
|
201
203
|
|
202
204
|
if len(files) > 1:
|
203
205
|
print(
|
204
|
-
f"[ERROR] Multiple xml dictionaries found in dictionary location {dict_dir}. Specify dictionary manually with --dictionary."
|
206
|
+
f"[ERROR] Multiple xml dictionaries found in dictionary location {dict_dir}. Specify dictionary manually with --dictionary.",
|
207
|
+
file=sys.stderr
|
205
208
|
)
|
206
209
|
sys.exit(-1)
|
207
210
|
|
fprime_gds/flask/app.py
CHANGED
@@ -30,7 +30,6 @@ import fprime_gds.flask.sequence
|
|
30
30
|
import fprime_gds.flask.stats
|
31
31
|
import fprime_gds.flask.updown
|
32
32
|
from fprime_gds.executables.cli import ParserBase, StandardPipelineParser
|
33
|
-
from fprime_gds.flask import flask_uploads
|
34
33
|
|
35
34
|
from . import components
|
36
35
|
|
@@ -49,8 +48,7 @@ def construct_app():
|
|
49
48
|
2. Setup JSON encoding for Flask and flask_restful to handle F prime types natively
|
50
49
|
3. Setup standard pipeline used throughout the system
|
51
50
|
4. Create Restful API for registering flask items
|
52
|
-
5.
|
53
|
-
6. Register all restful endpoints
|
51
|
+
5. Register all restful endpoints
|
54
52
|
|
55
53
|
:return: setup app
|
56
54
|
"""
|
@@ -77,9 +75,6 @@ def construct_app():
|
|
77
75
|
|
78
76
|
# Restful API registration
|
79
77
|
api = fprime_gds.flask.errors.setup_error_handling(app)
|
80
|
-
# File upload configuration, 1 set for everything
|
81
|
-
uplink_set = flask_uploads.UploadSet("uplink", flask_uploads.ALL)
|
82
|
-
flask_uploads.configure_uploads(app, [uplink_set])
|
83
78
|
|
84
79
|
# Application routes
|
85
80
|
api.add_resource(
|
@@ -137,7 +132,7 @@ def construct_app():
|
|
137
132
|
api.add_resource(
|
138
133
|
fprime_gds.flask.updown.FileUploads,
|
139
134
|
"/upload/files",
|
140
|
-
resource_class_args=[pipeline.files.uplinker,
|
135
|
+
resource_class_args=[pipeline.files.uplinker, pipeline.up_store],
|
141
136
|
)
|
142
137
|
api.add_resource(
|
143
138
|
fprime_gds.flask.updown.FileDownload,
|
@@ -150,9 +145,9 @@ def construct_app():
|
|
150
145
|
"/sequence",
|
151
146
|
resource_class_args=[
|
152
147
|
args_ns.dictionary,
|
153
|
-
|
148
|
+
pipeline.up_store,
|
154
149
|
pipeline.files.uplinker,
|
155
|
-
|
150
|
+
args_ns.remote_sequence_directory,
|
156
151
|
],
|
157
152
|
)
|
158
153
|
api.add_resource(
|
@@ -9,23 +9,12 @@
|
|
9
9
|
#
|
10
10
|
####
|
11
11
|
import os
|
12
|
-
import getpass
|
13
|
-
|
14
|
-
# Select uploads directory and create it
|
15
|
-
username = getpass.getuser()
|
16
|
-
uplink_dir = os.environ.get("UP_FILES_DIR", "/tmp/" + username + "/fprime-uplink/")
|
17
|
-
DOWNLINK_DIR = os.environ.get("DOWN_FILES_DIR", "/tmp/" + username + "/fprime-downlink/")
|
18
12
|
|
19
13
|
STANDARD_PIPELINE_ARGUMENTS = os.environ.get("STANDARD_PIPELINE_ARGUMENTS").split("|")
|
20
14
|
|
21
15
|
SERVE_LOGS = os.environ.get("SERVE_LOGS", "YES") == "YES"
|
22
|
-
UPLOADED_UPLINK_DEST = uplink_dir
|
23
|
-
UPLOADS_DEFAULT_DEST = uplink_dir
|
24
|
-
REMOTE_SEQ_DIRECTORY = "/seq"
|
25
|
-
MAX_CONTENT_LENGTH = 32 * 1024 * 1024 # Max length of request is 32MiB
|
26
16
|
|
17
|
+
MAX_CONTENT_LENGTH = 32 * 1024 * 1024 # Max length of request is 32MiB
|
27
18
|
|
28
|
-
for directory in [UPLOADED_UPLINK_DEST, UPLOADS_DEFAULT_DEST, DOWNLINK_DIR]:
|
29
|
-
os.makedirs(directory, exist_ok=True)
|
30
19
|
|
31
20
|
# TODO: load real config
|
@@ -102,8 +102,29 @@ Vue.component("command-history", {
|
|
102
102
|
cmd.full_name = template.full_name;
|
103
103
|
// Can only set command if it is a child of a command input
|
104
104
|
if (this.$parent.selectCmd) {
|
105
|
-
|
105
|
+
// command-input expects an array of strings as arguments
|
106
|
+
this.$parent.selectCmd(cmd.full_name, this.preprocess_args(cmd.args));
|
107
|
+
}
|
108
|
+
},
|
109
|
+
/**
|
110
|
+
* Process the arguments for a command. If the argument is (or contains) a number, it
|
111
|
+
* is converted to a string. Other types that should be pre-processed can be added here.
|
112
|
+
*
|
113
|
+
* @param {*} args
|
114
|
+
* @returns args processed for command input (numbers converted to strings)
|
115
|
+
*/
|
116
|
+
preprocess_args(args) {
|
117
|
+
if (Array.isArray(args)) {
|
118
|
+
return args.map(el => this.preprocess_args(el));
|
119
|
+
} else if (typeof args === 'object' && args !== null) {
|
120
|
+
return Object.fromEntries(
|
121
|
+
Object.entries(args).map(([key, value]) => [key, this.preprocess_args(value)])
|
122
|
+
);
|
123
|
+
} else if (typeof args === 'number') {
|
124
|
+
return args.toString();
|
125
|
+
} else {
|
126
|
+
return args;
|
106
127
|
}
|
107
128
|
}
|
108
129
|
}
|
109
|
-
});
|
130
|
+
});
|
fprime_gds/flask/updown.py
CHANGED
@@ -11,6 +11,9 @@ import os
|
|
11
11
|
|
12
12
|
import flask
|
13
13
|
import flask_restful
|
14
|
+
from werkzeug.datastructures import FileStorage
|
15
|
+
from werkzeug.utils import secure_filename
|
16
|
+
from pathlib import Path
|
14
17
|
|
15
18
|
|
16
19
|
class Destination(flask_restful.Resource):
|
@@ -51,12 +54,12 @@ class FileUploads(flask_restful.Resource):
|
|
51
54
|
A data model for the current uplinking file set.
|
52
55
|
"""
|
53
56
|
|
54
|
-
def __init__(self, uplinker,
|
57
|
+
def __init__(self, uplinker, dest_dir):
|
55
58
|
"""
|
56
59
|
Constructor: setup the uplinker and argument parsing
|
57
60
|
"""
|
58
61
|
self.uplinker = uplinker
|
59
|
-
self.
|
62
|
+
self.dest_dir = dest_dir
|
60
63
|
self.parser = flask_restful.reqparse.RequestParser()
|
61
64
|
self.parser.add_argument(
|
62
65
|
"action", required=True, help="Action to take against files"
|
@@ -99,11 +102,9 @@ class FileUploads(flask_restful.Resource):
|
|
99
102
|
failed = []
|
100
103
|
for key, file in flask.request.files.items():
|
101
104
|
try:
|
102
|
-
filename = self.
|
105
|
+
filename = self.save(file)
|
103
106
|
flask.current_app.logger.info(f"Received file. Saved to: {filename}")
|
104
|
-
self.uplinker.enqueue(
|
105
|
-
os.path.join(self.uplink_set.config.destination, filename)
|
106
|
-
)
|
107
|
+
self.uplinker.enqueue(os.path.join(self.dest_dir, filename))
|
107
108
|
successful.append(key)
|
108
109
|
except Exception as exc:
|
109
110
|
flask.current_app.logger.warning(
|
@@ -112,6 +113,56 @@ class FileUploads(flask_restful.Resource):
|
|
112
113
|
failed.append(key)
|
113
114
|
return {"successful": successful, "failed": failed}
|
114
115
|
|
116
|
+
def save(self, file_storage: FileStorage):
|
117
|
+
"""
|
118
|
+
This saves a `werkzeug.FileStorage` into this upload set.
|
119
|
+
|
120
|
+
:param file_storage: The uploaded file to save.
|
121
|
+
"""
|
122
|
+
if not isinstance(file_storage, FileStorage):
|
123
|
+
raise TypeError("file_storage must be a werkzeug.FileStorage")
|
124
|
+
|
125
|
+
filename = Path(secure_filename(file_storage.filename)).name
|
126
|
+
dest_dir = Path(self.dest_dir)
|
127
|
+
|
128
|
+
try:
|
129
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
130
|
+
except PermissionError:
|
131
|
+
raise PermissionError(
|
132
|
+
f"{dest_dir} is not writable. Fix permissions or change storage directory with --file-storage-directory."
|
133
|
+
)
|
134
|
+
|
135
|
+
# resolve conflict may not be needed
|
136
|
+
if (dest_dir / filename).exists():
|
137
|
+
filename = self.resolve_conflict(dest_dir, filename)
|
138
|
+
|
139
|
+
target = dest_dir / filename
|
140
|
+
file_storage.save(str(target))
|
141
|
+
|
142
|
+
return filename
|
143
|
+
|
144
|
+
def resolve_conflict(self, target_folder: Path, filename: str):
|
145
|
+
"""
|
146
|
+
If a file with the selected name already exists in the target folder,
|
147
|
+
this method is called to resolve the conflict. It should return a new
|
148
|
+
filename for the file.
|
149
|
+
|
150
|
+
The default implementation splits the name and extension and adds a
|
151
|
+
suffix to the name consisting of an underscore and a number, and tries
|
152
|
+
that until it finds one that doesn't exist.
|
153
|
+
|
154
|
+
:param target_folder: The absolute path to the target.
|
155
|
+
:param filename: The file's original filename.
|
156
|
+
"""
|
157
|
+
path = Path(filename)
|
158
|
+
name, ext = path.stem, path.suffix
|
159
|
+
count = 0
|
160
|
+
while True:
|
161
|
+
count = count + 1
|
162
|
+
newname = f"{name}_{count}{ext}"
|
163
|
+
if not (Path(target_folder) / newname).exists():
|
164
|
+
return newname
|
165
|
+
|
115
166
|
|
116
167
|
class FileDownload(flask_restful.Resource):
|
117
168
|
""" """
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fprime-gds
|
3
|
-
Version: 3.4.
|
3
|
+
Version: 3.4.2
|
4
4
|
Summary: F Prime Flight Software Ground Data System layer
|
5
5
|
Author-email: Michael Starch <Michael.D.Starch@jpl.nasa.gov>, Thomas Boyer-Chammard <Thomas.Boyer.Chammard@jpl.nasa.gov>
|
6
6
|
License:
|
@@ -38,12 +38,12 @@ fprime_gds/common/encoders/encoder.py,sha256=xgFFCi-qKEKG7T5Qfo-qIadSiY0NSnfDgQU
|
|
38
38
|
fprime_gds/common/encoders/event_encoder.py,sha256=Thpa6aNHK_Q_mO_u3PlGohbR1oiqxCyPsWKdHq6GQVs,3121
|
39
39
|
fprime_gds/common/encoders/file_encoder.py,sha256=G9uUXQP-oD2eW_GJuGNBrN7xPafKFhmgKiNi-zvZz-g,3830
|
40
40
|
fprime_gds/common/encoders/pkt_encoder.py,sha256=JthlbMIumPcaGDoDx7h9vI4rDb8Hvenr9pffTL7XCtU,3098
|
41
|
-
fprime_gds/common/encoders/seq_writer.py,sha256=
|
41
|
+
fprime_gds/common/encoders/seq_writer.py,sha256=Zmj482qe_IQVx3EDmfXlPKKTJcEyDMRCWk28yc4C4-8,6863
|
42
42
|
fprime_gds/common/files/File Decoder Documentation.txt,sha256=a-VYHcUMqh7mBGY2fqvMf1nd3gdg3cLdPKwrulNKXjc,5314
|
43
43
|
fprime_gds/common/files/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
44
44
|
fprime_gds/common/files/downlinker.py,sha256=CZPfhH0J9-LNqW5Cv_ryicLTuoedLSWK8OPQmmQDZZY,7498
|
45
45
|
fprime_gds/common/files/helpers.py,sha256=sGaxcczXmZ5_soawT7x_eJ_cC2PZ6KOGBfusAV4QC_g,7219
|
46
|
-
fprime_gds/common/files/uplinker.py,sha256
|
46
|
+
fprime_gds/common/files/uplinker.py,sha256=lgqhlgeipBt3Arx-ohzK8vCdS54fKpv9Rg7SUTocUX8,13244
|
47
47
|
fprime_gds/common/gds_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
48
48
|
fprime_gds/common/gds_cli/base_commands.py,sha256=RTR183kLmWPO-WXdsvPi9x1Zt7y4OJjpCJpy89Uom_Y,9410
|
49
49
|
fprime_gds/common/gds_cli/channels.py,sha256=S8y0Mo2JbBFPvMzeW22HFVJ8p-UC-tzpYM7rXqIGWi4,2136
|
@@ -66,7 +66,7 @@ fprime_gds/common/loaders/event_py_loader.py,sha256=m4KlDl0mXn8ZQr-IfpUg0KaGIOJU
|
|
66
66
|
fprime_gds/common/loaders/event_xml_loader.py,sha256=DXyriGpGiuzG1a5Eyb8PLH3x7X1-hNxiIpY_yC3UPgQ,2790
|
67
67
|
fprime_gds/common/loaders/pkt_xml_loader.py,sha256=ZS4qchqQnIBx0Tw69ehP8yqm1g_uYSQzmnijR3FxqJg,4795
|
68
68
|
fprime_gds/common/loaders/python_loader.py,sha256=FUNQbFy75bpqvss1JDu2UWZBMrtnMpFegM6mcglh42I,4858
|
69
|
-
fprime_gds/common/loaders/xml_loader.py,sha256=
|
69
|
+
fprime_gds/common/loaders/xml_loader.py,sha256=inrVbfhPOHzziGdZgdR1wgspdXYXvh-I9zgf_knPZeo,13914
|
70
70
|
fprime_gds/common/logger/__init__.py,sha256=YBrr9An0fZbp4kvphRl8nLfolkdBqFAsSGzEZXQiH6g,1448
|
71
71
|
fprime_gds/common/logger/data_logger.py,sha256=VjfhTGO1gGw954xNhSc0_zpw8JexCho5f8BlXDEYkL4,2505
|
72
72
|
fprime_gds/common/logger/test_logger.py,sha256=wL8Lq49sVmxGRALgv-ei6AnXFh79qlHFehmKJ1A8X28,6475
|
@@ -80,10 +80,10 @@ fprime_gds/common/parsers/seq_file_parser.py,sha256=6DZrA0jmt8IqsutfK7pdLtYn4oVH
|
|
80
80
|
fprime_gds/common/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
81
81
|
fprime_gds/common/pipeline/dictionaries.py,sha256=yWt-Q4bFGbP7yQoZGZzqLFVDnCY371TQFxZj4hSYGH0,5576
|
82
82
|
fprime_gds/common/pipeline/encoding.py,sha256=rMCBoZOrnLctl4QNlbMro_QiCQ4sapWjtcoFGfvO-WM,6631
|
83
|
-
fprime_gds/common/pipeline/files.py,sha256=
|
83
|
+
fprime_gds/common/pipeline/files.py,sha256=J2zm0sucvImtmSnv0iUp5uTpvUO8nlmz2lUdMuMC5aM,2244
|
84
84
|
fprime_gds/common/pipeline/histories.py,sha256=P1TN6mFOe9f5cZ_a-vCDN9o94vM7ax9n6fQogfUCce0,3548
|
85
85
|
fprime_gds/common/pipeline/router.py,sha256=-P1wI0KXEh_snOzDaq8CjEoWuM_zRm8vUMR1T0oY9qQ,2327
|
86
|
-
fprime_gds/common/pipeline/standard.py,sha256=
|
86
|
+
fprime_gds/common/pipeline/standard.py,sha256=k0QyPkRxjTMGFC-6O25LDxBQnAa5ibe1hj9cjjEPzOU,9325
|
87
87
|
fprime_gds/common/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
88
88
|
fprime_gds/common/templates/ch_template.py,sha256=1MoDZsia0dI_CvnIttwyKLhbQhum35OcJnFc50Xohuo,3893
|
89
89
|
fprime_gds/common/templates/cmd_template.py,sha256=Bdkfgjb9Yqw7zaZyS8fR9ZUebrkFsRofF0g7xpKtkE4,5180
|
@@ -91,7 +91,7 @@ fprime_gds/common/templates/data_template.py,sha256=U87d8oC-BDTDuBRZbNnPkXy6rI_P
|
|
91
91
|
fprime_gds/common/templates/event_template.py,sha256=L0hkWB_kEMhTNodPUqBAev76SMmWT9EWdcqxaaQX9ZE,4062
|
92
92
|
fprime_gds/common/templates/pkt_template.py,sha256=5Wi6389m5j8w7JITBGfeUnw6CYE1-hjcVJ42NJmLDcE,1794
|
93
93
|
fprime_gds/common/testing_fw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
94
|
-
fprime_gds/common/testing_fw/api.py,sha256=
|
94
|
+
fprime_gds/common/testing_fw/api.py,sha256=6rdgh6M0NkRVy69EeDh4z343Bjxp2CtBLj0b83U2ka8,60566
|
95
95
|
fprime_gds/common/testing_fw/predicates.py,sha256=CsHsVs_EVXCLQLd2NVOvy8MxmUQVxLMr3i1ouEUqOtQ,18371
|
96
96
|
fprime_gds/common/testing_fw/pytest_integration.py,sha256=FjbbsfLyScb9w4zypcQgahmNMgmh4yXwryfy9h2eaBY,4667
|
97
97
|
fprime_gds/common/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -102,28 +102,27 @@ fprime_gds/common/utils/data_desc_type.py,sha256=9GV8hV5q1dDxdfF-1-Wty5MBrFd94Eb
|
|
102
102
|
fprime_gds/common/utils/event_severity.py,sha256=7qPXHrDaM_REJ7sKBUEJTZIE0D4qVnVajsPDUuHg7sI,300
|
103
103
|
fprime_gds/common/utils/string_util.py,sha256=jqut5Dd0EjvTHMci1mvs_8KQ1Nq-38xZofeaaSoiJEY,3985
|
104
104
|
fprime_gds/executables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
105
|
-
fprime_gds/executables/cli.py,sha256=
|
106
|
-
fprime_gds/executables/comm.py,sha256=
|
105
|
+
fprime_gds/executables/cli.py,sha256=F4qkckoYCoGlh5T_WByOj6CDI3wm_nBM1K57wzkXmtM,31127
|
106
|
+
fprime_gds/executables/comm.py,sha256=86CO8eczu5y2lsdPnPPFtK1pEUVLrSyU9smC09bON-Q,5382
|
107
107
|
fprime_gds/executables/fprime_cli.py,sha256=GvvuUQuoDGBrqQB867bDjUR3Kn5yPUckAY2rdfTa8jo,12432
|
108
|
-
fprime_gds/executables/run_deployment.py,sha256=
|
108
|
+
fprime_gds/executables/run_deployment.py,sha256=4M5646tH-LdjEgQGwAIJ3mf10pfivHlDBPrUwhLkJWI,6157
|
109
109
|
fprime_gds/executables/tcpserver.py,sha256=KspVpu5YIuiWKOk5E6UDMKvqXYrRB1j9aX8CkMxysfw,17555
|
110
|
-
fprime_gds/executables/utils.py,sha256=
|
110
|
+
fprime_gds/executables/utils.py,sha256=CTw2gMO3vguqra8V8AEJodY6zcX18zO4AyA-EKwThmM,7195
|
111
111
|
fprime_gds/flask/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
112
|
-
fprime_gds/flask/app.py,sha256=
|
112
|
+
fprime_gds/flask/app.py,sha256=kJDCziri_BwZWKUszkR7u3RaNG_FWRzDkdCPsVDAtYM,6720
|
113
113
|
fprime_gds/flask/channels.py,sha256=sOeL-UmWPh2hqYvqj81STpABLlPcjdPgkRwjd3Qx77k,735
|
114
114
|
fprime_gds/flask/commands.py,sha256=pizE0AQ2Id5xAMBucdAf93ARVinnBQD8y5afAp2i5oo,3636
|
115
115
|
fprime_gds/flask/components.py,sha256=a-eG8XJfSrqR8MIzIc9StwbNwxcBqkxYMEYq46S2Bmk,4176
|
116
|
-
fprime_gds/flask/default_settings.py,sha256=
|
116
|
+
fprime_gds/flask/default_settings.py,sha256=SkNfd5R4tv59rcmPiHERIZNIEmzXP3KJcJZektgtZCA,603
|
117
117
|
fprime_gds/flask/errors.py,sha256=yN3jDsJd30jL6aOIF-SqbVoesvReHqPvXlIt8qWB87M,2133
|
118
118
|
fprime_gds/flask/events.py,sha256=BO9OUUwNDnRuOz4ZC6nFHMR7sJJ9P2P0xUxDluGd218,732
|
119
|
-
fprime_gds/flask/flask_uploads.py,sha256=DW3v3U7Rw7k7PLlcG9Yc9RCtM4tpvKUm8OWEM6ZQQZ0,19778
|
120
119
|
fprime_gds/flask/json.py,sha256=PBljX3afJzyE_04DvZS4OEFOQW_ldVmfWiTYfxPZiGo,5888
|
121
120
|
fprime_gds/flask/logs.py,sha256=CzHkXtv7_UG2b1c_z_cGIw-jJT088Sb7DggEB3c9D8U,1571
|
122
121
|
fprime_gds/flask/requirements.txt,sha256=K6y8h0MJ66Zq9Pz2ZIR721wV0EX1mYDfom2gmBobxb4,20
|
123
122
|
fprime_gds/flask/resource.py,sha256=h_zYGODshaInGQi2EpfU5ScDsQCR1RwqS8f7DzyuIOI,4131
|
124
123
|
fprime_gds/flask/sequence.py,sha256=P4yhxiM4dQxwCcAo9dT4J6bogxkBhWapqLxeaHurBBA,3553
|
125
124
|
fprime_gds/flask/stats.py,sha256=i62envu9V6WpNsRD_mlhwzB_2dGUOCTf1XpyC5HApzg,1177
|
126
|
-
fprime_gds/flask/updown.py,sha256=
|
125
|
+
fprime_gds/flask/updown.py,sha256=7za_zgOwQKHgm-3W1OBZqBgDLCmj7c0ziFEduhEuqdU,6176
|
127
126
|
fprime_gds/flask/static/favicon.ico,sha256=pwqBRx64V_wyrrXgXsBXPWWUmwqanuWAfesMpFfV1E4,15406
|
128
127
|
fprime_gds/flask/static/index.html,sha256=7SPp-p3ywyBtElPWEAAjrBuh-PNr8LUtd1JXw6emcVU,26087
|
129
128
|
fprime_gds/flask/static/.idea/.gitignore,sha256=NHNmSnP0xfbzJRKbluJMetE8N_J3-dXL-arvLWCrYTk,38
|
@@ -151,7 +150,7 @@ fprime_gds/flask/static/addons/commanding/addon.js,sha256=d027BtN0LTSIWh1vtL1LId
|
|
151
150
|
fprime_gds/flask/static/addons/commanding/argument-templates.js,sha256=vx5FP1ZCFT2w_Rc5fIIpaXnOoEq6vhy3tjIbOddwX3w,4528
|
152
151
|
fprime_gds/flask/static/addons/commanding/arguments.js,sha256=_pcoHumf8qJhw-qfAzZ0_AU1SNAuXSRt55LHYG9t8p4,13304
|
153
152
|
fprime_gds/flask/static/addons/commanding/command-history-template.js,sha256=2ak2B9eio3PLq6Bnie8iEsQN3HDJYokl0usMMP1D6lE,753
|
154
|
-
fprime_gds/flask/static/addons/commanding/command-history.js,sha256=
|
153
|
+
fprime_gds/flask/static/addons/commanding/command-history.js,sha256=iVyMFP_GkqxiMAkK_U2JeTqo96Q-nTghO8eKo3NFXQ0,4816
|
155
154
|
fprime_gds/flask/static/addons/commanding/command-input-template.js,sha256=Z3fHmPTaAnXDhHMu07bRMBse6wjJSexAStgV9pSeh8Q,2959
|
156
155
|
fprime_gds/flask/static/addons/commanding/command-input.js,sha256=l2sSqZbhLjrLdMGDdm1YPNTR1JCpXhR79A8exfcwHco,8842
|
157
156
|
fprime_gds/flask/static/addons/commanding/command-string-template.js,sha256=7Mq4BPcAS57WoyF9aAMdKLtMFN4DLbQVTAQ8YUQEITI,743
|
@@ -219,10 +218,10 @@ fprime_gds/flask/static/third-party/webfonts/fa-solid-900.svg,sha256=lnTrG9VQRxe
|
|
219
218
|
fprime_gds/flask/static/third-party/webfonts/fa-solid-900.ttf,sha256=r2OXUD_O-9YTl2whrVweNymMGLvgfQltsDzNOvbgW6g,202744
|
220
219
|
fprime_gds/flask/static/third-party/webfonts/fa-solid-900.woff,sha256=P200iM9lN09vZ2wxU0CwrCvoMr1VJAyAlEjjbvm5YyY,101648
|
221
220
|
fprime_gds/flask/static/third-party/webfonts/fa-solid-900.woff2,sha256=mDS4KtJuKjdYPSJnahLdLrD-fIA1aiEU0NsaqLOJlTc,78268
|
222
|
-
fprime_gds-3.4.
|
223
|
-
fprime_gds-3.4.
|
224
|
-
fprime_gds-3.4.
|
225
|
-
fprime_gds-3.4.
|
226
|
-
fprime_gds-3.4.
|
227
|
-
fprime_gds-3.4.
|
228
|
-
fprime_gds-3.4.
|
221
|
+
fprime_gds-3.4.2.dist-info/LICENSE.txt,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
222
|
+
fprime_gds-3.4.2.dist-info/METADATA,sha256=WUBi5_aQB8mc7eqrif1-T3rFL6pYeQLv6K5VC_BcE58,24755
|
223
|
+
fprime_gds-3.4.2.dist-info/NOTICE.txt,sha256=vXjA_xRcQhd83Vfk5D_vXg5kOjnnXvLuMi5vFKDEVmg,1612
|
224
|
+
fprime_gds-3.4.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
225
|
+
fprime_gds-3.4.2.dist-info/entry_points.txt,sha256=UisSXL905z4YEjwd7c-I2o6ZKmOw1xDDdO1mN0VPu6c,271
|
226
|
+
fprime_gds-3.4.2.dist-info/top_level.txt,sha256=6vzFLIX6ANfavKaXFHDMSLFtS94a6FaAsIWhjgYuSNE,27
|
227
|
+
fprime_gds-3.4.2.dist-info/RECORD,,
|
@@ -1,542 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
"""
|
3
|
-
flaskext.uploads
|
4
|
-
================
|
5
|
-
This module provides upload support for Flask. The basic pattern is to set up
|
6
|
-
an `UploadSet` object and upload your files to it.
|
7
|
-
|
8
|
-
:copyright: 2010 Matthew "LeafStorm" Frazier
|
9
|
-
:license: MIT/X11, see LICENSE for details
|
10
|
-
|
11
|
-
Note: originally from https://github.com/maxcountryman/flask-uploads
|
12
|
-
"""
|
13
|
-
|
14
|
-
import sys
|
15
|
-
|
16
|
-
PY3 = sys.version_info[0] == 3
|
17
|
-
|
18
|
-
if PY3:
|
19
|
-
string_types = (str,)
|
20
|
-
else:
|
21
|
-
string_types = (basestring,)
|
22
|
-
|
23
|
-
import os.path
|
24
|
-
import posixpath
|
25
|
-
from itertools import chain # lgtm [py/unused-import]
|
26
|
-
|
27
|
-
from flask import Blueprint, abort, current_app, send_from_directory, url_for
|
28
|
-
from werkzeug.datastructures import FileStorage
|
29
|
-
from werkzeug.utils import secure_filename
|
30
|
-
|
31
|
-
# Extension presets
|
32
|
-
|
33
|
-
#: This just contains plain text files (.txt).
|
34
|
-
TEXT = ("txt",)
|
35
|
-
|
36
|
-
#: This contains various office document formats (.rtf, .odf, .ods, .gnumeric,
|
37
|
-
#: .abw, .doc, .docx, .xls, .xlsx and .pdf). Note that the macro-enabled versions
|
38
|
-
#: of Microsoft Office 2007 files are not included.
|
39
|
-
DOCUMENTS = tuple("rtf odf ods gnumeric abw doc docx xls xlsx pdf".split())
|
40
|
-
|
41
|
-
#: This contains basic image types that are viewable from most browsers (.jpg,
|
42
|
-
#: .jpe, .jpeg, .png, .gif, .svg, .bmp and .webp).
|
43
|
-
IMAGES = tuple("jpg jpe jpeg png gif svg bmp webp".split())
|
44
|
-
|
45
|
-
#: This contains audio file types (.wav, .mp3, .aac, .ogg, .oga, and .flac).
|
46
|
-
AUDIO = tuple("wav mp3 aac ogg oga flac".split())
|
47
|
-
|
48
|
-
#: This is for structured data files (.csv, .ini, .json, .plist, .xml, .yaml,
|
49
|
-
#: and .yml).
|
50
|
-
DATA = tuple("csv ini json plist xml yaml yml".split())
|
51
|
-
|
52
|
-
#: This contains various types of scripts (.js, .php, .pl, .py .rb, and .sh).
|
53
|
-
#: If your Web server has PHP installed and set to auto-run, you might want to
|
54
|
-
#: add ``php`` to the DENY setting.
|
55
|
-
SCRIPTS = tuple("js php pl py rb sh".split())
|
56
|
-
|
57
|
-
#: This contains archive and compression formats (.gz, .bz2, .zip, .tar,
|
58
|
-
#: .tgz, .txz, and .7z).
|
59
|
-
ARCHIVES = tuple("gz bz2 zip tar tgz txz 7z".split())
|
60
|
-
|
61
|
-
#: This contains nonexecutable source files - those which need to be
|
62
|
-
#: compiled or assembled to binaries to be used. They are generally safe to
|
63
|
-
#: accept, as without an existing RCE vulnerability, they cannot be compiled,
|
64
|
-
#: assembled, linked, or executed. Supports C, C++, Ada, Rust, Go (Golang),
|
65
|
-
#: FORTRAN, D, Java, C Sharp, F Sharp (compiled only), COBOL, Haskell, and
|
66
|
-
#: assembly.
|
67
|
-
SOURCE = tuple(
|
68
|
-
(
|
69
|
-
"c cpp c++ h hpp h++ cxx hxx hdl " # C/C++
|
70
|
-
+ "ada " # Ada
|
71
|
-
+ "rs " # Rust
|
72
|
-
+ "go " # Go
|
73
|
-
+ "f for f90 f95 f03 " # FORTRAN
|
74
|
-
+ "d dd di " # D
|
75
|
-
+ "java " # Java
|
76
|
-
+ "hs " # Haskell
|
77
|
-
+ "cs " # C Sharp
|
78
|
-
+ "fs " # F Sharp compiled source (NOT .fsx, which is interactive-ready)
|
79
|
-
+ "cbl cob " # COBOL
|
80
|
-
+ "asm s " # Assembly
|
81
|
-
).split()
|
82
|
-
)
|
83
|
-
|
84
|
-
#: This contains shared libraries and executable files (.so, .exe and .dll).
|
85
|
-
#: Most of the time, you will not want to allow this - it's better suited for
|
86
|
-
#: use with `AllExcept`.
|
87
|
-
EXECUTABLES = tuple("so exe dll".split())
|
88
|
-
|
89
|
-
#: The default allowed extensions - `TEXT`, `DOCUMENTS`, `DATA`, and `IMAGES`.
|
90
|
-
DEFAULTS = TEXT + DOCUMENTS + IMAGES + DATA
|
91
|
-
|
92
|
-
|
93
|
-
class UploadNotAllowed(Exception):
|
94
|
-
"""
|
95
|
-
This exception is raised if the upload was not allowed. You should catch
|
96
|
-
it in your view code and display an appropriate message to the user.
|
97
|
-
"""
|
98
|
-
|
99
|
-
|
100
|
-
def tuple_from(*iters):
|
101
|
-
return tuple(itertools.chain(*iters))
|
102
|
-
|
103
|
-
|
104
|
-
def extension(filename):
|
105
|
-
ext = os.path.splitext(filename)[1]
|
106
|
-
if ext == "":
|
107
|
-
# add non-ascii filename support
|
108
|
-
ext = os.path.splitext(filename)[0]
|
109
|
-
if ext.startswith("."):
|
110
|
-
# os.path.splitext retains . separator
|
111
|
-
ext = ext[1:]
|
112
|
-
return ext
|
113
|
-
|
114
|
-
|
115
|
-
def lowercase_ext(filename):
|
116
|
-
"""
|
117
|
-
This is a helper used by UploadSet.save to provide lowercase extensions for
|
118
|
-
all processed files, to compare with configured extensions in the same
|
119
|
-
case.
|
120
|
-
|
121
|
-
.. versionchanged:: 0.1.4
|
122
|
-
Filenames without extensions are no longer lowercased, only the
|
123
|
-
extension is returned in lowercase, if an extension exists.
|
124
|
-
|
125
|
-
:param filename: The filename to ensure has a lowercase extension.
|
126
|
-
"""
|
127
|
-
if "." in filename:
|
128
|
-
main, ext = os.path.splitext(filename)
|
129
|
-
return main + ext.lower()
|
130
|
-
# For consistency with os.path.splitext,
|
131
|
-
# do not treat a filename without an extension as an extension.
|
132
|
-
# That is, do not return filename.lower().
|
133
|
-
return filename
|
134
|
-
|
135
|
-
|
136
|
-
def addslash(url):
|
137
|
-
return url if url.endswith("/") else f"{url}/"
|
138
|
-
|
139
|
-
|
140
|
-
def patch_request_class(app, size=64 * 1024 * 1024):
|
141
|
-
"""
|
142
|
-
By default, Flask will accept uploads to an arbitrary size. While Werkzeug
|
143
|
-
switches uploads from memory to a temporary file when they hit 500 KiB,
|
144
|
-
it's still possible for someone to overload your disk space with a
|
145
|
-
gigantic file.
|
146
|
-
|
147
|
-
This patches the app's request class's
|
148
|
-
`~werkzeug.BaseRequest.max_content_length` attribute so that any upload
|
149
|
-
larger than the given size is rejected with an HTTP error.
|
150
|
-
|
151
|
-
.. note::
|
152
|
-
|
153
|
-
In Flask 0.6, you can do this by setting the `MAX_CONTENT_LENGTH`
|
154
|
-
setting, without patching the request class. To emulate this behavior,
|
155
|
-
you can pass `None` as the size (you must pass it explicitly). That is
|
156
|
-
the best way to call this function, as it won't break the Flask 0.6
|
157
|
-
functionality if it exists.
|
158
|
-
|
159
|
-
.. versionchanged:: 0.1.1
|
160
|
-
|
161
|
-
:param app: The app to patch the request class of.
|
162
|
-
:param size: The maximum size to accept, in bytes. The default is 64 MiB.
|
163
|
-
If it is `None`, the app's `MAX_CONTENT_LENGTH` configuration
|
164
|
-
setting will be used to patch.
|
165
|
-
"""
|
166
|
-
if size is None:
|
167
|
-
if isinstance(app.request_class.__dict__["max_content_length"], property):
|
168
|
-
return
|
169
|
-
size = app.config.get("MAX_CONTENT_LENGTH")
|
170
|
-
reqclass = app.request_class
|
171
|
-
patched = type(reqclass.__name__, (reqclass,), {"max_content_length": size})
|
172
|
-
app.request_class = patched
|
173
|
-
|
174
|
-
|
175
|
-
def config_for_set(uset, app, defaults=None):
|
176
|
-
"""
|
177
|
-
This is a helper function for `configure_uploads` that extracts the
|
178
|
-
configuration for a single set.
|
179
|
-
|
180
|
-
:param uset: The upload set.
|
181
|
-
:param app: The app to load the configuration from.
|
182
|
-
:param defaults: A dict with keys `url` and `dest` from the
|
183
|
-
`UPLOADS_DEFAULT_DEST` and `DEFAULT_UPLOADS_URL`
|
184
|
-
settings.
|
185
|
-
"""
|
186
|
-
config = app.config
|
187
|
-
prefix = f'UPLOADED_{uset.name.upper()}_'
|
188
|
-
using_defaults = False
|
189
|
-
if defaults is None:
|
190
|
-
defaults = dict(dest=None, url=None)
|
191
|
-
|
192
|
-
allow_extns = tuple(config.get(prefix + "ALLOW", ()))
|
193
|
-
deny_extns = tuple(config.get(prefix + "DENY", ()))
|
194
|
-
destination = config.get(prefix + "DEST")
|
195
|
-
base_url = config.get(prefix + "URL")
|
196
|
-
|
197
|
-
if destination is None:
|
198
|
-
# the upload set's destination wasn't given
|
199
|
-
if uset.default_dest:
|
200
|
-
# use the "default_dest" callable
|
201
|
-
destination = uset.default_dest(app)
|
202
|
-
if destination is None: # still
|
203
|
-
# use the default dest from the config
|
204
|
-
if defaults["dest"] is not None:
|
205
|
-
using_defaults = True
|
206
|
-
destination = os.path.join(defaults["dest"], uset.name)
|
207
|
-
else:
|
208
|
-
msg = f"no destination for set {uset.name}"
|
209
|
-
raise RuntimeError(msg)
|
210
|
-
|
211
|
-
if base_url is None and using_defaults and defaults["url"]:
|
212
|
-
base_url = addslash(defaults["url"]) + uset.name + "/"
|
213
|
-
|
214
|
-
return UploadConfiguration(destination, base_url, allow_extns, deny_extns)
|
215
|
-
|
216
|
-
|
217
|
-
def configure_uploads(app, upload_sets):
|
218
|
-
"""
|
219
|
-
Call this after the app has been configured. It will go through all the
|
220
|
-
upload sets, get their configuration, and store the configuration on the
|
221
|
-
app. It will also register the uploads module if it hasn't been set. This
|
222
|
-
can be called multiple times with different upload sets.
|
223
|
-
|
224
|
-
.. versionchanged:: 0.1.3
|
225
|
-
The uploads module/blueprint will only be registered if it is needed
|
226
|
-
to serve the upload sets.
|
227
|
-
|
228
|
-
:param app: The `~flask.Flask` instance to get the configuration from.
|
229
|
-
:param upload_sets: The `UploadSet` instances to configure.
|
230
|
-
"""
|
231
|
-
if isinstance(upload_sets, UploadSet):
|
232
|
-
upload_sets = (upload_sets,)
|
233
|
-
|
234
|
-
if not hasattr(app, "upload_set_config"):
|
235
|
-
app.upload_set_config = {}
|
236
|
-
set_config = app.upload_set_config
|
237
|
-
defaults = dict(
|
238
|
-
dest=app.config.get("UPLOADS_DEFAULT_DEST"),
|
239
|
-
url=app.config.get("UPLOADS_DEFAULT_URL"),
|
240
|
-
)
|
241
|
-
|
242
|
-
for uset in upload_sets:
|
243
|
-
config = config_for_set(uset, app, defaults)
|
244
|
-
set_config[uset.name] = config
|
245
|
-
|
246
|
-
should_serve = any(s.base_url is None for s in set_config.values())
|
247
|
-
if "_uploads" not in app.blueprints and should_serve:
|
248
|
-
app.register_blueprint(uploads_mod)
|
249
|
-
|
250
|
-
|
251
|
-
class All(object):
|
252
|
-
"""
|
253
|
-
This type can be used to allow all extensions. There is a predefined
|
254
|
-
instance named `ALL`.
|
255
|
-
"""
|
256
|
-
|
257
|
-
def __contains__(self, item):
|
258
|
-
return True
|
259
|
-
|
260
|
-
|
261
|
-
#: This "contains" all items. You can use it to allow all extensions to be
|
262
|
-
#: uploaded.
|
263
|
-
ALL = All()
|
264
|
-
|
265
|
-
|
266
|
-
class AllExcept(object):
|
267
|
-
"""
|
268
|
-
This can be used to allow all file types except certain ones. For example,
|
269
|
-
to ban .exe and .iso files, pass::
|
270
|
-
|
271
|
-
AllExcept(('exe', 'iso'))
|
272
|
-
|
273
|
-
to the `UploadSet` constructor as `extensions`. You can use any container,
|
274
|
-
for example::
|
275
|
-
|
276
|
-
AllExcept(SCRIPTS + EXECUTABLES)
|
277
|
-
"""
|
278
|
-
|
279
|
-
def __init__(self, items):
|
280
|
-
self.items = items
|
281
|
-
|
282
|
-
def __contains__(self, item):
|
283
|
-
return item not in self.items
|
284
|
-
|
285
|
-
|
286
|
-
class UploadConfiguration(object):
|
287
|
-
"""
|
288
|
-
This holds the configuration for a single `UploadSet`. The constructor's
|
289
|
-
arguments are also the attributes.
|
290
|
-
|
291
|
-
:param destination: The directory to save files to.
|
292
|
-
:param base_url: The URL (ending with a /) that files can be downloaded
|
293
|
-
from. If this is `None`, Flask-Uploads will serve the
|
294
|
-
files itself.
|
295
|
-
:param allow: A list of extensions to allow, even if they're not in the
|
296
|
-
`UploadSet` extensions list.
|
297
|
-
:param deny: A list of extensions to deny, even if they are in the
|
298
|
-
`UploadSet` extensions list.
|
299
|
-
"""
|
300
|
-
|
301
|
-
def __init__(self, destination, base_url=None, allow=(), deny=()):
|
302
|
-
self.destination = destination
|
303
|
-
self.base_url = base_url
|
304
|
-
self.allow = allow
|
305
|
-
self.deny = deny
|
306
|
-
|
307
|
-
@property
|
308
|
-
def tuple(self):
|
309
|
-
return (self.destination, self.base_url, self.allow, self.deny)
|
310
|
-
|
311
|
-
def __eq__(self, other):
|
312
|
-
return self.tuple == other.tuple
|
313
|
-
|
314
|
-
|
315
|
-
class UploadSet(object):
|
316
|
-
"""
|
317
|
-
This represents a single set of uploaded files. Each upload set is
|
318
|
-
independent of the others. This can be reused across multiple application
|
319
|
-
instances, as all configuration is stored on the application object itself
|
320
|
-
and found with `flask.current_app`.
|
321
|
-
|
322
|
-
:param name: The name of this upload set. It defaults to ``files``, but
|
323
|
-
you can pick any alphanumeric name you want. (For simplicity,
|
324
|
-
it's best to use a plural noun.)
|
325
|
-
:param extensions: The extensions to allow uploading in this set. The
|
326
|
-
easiest way to do this is to add together the extension
|
327
|
-
presets (for example, ``TEXT + DOCUMENTS + IMAGES``).
|
328
|
-
It can be overridden by the configuration with the
|
329
|
-
`UPLOADED_X_ALLOW` and `UPLOADED_X_DENY` configuration
|
330
|
-
parameters. The default is `DEFAULTS`.
|
331
|
-
:param default_dest: If given, this should be a callable. If you call it
|
332
|
-
with the app, it should return the default upload
|
333
|
-
destination path for that app.
|
334
|
-
"""
|
335
|
-
|
336
|
-
def __init__(self, name="files", extensions=DEFAULTS, default_dest=None):
|
337
|
-
if not name.isalnum():
|
338
|
-
raise ValueError("Name must be alphanumeric (no underscores)")
|
339
|
-
self.name = name
|
340
|
-
self.extensions = extensions
|
341
|
-
self._config = None
|
342
|
-
self.default_dest = default_dest
|
343
|
-
|
344
|
-
@property
|
345
|
-
def config(self):
|
346
|
-
"""
|
347
|
-
This gets the current configuration. By default, it looks up the
|
348
|
-
current application and gets the configuration from there. But if you
|
349
|
-
don't want to go to the full effort of setting an application, or it's
|
350
|
-
otherwise outside of a request context, set the `_config` attribute to
|
351
|
-
an `UploadConfiguration` instance, then set it back to `None` when
|
352
|
-
you're done.
|
353
|
-
"""
|
354
|
-
if self._config is not None:
|
355
|
-
return self._config
|
356
|
-
try:
|
357
|
-
return current_app.upload_set_config[self.name]
|
358
|
-
except AttributeError:
|
359
|
-
raise RuntimeError("cannot access configuration outside request")
|
360
|
-
|
361
|
-
def url(self, filename):
|
362
|
-
"""
|
363
|
-
This function gets the URL a file uploaded to this set would be
|
364
|
-
accessed at. It doesn't check whether said file exists.
|
365
|
-
|
366
|
-
:param filename: The filename to return the URL for.
|
367
|
-
"""
|
368
|
-
base = self.config.base_url
|
369
|
-
if base is None:
|
370
|
-
return url_for(
|
371
|
-
"_uploads.uploaded_file",
|
372
|
-
setname=self.name,
|
373
|
-
filename=filename,
|
374
|
-
_external=True,
|
375
|
-
)
|
376
|
-
return base + filename
|
377
|
-
|
378
|
-
def path(self, filename, folder=None):
|
379
|
-
"""
|
380
|
-
This returns the absolute path of a file uploaded to this set. It
|
381
|
-
doesn't actually check whether said file exists.
|
382
|
-
|
383
|
-
:param filename: The filename to return the path for.
|
384
|
-
:param folder: The subfolder within the upload set previously used
|
385
|
-
to save to.
|
386
|
-
"""
|
387
|
-
if folder is not None:
|
388
|
-
target_folder = os.path.join(self.config.destination, folder)
|
389
|
-
else:
|
390
|
-
target_folder = self.config.destination
|
391
|
-
return os.path.join(target_folder, filename)
|
392
|
-
|
393
|
-
def file_allowed(self, storage, basename):
|
394
|
-
"""
|
395
|
-
This tells whether a file is allowed. It should return `True` if the
|
396
|
-
given `werkzeug.FileStorage` object can be saved with the given
|
397
|
-
basename, and `False` if it can't. The default implementation just
|
398
|
-
checks the extension, so you can override this if you want.
|
399
|
-
|
400
|
-
:param storage: The `werkzeug.FileStorage` to check.
|
401
|
-
:param basename: The basename it will be saved under.
|
402
|
-
"""
|
403
|
-
return self.extension_allowed(extension(basename))
|
404
|
-
|
405
|
-
def extension_allowed(self, ext):
|
406
|
-
"""
|
407
|
-
This determines whether a specific extension is allowed. It is called
|
408
|
-
by `file_allowed`, so if you override that but still want to check
|
409
|
-
extensions, call back into this.
|
410
|
-
|
411
|
-
:param ext: The extension to check, without the dot.
|
412
|
-
"""
|
413
|
-
return (ext in self.config.allow) or (
|
414
|
-
ext in self.extensions and ext not in self.config.deny
|
415
|
-
)
|
416
|
-
|
417
|
-
def get_basename(self, filename):
|
418
|
-
return lowercase_ext(secure_filename(filename))
|
419
|
-
|
420
|
-
def save(self, storage, folder=None, name=None):
|
421
|
-
"""
|
422
|
-
This saves a `werkzeug.FileStorage` into this upload set. If the
|
423
|
-
upload is not allowed, an `UploadNotAllowed` error will be raised.
|
424
|
-
Otherwise, the file will be saved and its name (including the folder)
|
425
|
-
will be returned.
|
426
|
-
|
427
|
-
:param storage: The uploaded file to save.
|
428
|
-
:param folder: The subfolder within the upload set to save to.
|
429
|
-
:param name: The name to save the file as. If it ends with a dot, the
|
430
|
-
file's extension will be appended to the end. (If you
|
431
|
-
are using `name`, you can include the folder in the
|
432
|
-
`name` instead of explicitly using `folder`, i.e.
|
433
|
-
``uset.save(file, name="someguy/photo_123.")``
|
434
|
-
"""
|
435
|
-
if not isinstance(storage, FileStorage):
|
436
|
-
raise TypeError("storage must be a werkzeug.FileStorage")
|
437
|
-
|
438
|
-
if folder is None and name is not None and "/" in name:
|
439
|
-
folder, name = os.path.split(name)
|
440
|
-
|
441
|
-
basename = self.get_basename(storage.filename)
|
442
|
-
|
443
|
-
if not self.file_allowed(storage, basename):
|
444
|
-
raise UploadNotAllowed()
|
445
|
-
|
446
|
-
if name:
|
447
|
-
basename = name + extension(basename) if name.endswith(".") else name
|
448
|
-
|
449
|
-
if folder:
|
450
|
-
target_folder = os.path.join(self.config.destination, folder)
|
451
|
-
else:
|
452
|
-
target_folder = self.config.destination
|
453
|
-
if not os.path.exists(target_folder):
|
454
|
-
os.makedirs(target_folder)
|
455
|
-
if os.path.exists(os.path.join(target_folder, basename)):
|
456
|
-
basename = self.resolve_conflict(target_folder, basename)
|
457
|
-
|
458
|
-
target = os.path.join(target_folder, basename)
|
459
|
-
storage.save(target)
|
460
|
-
return posixpath.join(folder, basename) if folder else basename
|
461
|
-
|
462
|
-
def resolve_conflict(self, target_folder, basename):
|
463
|
-
"""
|
464
|
-
If a file with the selected name already exists in the target folder,
|
465
|
-
this method is called to resolve the conflict. It should return a new
|
466
|
-
basename for the file.
|
467
|
-
|
468
|
-
The default implementation splits the name and extension and adds a
|
469
|
-
suffix to the name consisting of an underscore and a number, and tries
|
470
|
-
that until it finds one that doesn't exist.
|
471
|
-
|
472
|
-
:param target_folder: The absolute path to the target.
|
473
|
-
:param basename: The file's original basename.
|
474
|
-
"""
|
475
|
-
name, ext = os.path.splitext(basename)
|
476
|
-
count = 0
|
477
|
-
while True:
|
478
|
-
count = count + 1
|
479
|
-
newname = "%s_%d%s" % (name, count, ext)
|
480
|
-
if not os.path.exists(os.path.join(target_folder, newname)):
|
481
|
-
return newname
|
482
|
-
|
483
|
-
|
484
|
-
uploads_mod = Blueprint("_uploads", __name__, url_prefix="/_uploads")
|
485
|
-
|
486
|
-
|
487
|
-
@uploads_mod.route("/<setname>/<path:filename>")
|
488
|
-
def uploaded_file(setname, filename):
|
489
|
-
config = current_app.upload_set_config.get(setname)
|
490
|
-
if config is None:
|
491
|
-
abort(404)
|
492
|
-
return send_from_directory(config.destination, filename)
|
493
|
-
|
494
|
-
|
495
|
-
class TestingFileStorage(FileStorage):
|
496
|
-
"""
|
497
|
-
This is a helper for testing upload behavior in your application. You
|
498
|
-
can manually create it, and its save method is overloaded to set `saved`
|
499
|
-
to the name of the file it was saved to. All of these parameters are
|
500
|
-
optional, so only bother setting the ones relevant to your application.
|
501
|
-
|
502
|
-
:param stream: A stream. The default is an empty stream.
|
503
|
-
:param filename: The filename uploaded from the client. The default is the
|
504
|
-
stream's name.
|
505
|
-
:param name: The name of the form field it was loaded from. The default is
|
506
|
-
`None`.
|
507
|
-
:param content_type: The content type it was uploaded as. The default is
|
508
|
-
``application/octet-stream``.
|
509
|
-
:param content_length: How long it is. The default is -1.
|
510
|
-
:param headers: Multipart headers as a `werkzeug.Headers`. The default is
|
511
|
-
`None`.
|
512
|
-
"""
|
513
|
-
|
514
|
-
def __init__(
|
515
|
-
self,
|
516
|
-
stream=None,
|
517
|
-
filename=None,
|
518
|
-
name=None,
|
519
|
-
content_type="application/octet-stream",
|
520
|
-
content_length=-1,
|
521
|
-
headers=None,
|
522
|
-
):
|
523
|
-
FileStorage.__init__(
|
524
|
-
self,
|
525
|
-
stream,
|
526
|
-
filename,
|
527
|
-
name=name,
|
528
|
-
content_type=content_type,
|
529
|
-
content_length=content_length,
|
530
|
-
headers=None,
|
531
|
-
)
|
532
|
-
self.saved = None
|
533
|
-
|
534
|
-
def save(self, dst, buffer_size=16384):
|
535
|
-
"""
|
536
|
-
This marks the file as saved by setting the `saved` attribute to the
|
537
|
-
name of the file it was saved to.
|
538
|
-
|
539
|
-
:param dst: The file to save to.
|
540
|
-
:param buffer_size: Ignored.
|
541
|
-
"""
|
542
|
-
self.saved = dst if isinstance(dst, string_types) else dst.name
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|