aind-data-transfer-service 1.16.0__py3-none-any.whl → 1.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ aind_data_transfer_service/__init__.py,sha256=cCMYl-lxHhXlEMAHdJ7INgHPd0O6ETEp9wsqf6DT8TQ,272
2
+ aind_data_transfer_service/log_handler.py,sha256=c7a-gLmZeRpeCUBwCz6XsTszWXQeQdR7eKZtas4llXM,1700
3
+ aind_data_transfer_service/server.py,sha256=CQdOkX86qn8RQm36NqS5b0aUyNCjvaEbTIObaza6RY0,48910
4
+ aind_data_transfer_service/configs/__init__.py,sha256=9W5GTuso9Is1B9X16RXcdb_GxasZvj6qDzOBDv0AbTc,36
5
+ aind_data_transfer_service/configs/csv_handler.py,sha256=hCdfAYZW_49-l1rbua5On2Tw2ks674Z-MgB_NJlIkU4,5746
6
+ aind_data_transfer_service/configs/job_configs.py,sha256=T-h5N6lyY9xTZ_xg_5FxkyYuMdagApbE6xalxFQ-bqA,18848
7
+ aind_data_transfer_service/configs/job_upload_template.py,sha256=aC5m1uD_YcpbggFQ-yZ7ZJSUUGX1yQqQLF3SwmljrLk,5127
8
+ aind_data_transfer_service/hpc/__init__.py,sha256=YNc68YNlmXwKIPFMIViz_K4XzVVHkLPEBOFyO5DKMKI,53
9
+ aind_data_transfer_service/hpc/client.py,sha256=-WwNjpV2FAJMiXT3Y_JCllSbXPyvqrkPxpNWE0BAWzY,5084
10
+ aind_data_transfer_service/hpc/models.py,sha256=-7HhV16s_MUyKPy0x0FGIbnq8DPL2qJAzJO5G7003AE,16184
11
+ aind_data_transfer_service/models/__init__.py,sha256=Meym73bEZ9nQr4QoeyhQmV3nRTYtd_4kWKPNygsBfJg,25
12
+ aind_data_transfer_service/models/core.py,sha256=uXtPUqjxKalg-sE8MxaJr11w_T_KKBRBSJuUgwoMZlQ,10135
13
+ aind_data_transfer_service/models/internal.py,sha256=tWO1yRMu9hHLv0mt7QhOwTPWGbKApWmCf1wwajx5qjI,10681
14
+ aind_data_transfer_service-1.17.2.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
15
+ aind_data_transfer_service-1.17.2.dist-info/METADATA,sha256=DotVNzRhHX0woD5qMefPipnYAja4EbXje8luub37g9I,2478
16
+ aind_data_transfer_service-1.17.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
17
+ aind_data_transfer_service-1.17.2.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
18
+ aind_data_transfer_service-1.17.2.dist-info/RECORD,,
@@ -1,36 +0,0 @@
1
- <!DOCTYPE html>
2
- <html>
3
- <head>
4
- <meta charset="UTF-8">
5
- <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
6
- <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
7
- <title>{% block title %} {% endblock %} AIND Data Transfer Service Admin</title>
8
- <style>
9
- body {
10
- margin: 20px;
11
- font-family: arial, sans-serif;
12
- }
13
- nav {
14
- height: 40px;
15
- }
16
- </style>
17
- </head>
18
- <body>
19
- <nav>
20
- <a href="/">Submit Jobs</a> |
21
- <a href="/jobs">Job Status</a> |
22
- <a href="/job_params">Job Parameters</a> |
23
- <a title="Download job template as .xslx" href="/api/job_upload_template" download>Job Submit Template</a> |
24
- <a title="List of project names" href="{{ project_names_url }}" target="_blank">Project Names</a> |
25
- <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"
26
- target="_blank">Help</a> |
27
- <a href="/admin">Admin</a> |
28
- <a href="/logout">Log out</a>
29
- </nav>
30
- <div>
31
- <h3>Admin</h3>
32
- <div>Hello {{user_name}}, welcome to the admin page</div>
33
- <div>Email: {{user_email}}</div>
34
- </div>
35
- </body>
36
- </html>
@@ -1,258 +0,0 @@
1
- <!DOCTYPE html>
2
- <html>
3
- <head>
4
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.4/jquery.min.js"></script>
5
- <meta charset="UTF-8">
6
- <title>{% block title %} {% endblock %} AIND Data Transfer Service</title>
7
- <style>
8
- body {
9
- margin: 20px;
10
- font-family: arial, sans-serif;
11
- }
12
- fieldset {
13
- display:inline
14
- }
15
- table {
16
- border-collapse: collapse;
17
- width: 75%;
18
- }
19
- td, th {
20
- border: 1px solid #dddddd;
21
- text-align: left;
22
- padding: 8px;
23
- }
24
- th, tr:hover {
25
- background-color: #E8EAF6;
26
- color: #3F51B5;
27
- }
28
- #submit {
29
- border: none;
30
- border-radius: 6px;
31
- color: #ffffff;
32
- padding: 8px;
33
- width: 80px;
34
- font-size: medium;
35
- background-color: #5065df;
36
- &:hover { background-color: #3f51b5; }
37
- }
38
- #message {
39
- margin-top: 20px;
40
- &.success { color: green; }
41
- &.error { color: red; }
42
- }
43
- </style>
44
- </head>
45
- <body>
46
- <nav>
47
- <a href="/">Submit Jobs</a> |
48
- <a href="/jobs">Job Status</a> |
49
- <a href="/job_params">Job Parameters</a> |
50
- <a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
51
- <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
52
- <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io" target="_blank" >Help</a> |
53
- <a href="/admin">Admin</a>
54
- </nav>
55
- <br>
56
- <div>
57
- <p1>For instructions on how to submit jobs click on the Help tab</p1>
58
- </div>
59
- <h2>Submit Jobs</h2>
60
- <div>
61
- <fieldset>
62
- <legend>Mail Notifications (optional)</legend><br>
63
- <div>
64
- <label for="email" title="Optionally provide an allen institute email address to receive upload job status notifications">Allen Institute email:</label>
65
- <input type="email" id="email" pattern=".+@alleninstitute\.org" size="30" placeholder="@alleninstitute.org" /><br><br>
66
- </div>
67
- <div>
68
- <input type="checkbox" id="begin" name="begin" />
69
- <label for="begin">BEGIN</label> |
70
- <input type="checkbox" id="end" name="end" />
71
- <label for="end">END</label> |
72
- <input type="checkbox" id="fail" name="fail" checked />
73
- <label for="fail">FAIL</label> |
74
- <input type="checkbox" id="retry" name="retry" />
75
- <label for="retry">RETRY</label> |
76
- <input type="checkbox" id="all" name="all" />
77
- <label for="all">ALL</label>
78
- </div>
79
- </fieldset>
80
- </div><br><br>
81
- <form id="file_form" method="post" enctype="multipart/form-data">
82
- <label for="file">Please select a .csv or .xlsx file:</label>
83
- <input type="file" id="file" name="file" accept=".csv,.xlsx" onchange="validateJobs(this)" onclick="this.value=null"><br><br>
84
- </form>
85
- <button id="submit" type="button" onclick="submitJobs()">Submit</button>
86
- <div id="message"></div><br>
87
- <div id="response"></div>
88
- <script>
89
- var jobs = []
90
- var parsing_errors = []
91
- const msgTypes = {
92
- "validatePending": "Validating...",
93
- "validateSuccess": "Successfully validated jobs from file.",
94
- "validateError": "Error validating jobs from file.",
95
- "submitPending": "Submitting jobs. Please do not refresh or re-submit...",
96
- "submitSuccess": "Successfully submitted jobs.",
97
- "submitError": "Error submitting jobs."
98
- }
99
- setMessage = function(msgType) {
100
- $("#message").removeClass("success");
101
- $("#message").removeClass("error");
102
- $("#message").html(msgType);
103
- if (msgType == msgTypes.validateSuccess || msgType == msgTypes.submitSuccess) {
104
- $("#message").addClass("success");
105
- } else if (msgType == msgTypes.validateError || msgType == msgTypes.submitError) {
106
- $("#message").addClass("error");
107
- }
108
- };
109
- addTableRow = function(data, table, tr, td, isHeader) {
110
- tr = document.createElement('tr');
111
- for (var d of data) {
112
- td = document.createElement(isHeader ? 'th' : 'td');
113
- td.innerHTML = d.value ?? d;
114
- if (d.rowspan) {
115
- td.setAttribute("rowspan", d.rowspan);
116
- }
117
- tr.append(td);
118
- }
119
- table.appendChild(tr);
120
- };
121
- validateJobs = function(fileElement) {
122
- if (fileElement.files.length != 1) {
123
- // File attach was cancelled by user
124
- return;
125
- }
126
- if (![".csv", ".xlsx"].some(ext => fileElement.files[0].name.endsWith(ext))) {
127
- fileElement.value = null;
128
- alert("Invalid file type. Please attach a .csv or .xlsx file.");
129
- return;
130
- }
131
- var formData = new FormData(document.getElementById("file_form"));
132
- $.ajax({
133
- url: "/api/v2/validate_csv",
134
- type: "POST",
135
- data: formData,
136
- cache: false,
137
- contentType: false,
138
- processData: false,
139
- beforeSend: function() {
140
- setMessage(msgTypes.validatePending);
141
- $("#response").html("");
142
- },
143
- success: function(data) {
144
- setMessage(msgTypes.validateSuccess);
145
- jobs = data["data"]["jobs"];
146
- parsing_errors = []
147
- let jobsLength = jobs.length;
148
- var table = document.createElement('table'), tr, td, row;
149
- addTableRow(
150
- [ "job_type", "project_name", "s3_bucket", "platform", "subject_id", "acq_datetime", "metadata_dir", "modality", "modality.input_source" ],
151
- table, tr, td, true
152
- );
153
- for (row = 0; row < jobsLength; row++) {
154
- let job = jobs[row];
155
- let modalities = job.tasks?.modality_transformation_settings;
156
- if (modalities) {
157
- modalities = Object.entries(modalities).map(([key, value]) => ({
158
- abbreviation: key,
159
- input_source: value.job_settings?.input_source
160
- }))
161
- }
162
- let modalitiesLength = modalities ? modalities.length: 0;
163
- let metadata_dir = job.tasks?.gather_preliminary_metadata?.job_settings?.metadata_dir
164
- addTableRow(
165
- [
166
- { value: job.job_type, rowspan: modalitiesLength },
167
- { value: job.project_name, rowspan: modalitiesLength },
168
- { value: job.s3_bucket, rowspan: modalitiesLength },
169
- { value: job.platform.abbreviation, rowspan: modalitiesLength },
170
- { value: job.subject_id, rowspan: modalitiesLength },
171
- { value: job.acq_datetime, rowspan: modalitiesLength },
172
- { value: metadata_dir ?? "", rowspan: modalitiesLength },
173
- modalities ? modalities[0].abbreviation : "",
174
- modalities ? modalities[0].input_source : ""
175
- ], table, tr, td, false
176
- );
177
- for (mRow = 1; mRow < modalitiesLength; mRow++) {
178
- let modality = modalities[mRow]
179
- addTableRow(
180
- [ modality.abbreviation, modality.input_source ],
181
- table, tr, td, false
182
- );
183
- }
184
- }
185
- $("#response").html(table);
186
- },
187
- error: function(data) {
188
- jobs = []
189
- parsing_errors = data.responseJSON["data"]["errors"]
190
- setMessage(msgTypes.validateError);
191
- $("#response").html(parsing_errors.map((err) => {
192
- return `<li>${err}</li>`
193
- }));
194
- }
195
- });
196
- };
197
- submitJobs = function() {
198
- if(jobs.length > 0 && parsing_errors.length == 0){
199
- let job_settings = {};
200
- let mail_user = $("#email").val();
201
- if (mail_user !== "" && mail_user !== undefined) {
202
- job_settings["user_email"] = mail_user;
203
- };
204
- let mail_type = [];
205
- if ($("#all").is(":checked")) {
206
- mail_type = ["all"];
207
- };
208
- if ($("#begin").is(":checked")) {
209
- mail_type.push("begin");
210
- };
211
- if ($("#end").is(":checked")) {
212
- mail_type.push("end");
213
- };
214
- if ($("#fail").is(":checked")) {
215
- mail_type.push("fail");
216
- };
217
- if ($("#retry").is(":checked")) {
218
- mail_type.push("retry");
219
- };
220
- job_settings["email_notification_types"] = mail_type
221
- job_settings["upload_jobs"] = jobs
222
- $.ajax({
223
- url: "/api/v2/submit_jobs",
224
- type: "POST",
225
- data: JSON.stringify(job_settings),
226
- contentType: 'application/json; charset=utf-8',
227
- beforeSend: function() {
228
- setMessage(msgTypes.submitPending);
229
- $("#response").html("");
230
- },
231
- success: function (data) {
232
- jobs = []
233
- parsing_errors = []
234
- setMessage(msgTypes.submitSuccess);
235
- $("#response").html(data);
236
- },
237
- error: function(data) {
238
- jobs = []
239
- setMessage(msgTypes.submitError);
240
- let errors = data.responseJSON["data"]["errors"];
241
- try {
242
- parsing_errors = JSON.parse(errors).map((err) => JSON.stringify(err));
243
- } catch (e) {
244
- parsing_errors = (typeof errors == "string") ? [errors] : errors;
245
- }
246
- $("#response").html(parsing_errors.map((errStr) => {
247
- return `<li>${errStr}</li>`
248
- }));
249
- }
250
- });
251
- } else if (jobs.length == 0) {
252
- alert("No valid jobs to submit. Please attach a .csv or .xlsx file with valid jobs.");
253
- return;
254
- }
255
- };
256
- </script>
257
- </body>
258
- </html>
@@ -1,195 +0,0 @@
1
- <!DOCTYPE html>
2
- <html>
3
- <head>
4
- <meta charset="UTF-8">
5
- <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
6
- <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-icons/1.8.1/font/bootstrap-icons.min.css" rel="stylesheet">
7
- <link rel="stylesheet" href="https://cdn.datatables.net/2.1.8/css/dataTables.dataTables.css" />
8
- <link rel="stylesheet" href="https://cdn.datatables.net/searchpanes/2.3.3/css/searchPanes.dataTables.css" />
9
- <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
10
- <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.3/moment.min.js"></script>
11
- <script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
12
- <script src="https://cdn.datatables.net/2.1.8/js/dataTables.js"></script>
13
- <script src="https://cdn.datatables.net/searchpanes/2.3.3/js/dataTables.searchPanes.js"></script>
14
- <script src="https://cdn.datatables.net/select/3.0.0/js/dataTables.select.js"></script>
15
- <title>{% block title %} {% endblock %} AIND Data Transfer Service Job Parameters</title>
16
- <style>
17
- body {
18
- margin: 20px;
19
- font-family: arial, sans-serif;
20
- }
21
- nav {
22
- height: 40px;
23
- }
24
- .modal-body {
25
- max-height: calc(100vh - 200px);
26
- overflow: auto;
27
- }
28
- </style>
29
- </head>
30
- <body>
31
- <nav>
32
- <a href="/">Submit Jobs</a> |
33
- <a href="/jobs">Job Status</a> |
34
- <a href="/job_params">Job Parameters</a> |
35
- <a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
36
- <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
37
- <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io" target="_blank" >Help</a> |
38
- <a href="/admin">Admin</a>
39
- </nav>
40
- <div class="content">
41
- <h4 class="mb-2">
42
- <!-- dropdown to switch version -->
43
- <div id="version-dropdown" class="btn-group mb-2">
44
- <button id="version-button" type="button" class="btn btn-primary dropdown-toggle" data-bs-toggle="dropdown">
45
- {{default_version}}
46
- </button>
47
- <ul class="dropdown-menu">
48
- {% for v in versions %}
49
- <button class="dropdown-item" type="button">{{ v }}</button>
50
- {% endfor %}
51
- </ul>
52
- </div>
53
- <span>Job Parameters</span>
54
- </h4>
55
- <!-- job params table -->
56
- <div>
57
- <table id="job-params-table" class="display compact table table-bordered table-sm" style="font-size: small">
58
- <thead>
59
- <tr>
60
- <th>Job Type</th>
61
- <th>Task ID</th>
62
- <th>Modality</th>
63
- <th>Parameter Name</th>
64
- <th>Last Modified</th>
65
- </tr>
66
- </thead>
67
- </table>
68
- <!-- modal for displaying param value as json -->
69
- <div class="modal fade" id="param-modal" tabindex="-1" aria-labelledby="param-modal-label" aria-hidden="true">
70
- <div class="modal-dialog modal-xl">
71
- <div class="modal-content">
72
- <div class="modal-header">
73
- <h5 class="modal-title" id="param-modal-label"></h5>
74
- <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
75
- </div>
76
- <div class="modal-body">
77
- <pre id="param-modal-content" class="bg-light p-1 border rounded"></pre>
78
- </div>
79
- </div>
80
- </div>
81
- </div>
82
- </div>
83
- </div>
84
- <script>
85
- $(document).ready(function() {
86
- createJobParamsTable();
87
- // Event listeners for param value modal
88
- $('#param-modal').on('show.bs.modal', function(event) {
89
- var button = $(event.relatedTarget);
90
- var paramName = button.data('bs-param-name');
91
- var jobType = button.data('bs-job-type');
92
- var taskId = button.data('bs-task-id');
93
- var modality = button.data('bs-modality');
94
- // update the modal label and contents
95
- var modal = $(this);
96
- modal.find('#param-modal-label').text(paramName);
97
- var version = $('#version-button').text().trim();
98
- var getParameterUrl = `/api/${version}/parameters/job_types/${jobType}/tasks/${taskId}`;
99
- if (modality) {
100
- getParameterUrl += `/${modality}`;
101
- }
102
- $.ajax({
103
- url: getParameterUrl,
104
- type: 'GET',
105
- success: function(response) {
106
- jsonStr = JSON.stringify(response.data, null, 3);
107
- modal.find('#param-modal-content').text(jsonStr);
108
- },
109
- error: function(xhr, status, error) {
110
- modal.find('#param-modal-content').text(error);
111
- }
112
- });
113
- });
114
- $('#param-modal').on('hidden.bs.modal', function() {
115
- $(this).find('#param-modal-label').text('');
116
- $(this).find('#param-modal-content').text('');
117
- });
118
- // Event listener for version dropdown
119
- $('#version-dropdown .dropdown-item').on('click', function() {
120
- var version = $(this).text();
121
- if (version != $('#version-button').text().trim()) {
122
- $('#version-button').text(version);
123
- $('#job-params-table').DataTable().ajax.url(`/api/${version}/parameters`).load();
124
- }
125
- });
126
- });
127
- // Create DataTable for job params table
128
- function createJobParamsTable() {
129
- $('#job-params-table').DataTable({
130
- ajax: {
131
- url: "/api/{{default_version}}/parameters",
132
- dataSrc: 'data'
133
- },
134
- processing: true,
135
- columns: [
136
- { data: 'job_type', searchPanes: {show: true} },
137
- { data: 'task_id', searchPanes: {show: true} },
138
- { data: 'modality', searchPanes: {show: false} },
139
- { data: 'name', render: renderParameterButton },
140
- { data: 'last_modified', render: renderDatetime },
141
- ],
142
- order: [0, 'asc'],
143
- // preselect "default" job_type
144
- searchPanes: {
145
- preSelect: [ { rows: ['default'], column: 0 } ],
146
- },
147
- // options to match default jobs table
148
- pageLength: 25,
149
- layout: {
150
- topStart: null,
151
- topEnd: null,
152
- bottomStart: null,
153
- bottomEnd: null,
154
- top1Start: "searchPanes",
155
- top: [
156
- 'pageLength',
157
- 'info',
158
- { paging: { numbers: false } }
159
- ],
160
- },
161
- language: {
162
- info: "_START_ to _END_ of _TOTAL_",
163
- infoEmpty: "0 to 0 of 0",
164
- entries: { _: "parameters", 1: "parameter" },
165
- paginate: {
166
- first: '&laquo; First',
167
- previous: '&lsaquo; Prev',
168
- next: 'Next &rsaquo;',
169
- last: 'Last &raquo;'
170
- },
171
- },
172
- });
173
- }
174
- function renderDatetime(data, type, row) {
175
- return (type === 'display') ? moment.utc(data).local().format('YYYY-MM-DD h:mm:ss a') : data;
176
- }
177
- function renderParameterButton(data, type, row) {
178
- if (type == 'display') {
179
- return (
180
- `<button type="button" class="btn btn-link btn-sm"
181
- data-bs-toggle="modal"
182
- data-bs-target="#param-modal"
183
- data-bs-param-name=${data}
184
- data-bs-job-type=${row.job_type}
185
- data-bs-task-id=${row.task_id}
186
- data-bs-modality=${row.modality}
187
- >${data}
188
- </button>`
189
- );
190
- }
191
- return data;
192
- }
193
- </script>
194
- </body>
195
- </html>