aind-data-transfer-service 1.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of aind-data-transfer-service might be problematic. Click here for more details.
- aind_data_transfer_service/__init__.py +9 -0
- aind_data_transfer_service/configs/__init__.py +1 -0
- aind_data_transfer_service/configs/csv_handler.py +59 -0
- aind_data_transfer_service/configs/job_configs.py +545 -0
- aind_data_transfer_service/configs/job_upload_template.py +153 -0
- aind_data_transfer_service/hpc/__init__.py +1 -0
- aind_data_transfer_service/hpc/client.py +151 -0
- aind_data_transfer_service/hpc/models.py +492 -0
- aind_data_transfer_service/log_handler.py +58 -0
- aind_data_transfer_service/models/__init__.py +1 -0
- aind_data_transfer_service/models/core.py +300 -0
- aind_data_transfer_service/models/internal.py +277 -0
- aind_data_transfer_service/server.py +1125 -0
- aind_data_transfer_service/templates/index.html +245 -0
- aind_data_transfer_service/templates/job_params.html +194 -0
- aind_data_transfer_service/templates/job_status.html +323 -0
- aind_data_transfer_service/templates/job_tasks_table.html +146 -0
- aind_data_transfer_service/templates/task_logs.html +31 -0
- aind_data_transfer_service-1.12.0.dist-info/METADATA +49 -0
- aind_data_transfer_service-1.12.0.dist-info/RECORD +23 -0
- aind_data_transfer_service-1.12.0.dist-info/WHEEL +5 -0
- aind_data_transfer_service-1.12.0.dist-info/licenses/LICENSE +21 -0
- aind_data_transfer_service-1.12.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
|
|
6
|
+
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-icons/1.8.1/font/bootstrap-icons.min.css" rel="stylesheet">
|
|
7
|
+
<link rel="stylesheet" href="https://cdn.datatables.net/2.1.8/css/dataTables.dataTables.css" />
|
|
8
|
+
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
|
|
9
|
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.3/moment.min.js"></script>
|
|
10
|
+
<script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
|
|
11
|
+
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
|
|
12
|
+
<script src="https://cdn.datatables.net/2.1.8/js/dataTables.js"></script>
|
|
13
|
+
<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
|
|
14
|
+
<title>{% block title %} {% endblock %} AIND Data Transfer Service Jobs</title>
|
|
15
|
+
<style>
|
|
16
|
+
body {
|
|
17
|
+
margin: 20px;
|
|
18
|
+
font-family: arial, sans-serif;
|
|
19
|
+
}
|
|
20
|
+
nav {
|
|
21
|
+
height: 40px;
|
|
22
|
+
}
|
|
23
|
+
.modal-body {
|
|
24
|
+
height: calc(100vh - 100px);
|
|
25
|
+
}
|
|
26
|
+
</style>
|
|
27
|
+
</head>
|
|
28
|
+
<body>
|
|
29
|
+
<nav>
|
|
30
|
+
<a href="/">Submit Jobs</a> |
|
|
31
|
+
<a href="/jobs">Job Status</a> |
|
|
32
|
+
<a href="/job_params">Job Parameters</a> |
|
|
33
|
+
<a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
|
|
34
|
+
<a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
|
|
35
|
+
<a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io" target="_blank" >Help</a>
|
|
36
|
+
</nav>
|
|
37
|
+
<div class="content">
|
|
38
|
+
<!-- display total entries -->
|
|
39
|
+
<h4 class="mb-2">Jobs Submitted: <span id="total-entries"></span></h4>
|
|
40
|
+
<!-- filters for job status results-->
|
|
41
|
+
<div class="card mb-4 small" style="width:400px">
|
|
42
|
+
<div class="card-header py-1" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-filters" aria-expanded="false" aria-controls="collapse-filters">
|
|
43
|
+
<i class="bi bi-filter"></i><span class="ms-2">Filter by</span>
|
|
44
|
+
<i class="bi bi-chevron-expand float-end"></i>
|
|
45
|
+
</div>
|
|
46
|
+
<div id="collapse-filters" class="collapse card-body p-2">
|
|
47
|
+
<!-- filter by job status -->
|
|
48
|
+
<div class="input-group input-group-sm mb-1">
|
|
49
|
+
<span class="input-group-text" style="width:35%">Status</span>
|
|
50
|
+
<select class="form-select" onchange="filterJobsByColumn(4, this.value);this.blur();">
|
|
51
|
+
{% for s in [
|
|
52
|
+
{"label": "all", "value": "", "class": "text-dark"},
|
|
53
|
+
{"label": "queued", "value": "queued", "class": "text-secondary"},
|
|
54
|
+
{"label": "running", "value": "running", "class": "text-info"},
|
|
55
|
+
{"label": "failed", "value": "failed", "class": "text-danger"},
|
|
56
|
+
{"label": "success", "value": "success", "class": "text-success"},
|
|
57
|
+
] %}
|
|
58
|
+
<option class="{{ s.class }}" value="{{ s.value }}">{{ s.label }}</option>
|
|
59
|
+
{% endfor %}
|
|
60
|
+
</select>
|
|
61
|
+
</div>
|
|
62
|
+
<!-- filter by job submitted date range -->
|
|
63
|
+
<div class="input-group input-group-sm">
|
|
64
|
+
<span class="input-group-text" style="width:35%">Submit Time</span>
|
|
65
|
+
<input id="submit-date-range" class="form-select" type="text" />
|
|
66
|
+
</div>
|
|
67
|
+
<!-- filter by asset name -->
|
|
68
|
+
<hr class="flex-grow-1 border-secondary">
|
|
69
|
+
<div class="input-group input-group-sm mb-1">
|
|
70
|
+
<span class="input-group-text" style="width:35%">Asset Name</span>
|
|
71
|
+
<input id="filter-name-input" type="text" class="form-control" placeholder="asset name" oninput="filterJobsByColumn(0, this.value)">
|
|
72
|
+
<button id="filter-name-clear" class="btn btn-outline-secondary" type="button" title="Clear" onclick="clearFilterJobsByColumn(0, '#filter-name-input')">
|
|
73
|
+
<i class="bi bi-x-lg"></i>
|
|
74
|
+
</button>
|
|
75
|
+
</div>
|
|
76
|
+
<!-- filter by job id-->
|
|
77
|
+
<div class="input-group input-group-sm mb-1">
|
|
78
|
+
<span class="input-group-text" style="width:35%">Job ID</span>
|
|
79
|
+
<input id="filter-id-input" type="text" class="form-control" placeholder="job id" oninput="filterJobsByColumn(1, this.value)">
|
|
80
|
+
<button id="filter-id-clear" class="btn btn-outline-secondary" type="button" title="Clear" onclick="clearFilterJobsByColumn(1, '#filter-id-input')">
|
|
81
|
+
<i class="bi bi-x-lg"></i>
|
|
82
|
+
</button>
|
|
83
|
+
</div>
|
|
84
|
+
<!-- filter by job type-->
|
|
85
|
+
<div class="input-group input-group-sm mb-1">
|
|
86
|
+
<span class="input-group-text" style="width:35%">Job Type</span>
|
|
87
|
+
<input id="filter-job-type-input" type="text" class="form-control" placeholder="job type" oninput="filterJobsByColumn(2, this.value)">
|
|
88
|
+
<button id="filter-job-type-clear" class="btn btn-outline-secondary" type="button" title="Clear" onclick="clearFilterJobsByColumn(2, '#filter-job-type-input')">
|
|
89
|
+
<i class="bi bi-x-lg"></i>
|
|
90
|
+
</button>
|
|
91
|
+
</div>
|
|
92
|
+
<!-- filter by dag id -->
|
|
93
|
+
<div class="input-group input-group-sm mb-1">
|
|
94
|
+
<span class="input-group-text" style="width:35%">Dag ID</span>
|
|
95
|
+
<select class="form-select" onchange="filterJobsByColumn(3, this.value);this.blur();">
|
|
96
|
+
<option class="text-dark" value="">all</option>
|
|
97
|
+
{% for dag_id in dag_ids %}
|
|
98
|
+
<option class="text-secondary" value="^{{ dag_id }}$">{{ dag_id }}</option>
|
|
99
|
+
{% endfor %}
|
|
100
|
+
</select>
|
|
101
|
+
</div>
|
|
102
|
+
</div>
|
|
103
|
+
</div>
|
|
104
|
+
<!-- job status table -->
|
|
105
|
+
<div>
|
|
106
|
+
<table id="searchJobsTable" class="display compact table table-bordered table-sm" style="font-size: small">
|
|
107
|
+
<thead>
|
|
108
|
+
<tr>
|
|
109
|
+
<th>Asset Name</th>
|
|
110
|
+
<th>Job ID</th>
|
|
111
|
+
<th>Job Type</th>
|
|
112
|
+
<th>Dag ID</th>
|
|
113
|
+
<th>Status</th>
|
|
114
|
+
<th>Submit Time</th>
|
|
115
|
+
<th>Start Time</th>
|
|
116
|
+
<th>End Time</th>
|
|
117
|
+
<th>Comment</th>
|
|
118
|
+
<th>Tasks</th>
|
|
119
|
+
</tr>
|
|
120
|
+
</thead>
|
|
121
|
+
</table>
|
|
122
|
+
<!-- modal for displaying tasks per job from full jobs table -->
|
|
123
|
+
<div class="modal fade" id="tasks-modal-full" tabindex="-1" aria-labelledby="tasks-modal-full-label" aria-hidden="true">
|
|
124
|
+
<div class="modal-dialog modal-xl">
|
|
125
|
+
<div class="modal-content">
|
|
126
|
+
<div class="modal-header p-2">
|
|
127
|
+
<div class="modal-title fw-bold" id="tasks-modal-full-label" style="font-size: small">
|
|
128
|
+
<span id="modal-title-job-name" class="me-2"></span>
|
|
129
|
+
<span id="modal-title-job-id" class="me-2"></span>
|
|
130
|
+
<span id="modal-title-job-state" class="badge"></span>
|
|
131
|
+
</div>
|
|
132
|
+
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
133
|
+
</div>
|
|
134
|
+
<div class="modal-body p-2">
|
|
135
|
+
<iframe id="tasks-iframe" class="w-100 h-100" src=""></iframe>
|
|
136
|
+
</div>
|
|
137
|
+
</div>
|
|
138
|
+
</div>
|
|
139
|
+
</div>
|
|
140
|
+
</div>
|
|
141
|
+
</div>
|
|
142
|
+
<script>
|
|
143
|
+
// User can filter for jobs by status and submit time, and view results.
|
|
144
|
+
// Full Jobs Table: Loads all jobs using DataTables library. Enables searching across all jobs.
|
|
145
|
+
let tableUrl = new URL("{{ url_for('get_job_status_list') }}");
|
|
146
|
+
$(document).ready(function() {
|
|
147
|
+
const today = moment();
|
|
148
|
+
const twoWeeksAgo = moment().subtract(13, 'days');
|
|
149
|
+
|
|
150
|
+
// initialize daterangepicker for submit date filter
|
|
151
|
+
$('#submit-date-range').daterangepicker({
|
|
152
|
+
startDate: twoWeeksAgo,
|
|
153
|
+
endDate: today,
|
|
154
|
+
minDate: twoWeeksAgo,
|
|
155
|
+
maxDate: today,
|
|
156
|
+
ranges: {
|
|
157
|
+
'Today': [today, today],
|
|
158
|
+
'Last 3 Days': [moment().subtract(2, 'days'), today],
|
|
159
|
+
'Last 7 Days': [moment().subtract(6, 'days'), today],
|
|
160
|
+
'Last 14 Days': [twoWeeksAgo, today],
|
|
161
|
+
}
|
|
162
|
+
}, filterJobsBySubmitTimeRange);
|
|
163
|
+
|
|
164
|
+
// initialize job status table with default values
|
|
165
|
+
const initialParams = {
|
|
166
|
+
// set default submit date range client-side for browser's local time
|
|
167
|
+
execution_date_gte: twoWeeksAgo.startOf('day').toISOString(),
|
|
168
|
+
execution_date_lte: today.endOf('day').toISOString(),
|
|
169
|
+
};
|
|
170
|
+
updateJobStatusTable(initialParams);
|
|
171
|
+
|
|
172
|
+
// tasks modal for full jobs table
|
|
173
|
+
var tasksModal = document.getElementById('tasks-modal-full');
|
|
174
|
+
tasksModal.addEventListener('show.bs.modal', function (event) {
|
|
175
|
+
var sourceData = event.relatedTarget?.dataset;
|
|
176
|
+
updateJobTasksModal(sourceData?.dagId, sourceData?.jobId, sourceData?.jobName, sourceData?.jobState);
|
|
177
|
+
})
|
|
178
|
+
tasksModal.addEventListener('hidden.bs.modal', function (event) {
|
|
179
|
+
updateJobTasksModal(null, null, null, null);
|
|
180
|
+
})
|
|
181
|
+
});
|
|
182
|
+
// FULL JOBS TABLE -----------------------------------------------
|
|
183
|
+
// Helper functions for custom column rendering
|
|
184
|
+
function renderJobStatus(cell, cellData, rowData, rowIndex, colIndex) {
|
|
185
|
+
let customClass = cellData === 'success' ? 'table-success'
|
|
186
|
+
: cellData === 'failed' ? 'table-danger'
|
|
187
|
+
: cellData === 'running' ? 'table-info'
|
|
188
|
+
: cellData === 'queued' ? 'table-secondary'
|
|
189
|
+
: null;
|
|
190
|
+
if (customClass) {
|
|
191
|
+
cell.classList.add(customClass);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function renderDatetime(data, type, row) {
|
|
195
|
+
return (type === 'display') ? moment.utc(data).local().format('YYYY-MM-DD h:mm:ss a') : data;
|
|
196
|
+
}
|
|
197
|
+
function renderTasksButton(data, type, row) {
|
|
198
|
+
if (type == 'display') {
|
|
199
|
+
return (`<button type="button" class="btn btn-outline-primary btn-sm" data-bs-toggle="modal" data-bs-target="#tasks-modal-full" data-dag-id="${row.dag_id}" data-job-id="${data}" data-job-name="${row.name}" data-job-state="${row.job_state}">
|
|
200
|
+
<i class="bi bi-box-arrow-up-right" title="View tasks and logs"></i>
|
|
201
|
+
</button>`);
|
|
202
|
+
}
|
|
203
|
+
return data;
|
|
204
|
+
}
|
|
205
|
+
// Create DataTable for full jobs table
|
|
206
|
+
function createFullJobsTable() {
|
|
207
|
+
$('#searchJobsTable').DataTable({
|
|
208
|
+
ajax: {
|
|
209
|
+
url: tableUrl,
|
|
210
|
+
dataSrc: 'data.job_status_list'
|
|
211
|
+
},
|
|
212
|
+
processing: true,
|
|
213
|
+
columns: [
|
|
214
|
+
{ data: 'name', searchable: true },
|
|
215
|
+
{ data: 'job_id', searchable: true },
|
|
216
|
+
{ data: 'job_type', searchable: true },
|
|
217
|
+
{ data: 'dag_id', searchable: true },
|
|
218
|
+
{ data: 'job_state', searchable: true, createdCell: renderJobStatus },
|
|
219
|
+
{ data: 'submit_time', searchable: false, render: renderDatetime },
|
|
220
|
+
{ data: 'start_time', searchable: false, render: renderDatetime },
|
|
221
|
+
{ data: 'end_time', searchable: false, render: renderDatetime },
|
|
222
|
+
{ data: 'comment', searchable: false, defaultContent: 'None' },
|
|
223
|
+
{ data: 'job_id', searchable: false, render: renderTasksButton },
|
|
224
|
+
],
|
|
225
|
+
initComplete: (settings, json) => updateJobsCount(json),
|
|
226
|
+
// layout options
|
|
227
|
+
pageLength: 25,
|
|
228
|
+
order: [5, 'desc'], // submit time descending
|
|
229
|
+
layout: {
|
|
230
|
+
topStart: null,
|
|
231
|
+
topEnd: null,
|
|
232
|
+
bottomStart: null,
|
|
233
|
+
bottomEnd: null,
|
|
234
|
+
top: [
|
|
235
|
+
'pageLength',
|
|
236
|
+
'info',
|
|
237
|
+
{ paging: { numbers: false } }
|
|
238
|
+
],
|
|
239
|
+
},
|
|
240
|
+
language: {
|
|
241
|
+
info: "_START_ to _END_ of _TOTAL_",
|
|
242
|
+
infoEmpty: "0 to 0 of 0",
|
|
243
|
+
entries: { _: "jobs", 1: "job" },
|
|
244
|
+
paginate: {
|
|
245
|
+
first: '« First',
|
|
246
|
+
previous: '‹ Prev',
|
|
247
|
+
next: 'Next ›',
|
|
248
|
+
last: 'Last »'
|
|
249
|
+
},
|
|
250
|
+
},
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
// Modal for tasks per job from full jobs table
|
|
254
|
+
function updateJobTasksModal(dagId, jobId, jobName, jobState) {
|
|
255
|
+
// Update the modal header with the job id and name
|
|
256
|
+
document.getElementById('modal-title-job-id').textContent = `(${jobId})`;
|
|
257
|
+
document.getElementById('modal-title-job-name').textContent = jobName;
|
|
258
|
+
var modalTitleJobState = document.getElementById('modal-title-job-state');
|
|
259
|
+
modalTitleJobState.textContent = jobState;
|
|
260
|
+
if (jobState) {
|
|
261
|
+
modalTitleJobState.classList.add(
|
|
262
|
+
jobState === 'success' ? 'bg-success'
|
|
263
|
+
: jobState === 'failed' ? 'bg-danger'
|
|
264
|
+
: jobState === 'running' ? 'bg-info'
|
|
265
|
+
: 'bg-secondary'
|
|
266
|
+
);
|
|
267
|
+
} else {
|
|
268
|
+
modalTitleJobState.classList.value = 'badge';
|
|
269
|
+
}
|
|
270
|
+
// Update the iframe src with the job id
|
|
271
|
+
var tasksIframe = document.getElementById('tasks-iframe');
|
|
272
|
+
if (jobId) {
|
|
273
|
+
var url = new URL("{{ url_for('job_tasks_table') }}");
|
|
274
|
+
url.searchParams.append('dag_id', dagId);
|
|
275
|
+
url.searchParams.append('dag_run_id', jobId);
|
|
276
|
+
tasksIframe.src = url;
|
|
277
|
+
} else {
|
|
278
|
+
tasksIframe.src = "";
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// EVENT HANDLERS ------------------------------------------------
|
|
282
|
+
function updateJobStatusTable(newParams) {
|
|
283
|
+
Object.entries(newParams).forEach(([key, value]) => {
|
|
284
|
+
tableUrl.searchParams.set(key, value);
|
|
285
|
+
});
|
|
286
|
+
if (DataTable.isDataTable('#searchJobsTable')) {
|
|
287
|
+
// load the table with new params and update total entries
|
|
288
|
+
$('#searchJobsTable').DataTable().ajax.url(tableUrl.toString()).load(
|
|
289
|
+
(data) => updateJobsCount(data)
|
|
290
|
+
);
|
|
291
|
+
} else {
|
|
292
|
+
createFullJobsTable();
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
function updateJobsCount(data) {
|
|
296
|
+
// data is the response from the server after ajax call
|
|
297
|
+
$('#total-entries').text(data?.data?.total_entries);
|
|
298
|
+
}
|
|
299
|
+
// Filters
|
|
300
|
+
function filterJobsByColumn(columnIndex, value) {
|
|
301
|
+
if (columnIndex == 3 || columnIndex == 4) {
|
|
302
|
+
//exact match for dag_id and job_state dropdowns
|
|
303
|
+
$('#searchJobsTable').DataTable().columns(columnIndex).search(value.trim(), true, false).draw();
|
|
304
|
+
} else {
|
|
305
|
+
// smart match
|
|
306
|
+
$('#searchJobsTable').DataTable().columns(columnIndex).search(value.trim()).draw();
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
function clearFilterJobsByColumn(columnIndex, inputElementId) {
|
|
310
|
+
$(inputElementId).val('');
|
|
311
|
+
filterJobsByColumn(columnIndex, '');
|
|
312
|
+
}
|
|
313
|
+
function filterJobsBySubmitTimeRange(start, end) {
|
|
314
|
+
// This filter is the only one that sends a new ajax request to the server
|
|
315
|
+
// NOTE: daterangepicker already has 00:00:00.000 and 23:59:59.999 for start and end
|
|
316
|
+
updateJobStatusTable({
|
|
317
|
+
execution_date_gte: start.toISOString(),
|
|
318
|
+
execution_date_lte: end.toISOString(),
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
</script>
|
|
322
|
+
</body>
|
|
323
|
+
</html>
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="UTF-8">
|
|
6
|
+
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
|
|
7
|
+
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-icons/1.8.1/font/bootstrap-icons.min.css" rel="stylesheet">
|
|
8
|
+
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
|
|
9
|
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.3/moment.min.js"></script>
|
|
10
|
+
</head>
|
|
11
|
+
|
|
12
|
+
<body>
|
|
13
|
+
<!-- tasks table -->
|
|
14
|
+
<table class="table table-bordered table-striped table-hover table-sm" style="font-size: small">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Task ID</th>
|
|
17
|
+
<th>Try Number</th>
|
|
18
|
+
<th>Map Index</th>
|
|
19
|
+
<th>Status</th>
|
|
20
|
+
<th>Submit Time</th>
|
|
21
|
+
<th>Start Time</th>
|
|
22
|
+
<th>End Time</th>
|
|
23
|
+
<th>Duration</th>
|
|
24
|
+
<th>Logs</th>
|
|
25
|
+
</tr>
|
|
26
|
+
{% for job_task in job_tasks_list %}
|
|
27
|
+
<tr>
|
|
28
|
+
<td>{{job_task.task_id}}</td>
|
|
29
|
+
<td>{{job_task.try_number}}</td>
|
|
30
|
+
<td>
|
|
31
|
+
{% if job_task.map_index > -1 %}
|
|
32
|
+
<span>{{job_task.map_index}}</span>
|
|
33
|
+
{% else %}
|
|
34
|
+
<span></span>
|
|
35
|
+
{% endif %}
|
|
36
|
+
</td>
|
|
37
|
+
<td class="{% if job_task.task_state == 'success' %}table-success
|
|
38
|
+
{% elif job_task.task_state == 'failed' %}table-danger
|
|
39
|
+
{% elif job_task.task_state == 'running' %}table-info
|
|
40
|
+
{% elif job_task.task_state == 'queued' %}table-secondary
|
|
41
|
+
{% endif %}">{{job_task.task_state}}</td>
|
|
42
|
+
<td class="datetime_to_be_adjusted">{{job_task.submit_time}}</td>
|
|
43
|
+
<td class="datetime_to_be_adjusted">{{job_task.start_time}}</td>
|
|
44
|
+
<td class="datetime_to_be_adjusted">{{job_task.end_time}}</td>
|
|
45
|
+
<td>{{job_task.duration}}</td>
|
|
46
|
+
<td>
|
|
47
|
+
{% if job_task.try_number > 0 %}
|
|
48
|
+
<button type="button" class="btn btn-outline-primary btn-sm" data-bs-toggle="modal" data-bs-target="#logs-modal"
|
|
49
|
+
data-dag-id="{{ job_task.dag_id }}"
|
|
50
|
+
data-job-id="{{ job_task.job_id }}"
|
|
51
|
+
data-task-id="{{ job_task.task_id }}"
|
|
52
|
+
data-task-state="{{ job_task.task_state }}"
|
|
53
|
+
data-task-try-number="{{ job_task.try_number }}"
|
|
54
|
+
data-task-map-index="{{ job_task.map_index }}"
|
|
55
|
+
><i class="bi bi-box-arrow-up-right" title="View logs"></i>
|
|
56
|
+
</button>
|
|
57
|
+
{% endif %}
|
|
58
|
+
</td>
|
|
59
|
+
</tr>
|
|
60
|
+
{% endfor %}
|
|
61
|
+
</table>
|
|
62
|
+
<!-- modal for displaying logs per task -->
|
|
63
|
+
<div class="modal fade" id="logs-modal" tabindex="-1" aria-labelledby="logs-modal-label" aria-hidden="true">
|
|
64
|
+
<div class="modal-dialog modal-fullscreen">
|
|
65
|
+
<div class="modal-content">
|
|
66
|
+
<div class="modal-header p-2">
|
|
67
|
+
<div class="modal-title fw-bold" id="logs-modal-label" style="font-size: small">
|
|
68
|
+
<span id="modal-title-task" class="me-2"></span><span id="modal-title-task-state" class="badge"></span>
|
|
69
|
+
</div>
|
|
70
|
+
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
71
|
+
</div>
|
|
72
|
+
<div class="modal-body p-2">
|
|
73
|
+
<iframe id="logs-iframe" class="w-100 h-100" src=""></iframe>
|
|
74
|
+
</div>
|
|
75
|
+
</div>
|
|
76
|
+
</div>
|
|
77
|
+
</div>
|
|
78
|
+
<!-- display errors to user -->
|
|
79
|
+
{% if status_code != 200 %}
|
|
80
|
+
<div class="alert alert-danger" role="alert">
|
|
81
|
+
<h4 class="alert-heading">{{ message }}</h4>
|
|
82
|
+
<p>
|
|
83
|
+
{% for error in errors %}
|
|
84
|
+
{% if error is string %}{{ error }}
|
|
85
|
+
{% elif error is mapping %}
|
|
86
|
+
{% for key, value in error.items() %}
|
|
87
|
+
<strong>{{ key }}:</strong> {{ value }}<br>
|
|
88
|
+
{% endfor %}
|
|
89
|
+
{% endif %}
|
|
90
|
+
{% endfor %}
|
|
91
|
+
</p>
|
|
92
|
+
</div>
|
|
93
|
+
{% endif %}
|
|
94
|
+
<script>
|
|
95
|
+
window.onload = function () {
|
|
96
|
+
document.querySelectorAll(".datetime_to_be_adjusted").forEach(function (el) {
|
|
97
|
+
if (el.innerHTML !== "") {
|
|
98
|
+
var utcTime = moment.utc(el.innerText); // This is the time in UTC
|
|
99
|
+
utcTime.local(); // Switch to using the browser's local timezone
|
|
100
|
+
el.innerText = utcTime.format('YYYY-MM-DD h:mm:ss a'); // Write the local time back to the element
|
|
101
|
+
};
|
|
102
|
+
});
|
|
103
|
+
var logsModal = document.getElementById('logs-modal');
|
|
104
|
+
logsModal.addEventListener('show.bs.modal', function (event) {
|
|
105
|
+
var sourceData = event.relatedTarget?.dataset;
|
|
106
|
+
updateTaskLogsModal(sourceData?.dagId,sourceData?.jobId, sourceData?.taskId, sourceData?.taskState, sourceData?.taskTryNumber, sourceData?.taskMapIndex);
|
|
107
|
+
})
|
|
108
|
+
logsModal.addEventListener('hidden.bs.modal', function (event) {
|
|
109
|
+
updateTaskLogsModal(null, null, null, null, null, null);
|
|
110
|
+
})
|
|
111
|
+
}
|
|
112
|
+
function updateTaskLogsModal(dagId, jobId, taskId, taskState, taskTryNumber, taskMapIndex) {
|
|
113
|
+
console.log('updateTaskLogsModal', dagId, jobId, taskId, taskState, taskTryNumber, taskMapIndex);
|
|
114
|
+
// Update the modal header with the task id, try number, map index, and state
|
|
115
|
+
var tryMapLabel = taskMapIndex > -1 ? `${taskId} (try ${taskTryNumber}, map_index ${taskMapIndex})` : `${taskId} (try ${taskTryNumber})`;
|
|
116
|
+
document.getElementById('modal-title-task').textContent = tryMapLabel;
|
|
117
|
+
var modalTitleTaskState = document.getElementById('modal-title-task-state');
|
|
118
|
+
modalTitleTaskState.textContent = taskState;
|
|
119
|
+
if (taskState) {
|
|
120
|
+
modalTitleTaskState.classList.add(
|
|
121
|
+
taskState === 'success' ? 'bg-success'
|
|
122
|
+
: taskState === 'failed' ? 'bg-danger'
|
|
123
|
+
: taskState === 'running' ? 'bg-info'
|
|
124
|
+
: 'bg-secondary'
|
|
125
|
+
);
|
|
126
|
+
} else {
|
|
127
|
+
modalTitleTaskState.classList.value = 'badge';
|
|
128
|
+
}
|
|
129
|
+
// Update the iframe src to get the logs for the task
|
|
130
|
+
var logsIframe = document.getElementById('logs-iframe');
|
|
131
|
+
if (jobId) {
|
|
132
|
+
var url = new URL("{{ url_for('task_logs') }}");
|
|
133
|
+
url.searchParams.append('dag_id', dagId);
|
|
134
|
+
url.searchParams.append('dag_run_id', jobId);
|
|
135
|
+
url.searchParams.append('task_id', taskId);
|
|
136
|
+
url.searchParams.append('try_number', taskTryNumber);
|
|
137
|
+
url.searchParams.append('map_index', taskMapIndex);
|
|
138
|
+
logsIframe.src = url;
|
|
139
|
+
} else {
|
|
140
|
+
logsIframe.src = "";
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
</script>
|
|
144
|
+
</body>
|
|
145
|
+
|
|
146
|
+
</html>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
|
|
6
|
+
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
|
|
7
|
+
</head>
|
|
8
|
+
<body>
|
|
9
|
+
{% if status_code == 200 %}
|
|
10
|
+
<!-- display task logs -->
|
|
11
|
+
<div>
|
|
12
|
+
<pre class="bg-light p-1 border rounded" style="font-size: x-small">{{logs}}</pre>
|
|
13
|
+
</div>
|
|
14
|
+
{% else %}
|
|
15
|
+
<!-- display errors to user -->
|
|
16
|
+
<div class="alert alert-danger" role="alert">
|
|
17
|
+
<h4 class="alert-heading">{{ message }}</h4>
|
|
18
|
+
<p>
|
|
19
|
+
{% for error in errors %}
|
|
20
|
+
{% if error is string %}{{ error }}
|
|
21
|
+
{% elif error is mapping %}
|
|
22
|
+
{% for key, value in error.items() %}
|
|
23
|
+
<strong>{{ key }}:</strong> {{ value }}<br>
|
|
24
|
+
{% endfor %}
|
|
25
|
+
{% endif %}
|
|
26
|
+
{% endfor %}
|
|
27
|
+
</p>
|
|
28
|
+
</div>
|
|
29
|
+
{% endif %}
|
|
30
|
+
</body>
|
|
31
|
+
</html>
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aind-data-transfer-service
|
|
3
|
+
Version: 1.12.0
|
|
4
|
+
Summary: Service that handles requests to upload data to the cloud
|
|
5
|
+
Author: Allen Institute for Neural Dynamics
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: boto3
|
|
12
|
+
Requires-Dist: boto3-stubs[ssm]
|
|
13
|
+
Requires-Dist: pydantic<2.9,>=2.7
|
|
14
|
+
Requires-Dist: pydantic-settings>=2.0
|
|
15
|
+
Requires-Dist: aind-data-schema<2.0,>=1.0.0
|
|
16
|
+
Requires-Dist: aind-data-transfer-models==0.17.0
|
|
17
|
+
Requires-Dist: aind-metadata-mapper==0.23.0
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: aind-data-transfer-service[server]; extra == "dev"
|
|
20
|
+
Requires-Dist: black; extra == "dev"
|
|
21
|
+
Requires-Dist: coverage; extra == "dev"
|
|
22
|
+
Requires-Dist: flake8; extra == "dev"
|
|
23
|
+
Requires-Dist: interrogate; extra == "dev"
|
|
24
|
+
Requires-Dist: isort; extra == "dev"
|
|
25
|
+
Provides-Extra: docs
|
|
26
|
+
Requires-Dist: Sphinx; extra == "docs"
|
|
27
|
+
Requires-Dist: furo; extra == "docs"
|
|
28
|
+
Provides-Extra: server
|
|
29
|
+
Requires-Dist: fastapi; extra == "server"
|
|
30
|
+
Requires-Dist: httpx; extra == "server"
|
|
31
|
+
Requires-Dist: jinja2; extra == "server"
|
|
32
|
+
Requires-Dist: starlette; extra == "server"
|
|
33
|
+
Requires-Dist: starlette_wtf; extra == "server"
|
|
34
|
+
Requires-Dist: uvicorn[standard]; extra == "server"
|
|
35
|
+
Requires-Dist: wtforms; extra == "server"
|
|
36
|
+
Requires-Dist: requests==2.25.0; extra == "server"
|
|
37
|
+
Requires-Dist: openpyxl; extra == "server"
|
|
38
|
+
Requires-Dist: python-logging-loki; extra == "server"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# aind-data-transfer-service
|
|
42
|
+
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
|
|
45
|
+
[](https://github.com/semantic-release/semantic-release)
|
|
46
|
+
|
|
47
|
+
This service can be used to upload data stored in a VAST drive. It uses FastAPI to upload a job submission csv file that will be used to trigger a data transfer job in an on-prem HPC. Based on the information provided in the file, the data upload process fetches the appropriate metadata and starts the upload process.
|
|
48
|
+
|
|
49
|
+
More information can be found at [readthedocs](https://aind-data-transfer-service.readthedocs.io).
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
aind_data_transfer_service/__init__.py,sha256=x8KPuByORZYNVuata2ndOXEyYaqruRRd98591uYweUs,272
|
|
2
|
+
aind_data_transfer_service/log_handler.py,sha256=c7a-gLmZeRpeCUBwCz6XsTszWXQeQdR7eKZtas4llXM,1700
|
|
3
|
+
aind_data_transfer_service/server.py,sha256=OL53huEa654aCjOWEafP9rV1Qp4l_qSX6R_Tsis_ing,41252
|
|
4
|
+
aind_data_transfer_service/configs/__init__.py,sha256=9W5GTuso9Is1B9X16RXcdb_GxasZvj6qDzOBDv0AbTc,36
|
|
5
|
+
aind_data_transfer_service/configs/csv_handler.py,sha256=-iB_6tRoD5sKA9K3CCcyFHHmnNSLN7bVnmS-vLsMMRE,2085
|
|
6
|
+
aind_data_transfer_service/configs/job_configs.py,sha256=T-h5N6lyY9xTZ_xg_5FxkyYuMdagApbE6xalxFQ-bqA,18848
|
|
7
|
+
aind_data_transfer_service/configs/job_upload_template.py,sha256=l1pZd_jT3dml0pdVr9Tjml8KRi5-LkHbSGIEAjFMgps,5096
|
|
8
|
+
aind_data_transfer_service/hpc/__init__.py,sha256=YNc68YNlmXwKIPFMIViz_K4XzVVHkLPEBOFyO5DKMKI,53
|
|
9
|
+
aind_data_transfer_service/hpc/client.py,sha256=-JSxAWn96_XOIDwhsXAHK3TZAdckddUhtcCzRHnaTqA,4700
|
|
10
|
+
aind_data_transfer_service/hpc/models.py,sha256=-7HhV16s_MUyKPy0x0FGIbnq8DPL2qJAzJO5G7003AE,16184
|
|
11
|
+
aind_data_transfer_service/models/__init__.py,sha256=Meym73bEZ9nQr4QoeyhQmV3nRTYtd_4kWKPNygsBfJg,25
|
|
12
|
+
aind_data_transfer_service/models/core.py,sha256=rVOCOBOv101w6fMXt5o9anxWlywMHSTL0uQT4u_T6H0,9985
|
|
13
|
+
aind_data_transfer_service/models/internal.py,sha256=MGQrPuHrR21nn4toqdTCIEDW6MG7pWRajoPqD3j-ST0,9706
|
|
14
|
+
aind_data_transfer_service/templates/index.html,sha256=8I4QLC4pAPKD3UUrrt3sUDP8Ynopf7B8YZWC7_VabjI,10624
|
|
15
|
+
aind_data_transfer_service/templates/job_params.html,sha256=vqIdNQsZTM0kq3Wa9u-VjmmMa0UzBTpK02WpOSatXBQ,8817
|
|
16
|
+
aind_data_transfer_service/templates/job_status.html,sha256=vIOaJGJM78hOWTLTAzMfHjG9sNqPvS-muAyXYQtpnYI,16901
|
|
17
|
+
aind_data_transfer_service/templates/job_tasks_table.html,sha256=rWFukhjZ4dhPyabe372tmi4lbQS2fyELZ7Awbn5Un4g,6181
|
|
18
|
+
aind_data_transfer_service/templates/task_logs.html,sha256=y1GnQft0S50ghPb2xJDjAlefymB9a4zYdMikUFV7Tl4,918
|
|
19
|
+
aind_data_transfer_service-1.12.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
|
|
20
|
+
aind_data_transfer_service-1.12.0.dist-info/METADATA,sha256=5HtVeGrYgmhrX1EwaIrqHcfi0VYhWigLdmtZCV_d2vQ,2236
|
|
21
|
+
aind_data_transfer_service-1.12.0.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
|
|
22
|
+
aind_data_transfer_service-1.12.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
|
|
23
|
+
aind_data_transfer_service-1.12.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Allen Institute for Neural Dynamics
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
aind_data_transfer_service
|