harmonize-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harmonize/__init__.py +65 -0
- harmonize/__main__.py +39 -0
- harmonize/adapt.py +105 -0
- harmonize/array.py +588 -0
- harmonize/atomics.py +206 -0
- harmonize/codegen.py +409 -0
- harmonize/config.py +220 -0
- harmonize/cpp/async/future.h +64 -0
- harmonize/cpp/async/mod.h +9 -0
- harmonize/cpp/async/promise.h +747 -0
- harmonize/cpp/harmonize.h +4 -0
- harmonize/cpp/mem/mod.h +14 -0
- harmonize/cpp/mem/work.h +168 -0
- harmonize/cpp/preamble/adapt.h +162 -0
- harmonize/cpp/preamble/mod.h +133 -0
- harmonize/cpp/program/async.h +2923 -0
- harmonize/cpp/program/event.h +560 -0
- harmonize/cpp/program/mod.h +84 -0
- harmonize/cpp/sync/barrier.h +704 -0
- harmonize/cpp/sync/mod.h +9 -0
- harmonize/cpp/util/cli.h +510 -0
- harmonize/cpp/util/host.h +287 -0
- harmonize/cpp/util/iter.h +565 -0
- harmonize/cpp/util/mem.h +1050 -0
- harmonize/cpp/util/mod.h +148 -0
- harmonize/diagnostics.py +487 -0
- harmonize/errors.py +12 -0
- harmonize/logging.py +51 -0
- harmonize/pointer.py +108 -0
- harmonize/prim.py +59 -0
- harmonize/printing.py +104 -0
- harmonize/python/__init__.py +48 -0
- harmonize/runtime.py +1501 -0
- harmonize/templates.py +369 -0
- harmonize/timing.py +10 -0
- harmonize/types.py +16 -0
- harmonize_python-0.1.0.dist-info/METADATA +161 -0
- harmonize_python-0.1.0.dist-info/RECORD +40 -0
- harmonize_python-0.1.0.dist-info/WHEEL +4 -0
- harmonize_python-0.1.0.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,704 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
#ifndef HARMONIZE_SYNC_BARRIER
|
|
5
|
+
#define HARMONIZE_SYNC_BARRIER
|
|
6
|
+
|
|
7
|
+
#include "../async/mod.h"
|
|
8
|
+
|
|
9
|
+
//!
|
|
10
|
+
//! A `RemappingBarrier` coaleces promises into links in a lock-free manner.
|
|
11
|
+
//! Once released, all work in the queue is made available for work groups to
|
|
12
|
+
//! execute and all further appending operations will redirect promises to execution.
|
|
13
|
+
//! After being released, a queue may be reset to a non-released state.
|
|
14
|
+
//!
|
|
15
|
+
template<typename OP_SET, typename ADR_TYPE>
|
|
16
|
+
struct RemappingBarrier {
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
#if 0
|
|
20
|
+
using AdrType = typename PROGRAM::AdrType;
|
|
21
|
+
using LinkAdrType = typename PROGRAM::LinkAdrType;
|
|
22
|
+
using QueueType = typename PROGRAM::QueueType;
|
|
23
|
+
using ProgramType = PROGRAM;
|
|
24
|
+
using UnionType = PromiseUnion<OP_SET>;
|
|
25
|
+
using LinkType = typename PROGRAM::LinkType;
|
|
26
|
+
|
|
27
|
+
using PairPack = util::mem::PairPack<AdrType>;
|
|
28
|
+
using PairType = typename PairPack::PairType;
|
|
29
|
+
#else
|
|
30
|
+
using AdrType = ADR_TYPE;
|
|
31
|
+
using LinkAdrType = util::mem::Adr<AdrType>;
|
|
32
|
+
using QueueType = util::mem::PoolQueue<LinkAdrType>;
|
|
33
|
+
using UnionType = PromiseUnion<OP_SET>;
|
|
34
|
+
|
|
35
|
+
using PairPack = util::mem::PairPack<AdrType>;
|
|
36
|
+
using PairType = typename PairPack::PairType;
|
|
37
|
+
#endif
|
|
38
|
+
|
|
39
|
+
//! The number of operations contained within the barrier's
|
|
40
|
+
//! operation set.
|
|
41
|
+
static const size_t TYPE_COUNT = UnionType::Info::COUNT;
|
|
42
|
+
|
|
43
|
+
//! A tagged semaphore that communicates both the release state of the
|
|
44
|
+
//! barrier, but the type of the barrier as well.
|
|
45
|
+
TaggedSemaphore semaphore;
|
|
46
|
+
unsigned int count;
|
|
47
|
+
|
|
48
|
+
//! A queue that contains all full links created by coalescing promises
|
|
49
|
+
//! awaiting the release of the barrier.
|
|
50
|
+
QueueType full_list;
|
|
51
|
+
|
|
52
|
+
//! A table used to mediate the coalescing of links
|
|
53
|
+
PairPack partial_table[UnionType::Info::COUNT];
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
RemappingBarrier<OP_SET,ADR_TYPE>() = default;
|
|
57
|
+
|
|
58
|
+
//! Creates a new `RemappingBarrier` with an empty full queue and partial table
|
|
59
|
+
//! and with a semaphore value initialized to the supplied value.
|
|
60
|
+
static __host__ __device__ RemappingBarrier<OP_SET,ADR_TYPE> blank(unsigned int sem_val)
|
|
61
|
+
{
|
|
62
|
+
RemappingBarrier<OP_SET,ADR_TYPE> result;
|
|
63
|
+
result.semaphore = TaggedSemaphore(sem_val,1);
|
|
64
|
+
result.count = 0u;
|
|
65
|
+
result.full_list.pair.data = QueueType::null;
|
|
66
|
+
for(int i=0; i<TYPE_COUNT; i++){
|
|
67
|
+
result.partial_table[i] = PairPack(0,LinkAdrType::null);
|
|
68
|
+
}
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
//! Counts the number of links in the given queue. This is only used for debugging purposes.
|
|
74
|
+
template<typename PROGRAM>
|
|
75
|
+
__device__ unsigned int queue_count(PROGRAM program, QueueType queue) {
|
|
76
|
+
using ProgramType = PROGRAM;
|
|
77
|
+
using LinkType = typename ProgramType::LinkType;
|
|
78
|
+
|
|
79
|
+
unsigned int total = 0;
|
|
80
|
+
LinkAdrType iter = queue.get_head();
|
|
81
|
+
while( ! iter.is_null() ){
|
|
82
|
+
LinkType& the_link = program._dev_ctx.arena[iter.adr];
|
|
83
|
+
total += the_link.count;
|
|
84
|
+
//printf("( [%d,%d] @%d #%d -> %d )",blockIdx.x,threadIdx.x,iter.adr,the_link.count,the_link.next.adr);
|
|
85
|
+
iter = the_link.next;
|
|
86
|
+
}
|
|
87
|
+
return total;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
//! Releases a queue for execution.
|
|
92
|
+
template<typename PROGRAM>
|
|
93
|
+
__device__ void release_queue(PROGRAM program, QueueType queue, AdrType release_count) {
|
|
94
|
+
using ProgramType = PROGRAM;
|
|
95
|
+
using LinkType = typename ProgramType::LinkType;
|
|
96
|
+
__threadfence();
|
|
97
|
+
//unsigned int true_count = queue_count(program,queue);
|
|
98
|
+
//printf("[%d,%d]: Releasing queue (%d,%d) with count %d with delta %d\n",blockIdx.x,threadIdx.x,queue.get_head().adr,queue.get_tail().adr,true_count,release_count);
|
|
99
|
+
unsigned int index = util::random_uint(program._thd_ctx.rand_state)%ProgramType::FRAME_SIZE;
|
|
100
|
+
program.push_promises(0, index, queue, release_count);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
//! This does not truely release partial links, but will mark the link for dumping
|
|
105
|
+
//! by bumping the next semaphore by the index of the claimed pair plus 1. This means
|
|
106
|
+
//! that, after all true promise insertions have occured, the next semaphore will
|
|
107
|
+
//! be at GROUP_SIZE+1, a normally impossible value. Should the semaphore reach
|
|
108
|
+
//! GROUP_SIZE+1 directly after the incrementation (and should the semaphore not
|
|
109
|
+
//! have an original value of zero), this call will truely release the link. This
|
|
110
|
+
//! caveat to zero-valued initial values prevents double-queuing.
|
|
111
|
+
template<typename PROGRAM>
|
|
112
|
+
__device__ void release_partial(PROGRAM program, PairPack pair) {
|
|
113
|
+
using ProgramType = PROGRAM;
|
|
114
|
+
using LinkType = typename ProgramType::LinkType;
|
|
115
|
+
static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
|
|
116
|
+
|
|
117
|
+
AdrType index = pair.get_left ();
|
|
118
|
+
AdrType address = pair.get_right();
|
|
119
|
+
|
|
120
|
+
if( (address == LinkAdrType::null) || (index >= GROUP_SIZE) ){
|
|
121
|
+
//printf("[%d,%d]: Tried to mark invalid pair (%d,@%d) for release\n",blockIdx.x,threadIdx.x,index,address);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if( index == 0 ){
|
|
127
|
+
program.dump_spare_link(address);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
LinkType& link = program._dev_ctx.arena[address];
|
|
131
|
+
unsigned int delta = index+1;
|
|
132
|
+
unsigned int checkout = atomicAdd(&link.next.adr,delta);
|
|
133
|
+
|
|
134
|
+
//printf("[%d,%d]: Marking pair (%d,@%d) for release. delta(%d->%d)\n",blockIdx.x,threadIdx.x,index,address,checkout,checkout+delta);
|
|
135
|
+
|
|
136
|
+
if( ( (checkout+delta) == (GROUP_SIZE+1) ) && (checkout != 0) ) {
|
|
137
|
+
//printf("[%d,%d]: Instant release of (%d,@%d)\n",blockIdx.x,threadIdx.x,index,address);
|
|
138
|
+
atomicExch(&link.next.adr,LinkAdrType::null);
|
|
139
|
+
QueueType queue(address,address);
|
|
140
|
+
unsigned int total = atomicAdd(&link.count,0);
|
|
141
|
+
release_queue(program,queue,total);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
//! Releases the links in the full list.
|
|
149
|
+
template<typename PROGRAM>
|
|
150
|
+
__device__ void release_full(PROGRAM program) {
|
|
151
|
+
using ProgramType = PROGRAM;
|
|
152
|
+
using LinkType = typename ProgramType::LinkType;
|
|
153
|
+
|
|
154
|
+
QueueType queue;
|
|
155
|
+
queue.pair.data = atomicExch(&full_list.pair.data,QueueType::null);
|
|
156
|
+
AdrType dump_count = atomicExch(&count,0);
|
|
157
|
+
__threadfence();
|
|
158
|
+
if( (dump_count != 0) || (! queue.is_null() ) ){
|
|
159
|
+
release_queue(program,queue,dump_count);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
//! Sweeps through full list and partial slots, releasing any queues or links that
|
|
167
|
+
//! is found in the sweep.
|
|
168
|
+
template<typename PROGRAM>
|
|
169
|
+
__device__ void release_sweep(PROGRAM program) {
|
|
170
|
+
using ProgramType = PROGRAM;
|
|
171
|
+
using LinkType = typename ProgramType::LinkType;
|
|
172
|
+
|
|
173
|
+
//printf("[%d,%d]: Performing release\n",blockIdx.x,threadIdx.x);
|
|
174
|
+
release_full(program);
|
|
175
|
+
|
|
176
|
+
for( size_t i=0; i<TYPE_COUNT; i++ ) {
|
|
177
|
+
PairPack null_pair(0,LinkAdrType::null);
|
|
178
|
+
PairPack swap;
|
|
179
|
+
swap.data = atomicExch(&partial_table[i].data,null_pair.data);
|
|
180
|
+
release_partial(program,swap);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
//! Appends a full link at the given address to the full list.
|
|
186
|
+
template<typename PROGRAM>
|
|
187
|
+
__device__ void append_full(PROGRAM program, LinkAdrType address){
|
|
188
|
+
using ProgramType = PROGRAM;
|
|
189
|
+
using LinkType = typename ProgramType::LinkType;
|
|
190
|
+
static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
|
|
191
|
+
//printf("Appending full link @%d\n",address.adr);
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
if( ! address.is_null() ) {
|
|
195
|
+
LinkType& dst_link = program._dev_ctx.arena[address.adr];
|
|
196
|
+
unsigned int count = atomicAdd(&dst_link.count,0);
|
|
197
|
+
//printf("[%d,%d]:Appending full link @%d with count %d\n",blockIdx.x,threadIdx.x,address.adr,count);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
atomicAdd(&count,GROUP_SIZE);
|
|
201
|
+
program._dev_ctx.arena[address].next.adr = LinkAdrType::null;
|
|
202
|
+
QueueType src_queue = QueueType(address,address);
|
|
203
|
+
program.push_queue(full_list,src_queue);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
//!
|
|
208
|
+
//! Merges data from the link supplied as the third argument into the remaning space
|
|
209
|
+
//! available in the link supplied as the second argument. The link given by the third
|
|
210
|
+
//! argument must never have any merges in flight with it as the destination. The link
|
|
211
|
+
//! given by the second argument may have concurent merges in flight with it as the
|
|
212
|
+
//! destination. If the link given by the second argument was claimed from a partial
|
|
213
|
+
//! slot through an atomic exchange. If it has been claimed, the fourth argument MUST be
|
|
214
|
+
//! true. Likewise, if it has not been claimed, the fourth argument MUST NOT be true.
|
|
215
|
+
//! This is done to ensure that, if a link is merged into but is still not full, that
|
|
216
|
+
//! link can be safely used in future merging operations as a source.
|
|
217
|
+
//!
|
|
218
|
+
//! After merging, the count fields of the input pairs are updated to reflect the change
|
|
219
|
+
//! in occupancy. If the current thread is found to have custody of the destination
|
|
220
|
+
//! link, this function returns true. Otherwise, false.
|
|
221
|
+
//!
|
|
222
|
+
template<typename PROGRAM>
|
|
223
|
+
__device__ bool merge_links(PROGRAM program, PairPack& dst, PairPack& src, bool claimed){
|
|
224
|
+
using ProgramType = PROGRAM;
|
|
225
|
+
using LinkType = typename ProgramType::LinkType;
|
|
226
|
+
static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
|
|
227
|
+
|
|
228
|
+
AdrType dst_index = dst.get_left ();
|
|
229
|
+
AdrType dst_address = dst.get_right();
|
|
230
|
+
AdrType src_index = src.get_left ();
|
|
231
|
+
AdrType src_address = src.get_right();
|
|
232
|
+
|
|
233
|
+
if( src_index >= GROUP_SIZE ){
|
|
234
|
+
printf("\n\n\nVERY BAD\n\n\n");
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
unsigned int total = dst_index + src_index;
|
|
238
|
+
|
|
239
|
+
unsigned int dst_total = ( total >= GROUP_SIZE ) ? GROUP_SIZE : total;
|
|
240
|
+
unsigned int src_total = ( total >= GROUP_SIZE ) ? total - GROUP_SIZE : 0;
|
|
241
|
+
|
|
242
|
+
unsigned int dst_delta = dst_total - dst_index;
|
|
243
|
+
|
|
244
|
+
//printf("{[%d,%d]: src is (%d,@%d) }",blockIdx.x,threadIdx.x,src_index,src_address);
|
|
245
|
+
|
|
246
|
+
LinkType& dst_link = program._dev_ctx.arena[dst_address];
|
|
247
|
+
|
|
248
|
+
//if( claimed ) {
|
|
249
|
+
//unsigned int old_next = atomicAdd(&dst_link.next.adr,(AdrType)-dst_empty);
|
|
250
|
+
//printf("{[%d,%d]: claimed @%d}",blockIdx.x,threadIdx.x,dst_address);
|
|
251
|
+
//}
|
|
252
|
+
|
|
253
|
+
unsigned int dst_offset = atomicAdd(&dst_link.count,dst_delta);
|
|
254
|
+
unsigned int src_offset = src_index - dst_delta;
|
|
255
|
+
|
|
256
|
+
//printf("{[%d,%d]: count(%d->%d)}",blockIdx.x,threadIdx.x,dst_offset,dst_offset+dst_delta);
|
|
257
|
+
|
|
258
|
+
//printf("%d: src_offset=%d\tdst_offset=%d\tdst_delta=%d\n",blockIdx.x,src_offset,dst_offset,dst_delta);
|
|
259
|
+
|
|
260
|
+
if(src_address != LinkAdrType::null){
|
|
261
|
+
LinkType& src_link = program._dev_ctx.arena[src_address];
|
|
262
|
+
for(unsigned int i = 0; i < dst_delta; i++){
|
|
263
|
+
dst_link.promises[dst_offset+i] = src_link.promises[src_offset+i];
|
|
264
|
+
}
|
|
265
|
+
atomicAdd(&src_link.count ,-dst_delta);
|
|
266
|
+
atomicAdd(&src_link.next.adr, dst_delta);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
unsigned int checkout_delta = dst_delta;
|
|
270
|
+
if( claimed ){
|
|
271
|
+
checkout_delta += GROUP_SIZE - dst_total;
|
|
272
|
+
}
|
|
273
|
+
AdrType checkout = atomicAdd(&dst_link.next.adr,-checkout_delta);
|
|
274
|
+
//printf("{[%d,%d]: %d checkout(%d->%d)}",blockIdx.x,threadIdx.x,dst_address,checkout,checkout-checkout_delta);
|
|
275
|
+
src.set_left(src_total);
|
|
276
|
+
|
|
277
|
+
//! If checkout==0, this thread is the last thread to modify the link, and the
|
|
278
|
+
//! link has not been marked for dumping. This means we have custody of the link
|
|
279
|
+
//! and must manage getting it into the queue via a partial slot or the full
|
|
280
|
+
//! list.
|
|
281
|
+
if( (checkout-checkout_delta) == 0 ){
|
|
282
|
+
//printf("[%d-%d]: Got custody of link @%d for re-insertion.\n",blockIdx.x,threadIdx.x,dst_address);
|
|
283
|
+
unsigned int final_count = atomicAdd(&dst_link.count,0);
|
|
284
|
+
dst.set_left(final_count);
|
|
285
|
+
unsigned int reset_delta = (GROUP_SIZE-final_count);
|
|
286
|
+
unsigned int old_next = atomicAdd(&dst_link.next.adr,reset_delta);
|
|
287
|
+
//printf("[%d-%d]: Reset the next field (%d->%d) of link @%d.\n",blockIdx.x,threadIdx.x,old_next,old_next+reset_delta,dst_address);
|
|
288
|
+
__threadfence();
|
|
289
|
+
return true;
|
|
290
|
+
}
|
|
291
|
+
//! If checkout==(GROUP_SIZE+1), this thread is the last thread to modify the
|
|
292
|
+
//! link, and the link has been marked for dumping. This means we have custody
|
|
293
|
+
//! of the link and must release it.
|
|
294
|
+
else if ( (checkout-checkout_delta) == (GROUP_SIZE+1) ) {
|
|
295
|
+
//printf("[%d-%d]: Got custody of link @%d for immediate dumping.\n",blockIdx.x,threadIdx.x,dst_address);
|
|
296
|
+
atomicExch(&dst_link.next.adr,LinkAdrType::null);
|
|
297
|
+
QueueType queue(dst_address,dst_address);
|
|
298
|
+
unsigned int total = atomicAdd(&dst_link.count,0);
|
|
299
|
+
__threadfence();
|
|
300
|
+
release_queue(program,queue,total);
|
|
301
|
+
__threadfence();
|
|
302
|
+
return false;
|
|
303
|
+
}
|
|
304
|
+
//! In all other cases, we have no custody of the link.
|
|
305
|
+
else {
|
|
306
|
+
//printf("(checkout-checkout_delta) = %d\n",checkout-checkout_delta);
|
|
307
|
+
__threadfence();
|
|
308
|
+
return false;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
//! Awaits the barrier with a promise union, using the provided discriminant to determine
|
|
315
|
+
//! the type of the contained promise.
|
|
316
|
+
template<bool CAN_RELEASE=true, typename PROGRAM>
|
|
317
|
+
__device__ void union_await(PROGRAM program, OpDisc disc, typename PROGRAM::PromiseUnionType promise_union) {
|
|
318
|
+
using ProgramType = PROGRAM;
|
|
319
|
+
using LinkType = typename ProgramType::LinkType;
|
|
320
|
+
static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
|
|
321
|
+
|
|
322
|
+
//printf("Performing atomic append for type with discriminant %d.\n",disc);
|
|
323
|
+
|
|
324
|
+
PairPack& part_slot = partial_table[disc];
|
|
325
|
+
PairType inc_val = PairPack::RIGHT_MASK + 1;
|
|
326
|
+
bool optimistic = true;
|
|
327
|
+
|
|
328
|
+
//! We check the semaphore. If the semaphore is non-zero, the queue has been
|
|
329
|
+
//! released, and so the promise can be queued normally. This check does not
|
|
330
|
+
//! force a load with atomics (as is done later) because the benefits of
|
|
331
|
+
//! a forced load don't seem to make up for the overhead.
|
|
332
|
+
if( (semaphore.sem == 0) && CAN_RELEASE ){
|
|
333
|
+
//printf("early exit\n");
|
|
334
|
+
program.async_call(disc, 0, promise_union);
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
//! We start off with a link containing just our input promise. Depending
|
|
339
|
+
//! upon how merges transpire, this link will either fill up and be
|
|
340
|
+
//! successfully deposited into the queue, or will have its contents
|
|
341
|
+
//! drained into a different link and will be stored away for future use.
|
|
342
|
+
LinkAdrType first_link_adr = program.alloc_spare_link();
|
|
343
|
+
PairPack spare_pair(1u,first_link_adr.adr);
|
|
344
|
+
LinkType& start_link = program._dev_ctx.arena[first_link_adr];
|
|
345
|
+
start_link.id = disc;
|
|
346
|
+
start_link.count = 1;
|
|
347
|
+
start_link.next = GROUP_SIZE - 1;
|
|
348
|
+
start_link.promises[0] = promise_union;
|
|
349
|
+
__threadfence();
|
|
350
|
+
|
|
351
|
+
unsigned int opt_fail_count = 0;
|
|
352
|
+
bool had_custody = true;
|
|
353
|
+
|
|
354
|
+
//printf("Allocated spare link %d.\n",first_link_adr.adr);
|
|
355
|
+
|
|
356
|
+
while(true) {
|
|
357
|
+
PairPack dst_pair;
|
|
358
|
+
|
|
359
|
+
LinkAdrType spare_link_adr = spare_pair.get_right();
|
|
360
|
+
LinkType& spare_link = program._dev_ctx.arena[spare_link_adr];
|
|
361
|
+
|
|
362
|
+
//! Attempt to claim a slot in the link just by incrementing
|
|
363
|
+
//! the index of the index/address pair pack. This is quicker,
|
|
364
|
+
//! but only works if the incrementation reaches the field before
|
|
365
|
+
//! other threads claim the remaining promise slots.
|
|
366
|
+
if ( optimistic ) {
|
|
367
|
+
//printf("Today, we choose optimisim.\n");
|
|
368
|
+
dst_pair.data = atomicAdd(&part_slot.data,inc_val*spare_pair.get_left());
|
|
369
|
+
//printf("{[%d,%d] Optimistic (%d->%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_left()+spare_pair.get_left(),dst_pair.get_right());
|
|
370
|
+
}
|
|
371
|
+
//! Gain exclusive access to link via an atomic exchange. This is
|
|
372
|
+
//! slower, but has guaranteed progress.
|
|
373
|
+
else {
|
|
374
|
+
//printf("Today, we resort to pessimism.\n");
|
|
375
|
+
PairPack null_pair(0,LinkAdrType::null);
|
|
376
|
+
dst_pair.data = atomicExch(&part_slot.data,spare_pair.data);
|
|
377
|
+
spare_pair = PairPack(0,LinkAdrType::null);
|
|
378
|
+
//printf("{[%d,%d] Pessimistic (%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_right());
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
AdrType dst_index = dst_pair.get_left ();
|
|
383
|
+
AdrType dst_address = dst_pair.get_right();
|
|
384
|
+
//! Handle cases where represented link is null or already full
|
|
385
|
+
if ( (dst_address == LinkAdrType::null) || (dst_index >= GROUP_SIZE) ){
|
|
386
|
+
//! Optimistic queuing must retry, but pessimistic
|
|
387
|
+
//! queueing can get away with leaving its link behind
|
|
388
|
+
//! and doing nothing else.
|
|
389
|
+
if ( optimistic ) {
|
|
390
|
+
opt_fail_count += 1;
|
|
391
|
+
optimistic = (dst_index % GROUP_SIZE) > opt_fail_count;
|
|
392
|
+
continue;
|
|
393
|
+
} else {
|
|
394
|
+
break;
|
|
395
|
+
}
|
|
396
|
+
} else {
|
|
397
|
+
bool owns_dst = merge_links(program,dst_pair,spare_pair,!optimistic);
|
|
398
|
+
had_custody = owns_dst;
|
|
399
|
+
optimistic = (dst_index % GROUP_SIZE) != 0;
|
|
400
|
+
opt_fail_count = 0;
|
|
401
|
+
//printf("owns_dst=%d\n",owns_dst);
|
|
402
|
+
//! If the current thread has custody of the destination link,
|
|
403
|
+
//! it must handle appending it to the full list if it is full and
|
|
404
|
+
//! merging it into the partial slot if it is partial.
|
|
405
|
+
if ( owns_dst ){
|
|
406
|
+
//! Append full destination links to the full list.
|
|
407
|
+
//! DO NOT BREAK FROM THE LOOP. There may be a partial
|
|
408
|
+
//! source link that still needs to be merged in another
|
|
409
|
+
//! pass.
|
|
410
|
+
if ( dst_pair.get_left() == GROUP_SIZE ) {
|
|
411
|
+
append_full(program,dst_address);
|
|
412
|
+
}
|
|
413
|
+
//! Dump the current spare link and restart the merging
|
|
414
|
+
//! procedure with our new partial link
|
|
415
|
+
else if ( dst_pair.get_left() != 0 ) {
|
|
416
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
417
|
+
spare_pair = dst_pair;
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
//! This case should not happen, but it does not hurt to
|
|
421
|
+
//! include a branch to handle it, in case something
|
|
422
|
+
//! unexpected occurs.
|
|
423
|
+
else {
|
|
424
|
+
//printf("\n\nTHIS PRINT SHOULD BE UNREACHABLE\n\n");
|
|
425
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
426
|
+
program.dump_spare_link( dst_pair.get_right());
|
|
427
|
+
break;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
//! If the spare link is empty, dump it rather than try to merge
|
|
431
|
+
if (spare_pair.get_left() == 0) {
|
|
432
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
433
|
+
break;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
//! Double-check the sememaphore at the end. It is very important we do this double
|
|
441
|
+
//! check and that we do it at the very end of every append operation. Because
|
|
442
|
+
//! custody of partial links can be given to any append operation of the
|
|
443
|
+
//! corresponding operation type, and we don't know which append operation
|
|
444
|
+
//! comes last, we need to assume that, if the append has gotten this far, it
|
|
445
|
+
//! may be the last append and hence should make sure that no work is left
|
|
446
|
+
//! behind.
|
|
447
|
+
__threadfence();
|
|
448
|
+
unsigned int now_semaphore = atomicAdd(&semaphore.sem,0);
|
|
449
|
+
if( (now_semaphore == 0) && had_custody && CAN_RELEASE ){
|
|
450
|
+
//printf("[%d,%d]: Last-minute release required!\n",blockIdx.x,threadIdx.x);
|
|
451
|
+
release_sweep(program);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
//! Awaits the barrier with th supplied promise.
|
|
459
|
+
template<bool CAN_RELEASE=true, typename PROGRAM, typename OP_TYPE>
|
|
460
|
+
__device__ void await(PROGRAM program, Promise<OP_TYPE> promise) {
|
|
461
|
+
using ProgramType = PROGRAM;
|
|
462
|
+
using LinkType = typename ProgramType::LinkType;
|
|
463
|
+
static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
|
|
464
|
+
|
|
465
|
+
//! Guards invocations of the function to make sure invalid promise types
|
|
466
|
+
//! are not passed in.
|
|
467
|
+
static_assert(
|
|
468
|
+
UnionType::template Lookup<OP_TYPE>::type::CONTAINED,
|
|
469
|
+
"TYPE ERROR: Remapping work queue cannot queue promises of this type."
|
|
470
|
+
);
|
|
471
|
+
|
|
472
|
+
OpDisc disc = UnionType::template Lookup<OP_TYPE>::type::DISC;
|
|
473
|
+
|
|
474
|
+
//printf("Performing atomic append for type with discriminant %d.\n",disc);
|
|
475
|
+
|
|
476
|
+
PairPack& part_slot = partial_table[disc];
|
|
477
|
+
PairType inc_val = PairPack::RIGHT_MASK + 1;
|
|
478
|
+
bool optimistic = true;
|
|
479
|
+
|
|
480
|
+
//! We check the semaphore. If the semaphore is non-zero, the queue has been
|
|
481
|
+
//! released, and so the promise can be queued normally. This check does not
|
|
482
|
+
//! force a load with atomics (as is done later) because the benefits of
|
|
483
|
+
//! a forced load don't seem to make up for the overhead.
|
|
484
|
+
if( (semaphore.sem == 0) && CAN_RELEASE ){
|
|
485
|
+
//printf("early exit\n");
|
|
486
|
+
program.async_call_cast(0, promise);
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
//! We start off with a link containing just our input promise. Depending
|
|
491
|
+
//! upon how merges transpire, this link will either fill up and be
|
|
492
|
+
//! successfully deposited into the queue, or will have its contents
|
|
493
|
+
//! drained into a different link and will be stored away for future use.
|
|
494
|
+
LinkAdrType first_link_adr = program.alloc_spare_link();
|
|
495
|
+
PairPack spare_pair(1u,first_link_adr.adr);
|
|
496
|
+
LinkType& start_link = program._dev_ctx.arena[first_link_adr];
|
|
497
|
+
start_link.id = disc;
|
|
498
|
+
start_link.count = 1;
|
|
499
|
+
start_link.next = GROUP_SIZE - 1;
|
|
500
|
+
start_link.promises[0] = promise;
|
|
501
|
+
__threadfence();
|
|
502
|
+
|
|
503
|
+
unsigned int opt_fail_count = 0;
|
|
504
|
+
bool had_custody = true;
|
|
505
|
+
|
|
506
|
+
//printf("Allocated spare link %d.\n",first_link_adr.adr);
|
|
507
|
+
|
|
508
|
+
while(true) {
|
|
509
|
+
PairPack dst_pair;
|
|
510
|
+
|
|
511
|
+
LinkAdrType spare_link_adr = spare_pair.get_right();
|
|
512
|
+
LinkType& spare_link = program._dev_ctx.arena[spare_link_adr];
|
|
513
|
+
|
|
514
|
+
//! Attempt to claim a slot in the link just by incrementing
|
|
515
|
+
//! the index of the index/address pair pack. This is quicker,
|
|
516
|
+
//! but only works if the incrementation reaches the field before
|
|
517
|
+
//! other threads claim the remaining promise slots.
|
|
518
|
+
if ( optimistic ) {
|
|
519
|
+
//printf("Today, we choose optimisim.\n");
|
|
520
|
+
dst_pair.data = atomicAdd(&part_slot.data,inc_val*spare_pair.get_left());
|
|
521
|
+
//printf("{[%d,%d] Optimistic (%d->%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_left()+spare_pair.get_left(),dst_pair.get_right());
|
|
522
|
+
}
|
|
523
|
+
//! Gain exclusive access to link via an atomic exchange. This is
|
|
524
|
+
//! slower, but has guaranteed progress.
|
|
525
|
+
else {
|
|
526
|
+
//printf("Today, we resort to pessimism.\n");
|
|
527
|
+
PairPack null_pair(0,LinkAdrType::null);
|
|
528
|
+
dst_pair.data = atomicExch(&part_slot.data,spare_pair.data);
|
|
529
|
+
spare_pair = PairPack(0,LinkAdrType::null);
|
|
530
|
+
//printf("{[%d,%d] Pessimistic (%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_right());
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
AdrType dst_index = dst_pair.get_left ();
|
|
535
|
+
AdrType dst_address = dst_pair.get_right();
|
|
536
|
+
//! Handle cases where represented link is null or already full
|
|
537
|
+
if ( (dst_address == LinkAdrType::null) || (dst_index >= GROUP_SIZE) ){
|
|
538
|
+
//! Optimistic queuing must retry, but pessimistic
|
|
539
|
+
//! queueing can get away with leaving its link behind
|
|
540
|
+
//! and doing nothing else.
|
|
541
|
+
if ( optimistic ) {
|
|
542
|
+
opt_fail_count += 1;
|
|
543
|
+
optimistic = (dst_index % GROUP_SIZE) > opt_fail_count;
|
|
544
|
+
continue;
|
|
545
|
+
} else {
|
|
546
|
+
break;
|
|
547
|
+
}
|
|
548
|
+
} else {
|
|
549
|
+
bool owns_dst = merge_links(program,dst_pair,spare_pair,!optimistic);
|
|
550
|
+
had_custody = owns_dst;
|
|
551
|
+
optimistic = (dst_index % GROUP_SIZE) != 0;
|
|
552
|
+
opt_fail_count = 0;
|
|
553
|
+
//printf("owns_dst=%d\n",owns_dst);
|
|
554
|
+
//! If the current thread has custody of the destination link,
|
|
555
|
+
//! it must handle appending it to the full list if it is full and
|
|
556
|
+
//! merging it into the partial slot if it is partial.
|
|
557
|
+
if ( owns_dst ){
|
|
558
|
+
//! Append full destination links to the full list.
|
|
559
|
+
//! DO NOT BREAK FROM THE LOOP. There may be a partial
|
|
560
|
+
//! source link that still needs to be merged in another
|
|
561
|
+
//! pass.
|
|
562
|
+
if ( dst_pair.get_left() == GROUP_SIZE ) {
|
|
563
|
+
append_full(program,dst_address);
|
|
564
|
+
}
|
|
565
|
+
//! Dump the current spare link and restart the merging
|
|
566
|
+
//! procedure with our new partial link
|
|
567
|
+
else if ( dst_pair.get_left() != 0 ) {
|
|
568
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
569
|
+
spare_pair = dst_pair;
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
//! This case should not happen, but it does not hurt to
|
|
573
|
+
//! include a branch to handle it, in case something
|
|
574
|
+
//! unexpected occurs.
|
|
575
|
+
else {
|
|
576
|
+
//printf("\n\nTHIS PRINT SHOULD BE UNREACHABLE\n\n");
|
|
577
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
578
|
+
program.dump_spare_link( dst_pair.get_right());
|
|
579
|
+
break;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
//! If the spare link is empty, dump it rather than try to merge
|
|
583
|
+
if (spare_pair.get_left() == 0) {
|
|
584
|
+
program.dump_spare_link(spare_pair.get_right());
|
|
585
|
+
break;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
//! Double-check the sememaphore at the end. It is very important we do this double
|
|
595
|
+
//! check and that we do it at the very end of every append operation. Because
|
|
596
|
+
//! custody of partial links can be given to any append operation of the
|
|
597
|
+
//! corresponding operation type, and we don't know which append operation
|
|
598
|
+
//! comes last, we need to assume that, if the append has gotten this far, it
|
|
599
|
+
//! may be the last append and hence should make sure that no work is left
|
|
600
|
+
//! behind.
|
|
601
|
+
__threadfence();
|
|
602
|
+
unsigned int now_semaphore = atomicAdd(&semaphore.sem,0);
|
|
603
|
+
if( (now_semaphore == 0) && had_custody && CAN_RELEASE ){
|
|
604
|
+
//printf("[%d,%d]: Last-minute release required!\n",blockIdx.x,threadIdx.x);
|
|
605
|
+
release_sweep(program);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
//! Sets the semaphore to zero and performs a release sweep. The sweep is necessary, because
|
|
612
|
+
//! it is possible for no append operations to occur after a release operation.
|
|
613
|
+
template<typename PROGRAM>
|
|
614
|
+
__device__ void release(PROGRAM program) {
|
|
615
|
+
unsigned int now_semaphore = atomicExch(&semaphore.sem,0);
|
|
616
|
+
if( now_semaphore != 0 ) {
|
|
617
|
+
release_sweep(program);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
//! Adds the supplied delta value to the semaphore and performs a release sweep if the
|
|
622
|
+
//! result value is zero.
|
|
623
|
+
template<typename PROGRAM>
|
|
624
|
+
__device__ void add_semaphore(PROGRAM program,unsigned int delta) {
|
|
625
|
+
unsigned int now_semaphore = atomicAdd(&semaphore.sem,delta);
|
|
626
|
+
if( (now_semaphore+delta) == 0 ) {
|
|
627
|
+
release_sweep(program);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
//! Subtracts the supplied delta value from the semaphore and performs a release sweep
|
|
632
|
+
//! if the result value is zero.
|
|
633
|
+
template<typename PROGRAM>
|
|
634
|
+
__device__ void sub_semaphore(PROGRAM program,unsigned int delta) {
|
|
635
|
+
unsigned int now_semaphore = atomicAdd(&semaphore.sem,(unsigned int) -delta);
|
|
636
|
+
if( (now_semaphore-delta) == 0 ) {
|
|
637
|
+
release_sweep(program);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
//! The `UnitBarrier` template struct acts as a barrier for a single await. This is useful for
|
|
647
|
+
//! setting up additional layers of resolution for multi-dependency awaits.
|
|
648
|
+
template<typename OP_SET, typename ADR_TYPE>
|
|
649
|
+
struct UnitBarrier {
|
|
650
|
+
|
|
651
|
+
using AdrType = ADR_TYPE;
|
|
652
|
+
|
|
653
|
+
TaggedSemaphore semaphore;
|
|
654
|
+
PromiseEnum<OP_SET> promise;
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
UnitBarrier<OP_SET,ADR_TYPE>() = default;
|
|
658
|
+
|
|
659
|
+
UnitBarrier<OP_SET,ADR_TYPE>(PromiseEnum<OP_SET> promise_value, unsigned int semaphore_value)
|
|
660
|
+
: semaphore(semaphore_value,0)
|
|
661
|
+
, promise(promise_value)
|
|
662
|
+
{}
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
//! Sets the semaphore value to zero and, if it was not already zero, releases the
|
|
668
|
+
//! promise for execution
|
|
669
|
+
template<typename PROGRAM>
|
|
670
|
+
__device__ void release(PROGRAM program) {
|
|
671
|
+
unsigned int old_val = atomicExch(&semaphore.sem,0);
|
|
672
|
+
if( old_val != 0 ) {
|
|
673
|
+
program.async_call(promise);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
//! Adds the supplied delta value to the semaphore and performs a release sweep if the
|
|
678
|
+
//! result value is zero and the previous value was not zero.
|
|
679
|
+
template<typename PROGRAM>
|
|
680
|
+
__device__ void add_semaphore(PROGRAM program,unsigned int delta) {
|
|
681
|
+
unsigned int old_val = atomicAdd(&semaphore.sem,delta);
|
|
682
|
+
if( ((old_val+delta) == 0) && (old_val != 0) ) {
|
|
683
|
+
program.async_call(promise);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
//! Subtracts the supplied delta value from the semaphore and performs a release sweep
|
|
688
|
+
//! if the result value is zero and the previous value was not zero.
|
|
689
|
+
template<typename PROGRAM>
|
|
690
|
+
__device__ void sub_semaphore(PROGRAM program,unsigned int delta) {
|
|
691
|
+
unsigned int old_val = atomicAdd(&semaphore.sem,(unsigned int) -delta);
|
|
692
|
+
if( ((old_val-delta) == 0) && (old_val != 0) ) {
|
|
693
|
+
program.async_call(promise);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
};
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
#endif
|
|
703
|
+
|
|
704
|
+
|