harmonize-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,704 @@
1
+
2
+
3
+
4
+ #ifndef HARMONIZE_SYNC_BARRIER
5
+ #define HARMONIZE_SYNC_BARRIER
6
+
7
+ #include "../async/mod.h"
8
+
9
+ //!
10
+ //! A `RemappingBarrier` coaleces promises into links in a lock-free manner.
11
+ //! Once released, all work in the queue is made available for work groups to
12
+ //! execute and all further appending operations will redirect promises to execution.
13
+ //! After being released, a queue may be reset to a non-released state.
14
+ //!
15
+ template<typename OP_SET, typename ADR_TYPE>
16
+ struct RemappingBarrier {
17
+
18
+
19
+ #if 0
20
+ using AdrType = typename PROGRAM::AdrType;
21
+ using LinkAdrType = typename PROGRAM::LinkAdrType;
22
+ using QueueType = typename PROGRAM::QueueType;
23
+ using ProgramType = PROGRAM;
24
+ using UnionType = PromiseUnion<OP_SET>;
25
+ using LinkType = typename PROGRAM::LinkType;
26
+
27
+ using PairPack = util::mem::PairPack<AdrType>;
28
+ using PairType = typename PairPack::PairType;
29
+ #else
30
+ using AdrType = ADR_TYPE;
31
+ using LinkAdrType = util::mem::Adr<AdrType>;
32
+ using QueueType = util::mem::PoolQueue<LinkAdrType>;
33
+ using UnionType = PromiseUnion<OP_SET>;
34
+
35
+ using PairPack = util::mem::PairPack<AdrType>;
36
+ using PairType = typename PairPack::PairType;
37
+ #endif
38
+
39
+ //! The number of operations contained within the barrier's
40
+ //! operation set.
41
+ static const size_t TYPE_COUNT = UnionType::Info::COUNT;
42
+
43
+ //! A tagged semaphore that communicates both the release state of the
44
+ //! barrier, but the type of the barrier as well.
45
+ TaggedSemaphore semaphore;
46
+ unsigned int count;
47
+
48
+ //! A queue that contains all full links created by coalescing promises
49
+ //! awaiting the release of the barrier.
50
+ QueueType full_list;
51
+
52
+ //! A table used to mediate the coalescing of links
53
+ PairPack partial_table[UnionType::Info::COUNT];
54
+
55
+
56
+ RemappingBarrier<OP_SET,ADR_TYPE>() = default;
57
+
58
+ //! Creates a new `RemappingBarrier` with an empty full queue and partial table
59
+ //! and with a semaphore value initialized to the supplied value.
60
+ static __host__ __device__ RemappingBarrier<OP_SET,ADR_TYPE> blank(unsigned int sem_val)
61
+ {
62
+ RemappingBarrier<OP_SET,ADR_TYPE> result;
63
+ result.semaphore = TaggedSemaphore(sem_val,1);
64
+ result.count = 0u;
65
+ result.full_list.pair.data = QueueType::null;
66
+ for(int i=0; i<TYPE_COUNT; i++){
67
+ result.partial_table[i] = PairPack(0,LinkAdrType::null);
68
+ }
69
+ return result;
70
+ }
71
+
72
+
73
+ //! Counts the number of links in the given queue. This is only used for debugging purposes.
74
+ template<typename PROGRAM>
75
+ __device__ unsigned int queue_count(PROGRAM program, QueueType queue) {
76
+ using ProgramType = PROGRAM;
77
+ using LinkType = typename ProgramType::LinkType;
78
+
79
+ unsigned int total = 0;
80
+ LinkAdrType iter = queue.get_head();
81
+ while( ! iter.is_null() ){
82
+ LinkType& the_link = program._dev_ctx.arena[iter.adr];
83
+ total += the_link.count;
84
+ //printf("( [%d,%d] @%d #%d -> %d )",blockIdx.x,threadIdx.x,iter.adr,the_link.count,the_link.next.adr);
85
+ iter = the_link.next;
86
+ }
87
+ return total;
88
+ }
89
+
90
+
91
+ //! Releases a queue for execution.
92
+ template<typename PROGRAM>
93
+ __device__ void release_queue(PROGRAM program, QueueType queue, AdrType release_count) {
94
+ using ProgramType = PROGRAM;
95
+ using LinkType = typename ProgramType::LinkType;
96
+ __threadfence();
97
+ //unsigned int true_count = queue_count(program,queue);
98
+ //printf("[%d,%d]: Releasing queue (%d,%d) with count %d with delta %d\n",blockIdx.x,threadIdx.x,queue.get_head().adr,queue.get_tail().adr,true_count,release_count);
99
+ unsigned int index = util::random_uint(program._thd_ctx.rand_state)%ProgramType::FRAME_SIZE;
100
+ program.push_promises(0, index, queue, release_count);
101
+ }
102
+
103
+
104
+ //! This does not truely release partial links, but will mark the link for dumping
105
+ //! by bumping the next semaphore by the index of the claimed pair plus 1. This means
106
+ //! that, after all true promise insertions have occured, the next semaphore will
107
+ //! be at GROUP_SIZE+1, a normally impossible value. Should the semaphore reach
108
+ //! GROUP_SIZE+1 directly after the incrementation (and should the semaphore not
109
+ //! have an original value of zero), this call will truely release the link. This
110
+ //! caveat to zero-valued initial values prevents double-queuing.
111
+ template<typename PROGRAM>
112
+ __device__ void release_partial(PROGRAM program, PairPack pair) {
113
+ using ProgramType = PROGRAM;
114
+ using LinkType = typename ProgramType::LinkType;
115
+ static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
116
+
117
+ AdrType index = pair.get_left ();
118
+ AdrType address = pair.get_right();
119
+
120
+ if( (address == LinkAdrType::null) || (index >= GROUP_SIZE) ){
121
+ //printf("[%d,%d]: Tried to mark invalid pair (%d,@%d) for release\n",blockIdx.x,threadIdx.x,index,address);
122
+ return;
123
+ }
124
+
125
+
126
+ if( index == 0 ){
127
+ program.dump_spare_link(address);
128
+ }
129
+
130
+ LinkType& link = program._dev_ctx.arena[address];
131
+ unsigned int delta = index+1;
132
+ unsigned int checkout = atomicAdd(&link.next.adr,delta);
133
+
134
+ //printf("[%d,%d]: Marking pair (%d,@%d) for release. delta(%d->%d)\n",blockIdx.x,threadIdx.x,index,address,checkout,checkout+delta);
135
+
136
+ if( ( (checkout+delta) == (GROUP_SIZE+1) ) && (checkout != 0) ) {
137
+ //printf("[%d,%d]: Instant release of (%d,@%d)\n",blockIdx.x,threadIdx.x,index,address);
138
+ atomicExch(&link.next.adr,LinkAdrType::null);
139
+ QueueType queue(address,address);
140
+ unsigned int total = atomicAdd(&link.count,0);
141
+ release_queue(program,queue,total);
142
+ }
143
+
144
+
145
+ }
146
+
147
+
148
+ //! Releases the links in the full list.
149
+ template<typename PROGRAM>
150
+ __device__ void release_full(PROGRAM program) {
151
+ using ProgramType = PROGRAM;
152
+ using LinkType = typename ProgramType::LinkType;
153
+
154
+ QueueType queue;
155
+ queue.pair.data = atomicExch(&full_list.pair.data,QueueType::null);
156
+ AdrType dump_count = atomicExch(&count,0);
157
+ __threadfence();
158
+ if( (dump_count != 0) || (! queue.is_null() ) ){
159
+ release_queue(program,queue,dump_count);
160
+ }
161
+
162
+
163
+ }
164
+
165
+
166
+ //! Sweeps through full list and partial slots, releasing any queues or links that
167
+ //! is found in the sweep.
168
+ template<typename PROGRAM>
169
+ __device__ void release_sweep(PROGRAM program) {
170
+ using ProgramType = PROGRAM;
171
+ using LinkType = typename ProgramType::LinkType;
172
+
173
+ //printf("[%d,%d]: Performing release\n",blockIdx.x,threadIdx.x);
174
+ release_full(program);
175
+
176
+ for( size_t i=0; i<TYPE_COUNT; i++ ) {
177
+ PairPack null_pair(0,LinkAdrType::null);
178
+ PairPack swap;
179
+ swap.data = atomicExch(&partial_table[i].data,null_pair.data);
180
+ release_partial(program,swap);
181
+ }
182
+ }
183
+
184
+
185
+ //! Appends a full link at the given address to the full list.
186
+ template<typename PROGRAM>
187
+ __device__ void append_full(PROGRAM program, LinkAdrType address){
188
+ using ProgramType = PROGRAM;
189
+ using LinkType = typename ProgramType::LinkType;
190
+ static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
191
+ //printf("Appending full link @%d\n",address.adr);
192
+
193
+
194
+ if( ! address.is_null() ) {
195
+ LinkType& dst_link = program._dev_ctx.arena[address.adr];
196
+ unsigned int count = atomicAdd(&dst_link.count,0);
197
+ //printf("[%d,%d]:Appending full link @%d with count %d\n",blockIdx.x,threadIdx.x,address.adr,count);
198
+ }
199
+
200
+ atomicAdd(&count,GROUP_SIZE);
201
+ program._dev_ctx.arena[address].next.adr = LinkAdrType::null;
202
+ QueueType src_queue = QueueType(address,address);
203
+ program.push_queue(full_list,src_queue);
204
+ }
205
+
206
+
207
+ //!
208
+ //! Merges data from the link supplied as the third argument into the remaning space
209
+ //! available in the link supplied as the second argument. The link given by the third
210
+ //! argument must never have any merges in flight with it as the destination. The link
211
+ //! given by the second argument may have concurent merges in flight with it as the
212
+ //! destination. If the link given by the second argument was claimed from a partial
213
+ //! slot through an atomic exchange. If it has been claimed, the fourth argument MUST be
214
+ //! true. Likewise, if it has not been claimed, the fourth argument MUST NOT be true.
215
+ //! This is done to ensure that, if a link is merged into but is still not full, that
216
+ //! link can be safely used in future merging operations as a source.
217
+ //!
218
+ //! After merging, the count fields of the input pairs are updated to reflect the change
219
+ //! in occupancy. If the current thread is found to have custody of the destination
220
+ //! link, this function returns true. Otherwise, false.
221
+ //!
222
+ template<typename PROGRAM>
223
+ __device__ bool merge_links(PROGRAM program, PairPack& dst, PairPack& src, bool claimed){
224
+ using ProgramType = PROGRAM;
225
+ using LinkType = typename ProgramType::LinkType;
226
+ static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
227
+
228
+ AdrType dst_index = dst.get_left ();
229
+ AdrType dst_address = dst.get_right();
230
+ AdrType src_index = src.get_left ();
231
+ AdrType src_address = src.get_right();
232
+
233
+ if( src_index >= GROUP_SIZE ){
234
+ printf("\n\n\nVERY BAD\n\n\n");
235
+ }
236
+
237
+ unsigned int total = dst_index + src_index;
238
+
239
+ unsigned int dst_total = ( total >= GROUP_SIZE ) ? GROUP_SIZE : total;
240
+ unsigned int src_total = ( total >= GROUP_SIZE ) ? total - GROUP_SIZE : 0;
241
+
242
+ unsigned int dst_delta = dst_total - dst_index;
243
+
244
+ //printf("{[%d,%d]: src is (%d,@%d) }",blockIdx.x,threadIdx.x,src_index,src_address);
245
+
246
+ LinkType& dst_link = program._dev_ctx.arena[dst_address];
247
+
248
+ //if( claimed ) {
249
+ //unsigned int old_next = atomicAdd(&dst_link.next.adr,(AdrType)-dst_empty);
250
+ //printf("{[%d,%d]: claimed @%d}",blockIdx.x,threadIdx.x,dst_address);
251
+ //}
252
+
253
+ unsigned int dst_offset = atomicAdd(&dst_link.count,dst_delta);
254
+ unsigned int src_offset = src_index - dst_delta;
255
+
256
+ //printf("{[%d,%d]: count(%d->%d)}",blockIdx.x,threadIdx.x,dst_offset,dst_offset+dst_delta);
257
+
258
+ //printf("%d: src_offset=%d\tdst_offset=%d\tdst_delta=%d\n",blockIdx.x,src_offset,dst_offset,dst_delta);
259
+
260
+ if(src_address != LinkAdrType::null){
261
+ LinkType& src_link = program._dev_ctx.arena[src_address];
262
+ for(unsigned int i = 0; i < dst_delta; i++){
263
+ dst_link.promises[dst_offset+i] = src_link.promises[src_offset+i];
264
+ }
265
+ atomicAdd(&src_link.count ,-dst_delta);
266
+ atomicAdd(&src_link.next.adr, dst_delta);
267
+ }
268
+
269
+ unsigned int checkout_delta = dst_delta;
270
+ if( claimed ){
271
+ checkout_delta += GROUP_SIZE - dst_total;
272
+ }
273
+ AdrType checkout = atomicAdd(&dst_link.next.adr,-checkout_delta);
274
+ //printf("{[%d,%d]: %d checkout(%d->%d)}",blockIdx.x,threadIdx.x,dst_address,checkout,checkout-checkout_delta);
275
+ src.set_left(src_total);
276
+
277
+ //! If checkout==0, this thread is the last thread to modify the link, and the
278
+ //! link has not been marked for dumping. This means we have custody of the link
279
+ //! and must manage getting it into the queue via a partial slot or the full
280
+ //! list.
281
+ if( (checkout-checkout_delta) == 0 ){
282
+ //printf("[%d-%d]: Got custody of link @%d for re-insertion.\n",blockIdx.x,threadIdx.x,dst_address);
283
+ unsigned int final_count = atomicAdd(&dst_link.count,0);
284
+ dst.set_left(final_count);
285
+ unsigned int reset_delta = (GROUP_SIZE-final_count);
286
+ unsigned int old_next = atomicAdd(&dst_link.next.adr,reset_delta);
287
+ //printf("[%d-%d]: Reset the next field (%d->%d) of link @%d.\n",blockIdx.x,threadIdx.x,old_next,old_next+reset_delta,dst_address);
288
+ __threadfence();
289
+ return true;
290
+ }
291
+ //! If checkout==(GROUP_SIZE+1), this thread is the last thread to modify the
292
+ //! link, and the link has been marked for dumping. This means we have custody
293
+ //! of the link and must release it.
294
+ else if ( (checkout-checkout_delta) == (GROUP_SIZE+1) ) {
295
+ //printf("[%d-%d]: Got custody of link @%d for immediate dumping.\n",blockIdx.x,threadIdx.x,dst_address);
296
+ atomicExch(&dst_link.next.adr,LinkAdrType::null);
297
+ QueueType queue(dst_address,dst_address);
298
+ unsigned int total = atomicAdd(&dst_link.count,0);
299
+ __threadfence();
300
+ release_queue(program,queue,total);
301
+ __threadfence();
302
+ return false;
303
+ }
304
+ //! In all other cases, we have no custody of the link.
305
+ else {
306
+ //printf("(checkout-checkout_delta) = %d\n",checkout-checkout_delta);
307
+ __threadfence();
308
+ return false;
309
+ }
310
+
311
+ }
312
+
313
+
314
+ //! Awaits the barrier with a promise union, using the provided discriminant to determine
315
+ //! the type of the contained promise.
316
+ template<bool CAN_RELEASE=true, typename PROGRAM>
317
+ __device__ void union_await(PROGRAM program, OpDisc disc, typename PROGRAM::PromiseUnionType promise_union) {
318
+ using ProgramType = PROGRAM;
319
+ using LinkType = typename ProgramType::LinkType;
320
+ static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
321
+
322
+ //printf("Performing atomic append for type with discriminant %d.\n",disc);
323
+
324
+ PairPack& part_slot = partial_table[disc];
325
+ PairType inc_val = PairPack::RIGHT_MASK + 1;
326
+ bool optimistic = true;
327
+
328
+ //! We check the semaphore. If the semaphore is non-zero, the queue has been
329
+ //! released, and so the promise can be queued normally. This check does not
330
+ //! force a load with atomics (as is done later) because the benefits of
331
+ //! a forced load don't seem to make up for the overhead.
332
+ if( (semaphore.sem == 0) && CAN_RELEASE ){
333
+ //printf("early exit\n");
334
+ program.async_call(disc, 0, promise_union);
335
+ return;
336
+ }
337
+
338
+ //! We start off with a link containing just our input promise. Depending
339
+ //! upon how merges transpire, this link will either fill up and be
340
+ //! successfully deposited into the queue, or will have its contents
341
+ //! drained into a different link and will be stored away for future use.
342
+ LinkAdrType first_link_adr = program.alloc_spare_link();
343
+ PairPack spare_pair(1u,first_link_adr.adr);
344
+ LinkType& start_link = program._dev_ctx.arena[first_link_adr];
345
+ start_link.id = disc;
346
+ start_link.count = 1;
347
+ start_link.next = GROUP_SIZE - 1;
348
+ start_link.promises[0] = promise_union;
349
+ __threadfence();
350
+
351
+ unsigned int opt_fail_count = 0;
352
+ bool had_custody = true;
353
+
354
+ //printf("Allocated spare link %d.\n",first_link_adr.adr);
355
+
356
+ while(true) {
357
+ PairPack dst_pair;
358
+
359
+ LinkAdrType spare_link_adr = spare_pair.get_right();
360
+ LinkType& spare_link = program._dev_ctx.arena[spare_link_adr];
361
+
362
+ //! Attempt to claim a slot in the link just by incrementing
363
+ //! the index of the index/address pair pack. This is quicker,
364
+ //! but only works if the incrementation reaches the field before
365
+ //! other threads claim the remaining promise slots.
366
+ if ( optimistic ) {
367
+ //printf("Today, we choose optimisim.\n");
368
+ dst_pair.data = atomicAdd(&part_slot.data,inc_val*spare_pair.get_left());
369
+ //printf("{[%d,%d] Optimistic (%d->%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_left()+spare_pair.get_left(),dst_pair.get_right());
370
+ }
371
+ //! Gain exclusive access to link via an atomic exchange. This is
372
+ //! slower, but has guaranteed progress.
373
+ else {
374
+ //printf("Today, we resort to pessimism.\n");
375
+ PairPack null_pair(0,LinkAdrType::null);
376
+ dst_pair.data = atomicExch(&part_slot.data,spare_pair.data);
377
+ spare_pair = PairPack(0,LinkAdrType::null);
378
+ //printf("{[%d,%d] Pessimistic (%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_right());
379
+ }
380
+
381
+
382
+ AdrType dst_index = dst_pair.get_left ();
383
+ AdrType dst_address = dst_pair.get_right();
384
+ //! Handle cases where represented link is null or already full
385
+ if ( (dst_address == LinkAdrType::null) || (dst_index >= GROUP_SIZE) ){
386
+ //! Optimistic queuing must retry, but pessimistic
387
+ //! queueing can get away with leaving its link behind
388
+ //! and doing nothing else.
389
+ if ( optimistic ) {
390
+ opt_fail_count += 1;
391
+ optimistic = (dst_index % GROUP_SIZE) > opt_fail_count;
392
+ continue;
393
+ } else {
394
+ break;
395
+ }
396
+ } else {
397
+ bool owns_dst = merge_links(program,dst_pair,spare_pair,!optimistic);
398
+ had_custody = owns_dst;
399
+ optimistic = (dst_index % GROUP_SIZE) != 0;
400
+ opt_fail_count = 0;
401
+ //printf("owns_dst=%d\n",owns_dst);
402
+ //! If the current thread has custody of the destination link,
403
+ //! it must handle appending it to the full list if it is full and
404
+ //! merging it into the partial slot if it is partial.
405
+ if ( owns_dst ){
406
+ //! Append full destination links to the full list.
407
+ //! DO NOT BREAK FROM THE LOOP. There may be a partial
408
+ //! source link that still needs to be merged in another
409
+ //! pass.
410
+ if ( dst_pair.get_left() == GROUP_SIZE ) {
411
+ append_full(program,dst_address);
412
+ }
413
+ //! Dump the current spare link and restart the merging
414
+ //! procedure with our new partial link
415
+ else if ( dst_pair.get_left() != 0 ) {
416
+ program.dump_spare_link(spare_pair.get_right());
417
+ spare_pair = dst_pair;
418
+ continue;
419
+ }
420
+ //! This case should not happen, but it does not hurt to
421
+ //! include a branch to handle it, in case something
422
+ //! unexpected occurs.
423
+ else {
424
+ //printf("\n\nTHIS PRINT SHOULD BE UNREACHABLE\n\n");
425
+ program.dump_spare_link(spare_pair.get_right());
426
+ program.dump_spare_link( dst_pair.get_right());
427
+ break;
428
+ }
429
+ }
430
+ //! If the spare link is empty, dump it rather than try to merge
431
+ if (spare_pair.get_left() == 0) {
432
+ program.dump_spare_link(spare_pair.get_right());
433
+ break;
434
+ }
435
+ }
436
+
437
+ }
438
+
439
+
440
+ //! Double-check the sememaphore at the end. It is very important we do this double
441
+ //! check and that we do it at the very end of every append operation. Because
442
+ //! custody of partial links can be given to any append operation of the
443
+ //! corresponding operation type, and we don't know which append operation
444
+ //! comes last, we need to assume that, if the append has gotten this far, it
445
+ //! may be the last append and hence should make sure that no work is left
446
+ //! behind.
447
+ __threadfence();
448
+ unsigned int now_semaphore = atomicAdd(&semaphore.sem,0);
449
+ if( (now_semaphore == 0) && had_custody && CAN_RELEASE ){
450
+ //printf("[%d,%d]: Last-minute release required!\n",blockIdx.x,threadIdx.x);
451
+ release_sweep(program);
452
+ }
453
+
454
+
455
+ }
456
+
457
+
458
+ //! Awaits the barrier with th supplied promise.
459
+ template<bool CAN_RELEASE=true, typename PROGRAM, typename OP_TYPE>
460
+ __device__ void await(PROGRAM program, Promise<OP_TYPE> promise) {
461
+ using ProgramType = PROGRAM;
462
+ using LinkType = typename ProgramType::LinkType;
463
+ static const size_t GROUP_SIZE = ProgramType::GROUP_SIZE;
464
+
465
+ //! Guards invocations of the function to make sure invalid promise types
466
+ //! are not passed in.
467
+ static_assert(
468
+ UnionType::template Lookup<OP_TYPE>::type::CONTAINED,
469
+ "TYPE ERROR: Remapping work queue cannot queue promises of this type."
470
+ );
471
+
472
+ OpDisc disc = UnionType::template Lookup<OP_TYPE>::type::DISC;
473
+
474
+ //printf("Performing atomic append for type with discriminant %d.\n",disc);
475
+
476
+ PairPack& part_slot = partial_table[disc];
477
+ PairType inc_val = PairPack::RIGHT_MASK + 1;
478
+ bool optimistic = true;
479
+
480
+ //! We check the semaphore. If the semaphore is non-zero, the queue has been
481
+ //! released, and so the promise can be queued normally. This check does not
482
+ //! force a load with atomics (as is done later) because the benefits of
483
+ //! a forced load don't seem to make up for the overhead.
484
+ if( (semaphore.sem == 0) && CAN_RELEASE ){
485
+ //printf("early exit\n");
486
+ program.async_call_cast(0, promise);
487
+ return;
488
+ }
489
+
490
+ //! We start off with a link containing just our input promise. Depending
491
+ //! upon how merges transpire, this link will either fill up and be
492
+ //! successfully deposited into the queue, or will have its contents
493
+ //! drained into a different link and will be stored away for future use.
494
+ LinkAdrType first_link_adr = program.alloc_spare_link();
495
+ PairPack spare_pair(1u,first_link_adr.adr);
496
+ LinkType& start_link = program._dev_ctx.arena[first_link_adr];
497
+ start_link.id = disc;
498
+ start_link.count = 1;
499
+ start_link.next = GROUP_SIZE - 1;
500
+ start_link.promises[0] = promise;
501
+ __threadfence();
502
+
503
+ unsigned int opt_fail_count = 0;
504
+ bool had_custody = true;
505
+
506
+ //printf("Allocated spare link %d.\n",first_link_adr.adr);
507
+
508
+ while(true) {
509
+ PairPack dst_pair;
510
+
511
+ LinkAdrType spare_link_adr = spare_pair.get_right();
512
+ LinkType& spare_link = program._dev_ctx.arena[spare_link_adr];
513
+
514
+ //! Attempt to claim a slot in the link just by incrementing
515
+ //! the index of the index/address pair pack. This is quicker,
516
+ //! but only works if the incrementation reaches the field before
517
+ //! other threads claim the remaining promise slots.
518
+ if ( optimistic ) {
519
+ //printf("Today, we choose optimisim.\n");
520
+ dst_pair.data = atomicAdd(&part_slot.data,inc_val*spare_pair.get_left());
521
+ //printf("{[%d,%d] Optimistic (%d->%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_left()+spare_pair.get_left(),dst_pair.get_right());
522
+ }
523
+ //! Gain exclusive access to link via an atomic exchange. This is
524
+ //! slower, but has guaranteed progress.
525
+ else {
526
+ //printf("Today, we resort to pessimism.\n");
527
+ PairPack null_pair(0,LinkAdrType::null);
528
+ dst_pair.data = atomicExch(&part_slot.data,spare_pair.data);
529
+ spare_pair = PairPack(0,LinkAdrType::null);
530
+ //printf("{[%d,%d] Pessimistic (%d,@%d)}",blockIdx.x,threadIdx.x,dst_pair.get_left(),dst_pair.get_right());
531
+ }
532
+
533
+
534
+ AdrType dst_index = dst_pair.get_left ();
535
+ AdrType dst_address = dst_pair.get_right();
536
+ //! Handle cases where represented link is null or already full
537
+ if ( (dst_address == LinkAdrType::null) || (dst_index >= GROUP_SIZE) ){
538
+ //! Optimistic queuing must retry, but pessimistic
539
+ //! queueing can get away with leaving its link behind
540
+ //! and doing nothing else.
541
+ if ( optimistic ) {
542
+ opt_fail_count += 1;
543
+ optimistic = (dst_index % GROUP_SIZE) > opt_fail_count;
544
+ continue;
545
+ } else {
546
+ break;
547
+ }
548
+ } else {
549
+ bool owns_dst = merge_links(program,dst_pair,spare_pair,!optimistic);
550
+ had_custody = owns_dst;
551
+ optimistic = (dst_index % GROUP_SIZE) != 0;
552
+ opt_fail_count = 0;
553
+ //printf("owns_dst=%d\n",owns_dst);
554
+ //! If the current thread has custody of the destination link,
555
+ //! it must handle appending it to the full list if it is full and
556
+ //! merging it into the partial slot if it is partial.
557
+ if ( owns_dst ){
558
+ //! Append full destination links to the full list.
559
+ //! DO NOT BREAK FROM THE LOOP. There may be a partial
560
+ //! source link that still needs to be merged in another
561
+ //! pass.
562
+ if ( dst_pair.get_left() == GROUP_SIZE ) {
563
+ append_full(program,dst_address);
564
+ }
565
+ //! Dump the current spare link and restart the merging
566
+ //! procedure with our new partial link
567
+ else if ( dst_pair.get_left() != 0 ) {
568
+ program.dump_spare_link(spare_pair.get_right());
569
+ spare_pair = dst_pair;
570
+ continue;
571
+ }
572
+ //! This case should not happen, but it does not hurt to
573
+ //! include a branch to handle it, in case something
574
+ //! unexpected occurs.
575
+ else {
576
+ //printf("\n\nTHIS PRINT SHOULD BE UNREACHABLE\n\n");
577
+ program.dump_spare_link(spare_pair.get_right());
578
+ program.dump_spare_link( dst_pair.get_right());
579
+ break;
580
+ }
581
+ }
582
+ //! If the spare link is empty, dump it rather than try to merge
583
+ if (spare_pair.get_left() == 0) {
584
+ program.dump_spare_link(spare_pair.get_right());
585
+ break;
586
+ }
587
+ }
588
+
589
+ }
590
+
591
+
592
+
593
+
594
+ //! Double-check the sememaphore at the end. It is very important we do this double
595
+ //! check and that we do it at the very end of every append operation. Because
596
+ //! custody of partial links can be given to any append operation of the
597
+ //! corresponding operation type, and we don't know which append operation
598
+ //! comes last, we need to assume that, if the append has gotten this far, it
599
+ //! may be the last append and hence should make sure that no work is left
600
+ //! behind.
601
+ __threadfence();
602
+ unsigned int now_semaphore = atomicAdd(&semaphore.sem,0);
603
+ if( (now_semaphore == 0) && had_custody && CAN_RELEASE ){
604
+ //printf("[%d,%d]: Last-minute release required!\n",blockIdx.x,threadIdx.x);
605
+ release_sweep(program);
606
+ }
607
+
608
+
609
+ }
610
+
611
+ //! Sets the semaphore to zero and performs a release sweep. The sweep is necessary, because
612
+ //! it is possible for no append operations to occur after a release operation.
613
+ template<typename PROGRAM>
614
+ __device__ void release(PROGRAM program) {
615
+ unsigned int now_semaphore = atomicExch(&semaphore.sem,0);
616
+ if( now_semaphore != 0 ) {
617
+ release_sweep(program);
618
+ }
619
+ }
620
+
621
+ //! Adds the supplied delta value to the semaphore and performs a release sweep if the
622
+ //! result value is zero.
623
+ template<typename PROGRAM>
624
+ __device__ void add_semaphore(PROGRAM program,unsigned int delta) {
625
+ unsigned int now_semaphore = atomicAdd(&semaphore.sem,delta);
626
+ if( (now_semaphore+delta) == 0 ) {
627
+ release_sweep(program);
628
+ }
629
+ }
630
+
631
+ //! Subtracts the supplied delta value from the semaphore and performs a release sweep
632
+ //! if the result value is zero.
633
+ template<typename PROGRAM>
634
+ __device__ void sub_semaphore(PROGRAM program,unsigned int delta) {
635
+ unsigned int now_semaphore = atomicAdd(&semaphore.sem,(unsigned int) -delta);
636
+ if( (now_semaphore-delta) == 0 ) {
637
+ release_sweep(program);
638
+ }
639
+ }
640
+
641
+ };
642
+
643
+
644
+
645
+
646
+ //! The `UnitBarrier` template struct acts as a barrier for a single await. This is useful for
647
+ //! setting up additional layers of resolution for multi-dependency awaits.
648
+ template<typename OP_SET, typename ADR_TYPE>
649
+ struct UnitBarrier {
650
+
651
+ using AdrType = ADR_TYPE;
652
+
653
+ TaggedSemaphore semaphore;
654
+ PromiseEnum<OP_SET> promise;
655
+
656
+
657
+ UnitBarrier<OP_SET,ADR_TYPE>() = default;
658
+
659
+ UnitBarrier<OP_SET,ADR_TYPE>(PromiseEnum<OP_SET> promise_value, unsigned int semaphore_value)
660
+ : semaphore(semaphore_value,0)
661
+ , promise(promise_value)
662
+ {}
663
+
664
+
665
+
666
+
667
+ //! Sets the semaphore value to zero and, if it was not already zero, releases the
668
+ //! promise for execution
669
+ template<typename PROGRAM>
670
+ __device__ void release(PROGRAM program) {
671
+ unsigned int old_val = atomicExch(&semaphore.sem,0);
672
+ if( old_val != 0 ) {
673
+ program.async_call(promise);
674
+ }
675
+ }
676
+
677
+ //! Adds the supplied delta value to the semaphore and performs a release sweep if the
678
+ //! result value is zero and the previous value was not zero.
679
+ template<typename PROGRAM>
680
+ __device__ void add_semaphore(PROGRAM program,unsigned int delta) {
681
+ unsigned int old_val = atomicAdd(&semaphore.sem,delta);
682
+ if( ((old_val+delta) == 0) && (old_val != 0) ) {
683
+ program.async_call(promise);
684
+ }
685
+ }
686
+
687
+ //! Subtracts the supplied delta value from the semaphore and performs a release sweep
688
+ //! if the result value is zero and the previous value was not zero.
689
+ template<typename PROGRAM>
690
+ __device__ void sub_semaphore(PROGRAM program,unsigned int delta) {
691
+ unsigned int old_val = atomicAdd(&semaphore.sem,(unsigned int) -delta);
692
+ if( ((old_val-delta) == 0) && (old_val != 0) ) {
693
+ program.async_call(promise);
694
+ }
695
+ }
696
+
697
+
698
+
699
+ };
700
+
701
+
702
+ #endif
703
+
704
+