harmonize-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2923 @@
1
+
2
+
3
+
4
+ #ifndef HARMONIZE_PROGRAM_ASYNC
5
+ #define HARMONIZE_PROGRAM_ASYNC
6
+
7
+
8
+
9
+ #include "../async/mod.h"
10
+
11
+ #include "../mem/mod.h"
12
+
13
+
14
+ //! This macro is currently unused and partially implemented. When finished, it will
15
+ //! operate like the `MEMBER_SWITCH` macro, but for internal templates.
16
+ #define TEMPLATE_MEMBER_SWITCH(NAME,DEFAULT_TYPE) \
17
+ template<class PROG,class DEFAULT> static auto NAME##Lookup (int) -> DEFAULT; \
18
+ template<class PROG,class DEFAULT> static auto NAME##Lookup (bool) -> typename Specializer<typename PROG::NAME>::Type; \
19
+ typedef decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_TYPE>(true)) NAME;
20
+
21
+
22
+ //! This macro inserts code that detects whether or not an internal type is defined by
23
+ //! the type `PROGRAM_SPEC` (the parameter name used to refer to a program specification)
24
+ //! and either defines an internal type that duplicates the internal type of `PROGRAM_SPEC`
25
+ //! or defines an internal type based off of a default.
26
+ #define MEMBER_SWITCH(NAME,DEFAULT_TYPE) \
27
+ template<class PROG,class DEFAULT> static auto NAME##Lookup (int) -> DEFAULT; \
28
+ template<class PROG,class DEFAULT> static auto NAME##Lookup (bool) -> typename PROG::NAME; \
29
+ typedef decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_TYPE>(true)) NAME;
30
+
31
+ //! This macro behaves the same as `MEMBER_SWITCH`, but for internal static constants,
32
+ //! rather than for types
33
+ #define CONST_SWITCH(TYPE,NAME,DEFAULT_VAL) \
34
+ template<class PROG,TYPE DEFAULT> static auto NAME##Lookup (int) -> std::integral_constant<TYPE, DEFAULT>; \
35
+ template<class PROG,TYPE DEFAULT> static auto NAME##Lookup (bool) -> std::integral_constant<TYPE, PROG::NAME>; \
36
+ static const size_t NAME = decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_VAL>(true))::value;
37
+
38
+
39
+
40
+
41
+ ////////////////////////////////////////////////////////////////////////////////////
42
+ // //
43
+ // To have a well-defined Harmonize program, we need to define: //
44
+ // //
45
+ // - the set of valid async function identifiers (in the PROMISE_UNION) //
46
+ // - the number of threads within a work group (and hence the number of //
47
+ // promises per link). Only a work group size of 32 is currently supported. //
48
+ // - the type used to index links in their resident arena //
49
+ // - the number of queues present in pools and stack frames, respectively //
50
+ // - the height of the stack (currently, only a flat stack is supported) //
51
+ // - the states that should be tracked per-group and per-thread //
52
+ // //
53
+ ////////////////////////////////////////////////////////////////////////////////////
54
+ // //
55
+ // .-->(QUEUES_PER_POOL) .--> (PROMISE_UNION) //
56
+ // (GROUP/THREAD_STATE) / / //
57
+ // A .-----------. / .--> LINK ----> (GROUP_SIZE) //
58
+ // | / X / \ //
59
+ // | / / `-> ARENA------------. \ //
60
+ // PROG ------> POOL --------------. \ \ //
61
+ // | \ \ \ \ //
62
+ // | `--> STACK ----> FRAME -----> QUEUE -------> ADR --> (ADR_TYPE) //
63
+ // | \ \ //
64
+ // | \ `--> (QUEUE_PER_FRAME) //
65
+ // V \ //
66
+ // (STASH_SIZE) `--> (STACK_SIZE) //
67
+ // //
68
+ ////////////////////////////////////////////////////////////////////////////////////
69
+
70
+
71
+ //! Switches whether or not lazy allocation is enabled for the allocation of a `WorkLink`
72
+ #define LAZY_LINK
73
+
74
+
75
+
76
+
77
+ //! The class that defines an asynchronous program and all of its types.
78
+ template< typename PROGRAM_SPEC >
79
+ class AsyncProgram
80
+ {
81
+
82
+
83
+ public:
84
+
85
+ typedef AsyncProgram<PROGRAM_SPEC> ProgramType;
86
+
87
+ //! Templates for use by `TEMPLATE_MEMBER_SWITCH`
88
+ template<typename BASE>
89
+ struct Specializer
90
+ {
91
+ using Type = BASE;
92
+ };
93
+
94
+ template<template<typename>typename BASE>
95
+ struct Specializer<BASE<ProgramType>>
96
+ {
97
+ using Type = BASE<ProgramType>;
98
+ };
99
+
100
+ //! Define the type used to address work links
101
+ MEMBER_SWITCH( AdrType,unsigned int)
102
+ //! Define the set of operations
103
+ MEMBER_SWITCH( OpSet, OpUnion<>)
104
+
105
+ template<typename OP_SET, typename ADR_TYPE>
106
+ friend class RemappingBarrier;
107
+
108
+ template<typename... TYPES>
109
+ friend class ArgTuple;
110
+
111
+ //! Define the states stored in global, shared, and private memory
112
+ MEMBER_SWITCH(DeviceState, VoidState)
113
+ MEMBER_SWITCH( GroupState, VoidState)
114
+ MEMBER_SWITCH(ThreadState, VoidState)
115
+
116
+
117
+ //! Define the type of `PromiseUnion` used by the program.
118
+ typedef PromiseUnion<OpSet> PromiseUnionType;
119
+
120
+
121
+ //! Used to look up information about the primary `PromiseUnion` type used
122
+ template<typename TYPE>
123
+ struct Lookup { typedef typename PromiseUnionType::template Lookup<TYPE>::type type; };
124
+
125
+ //! Define internal constants based off of the program specification, or
126
+ //! fall back onto defaults.
127
+ CONST_SWITCH(size_t,STASH_SIZE,16)
128
+ CONST_SWITCH(size_t,FRAME_SIZE,32)
129
+ CONST_SWITCH(size_t, POOL_SIZE,32)
130
+ CONST_SWITCH(size_t,STACK_SIZE, 0)
131
+ CONST_SWITCH(size_t,GROUP_SIZE,32)
132
+
133
+
134
+ //! Constants used to determine when to spill or fill the stash, and
135
+ //! by how much
136
+ static const size_t STASH_MARGIN = 2;
137
+ static const size_t STASH_HIGH_WATER = STASH_SIZE-STASH_MARGIN;
138
+
139
+ //! The number of async functions present in the program.
140
+ static const unsigned char FN_ID_COUNT = PromiseUnionType::Info::COUNT;
141
+
142
+
143
+ //! During system verification/debugging, this will be used as a cutoff to prevent infinite
144
+ //! looping
145
+ static const unsigned int PUSH_QUEUE_RETRY_LIMIT = 32;
146
+ static const unsigned int FILL_STASH_RETRY_LIMIT = 1;
147
+ static const unsigned int FILL_STASH_LINKS_RETRY_LIMIT = 32;
148
+
149
+ static const size_t WORK_GROUP_SIZE = GROUP_SIZE;
150
+
151
+ //! A set of halting condition flags
152
+ static const unsigned int BAD_FUNC_ID_FLAG = 0x00000001;
153
+ static const unsigned int STASH_FAIL_FLAG = 0x00000002;
154
+ static const unsigned int COMPLETION_FLAG = 0x80000000;
155
+ static const unsigned int EARLY_HALT_FLAG = 0x40000000;
156
+
157
+ //! Defining a set of internal short-hand names for the specializaions used by the class
158
+ typedef util::mem::Adr <AdrType> LinkAdrType;
159
+ typedef util::mem::PoolQueue <LinkAdrType> QueueType;
160
+ typedef WorkFrame <QueueType,FRAME_SIZE> FrameType;
161
+ typedef WorkStack <FrameType,STACK_SIZE> StackType;
162
+ typedef WorkPool <QueueType,POOL_SIZE> PoolType;
163
+
164
+ typedef WorkLink <OpSet, LinkAdrType, WORK_GROUP_SIZE> LinkType;
165
+
166
+ typedef WorkArena <LinkAdrType,LinkType> ArenaType;
167
+
168
+
169
+
170
+ //! The depth of the partial table (1 if stack is flat, 3 otherwise).
171
+ static const unsigned char PART_TABLE_DEPTH = StackType::PART_MULT;
172
+ static const unsigned char PART_ENTRY_COUNT = FN_ID_COUNT*PART_TABLE_DEPTH;
173
+
174
+
175
+ static const AdrType SPARE_LINK_COUNT = 2u;
176
+
177
+
178
+ //! This struct represents the entire set of data structures that must be stored in thread
179
+ //! memory to track te state of the program defined by the developer as well as the state of
180
+ //! the context which is driving exection.
181
+ struct ThreadContext {
182
+
183
+ unsigned int thread_id;
184
+ unsigned int rand_state;
185
+ unsigned int spare_index;
186
+ LinkAdrType spare_links[SPARE_LINK_COUNT];
187
+
188
+ };
189
+
190
+
191
+ //! A non-atomic promise coalescing structure used to track information about full and
192
+ //! partial work links.
193
+ struct RemapQueue {
194
+ unsigned char count;
195
+ unsigned char full_head;
196
+ unsigned char partial_map[PART_ENTRY_COUNT];
197
+ };
198
+
199
+
200
+ //! This struct represents the entire set of data structures that must be stored in group
201
+ //! memory to track te state of the program defined by the developer as well as the state of
202
+ //! the context which is driving exection.
203
+ struct GroupContext {
204
+
205
+ size_t level; // Current level being run
206
+
207
+ bool keep_running;
208
+ bool busy;
209
+ bool can_make_work;
210
+ bool scarce_work;
211
+
212
+ unsigned char exec_head; // Indexes the link that is/will be evaluated next
213
+ unsigned char empty_head; // Head of the linked list of empty links
214
+
215
+ RemapQueue main_queue;
216
+
217
+ #ifdef ASYNC_LOADS
218
+ RemapQueue load_queue;
219
+ adapt::GPUrt::barrier<adapt::GPUrt::thread_scope_system> load_barrier;
220
+ #endif
221
+
222
+ unsigned char link_stash_count; // Number of device-space links stored
223
+
224
+ LinkType stash[STASH_SIZE];
225
+ LinkAdrType link_stash[STASH_SIZE];
226
+
227
+
228
+ int SM_promise_delta;
229
+ unsigned long long int work_iterator;
230
+
231
+ #ifdef BARRIER_SPILL
232
+ RemappingBarrier<OpSet> spill_barrier;
233
+ #endif
234
+
235
+ #ifdef HRM_TIME
236
+ unsigned long long int time_totals[HRM_TIME];
237
+ #endif
238
+
239
+
240
+ };
241
+
242
+
243
+ //! This struct represents the entire set of data structures that must be stored in device
244
+ //! memory to track the state of the program defined by the developer as well as the state
245
+ //! of the context which is driving execution.
246
+ struct DeviceContext {
247
+
248
+ typedef ProgramType ParentProgramType;
249
+
250
+ #ifdef LAZY_LINK
251
+ AdrType* claim_count;
252
+ #endif
253
+
254
+ #ifdef HRM_TIME
255
+ unsigned long long int* time_totals;
256
+ #endif
257
+
258
+ ArenaType arena;
259
+ PoolType* pool;
260
+ StackType* stack;
261
+
262
+ };
263
+
264
+
265
+ protected:
266
+ public:
267
+
268
+ DeviceContext & _dev_ctx;
269
+ GroupContext & _grp_ctx;
270
+ ThreadContext & _thd_ctx;
271
+
272
+
273
+ DeviceState & device;
274
+ GroupState & group;
275
+ ThreadState & thread;
276
+
277
+ __device__ AsyncProgram<PROGRAM_SPEC> (
278
+ DeviceContext & d_c,
279
+ GroupContext & g_c,
280
+ ThreadContext & t_c,
281
+
282
+ DeviceState & d,
283
+ GroupState & g,
284
+ ThreadState & t
285
+ )
286
+ : _dev_ctx(d_c)
287
+ , _grp_ctx(g_c)
288
+ , _thd_ctx(t_c)
289
+ , device (d)
290
+ , group (g)
291
+ , thread (t)
292
+ {}
293
+
294
+
295
+
296
+
297
+
298
+
299
+ //! Instances wrap around their program scope's DeviceContext. These differ from a program's
300
+ //! DeviceContext object in that they perform automatic deallocation as soon as they drop
301
+ //! out of scope.
302
+ struct Instance {
303
+
304
+
305
+ AdrType arena_size;
306
+ #ifdef LAZY_LINK
307
+ util::host::DevBuf<AdrType> claim_count;
308
+ #endif
309
+ #ifdef HRM_TIME
310
+ util::host::DevBuf<unsigned long long int> time_totals;
311
+ #endif
312
+ util::host::DevBuf<LinkType> arena;
313
+ util::host::DevBuf<PoolType> pool;
314
+ util::host::DevBuf<StackType> stack;
315
+ DeviceState device_state;
316
+
317
+ __host__ Instance (AdrType arsize, DeviceState gs)
318
+ : arena(arsize)
319
+ , pool (1)
320
+ , stack(1)
321
+ , arena_size(arsize)
322
+ , device_state(gs)
323
+ #ifdef LAZY_LINK
324
+ , claim_count(0u)
325
+ #endif
326
+ #ifdef HRM_TIME
327
+ , time_totals((size_t)HRM_TIME)
328
+ #endif
329
+ {
330
+ #ifdef HRM_TIME
331
+ adapt::GPUrtMemset( time_totals, 0, sizeof(unsigned long long int) * HRM_TIME );
332
+ #endif
333
+ }
334
+
335
+ __host__ DeviceContext to_context(){
336
+
337
+ DeviceContext result;
338
+
339
+ #ifdef LAZY_LINK
340
+ result.claim_count = claim_count;
341
+ #endif
342
+ #ifdef HRM_TIME
343
+ result.time_totals = time_totals;
344
+ #endif
345
+
346
+ result.arena.size = arena_size;
347
+ result.arena.links = arena;
348
+ result.pool = pool ;
349
+ result.stack = stack;
350
+
351
+ return result;
352
+
353
+ }
354
+
355
+ #ifdef HRM_TIME
356
+ __host__ void print_times(){
357
+ std::vector<unsigned long long int> times;
358
+ time_totals >> times;
359
+ double total = times[0];
360
+ for(unsigned int i=0; i<times.size(); i++){
361
+ double the_time = times[i];
362
+ double prop = 100.0 * (the_time / total);
363
+ printf("T%d: %llu (~%f%)\n",i,times[i],prop );
364
+ }
365
+ }
366
+ #endif
367
+
368
+ __host__ bool complete(){
369
+
370
+ unsigned int* flags_ptr = &(((StackType*)stack)->status_flags);
371
+ unsigned int flags = 0;
372
+ util::host::auto_throw(adapt::GPUrtMemcpy(&flags,flags_ptr,sizeof(unsigned int),adapt::GPUrtMemcpyDeviceToHost));
373
+ check_error();
374
+ return ((flags & COMPLETION_FLAG) != 0);
375
+ }
376
+
377
+ __host__ void clear_flags(){
378
+ unsigned int flags = 0;
379
+ unsigned int* flags_ptr = &(((StackType*)stack)->status_flags);
380
+ util::host::auto_throw(adapt::GPUrtMemcpy(flags_ptr,&flags,sizeof(unsigned int),adapt::GPUrtMemcpyHostToDevice));
381
+ check_error();
382
+ }
383
+
384
+ };
385
+
386
+
387
+
388
+
389
+
390
+ protected:
391
+
392
+ //! Returns true only if the stash cannot support the allocation of any more work links
393
+ __device__ bool stash_overfilled(){
394
+ #ifdef ASYNC_LOADS
395
+ return (_grp_ctx.main_queue.count+_grp_ctx.load_queue.count) >= STASH_HIGH_WATER;
396
+ #else
397
+ return _grp_ctx.main_queue.count >= STASH_HIGH_WATER;
398
+ #endif
399
+ }
400
+
401
+ //! Returns an index into the partial map of a group based off of a function id and a depth. If
402
+ //! an invalid depth or function id is used, PART_ENTRY_COUNT is returned.
403
+ __device__ unsigned int partial_map_index(
404
+ OpDisc func_id,
405
+ unsigned int depth,
406
+ unsigned int current_level
407
+ ){
408
+
409
+ unsigned int the_id = static_cast<unsigned int>(func_id);
410
+
411
+
412
+ if( the_id >= FN_ID_COUNT){
413
+ return PART_ENTRY_COUNT;
414
+ }
415
+
416
+ unsigned int result = the_id;
417
+
418
+ if( ! StackType::FLAT ){
419
+ result *= PART_TABLE_DEPTH;
420
+ if( depth == current_level ){
421
+ result += 1;
422
+ } else if ( depth == (current_level+1) ){
423
+ result += 2;
424
+ } else if ( depth != (current_level-1) ){
425
+ result = PART_ENTRY_COUNT;
426
+ }
427
+ }
428
+
429
+ return result;
430
+
431
+ }
432
+
433
+
434
+ //! Initializes the shared state of a work group, which is stored as a ctx_shared struct. This
435
+ //! is mainly done by initializing handles to the arena, pool, and stack, setting the current
436
+ //! level to null, setting the stash iterator to null, and zeroing the stash.
437
+ __device__ void init_group(){
438
+
439
+ //printf("Initing group\n");
440
+ unsigned int active = __activemask();
441
+
442
+ __syncwarp(active);
443
+
444
+ if(util::current_leader()){
445
+
446
+ if( StackType::FLAT ){
447
+ _grp_ctx.level = 0;
448
+ } else {
449
+ _grp_ctx.level = StackType::NULL_LEVEL;
450
+ }
451
+
452
+ _grp_ctx.main_queue.count = 0;
453
+ _grp_ctx.link_stash_count = 0;
454
+ _grp_ctx.keep_running = true;
455
+ _grp_ctx.busy = false;
456
+ _grp_ctx.can_make_work= true;
457
+ _grp_ctx.exec_head = STASH_SIZE;
458
+ _grp_ctx.main_queue.full_head = STASH_SIZE;
459
+ _grp_ctx.empty_head = 0;
460
+ _grp_ctx.work_iterator= 0;
461
+ _grp_ctx.scarce_work = false;
462
+
463
+ #ifdef BARRIER_SPILL
464
+ _grp_ctx.spill_barrier = RemappingBarrier<OpSet>::blank(1);
465
+ #endif
466
+
467
+ for(unsigned int i=0; i<STASH_SIZE; i++){
468
+ _grp_ctx.stash[i].empty(i+1);
469
+ }
470
+
471
+ for(unsigned int i=0; i<PART_ENTRY_COUNT; i++){
472
+ _grp_ctx.main_queue.partial_map[i] = STASH_SIZE;
473
+ }
474
+
475
+ #ifdef ASYNC_LOADS
476
+ _grp_ctx.load_queue.count = 0;
477
+ _grp_ctx.load_queue.full_head = STASH_SIZE;
478
+ for(unsigned int i=0; i<PART_ENTRY_COUNT; i++){
479
+ _grp_ctx.load_queue.partial_map[i] = STASH_SIZE;
480
+ }
481
+ init(&_grp_ctx.load_barrier,WORK_GROUP_SIZE);
482
+ #endif
483
+
484
+ _grp_ctx.SM_promise_delta = 0;
485
+
486
+ #ifdef HRM_TIME
487
+ for(unsigned int i=0; i<HRM_TIME; i++){
488
+ _grp_ctx.time_totals[i] = 0;
489
+ }
490
+ beg_time(0);
491
+ #endif
492
+
493
+ }
494
+
495
+ __syncwarp(active);
496
+
497
+ }
498
+
499
+ //! Initializes the local state of a thread, which is just the device id of the thread and the
500
+ //! state used by the thread to generate random numbers for stochastic choices needed to manage
501
+ //! the runtime state.
502
+ __device__ void init_thread(){
503
+
504
+ //printf("Initing thread %d\n",threadIdx.x);
505
+ _thd_ctx.thread_id = (blockIdx.x * blockDim.x) + threadIdx.x;
506
+ _thd_ctx.rand_state = _thd_ctx.thread_id;
507
+ _thd_ctx.spare_index = 0;
508
+ for(size_t i=0; i<SPARE_LINK_COUNT; i++){
509
+ _thd_ctx.spare_links[i] = LinkAdrType::null;
510
+ }
511
+
512
+ }
513
+
514
+
515
+ //! Sets the bits in the status_flags field of the stack according to the given flag bits.
516
+ __device__ void set_flags(unsigned int flag_bits){
517
+
518
+ atomicOr(&_dev_ctx.stack->status_flags,flag_bits);
519
+
520
+ }
521
+
522
+
523
+ //! Unsets the bits in the status_flags field of the stack according to the given flag bits.
524
+ __device__ void unset_flags(unsigned int flag_bits){
525
+
526
+ atomicAnd(&_dev_ctx.stack->status_flags,~flag_bits);
527
+
528
+ }
529
+
530
+ //! Returns the current highest level in the stack. Given that this program is highly parallel,
531
+ //! this number inherently cannot be trusted. By the time the value is fetched, the stack could
532
+ //! have a different height or the thread that set the height may not have deposited links in the
533
+ //! corresponding level yet.
534
+ __device__ unsigned int highest_level(){
535
+
536
+ return left_half(_dev_ctx.stack->depth_live);
537
+
538
+ }
539
+
540
+
541
+ //! Returns a reference to the frame at the requested level in the stack.
542
+ __device__ FrameType& get_frame(unsigned int level){
543
+
544
+ return _dev_ctx.stack->frames[level];
545
+
546
+ }
547
+
548
+
549
+ //! Joins two queues such that the right queue is now at the end of the left queue.
550
+ //!
551
+ //! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
552
+ //! atomically. If not, one of the queues manipulated with this function will almost certainly
553
+ //! become malformed at some point.
554
+ __device__ QueueType join_queues(QueueType left_queue, QueueType right_queue){
555
+
556
+ QueueType result;
557
+
558
+ /*
559
+ // If either input queue is null, we can simply return the other queue.
560
+ */
561
+ if( left_queue.is_null() ){
562
+ result = right_queue;
563
+ } else if ( right_queue.is_null() ){
564
+ result = left_queue;
565
+ } else {
566
+
567
+ LinkAdrType left_tail_adr = left_queue .get_tail();
568
+ LinkAdrType right_head_adr = right_queue.get_head();
569
+ LinkAdrType right_tail_adr = right_queue.get_tail();
570
+
571
+ //! Find last link in the queue referenced by left_queue.
572
+ LinkType& left_tail = _dev_ctx.arena[left_tail_adr];
573
+
574
+ //! Set the index for the tail's successor to the head of the queue referenced by
575
+ //! right_queue.
576
+ left_tail.next = right_head_adr;
577
+
578
+ //! Set the right half of the left_queue handle to index the new tail.
579
+ left_queue.set_tail(right_tail_adr);
580
+
581
+ result = left_queue;
582
+
583
+ }
584
+
585
+ return result;
586
+
587
+ }
588
+
589
+
590
+ //! Takes the first link off of the queue and returns the index of the link in the arena. If the
591
+ //! queue is empty, a null address is returned instead.
592
+ //!
593
+ //! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
594
+ //! atomically. If not, one of the queues manipulated with this function will almost certainly
595
+ //! become malformed at some point.
596
+ __device__ LinkAdrType pop_front(QueueType& queue){
597
+
598
+ LinkAdrType result;
599
+
600
+ //! Don't try unless the queue is non-null
601
+ if( queue.is_null() ){
602
+ result.adr = LinkAdrType::null;
603
+ } else {
604
+ result = queue.get_head();
605
+ LinkAdrType next = _dev_ctx.arena[result].next;
606
+ queue.set_head(next);
607
+ if(next.adr == LinkAdrType::null){
608
+ queue.set_tail(next);
609
+ } else if ( queue.get_tail() == result ){
610
+ //printf("ERROR: Final link does not have a null next.\n");
611
+ queue.pair.data = QueueType::null;
612
+ return result;
613
+ }
614
+ }
615
+ return result;
616
+
617
+ }
618
+
619
+
620
+ //! Adds the given link to the end of the given queue. This link can NOT be part of another queue,
621
+ //! and its next pointer will be automatically nulled before it is appended. If you need to merge
622
+ //! two queues together, use join_queues.
623
+ //!
624
+ //! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
625
+ //! atomically. If not, one of the queues manipulated with this function will almost certainly
626
+ //! become malformed at some point.
627
+ __device__ void push_back(QueueType& queue, LinkAdrType link_adr){
628
+
629
+ _dev_ctx.arena[link_adr].next = LinkAdrType::null;
630
+ if( queue.is_null() ){
631
+ queue = QueueType(link_adr,link_adr);
632
+ } else {
633
+ LinkAdrType tail = queue.get_tail();
634
+ _dev_ctx.arena[tail].next = link_adr;
635
+ //atomicExch( &(_dev_ctx.arena[tail].next.adr),link_adr.adr);
636
+ queue.set_tail(link_adr);
637
+ }
638
+
639
+ }
640
+
641
+
642
+
643
+ //! Attempts to pull a queue from a range of queue slots, trying each slot starting from the given
644
+ //! starting index onto the end of the range and then looping back from the beginning. If, after
645
+ //! trying every slot in the range, no non-null queue was obtained, a QueueType::null value is returned.
646
+ __device__ QueueType pull_queue(QueueType* src, unsigned int start_index, unsigned int range_size, unsigned int& src_index){
647
+
648
+ QueueType result;
649
+
650
+ __threadfence();
651
+ //! First iterate from the starting index to the end of the queue range, attempting to
652
+ //! claim a non-null queue until either there are no more slots to try, or the atomic
653
+ //! swap successfuly retrieves something.
654
+ for(unsigned int i=start_index; i < range_size; i++){
655
+ if( src[i].pair.data != QueueType::null ) {
656
+ result.pair.data = atomicExch(&(src[i].pair.data),QueueType::null);
657
+ if( ! result.is_null() ){
658
+ src_index = i;
659
+ __threadfence();
660
+ return result;
661
+ }
662
+ }
663
+ }
664
+
665
+ //! Continue searching from the beginning of the range to just before the beginning of the
666
+ //! previous scan.
667
+ for(unsigned int i=0; i < start_index; i++){
668
+ if( src[i].pair.data != QueueType::null ) {
669
+ result.pair.data = atomicExch(&(src[i].pair.data),QueueType::null);
670
+ if( ! result.is_null() ){
671
+ src_index = i;
672
+ __threadfence();
673
+ return result;
674
+ }
675
+ }
676
+ }
677
+
678
+ q_printf("COULD NOT PULL QUEUE\n");
679
+ //! Return QueueType::null if nothing is found
680
+ result.pair.data = QueueType::null;
681
+ return result;
682
+
683
+
684
+ }
685
+
686
+
687
+
688
+
689
+ //! Repeatedly tries to push a queue to a destination queue slot by atomic exchanges. If a non
690
+ //! null queue is ever returned by the exchange, it attempts to merge with a subsequent exchange.
691
+ //!
692
+ //! If an invalid queue (one half null, the other half non-null) is ever found at the destination,
693
+ //! the value of that invalid queue is returned, the invalid value is replaced into the queue, and
694
+ //! any work gathered from the replacement is assigned to the second argument. Otherwise, the second
695
+ //! argument is set to a null state.
696
+ __device__ QueueType push_queue(QueueType& dest, QueueType& queue){
697
+
698
+ if( queue.is_null() ){
699
+ return queue;
700
+ }
701
+ #ifdef INF_LOOP_SAFE
702
+ while(true)
703
+ #else
704
+ for(int i=0; i<PUSH_QUEUE_RETRY_LIMIT; i++)
705
+ #endif
706
+ {
707
+ __threadfence();
708
+ //printf("%d: Pushing queue (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
709
+
710
+ QueueType swap;
711
+ swap.pair.data = atomicExch(&dest.pair.data,queue.pair.data);
712
+ //! If our swap returns a non-null queue, we are still stuck with a queue that
713
+ //! needs to be offloaded to the stack. In this case, claim the queue from the
714
+ //! slot just swapped with, merge the two, and attempt again to place the queue
715
+ //! back. With this method, swap failures are bounded by the number of pushes to
716
+ //! the queue slot, with at most one failure per push_queue call, but no guarantee
717
+ //! of which attempt from which call will suffer from an incurred failure.
718
+ if( ! swap.is_null() ){
719
+ //printf("%d: We got queue (%d,%d) when trying to push a queue\n",blockIdx.x,swap.get_head().adr,swap.get_tail().adr);
720
+ QueueType other_swap;
721
+ if( swap.get_head().is_null() || swap.get_tail().is_null() ){
722
+ other_swap.pair.data = atomicExch(&dest.pair.data,swap.pair.data);
723
+ queue = other_swap;
724
+ return swap;
725
+ } else {
726
+ other_swap.pair.data = atomicExch(&dest.pair.data,QueueType::null);
727
+ queue = join_queues(other_swap,swap);
728
+ //printf("%d: Merged it to form queue (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
729
+ }
730
+ } else {
731
+ //printf("%d: Finally pushed (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
732
+ break;
733
+ }
734
+ }
735
+ return queue;
736
+
737
+ }
738
+
739
+
740
+
741
+
742
+ //! Claims a link from the link stash. If no link exists in the stash, LinkAdrType::null is returned.
743
+ __device__ LinkAdrType claim_stash_link(){
744
+
745
+ LinkAdrType link = LinkAdrType::null;
746
+ unsigned int count = _grp_ctx.link_stash_count;
747
+ if(count > 0){
748
+ link = _grp_ctx.link_stash[count-1];
749
+ _grp_ctx.link_stash_count = count - 1;
750
+ }
751
+ q_printf("New link stash count: %d\n",_grp_ctx.link_stash_count);
752
+ return link;
753
+
754
+ }
755
+
756
+
757
+
758
+ //! Inserts an empty slot into the stash. This should only be called if there is enough space in
759
+ //! the link stash.
760
+ __device__ void insert_stash_link(LinkAdrType link){
761
+
762
+ unsigned int count = _grp_ctx.link_stash_count;
763
+ _grp_ctx.link_stash[count] = link;
764
+ _grp_ctx.link_stash_count = count + 1;
765
+ q_printf("New link stash count: %d\n",_grp_ctx.link_stash_count);
766
+
767
+ }
768
+
769
+
770
+
771
+
772
+ //! Claims an empty slot from the stash and returns its index. If no empty slot exists in the stash,
773
+ //! then STASH_SIZE is returned.
774
+ __device__ unsigned int claim_empty_slot(){
775
+
776
+ unsigned int slot = _grp_ctx.empty_head;
777
+ if(slot != STASH_SIZE){
778
+ _grp_ctx.empty_head = _grp_ctx.stash[slot].next.adr;
779
+ db_printf("EMPTY: %d << %d\n",slot,_grp_ctx.empty_head);
780
+ }
781
+ return slot;
782
+
783
+ }
784
+
785
+
786
+ //! Inserts an empty slot into the stash. This should only be called if there is enough space in
787
+ //! the link stash.
788
+ __device__ void insert_empty_slot(unsigned int slot){
789
+
790
+ _grp_ctx.stash[slot].next.adr = _grp_ctx.empty_head;
791
+ db_printf("EMPTY: >> %d -> %d\n",slot,_grp_ctx.empty_head);
792
+ _grp_ctx.empty_head = slot;
793
+
794
+ }
795
+
796
+
797
+ //! Claims a full slot from the stash and returns its index. If no empty slot exists in the stash,
798
+ //! then STASH_SIZE is returned.
799
+ __device__ unsigned int claim_full_slot(){
800
+
801
+ unsigned int slot = _grp_ctx.main_queue.full_head;
802
+ if(slot != STASH_SIZE){
803
+ _grp_ctx.main_queue.full_head = _grp_ctx.stash[slot].next.adr;
804
+ db_printf("FULL : %d << %d\n",slot,_grp_ctx.main_queue.full_head);
805
+ }
806
+ return slot;
807
+
808
+ }
809
+
810
+
811
+ //! Inserts a full slot into the stash. This should only be called if there is enough space in
812
+ //! the link stash.
813
+ __device__ void insert_full_slot(RemapQueue& queue, unsigned int slot){
814
+
815
+ _grp_ctx.stash[slot].next.adr = queue.full_head;
816
+ db_printf("FULL : >> %d -> %d\n",slot,queue.full_head);
817
+ queue.full_head = slot;
818
+
819
+ }
820
+
821
+
822
+ __device__ void dealloc_links(LinkAdrType* src, size_t count) {
823
+
824
+
825
+ QueueType queue;
826
+ queue.pair.data = QueueType::null;
827
+
828
+ //! Connect all links into a queue
829
+ for(unsigned int i=0; i < count; i++){
830
+
831
+ push_back(queue,src[i]);
832
+
833
+ }
834
+
835
+ //! Push out the queue to the pool
836
+ q_printf("Pushing queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
837
+ unsigned int dest_idx = util::random_uint(_thd_ctx.rand_state) % POOL_SIZE;
838
+ push_queue(_dev_ctx.pool->queues[dest_idx],queue);
839
+
840
+ }
841
+
842
+
843
+ //!
844
+ //! A thread-safe way of allocating links from global memory. This function is used
845
+ //! by the warp leader to restock the link stash in between processing promises and
846
+ //! used by individual threads if its current promise needs to use a link while the
847
+ //! link stash is empty.
848
+ //!
849
+ __device__ size_t alloc_links(LinkAdrType* dst, size_t req_count) {
850
+
851
+ size_t alloc_count = 0;
852
+ #ifdef LAZY_LINK
853
+
854
+ #if 1
855
+ if( (alloc_count < req_count) && ((*_dev_ctx.claim_count) < _dev_ctx.arena.size) ){
856
+ AdrType claim_offset = atomicAdd(_dev_ctx.claim_count,req_count);
857
+ unsigned int received = 0;
858
+ if( claim_offset <= (_dev_ctx.arena.size - req_count) ){
859
+ received = req_count;
860
+ } else if ( claim_offset >= _dev_ctx.arena.size ) {
861
+ received = 0;
862
+ } else {
863
+ received = _dev_ctx.arena.size - claim_offset;
864
+ }
865
+ for( unsigned int index=0; index < received; index++){
866
+ //insert_stash_link(LinkAdrType(claim_offset+index));
867
+ dst[alloc_count] = LinkAdrType(claim_offset+index);
868
+ alloc_count++;
869
+ }
870
+ }
871
+ #else
872
+ for(int i=_grp_ctx.link_stash_count; i < threashold; i++){
873
+ if((*_dev_ctx.claim_count) < _dev_ctx.arena.size ){
874
+ AdrType claim_index = atomicAdd(_dev_ctx.claim_count,1);
875
+ if( claim_index < _dev_ctx.arena.size ){
876
+ //_dev_ctx.arena[claim_index].empty(LinkAdrType::null);
877
+ insert_stash_link(LinkAdrType(claim_index));
878
+ }
879
+ } else {
880
+ break;
881
+ }
882
+ }
883
+ #endif
884
+
885
+ #endif
886
+
887
+
888
+ for(int try_itr=0; try_itr < FILL_STASH_LINKS_RETRY_LIMIT; try_itr++){
889
+
890
+ if(alloc_count >= req_count){
891
+ break;
892
+ }
893
+ //! Attempt to pull a queue from the pool. This should be very unlikely to fail unless
894
+ //! almost all links have been exhausted or the pool size is disproportionately small
895
+ //! relative to the number of work groups. In the worst case, this should simply not
896
+ //! al_thd_ctxate any links, and the return value shall report this.
897
+ unsigned int src_index = LinkAdrType::null;
898
+ unsigned int start = util::random_uint(_thd_ctx.rand_state)%POOL_SIZE;
899
+ QueueType queue = pull_queue(_dev_ctx.pool->queues,start,POOL_SIZE,src_index);
900
+ q_printf("Pulled queue (%d,%d) from pool %d\n",queue.get_head().adr,queue.get_tail().adr,src_index);
901
+
902
+ //! Keep popping links from the queue until the full number of links have been added or
903
+ //! the queue runs out of links.
904
+ for(int i=alloc_count; i < req_count; i++){
905
+ LinkAdrType link = pop_front(queue);
906
+ if( ! link.is_null() ){
907
+ //insert_stash_link(link);
908
+ dst[alloc_count] = link;
909
+ alloc_count++;
910
+ q_printf("Inserted link %d into link stash\n",link.adr);
911
+ } else {
912
+ break;
913
+ }
914
+ }
915
+ push_queue(_dev_ctx.pool->queues[src_index],queue);
916
+ q_printf("Pushed queue (%d,%d) to pool %d\n",queue.get_head().adr,queue.get_tail().adr,src_index);
917
+
918
+ }
919
+
920
+ return alloc_count;
921
+
922
+ }
923
+
924
+ __device__ LinkAdrType alloc_spare_link(){
925
+ if(_thd_ctx.spare_index > 0){
926
+ _thd_ctx.spare_index--;
927
+ LinkAdrType result;
928
+ result = _thd_ctx.spare_links[_thd_ctx.spare_index];
929
+ //printf("Re-used spare alloc (%d)\n",result.adr);
930
+ return result;
931
+ } else {
932
+ LinkAdrType result = LinkAdrType::null;
933
+ alloc_links(&result,1);
934
+ //printf("Fresh spare alloc (%d)\n",result.adr);
935
+ return result;
936
+ }
937
+ }
938
+
939
+ __device__ void dump_spare_link(LinkAdrType link_adr){
940
+ if( link_adr.is_null() ){
941
+ return;
942
+ }
943
+ if(_thd_ctx.spare_index < SPARE_LINK_COUNT){
944
+ //printf("Saving link (%d) for later\n",link_adr.adr);
945
+ _thd_ctx.spare_links[_thd_ctx.spare_index] = link_adr;
946
+ _thd_ctx.spare_index++;
947
+ //printf("New spare index of %d is %d\n",threadIdx.x,_thd_ctx.spare_index);
948
+ } else {
949
+ //printf("Spilling spare link (%d)\n",link_adr.adr);
950
+ dealloc_links(&link_adr,1);
951
+ }
952
+ }
953
+
954
+ //! Attempts to fill the link stash to the given threshold with links. This should only ever
955
+ //! be called in a single-threaded manner.
956
+ __device__ void fill_stash_links(unsigned int threashold){
957
+
958
+ unsigned int active = __activemask();
959
+ __syncwarp(active);
960
+
961
+ if( util::current_leader() ){
962
+
963
+ LinkAdrType *dst = _grp_ctx.link_stash+_grp_ctx.link_stash_count;
964
+ size_t count = 0;
965
+ if( threashold < _grp_ctx.link_stash_count ){
966
+ count = 0;
967
+ } else if ( threashold >= STASH_SIZE ){
968
+ count = STASH_SIZE - _grp_ctx.link_stash_count;
969
+ } else {
970
+ count = threashold - _grp_ctx.link_stash_count;
971
+ }
972
+ size_t added = alloc_links(dst,count);
973
+ _grp_ctx.link_stash_count += added;
974
+
975
+ }
976
+ __syncwarp(active);
977
+
978
+ }
979
+
980
+
981
+
982
+
983
+ //! If the number of links in the link stash exceeds the given threshold value, this function frees
984
+ //! enough links to bring the number of links down to the threshold. This should only ever be
985
+ //! called in a single_threaded manner.
986
+ __device__ void spill_stash_links(unsigned int threashold){
987
+
988
+ //! Do not even try if no links can be or need to be removed
989
+ if(threashold >= _grp_ctx.link_stash_count){
990
+ q_printf("Nothing to spill...\n");
991
+ return;
992
+ }
993
+
994
+ //! Find where in the link stash to begin removing links
995
+
996
+ QueueType queue;
997
+ queue.pair.data = QueueType::null;
998
+
999
+ //! Connect all links into a queue
1000
+ unsigned int spill_count = _grp_ctx.link_stash_count - threashold;
1001
+ for(unsigned int i=0; i < spill_count; i++){
1002
+
1003
+ LinkAdrType link = claim_stash_link();
1004
+ q_printf("Claimed link %d from link stash\n",link.adr);
1005
+ push_back(queue,link);
1006
+
1007
+ }
1008
+
1009
+ _grp_ctx.link_stash_count = threashold;
1010
+
1011
+
1012
+ //! Push out the queue to the pool
1013
+ q_printf("Pushing queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
1014
+ unsigned int dest_idx = util::random_uint(_thd_ctx.rand_state) % POOL_SIZE;
1015
+ push_queue(_dev_ctx.pool->queues[dest_idx],queue);
1016
+
1017
+ q_printf("Pushed queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
1018
+
1019
+ }
1020
+
1021
+
1022
+
1023
+
1024
+ //! Decrements the child and resident counter of each frame corresponding to a call at level
1025
+ //! start_level in the stack returning to a continuation at level end_level in the stack. To reduce
1026
+ //! overall contention, decrementations are first pooled through a shared atomic operation before
1027
+ //! being applied to the stack.
1028
+ //
1029
+ //! A call without a continuation should use this function with start_level == end_level, which
1030
+ //! simply decrements the resident counter at the call's frame.
1031
+ __device__ void pop_frame_counters(unsigned int start_level, unsigned int end_level){
1032
+
1033
+
1034
+ unsigned int depth_dec = 0;
1035
+ PromiseCountPair delta;
1036
+ PromiseCountPair result;
1037
+
1038
+ FrameType& frame = _dev_ctx.stack->frames[start_level];
1039
+
1040
+ //! Decrement the residents counter for the start level
1041
+ delta = util::active_count();
1042
+ if(util::current_leader()){
1043
+ result = atomicSub(&frame.children_residents.data,delta);
1044
+ if(result == 0u){
1045
+ depth_dec += 1;
1046
+ }
1047
+ }
1048
+
1049
+ //! Decrement the children counter for the remaining levels
1050
+ for(int d=(start_level-1); d >= end_level; d--){
1051
+ FrameType& frame = _dev_ctx.stack->frames[d];
1052
+ delta = util::active_count();
1053
+ if(util::current_leader()){
1054
+ result = atomicSub(&frame.children_residents.data,delta);
1055
+ if(result == 0u){
1056
+ depth_dec += 1;
1057
+ }
1058
+ }
1059
+ }
1060
+
1061
+ //! Update the stack base once all other counters have been updated.
1062
+ if(util::current_leader()){
1063
+ result = atomicSub(&(_dev_ctx.stack->depth_live),depth_dec);
1064
+ if(result == 0){
1065
+ set_flags(_grp_ctx,COMPLETION_FLAG);
1066
+ }
1067
+ }
1068
+
1069
+ }
1070
+
1071
+
1072
+ //! Repetitively tries to merge the given queue of promises with the queue at the given index in the
1073
+ //! frame at the given level on the stack. This function currently aborts if an error flag is set
1074
+ //! or if too many merge failures occur, however, once its correctness is verified, this function
1075
+ //! will run forever until the merge is successful, as success is essentially guaranteed by
1076
+ //! the nature of the process.
1077
+ __device__ void push_promises(unsigned int level, unsigned int index, QueueType queue, int promise_delta) {
1078
+
1079
+
1080
+ LinkAdrType tail = queue.get_tail();
1081
+ LinkAdrType head = queue.get_head();
1082
+ rc_printf("SM %d: push_promises(level:%d,index:%d,queue:(%d,%d),delta:%d)\n",threadIdx.x,level,index,tail.adr,head.adr,promise_delta);
1083
+ //! Do not bother pushing a null queue if there is no delta to report
1084
+ if( ( ! queue.is_null() ) || (promise_delta != 0) ){
1085
+
1086
+ //! Change the resident counter of the destination frame by the number of promises
1087
+ //! that have been added to or removed from the given queue
1088
+ FrameType &dest = get_frame(level);
1089
+ unsigned int old_count;
1090
+ unsigned int new_count;
1091
+ if(promise_delta >= 0) {
1092
+ old_count = atomicAdd(&dest.children_residents.data,(PromiseCountPair) promise_delta);
1093
+ new_count = old_count + (unsigned int) promise_delta;
1094
+ if(old_count > new_count){
1095
+ rc_printf("\n\nOVERFLOW\n\n");
1096
+ }
1097
+ } else {
1098
+ PromiseCountPair neg_delta = -((PromiseCountPair) (-promise_delta));
1099
+ old_count = atomicAdd(&dest.children_residents.data,neg_delta);
1100
+ new_count = old_count + neg_delta;
1101
+ if(old_count < new_count){
1102
+ rc_printf("\n\nUNDERFLOW\n\n");
1103
+ }
1104
+ }
1105
+
1106
+
1107
+ _grp_ctx.SM_promise_delta += promise_delta;
1108
+ rc_printf("SM %d-%d: Old count: %d, New count: %d, Delta: %d\n",blockIdx.x,threadIdx.x,old_count,new_count,promise_delta);
1109
+
1110
+
1111
+ rc_printf("SM %d-%d: frame zero resident count is: %d\n",blockIdx.x,threadIdx.x,_dev_ctx.stack->frames[0].children_residents.data);
1112
+ //! If the addition caused a frame to change from empty to non-empty or vice-versa,
1113
+ //! make an appropriate incrementation or decrementation at the stack base.
1114
+ if( (old_count == 0) && (new_count != 0) ){
1115
+ atomicAdd(&(_dev_ctx.stack->depth_live),0x00010000u);
1116
+ } else if( (old_count != 0) && (new_count == 0) ){
1117
+ atomicSub(&(_dev_ctx.stack->depth_live),0x00010000u);
1118
+ } else {
1119
+ rc_printf("SM %d: No change!\n",threadIdx.x);
1120
+ }
1121
+
1122
+ //! Finally, push the queues
1123
+ push_queue(dest.pool.queues[index],queue);
1124
+ rc_printf("SM %d: Pushed queue (%d,%d) to stack at index %d\n",threadIdx.x,queue.get_head().adr,queue.get_tail().adr,index);
1125
+
1126
+ if( (_dev_ctx.stack->frames[0].children_residents.get_right()) == 0 ) {
1127
+ rc_printf("SM %d-%d: After queue pushed to stack, frame zero resident count is: %d\n",blockIdx.x,threadIdx.x,_dev_ctx.stack->frames[0].children_residents.data);
1128
+ }
1129
+
1130
+ }
1131
+ rc_printf("(%d) the delta: %d\n",threadIdx.x,promise_delta);
1132
+
1133
+ }
1134
+
1135
+
1136
+
1137
+ //! Attempts to pull a queue of promises from the frame in the stack of the given level, starting the
1138
+ //! pull attempt at the given index in the frame. If no queue could be pulled after attempting a
1139
+ //! pull at each queue in the given frame, a QueueType::null value is returned.
1140
+ __device__ QueueType pull_promises(unsigned int level, unsigned int& source_index) {
1141
+
1142
+
1143
+ rc_printf("SM %d: pull_promises(level:%d)\n",threadIdx.x,level);
1144
+ unsigned int src_idx = util::random_uint(_thd_ctx.rand_state) % FRAME_SIZE;
1145
+
1146
+ __threadfence();
1147
+ FrameType &src = get_frame(level);
1148
+
1149
+ QueueType queue = pull_queue(src.pool.queues,src_idx,FRAME_SIZE,source_index);
1150
+
1151
+ if( ! queue.is_null() ){
1152
+ q_printf("SM %d: Pulled queue (%d,%d) from stack at index %d\n",blockIdx.x,queue.get_tail().adr,queue.get_head().adr, src_idx);
1153
+ } else {
1154
+ q_printf("SM %d: Failed to pull queue from stack starting at index %d\n",blockIdx.x, src_idx);
1155
+ }
1156
+ __threadfence();
1157
+ return queue;
1158
+
1159
+ }
1160
+
1161
+
1162
+ //! Attempts to pull a queue from any frame in the stack, starting from the highest and working
1163
+ //! its way down. If no queue could be pulled, a QueueType::null value is returned.
1164
+ __device__ QueueType pull_promises_any_level(unsigned int& level, unsigned int& source_index){
1165
+
1166
+
1167
+ QueueType result;
1168
+ result.data = QueueType::null;
1169
+ unsigned int start_level = highest_level();
1170
+ for(int level_itr = start_level; level_itr>=0; level_itr--){
1171
+ q_printf("Pulling promises at level %d for pull_promises_any_level\n",level_itr);
1172
+ result = pull_promises(level_itr,source_index);
1173
+ if( ! result.is_null() ){
1174
+ level = level_itr;
1175
+ return result;
1176
+ }
1177
+ }
1178
+ result.data = QueueType::null;
1179
+ return result;
1180
+
1181
+ }
1182
+
1183
+
1184
+
1185
+
1186
+
1187
+ //! Adds the contents of the stash slot at the given index to a link and returns the index of the
1188
+ //! link in the arena. This should only ever be called if there is both a link available to store
1189
+ //! the data and if the index is pointing at a non-empty slot. This also should only ever be
1190
+ //! called in a single-threaded context.
1191
+ __device__ LinkAdrType produce_link(unsigned int slot_index ){
1192
+
1193
+
1194
+ __shared__ LinkAdrType result;
1195
+
1196
+ unsigned int active = __activemask();
1197
+
1198
+
1199
+ //__syncwarp(active);
1200
+
1201
+ //if(util::current_leader()){
1202
+ LinkAdrType link_index = claim_stash_link();
1203
+ q_printf("Claimed link %d from stash\n",link_index.adr);
1204
+ LinkType& the_link = _dev_ctx.arena[link_index];
1205
+ //_grp_ctx.SM_promise_delta += _grp_ctx.stash[slot_index].count;
1206
+ _grp_ctx.stash[slot_index].next = LinkAdrType::null;
1207
+ the_link = _grp_ctx.stash[slot_index];
1208
+ db_printf("Link has count %d and next %d in main memory",the_link.count, the_link.next);
1209
+ result = link_index;
1210
+ _grp_ctx.main_queue.count -= 1;
1211
+ //}
1212
+
1213
+
1214
+ //__syncwarp(active);
1215
+ return result;
1216
+
1217
+ }
1218
+
1219
+
1220
+
1221
+
1222
+
1223
+
1224
+
1225
+
1226
+ //! Removes all promises in the stash that do not correspond to the given level, or to the levels
1227
+ //! immediately above or below (level+1) and (level-1).
1228
+ __device__ void relevel_stash(unsigned int level){
1229
+
1230
+ if( ! StackType::FLAT ){
1231
+
1232
+ // TODO: Implement for non-flat stack
1233
+
1234
+ }
1235
+
1236
+ }
1237
+
1238
+
1239
+
1240
+
1241
+
1242
+ //! Dumps all full links not corresponding to the current execution level. Furthermore, should the
1243
+ //! remaining links still put the stash over the given threshold occupancy, links will be further
1244
+ //! removed in the order: full links at the current level, partial links not at the current level,
1245
+ //! partial links at the current level.
1246
+ __device__ void spill_stash(unsigned int threashold){
1247
+
1248
+ unsigned int active =__activemask();
1249
+ __syncwarp(active);
1250
+
1251
+
1252
+
1253
+ #if DEF_STACK_MODE == 0
1254
+
1255
+
1256
+
1257
+ if(util::current_leader() && (_grp_ctx.main_queue.count > threashold)){
1258
+
1259
+
1260
+ unsigned int spill_count = _grp_ctx.main_queue.count - threashold;
1261
+ int delta = 0;
1262
+ fill_stash_links(spill_count);
1263
+
1264
+ QueueType queue;
1265
+ queue.pair.data = QueueType::null;
1266
+ unsigned int partial_iter = 0;
1267
+ bool has_full_slots = true;
1268
+ q_printf("{Spilling to %d}",threashold);
1269
+ for(unsigned int i=0; i < spill_count; i++){
1270
+ unsigned int slot = STASH_SIZE;
1271
+ if(has_full_slots){
1272
+ slot = claim_full_slot();
1273
+ if(slot == STASH_SIZE){
1274
+ has_full_slots = false;
1275
+ }
1276
+ }
1277
+ if(! has_full_slots){
1278
+ for(;partial_iter < FN_ID_COUNT; partial_iter++){
1279
+ db_printf("%d",partial_iter);
1280
+ if(_grp_ctx.main_queue.partial_map[partial_iter] != STASH_SIZE){
1281
+ slot = _grp_ctx.main_queue.partial_map[partial_iter];
1282
+ q_printf("{Spilling partial}");
1283
+ partial_iter++;
1284
+ break;
1285
+ }
1286
+ }
1287
+ }
1288
+ if(slot == STASH_SIZE){
1289
+ break;
1290
+ }
1291
+
1292
+ delta += _grp_ctx.stash[slot].count;
1293
+ q_printf("Slot for production (%d) has %d promises\n",slot,_grp_ctx.stash[slot].count);
1294
+ LinkAdrType link = produce_link(slot);
1295
+ push_back(queue,link);
1296
+ insert_empty_slot(slot);
1297
+ if(_grp_ctx.main_queue.count <= threashold){
1298
+ break;
1299
+ }
1300
+ }
1301
+
1302
+ unsigned int push_index = util::random_uint(_thd_ctx.rand_state)%FRAME_SIZE;
1303
+ q_printf("Pushing promises in (%d,%d) for spilling\n",queue.get_head().adr,queue.get_tail().adr);
1304
+ push_promises(0,push_index,queue,delta);
1305
+ q_printf("Pushed queue (%d,%d) to stack\n",queue.get_head().adr,queue.get_tail().adr);
1306
+
1307
+
1308
+ }
1309
+
1310
+
1311
+ #else
1312
+
1313
+ __shared__ unsigned int queues[3];
1314
+ __shared__ unsigned int counts[3];
1315
+ __shared__ unsigned int bucket[4];
1316
+
1317
+ /*
1318
+ // Currently implemented in a single-threaded manner per work group to simplify the initial
1319
+ // correctness checking process. This can later be changed to take advantage of in-group
1320
+ // parallelism.
1321
+ */
1322
+ if(util::current_leader()){
1323
+
1324
+ /*
1325
+ // Zero the counters and null the queues
1326
+ */
1327
+ for(unsigned int i=0; i < 3; i++){
1328
+ queues[i] = QueueType::null;
1329
+ counts[i] = 0;
1330
+ }
1331
+
1332
+ for(unsigned int i=0; i < 4; i++){
1333
+ bucket[i] = 0;
1334
+ }
1335
+
1336
+
1337
+ /*
1338
+ // Count up each type of link
1339
+ */
1340
+ for(unsigned int i=0; i < STASH_SIZE; i++){
1341
+ unsigned int depth = _grp_ctx.stash[i].depth;
1342
+ unsigned int size = _grp_ctx.stash[i].size;
1343
+ unsigned int idx = (depth != level) ? 0 : 1;
1344
+ idx += (size >= WARP_COUNT) ? 0 : 2;
1345
+ bucket[idx] += 1;
1346
+ }
1347
+
1348
+ /*
1349
+ // Determine how much of which type of link needs to be dumped
1350
+ */
1351
+ unsigned int dump_total = bucket[0];
1352
+ unsigned int dump_count = (_grp_ctx.main_queue.count > threshold) ? _grp_ctx.main_queue.count - threshold : 0;
1353
+ dump_count = (dump_count <= bucket[0]) : 0 ? dump_count - bucket[0];
1354
+ for(unsigned int i=1; i< 4; i++){
1355
+ unsigned int delta = (bucket[i] <= dump_count) ? bucket[i] : dump_count;
1356
+ dump_count -= delta;
1357
+ bucket[i] = delta;
1358
+ dump_total += delta;
1359
+ }
1360
+
1361
+ /*
1362
+ // Dump the corresponding number of each type of link
1363
+ */
1364
+ for(unsigned int i=0; i < _grp_ctx.main_queue.count; i++){
1365
+ unsigned int depth = _grp_ctx.stash[i].depth;
1366
+ unsigned int size = _grp_ctx.stash[i].size;
1367
+ unsigned int bucket_idx = (depth != level) ? 0 : 1;
1368
+ bucket_idx += (size >= WARP_COUNT) ? 0 : 2;
1369
+ if(bucket[bucket_idx] == 0){
1370
+ continue;
1371
+ }
1372
+ LinkAdrType link = _grp_ctx.link_stash[_grp_ctx.link_stash_count];
1373
+ _grp_ctx.link_stash_count -= 1;
1374
+
1375
+ copy_link(_dev_ctx.arena[link], _grp_ctx.stash[i]);
1376
+
1377
+ unsigned int level_index = level+1-depth;
1378
+ counts[level_index] += size;
1379
+ push_back(queues[level_index],link);
1380
+
1381
+ _grp_ctx.stash[i].size = 0;
1382
+ }
1383
+ }
1384
+
1385
+ #endif
1386
+
1387
+ __syncwarp(active);
1388
+
1389
+
1390
+
1391
+ }
1392
+
1393
+
1394
+ //! Spills from stash in advance of inserting more work
1395
+ __device__ void async_call_stash_dump(OpDisc func_id, int depth_delta, unsigned int delta){
1396
+
1397
+ /*
1398
+ // Make room to queue incoming promises, if there isn't enough room already.
1399
+ */
1400
+ #if 0
1401
+ if(_grp_ctx.link_stash_count < 2){
1402
+ fill_stash_links(2);
1403
+ }
1404
+
1405
+ if(_grp_ctx.main_queue.count >= (STASH_SIZE-2)){
1406
+ spill_stash(STASH_SIZE-3);
1407
+ }
1408
+ #else
1409
+ /*
1410
+ unsigned int depth = (unsigned int) (_grp_ctx.level + depth_delta);
1411
+ unsigned int left_jump = partial_map_index(func_id,depth,_grp_ctx.level);
1412
+ unsigned int space = 0;
1413
+ if( left_jump != PART_ENTRY_COUNT ){
1414
+ unsigned int left_idx = _grp_ctx.main_queue.partial_map[left_jump];
1415
+ if( left_idx != STASH_SIZE ){
1416
+ space = WORK_GROUP_SIZE - _grp_ctx.stash[left_idx].count;
1417
+ }
1418
+ }
1419
+ */
1420
+ if( stash_overfilled() ) { //&& (space < delta) ){
1421
+ if(_grp_ctx.link_stash_count < STASH_MARGIN){
1422
+ fill_stash_links(STASH_MARGIN);
1423
+ }
1424
+ //printf("{Spilling for call.}");
1425
+ spill_stash(STASH_HIGH_WATER-1);
1426
+ }
1427
+ #endif
1428
+
1429
+ }
1430
+
1431
+
1432
+ //! Prepares the stash for the insertion of work
1433
+ __device__ void async_call_stash_prep(RemapQueue& dst, OpDisc func_id, int depth_delta, unsigned int delta,
1434
+ unsigned int &left, unsigned int &left_start, unsigned int &right
1435
+ ){
1436
+
1437
+ /*
1438
+ // Locate the destination links in the stash that the promises will be written to. For now,
1439
+ // like many other parts of the code, this will be single-threaded within the work group
1440
+ // to make validation easier but will be optimized for group-level parallelism later.
1441
+ */
1442
+ if( util::current_leader() ){
1443
+
1444
+ db_printf("{Queueing %d promises of type %d}",delta,func_id);
1445
+ /*
1446
+ // Null out the right index. This index should not be used unless the number of
1447
+ // promises queued spills over beyond the first link being written to (the left one)
1448
+ */
1449
+ right = STASH_SIZE;
1450
+
1451
+ /*
1452
+ // Find the index of the partial link in the stash corresponding to the id and
1453
+ // depth of the calls being queued (if it exists).
1454
+ */
1455
+ unsigned int depth = (unsigned int) (_grp_ctx.level + depth_delta);
1456
+ unsigned int left_jump = partial_map_index(func_id,depth,_grp_ctx.level);
1457
+
1458
+ /*
1459
+ // If there is a partially filled link to be filled, assign that to the left index
1460
+ */
1461
+ if(left_jump != PART_ENTRY_COUNT){
1462
+ db_printf("(%d:A)",blockIdx.x);
1463
+ left = dst.partial_map[left_jump];
1464
+ }
1465
+
1466
+ unsigned int left_count;
1467
+ if(left == STASH_SIZE){
1468
+ left = claim_empty_slot();
1469
+ dst.count += 1;
1470
+ db_printf("(%d:B+1->%d)",blockIdx.x,dst.count);
1471
+ db_printf("Updated stash count: %d\n",dst.count);
1472
+ _grp_ctx.stash[left].id = func_id;
1473
+ dst.partial_map[left_jump] = left;
1474
+ left_count = 0;
1475
+ } else {
1476
+ left_count = _grp_ctx.stash[left].count;
1477
+ }
1478
+
1479
+ if ( (left_count + delta) > WORK_GROUP_SIZE ){
1480
+ right = claim_empty_slot();
1481
+ dst.count += 1;
1482
+ db_printf("(%d:C+1->%d)",blockIdx.x,dst.count);
1483
+ _grp_ctx.stash[right].count = left_count+delta - WORK_GROUP_SIZE;
1484
+ _grp_ctx.stash[right].id = func_id;
1485
+ insert_full_slot(dst,left);
1486
+ dst.partial_map[left_jump] = right;
1487
+ _grp_ctx.stash[left].count = WORK_GROUP_SIZE;
1488
+ } else if ( (left_count + delta) == WORK_GROUP_SIZE ){
1489
+ db_printf("(%d:D)",blockIdx.x);
1490
+ dst.partial_map[left_jump] = STASH_SIZE;
1491
+ insert_full_slot(dst,left);
1492
+ _grp_ctx.stash[left].count = WORK_GROUP_SIZE;
1493
+ } else {
1494
+ db_printf("(%d:E)",blockIdx.x);
1495
+ _grp_ctx.stash[left].count = left_count + delta;
1496
+ }
1497
+
1498
+ left_start = left_count;
1499
+
1500
+
1501
+ }
1502
+
1503
+ }
1504
+
1505
+
1506
+ /*
1507
+ // Queues the input promise into a corresponding local link according to the given function id and
1508
+ // at a level corresponding to the current level of he promises being evaluated plus the value of
1509
+ // depth_delta. This scheme ensures that the function being called and the depth of the promises
1510
+ // being created for those calls are consistent across the warp.
1511
+ */
1512
+ __device__ void async_call(OpDisc func_id, int depth_delta, PromiseUnionType& promise){
1513
+
1514
+ unsigned int active = __activemask();
1515
+
1516
+ /*
1517
+ // Calculate how many promises are being queued as well as the assigned index of the
1518
+ // current thread's promise in the write to the stash.
1519
+ */
1520
+ unsigned int index = util::warp_inc_scan();
1521
+ unsigned int delta = util::active_count();
1522
+
1523
+
1524
+ #ifdef BARRIER_SPILL
1525
+ if( stash_overfilled() ) { //&& (space < delta) ){
1526
+ _grp_ctx.spill_barrier.union_await<false>(*this,func_id,promise);
1527
+ return;
1528
+ }
1529
+ #else
1530
+ async_call_stash_dump(func_id, depth_delta, delta);
1531
+ #endif
1532
+
1533
+
1534
+ __shared__ unsigned int left, left_start, right;
1535
+
1536
+
1537
+ async_call_stash_prep(_grp_ctx.main_queue,func_id,depth_delta,delta,left,left_start,right);
1538
+
1539
+
1540
+ /*
1541
+ // Write the promise into the appropriate part of the stash, writing into the left link
1542
+ // when possible and spilling over into the right link when necessary.
1543
+ */
1544
+ __syncwarp(active);
1545
+ if( (left_start + index) >= WORK_GROUP_SIZE ){
1546
+ //db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1547
+ _grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].dyn_copy_as(func_id,promise);
1548
+ //_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE] = promise;
1549
+ } else {
1550
+ //db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1551
+ _grp_ctx.stash[left].promises[left_start+index].dyn_copy_as(func_id,promise);
1552
+ //_grp_ctx.stash[left].promises[left_start+index] = promise;
1553
+ }
1554
+ __syncwarp(active);
1555
+
1556
+ }
1557
+
1558
+ /*
1559
+ // Like async_call, but allows for one to hand in the underlying type corresponding to a function id directly
1560
+ */
1561
+ template<typename TYPE, typename... ARGS>
1562
+ __device__ void async_call_cast(int depth_delta, ARGS... args){
1563
+
1564
+ beg_time(7);
1565
+ unsigned int active = __activemask();
1566
+
1567
+ /*
1568
+ // Calculate how many promises are being queued as well as the assigned index of the
1569
+ // current thread's promise in the write to the stash.
1570
+ */
1571
+ unsigned int index = util::warp_inc_scan();
1572
+ unsigned int delta = util::active_count();
1573
+
1574
+ #ifdef BARRIER_SPILL
1575
+ if( stash_overfilled() ) { //&& (space < delta) ){
1576
+ beg_time(8);
1577
+ _grp_ctx.spill_barrier.await<false>(*this,Promise<TYPE>(args...));
1578
+ end_time(8);
1579
+ return;
1580
+ }
1581
+ #else
1582
+ beg_time(8);
1583
+ async_call_stash_dump(Lookup<TYPE>::type::DISC, depth_delta, delta);
1584
+ end_time(8);
1585
+ #endif
1586
+
1587
+ __shared__ unsigned int left, left_start, right;
1588
+
1589
+
1590
+ #ifdef ASYNC_LOADS
1591
+ RemapQueue& dst_queue = has_load(args...) ? _grp_ctx.load_queue : _grp_ctx.main_queue;
1592
+ #else
1593
+ RemapQueue& dst_queue = _grp_ctx.main_queue;
1594
+ #endif
1595
+
1596
+ beg_time(9);
1597
+ async_call_stash_prep(dst_queue,Lookup<TYPE>::type::DISC,depth_delta,delta,left,left_start,right);
1598
+ end_time(9);
1599
+
1600
+ /*
1601
+ // Write the promise into the appropriate part of the stash, writing into the left link
1602
+ // when possible and spilling over into the right link when necessary.
1603
+ */
1604
+ __syncwarp(active);
1605
+ if( (left_start + index) >= WORK_GROUP_SIZE ){
1606
+ //db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1607
+ #ifdef ASYNC_LOADS
1608
+ _grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>().async_load(*this,args...);
1609
+ #else
1610
+ _grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>() = Promise<TYPE>(args...);
1611
+ #endif
1612
+ } else {
1613
+ //db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1614
+ #ifdef ASYNC_LOADS
1615
+ _grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>().async_load(*this,args...);
1616
+ #else
1617
+ _grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>() = Promise<TYPE>(args...);
1618
+ #endif
1619
+ }
1620
+ __syncwarp(active);
1621
+ end_time(7);
1622
+
1623
+ }
1624
+
1625
+
1626
+ template<typename TYPE>
1627
+ __device__ void async_call_cast(int depth_delta, Promise<TYPE> promise){
1628
+ beg_time(7);
1629
+ unsigned int active = __activemask();
1630
+
1631
+ /*
1632
+ // Calculate how many promises are being queued as well as the assigned index of the
1633
+ // current thread's promise in the write to the stash.
1634
+ */
1635
+ unsigned int index = util::warp_inc_scan();
1636
+ unsigned int delta = util::active_count();
1637
+
1638
+
1639
+
1640
+
1641
+ #ifdef BARRIER_SPILL
1642
+ if( stash_overfilled() ) { //&& (space < delta) ){
1643
+ beg_time(8);
1644
+ _grp_ctx.spill_barrier.await<false>(*this,promise);
1645
+ end_time(8);
1646
+ return;
1647
+ }
1648
+ #else
1649
+ beg_time(8);
1650
+ async_call_stash_dump(Lookup<TYPE>::type::DISC, depth_delta, delta);
1651
+ end_time(8);
1652
+ #endif
1653
+
1654
+ __shared__ unsigned int left, left_start, right;
1655
+
1656
+
1657
+ beg_time(9);
1658
+ async_call_stash_prep(_grp_ctx.main_queue,Lookup<TYPE>::type::DISC,depth_delta,delta,left,left_start,right);
1659
+ end_time(9);
1660
+
1661
+ /*
1662
+ // Write the promise into the appropriate part of the stash, writing into the left link
1663
+ // when possible and spilling over into the right link when necessary.
1664
+ */
1665
+ __syncwarp(active);
1666
+ if( (left_start + index) >= WORK_GROUP_SIZE ){
1667
+ //db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1668
+ _grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>() = promise;
1669
+ } else {
1670
+ //db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
1671
+ _grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>() = promise;
1672
+ }
1673
+ __syncwarp(active);
1674
+ end_time(7);
1675
+
1676
+ }
1677
+
1678
+
1679
+ template<typename TYPE>
1680
+ __device__ void immediate_call_cast(Promise<TYPE> param_value){
1681
+ PromiseUnionType promise;
1682
+ promise.template cast<TYPE>() = param_value;
1683
+ promise.template rigid_eval<ProgramType,TYPE>(*this);
1684
+ //promise_eval<ProgramType,FUNC_ID>(param_value);
1685
+
1686
+ }
1687
+
1688
+
1689
+
1690
+ #define PARACON
1691
+
1692
+ /*
1693
+ // Adds the contents of the link at the given index to the stash and adds the given link to link
1694
+ // stash. Once complete, it returns the number of promises added to the stash by the operation.
1695
+ // This should only ever be called if there is enough space to store the extra work and link.
1696
+ */
1697
+ __device__ unsigned int consume_link(LinkAdrType link_index ){
1698
+
1699
+
1700
+ #if 0 //def PARACON
1701
+ __shared__ LinkAdrType the_index;
1702
+ __shared__ unsigned int add_count;
1703
+ __shared__ OpDisc func_id;
1704
+
1705
+ unsigned int active = __activemask();
1706
+
1707
+ __syncwarp(active);
1708
+
1709
+ if(util::current_leader()){
1710
+
1711
+ q_printf("Consuming link %d\n",link_index.adr);
1712
+
1713
+ the_index = link_index;
1714
+ add_count = _dev_ctx.arena[link_index].count;
1715
+ func_id = _dev_ctx.arena[link_index].id;
1716
+
1717
+ }
1718
+
1719
+
1720
+ __syncwarp(active);
1721
+
1722
+ #if 0
1723
+ if(threadIdx.x < add_count){
1724
+ async_call(func_id,0,_dev_ctx.arena[the_index].promises[threadIdx.x]);
1725
+ }
1726
+ #else
1727
+ unsigned int idx = util::warp_inc_scan();
1728
+ unsigned int tot = util::active_count();
1729
+ for(unsigned int i=idx; i<add_count; i+=tot){
1730
+ async_call(func_id,0,_dev_ctx.arena[the_index].promises[i]);
1731
+ }
1732
+ #endif
1733
+
1734
+
1735
+ __syncwarp(active);
1736
+
1737
+
1738
+ if(util::current_leader()){
1739
+ insert_stash_link(link_index);
1740
+ }
1741
+
1742
+ return add_count;
1743
+
1744
+
1745
+ #else
1746
+
1747
+ LinkAdrType the_index;
1748
+ unsigned int add_count;
1749
+ OpDisc func_id;
1750
+
1751
+ unsigned int active = __activemask();
1752
+ unsigned int acount = util::active_count();
1753
+
1754
+
1755
+
1756
+ the_index = link_index;
1757
+ add_count = _dev_ctx.arena[link_index].count;
1758
+ func_id = _dev_ctx.arena[link_index].id;
1759
+
1760
+ //_grp_ctx.SM_promise_delta -= add_count;
1761
+
1762
+ db_printf("active count: %d, add count: %d\n",acount,add_count);
1763
+
1764
+
1765
+ db_printf("\n\nprior stash count: %d\n\n\n",_grp_ctx.main_queue.count);
1766
+ //*
1767
+ for(unsigned int i=0; i< add_count; i++){
1768
+ //PromiseUnionType promise = _dev_ctx.arena[the_index].promises[i];
1769
+ //async_call(func_id,0,promise);
1770
+ async_call(func_id,0, _dev_ctx.arena[the_index].promises[i] );
1771
+ }
1772
+ // */
1773
+ //PromiseType promise = _dev_ctx.arena[the_index].data.data[0];
1774
+ //async_call(func_id,0,promise);
1775
+
1776
+ db_printf("\n\nafter stash count: %d\n\n\n",_grp_ctx.main_queue.count);
1777
+
1778
+
1779
+ insert_stash_link(link_index);
1780
+
1781
+ return add_count;
1782
+
1783
+
1784
+
1785
+ #endif
1786
+
1787
+
1788
+ }
1789
+
1790
+
1791
+
1792
+
1793
+
1794
+
1795
+ /*
1796
+ // Tries to transfer links from the stack into the stash of the work group until the stash
1797
+ // is filled to the given threashold. If a halting condition is reached, this function will set
1798
+ // the keep_running value in the shared context to false.
1799
+ */
1800
+ __device__ void fill_stash(unsigned int threashold, bool halt_on_fail){
1801
+
1802
+ unsigned int active =__activemask();
1803
+ __syncwarp(active);
1804
+
1805
+
1806
+ #ifdef PARACON
1807
+ __shared__ unsigned int link_count;
1808
+ __shared__ LinkAdrType links[STASH_SIZE];
1809
+ #endif
1810
+
1811
+ /*
1812
+ // Currently implemented in a single-threaded manner per work group to simplify the initial
1813
+ // correctness checking process. This can later be changed to take advantage of in-group
1814
+ // parallelism.
1815
+ */
1816
+ if(util::current_leader()){
1817
+
1818
+ //db_printf("Filling stash...\n");
1819
+
1820
+ unsigned int taken = 0;
1821
+
1822
+ beg_time(12);
1823
+ threashold = (threashold > STASH_SIZE) ? STASH_SIZE : threashold;
1824
+ //printf("{Filling to %d @ %d}",threashold,blockIdx.x);
1825
+
1826
+ unsigned int gather_count = (threashold < _grp_ctx.main_queue.count) ? 0 : threashold - _grp_ctx.main_queue.count;
1827
+ if( (STASH_SIZE - _grp_ctx.link_stash_count) < gather_count){
1828
+ unsigned int spill_thresh = STASH_SIZE - gather_count;
1829
+ spill_stash_links(spill_thresh);
1830
+ }
1831
+ end_time(12);
1832
+
1833
+
1834
+ #ifdef PARACON
1835
+ unsigned int _thd_ctx_link_count = 0;
1836
+ #endif
1837
+
1838
+ #ifdef RACE_COND_PRINT
1839
+ unsigned int p_depth_live = _dev_ctx.stack->depth_live;
1840
+ rc_printf("SM %d: depth_live is (%d,%d)\n",threadIdx.x,(p_depth_live&0xFFFF0000)>>16,p_depth_live&0xFFFF);
1841
+ #endif
1842
+
1843
+ for(unsigned int i = 0; i < FILL_STASH_RETRY_LIMIT; i++){
1844
+
1845
+ /* If the stack is empty or a flag is set, return false */
1846
+ unsigned int depth_live = _dev_ctx.stack->depth_live;
1847
+ if( (depth_live == 0u) || ( _dev_ctx.stack->status_flags != 0u) ){
1848
+ if( halt_on_fail || ( _dev_ctx.stack->status_flags != 0u) ) {
1849
+ _grp_ctx.keep_running = false;
1850
+ }
1851
+ break;
1852
+ }
1853
+
1854
+
1855
+ unsigned int src_index;
1856
+ QueueType queue;
1857
+
1858
+ beg_time(3);
1859
+ #if DEF_STACK_MODE == 0
1860
+
1861
+ db_printf("STACK MODE ZERO\n");
1862
+ q_printf("%dth try pulling promises for fill\n",i+1);
1863
+ if( get_frame(_grp_ctx.level).children_residents.data != 0 ){
1864
+ queue = pull_promises(_grp_ctx.level,src_index);
1865
+ } else {
1866
+ queue.pair.data = QueueType::null;
1867
+ }
1868
+
1869
+ #else
1870
+ /*
1871
+ // Determine whether or not to pull from the current level in the stack
1872
+ */
1873
+ unsigned int depth = left_half(depth_live);
1874
+ bool pull_any = (depth < _grp_ctx.level);
1875
+ FrameType &current_frame = get_frame(depth);
1876
+ if(!pull_any){
1877
+ pull_any = ((current_frame.children_residents.get_right()) == 0);
1878
+ }
1879
+
1880
+
1881
+ /*
1882
+ // Retrieve a queue from the stack.
1883
+ */
1884
+
1885
+ if(pull_any){
1886
+ unsigned int new_level;
1887
+ queue = pull_promises_any_level(new_level,src_index);
1888
+ relevel_stash(new_level);
1889
+ } else {
1890
+ queue = pull_promises(_grp_ctx.level,src_index);
1891
+ }
1892
+ #endif
1893
+ end_time(3);
1894
+
1895
+
1896
+ beg_time(11);
1897
+ #ifdef PARACON
1898
+ db_printf("About to pop promises\n");
1899
+ while( ( ! queue.is_null() )
1900
+ && (_thd_ctx_link_count < gather_count)
1901
+ && (_grp_ctx.link_stash_count < STASH_SIZE)
1902
+ ){
1903
+ beg_time(13);
1904
+ LinkAdrType link = pop_front(queue);
1905
+ end_time(13);
1906
+ if( ! link.is_null() ){
1907
+ beg_time(14);
1908
+ db_printf("Popping front %d\n",link);
1909
+ links[_thd_ctx_link_count] = link;
1910
+ taken += _dev_ctx.arena[link].count;
1911
+ _thd_ctx_link_count++;
1912
+ end_time(14);
1913
+ } else {
1914
+ break;
1915
+ }
1916
+ }
1917
+ #else
1918
+ db_printf("About to pop promises\n");
1919
+ while( ( ! queue.is_null() )
1920
+ && (_grp_ctx.main_queue.count < threashold)
1921
+ && (_grp_ctx.link_stash_count < STASH_SIZE)
1922
+ ){
1923
+ beg_time(13);
1924
+ LinkAdrType link = pop_front(queue);
1925
+ end_time(13);
1926
+
1927
+ q_printf("Popping front %d. Q is now (%d,%d)\n",link.adr,queue.get_head().adr,queue.get_tail().adr);
1928
+
1929
+ if( ! link.is_null() ){
1930
+ beg_time(14);
1931
+ taken += consume_link(link);
1932
+ end_time(14);
1933
+ } else {
1934
+ break;
1935
+ }
1936
+ }
1937
+ #endif
1938
+ end_time(11);
1939
+
1940
+ db_printf("Popped promises\n");
1941
+ if(taken != 0){
1942
+ if(!_grp_ctx.busy){
1943
+ atomicAdd(&(_dev_ctx.stack->depth_live),1);
1944
+ _grp_ctx.busy = true;
1945
+ //printf("{got busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
1946
+ rc_printf("SM %d: Incremented depth value\n",threadIdx.x);
1947
+ }
1948
+ rc_printf("Pushing promises for filling\n");
1949
+ push_promises(_grp_ctx.level,src_index,queue,-taken);
1950
+ break;
1951
+ }
1952
+
1953
+ #ifdef PARACON
1954
+ if( _thd_ctx_link_count >= gather_count ){
1955
+ break;
1956
+ }
1957
+ #else
1958
+ if( _grp_ctx.main_queue.count >= threashold ){
1959
+ break;
1960
+ }
1961
+ #endif
1962
+
1963
+ }
1964
+
1965
+
1966
+
1967
+
1968
+
1969
+ #ifdef PARACON
1970
+ if(_grp_ctx.busy && (_grp_ctx.main_queue.count == 0) && (taken == 0) ){
1971
+ #else
1972
+ if(_grp_ctx.busy && (_grp_ctx.main_queue.count == 0)){
1973
+ #endif
1974
+ unsigned int depth_live = atomicSub(&(_dev_ctx.stack->depth_live),1);
1975
+ //printf("{unbusy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
1976
+ rc_printf("SM %d: Decremented depth value\n",threadIdx.x);
1977
+ _grp_ctx.busy = false;
1978
+ }
1979
+
1980
+
1981
+ #ifdef PARACON
1982
+ link_count = _thd_ctx_link_count;
1983
+ #endif
1984
+
1985
+
1986
+ }
1987
+
1988
+ __syncwarp(active);
1989
+
1990
+
1991
+
1992
+ #ifdef PARACON
1993
+
1994
+ __threadfence();
1995
+ beg_time(15);
1996
+ if(util::current_leader()){
1997
+ for(int i=0; i<link_count;i++){
1998
+ consume_link(links[i]);
1999
+ }
2000
+ }
2001
+ end_time(15);
2002
+
2003
+
2004
+ __syncwarp(active);
2005
+ __threadfence();
2006
+ #endif
2007
+
2008
+
2009
+
2010
+ }
2011
+
2012
+
2013
+
2014
+
2015
+ __device__ void clear_exec_head(){
2016
+
2017
+
2018
+ if( util::current_leader() && (_grp_ctx.exec_head != STASH_SIZE) ){
2019
+ insert_empty_slot(_grp_ctx.exec_head);
2020
+ _grp_ctx.exec_head = STASH_SIZE;
2021
+ }
2022
+ __syncwarp();
2023
+
2024
+ }
2025
+
2026
+
2027
+
2028
+
2029
+ /*
2030
+ // Selects the next link in the stash. This selection process could become more sophisticated
2031
+ // in later version to account for the average branching factor of each async function. For now,
2032
+ // it selects the fullest slot of the current level if it can. If no slots with promises for the
2033
+ // current level exist in the stash, the function returns false.
2034
+ */
2035
+ __device__ bool advance_stash_iter(){
2036
+
2037
+ __shared__ bool result;
2038
+ unsigned int active =__activemask();
2039
+ __syncwarp(active);
2040
+
2041
+
2042
+ if(util::current_leader()){
2043
+
2044
+ if(_grp_ctx.main_queue.full_head != STASH_SIZE){
2045
+ _grp_ctx.exec_head = claim_full_slot();
2046
+ _grp_ctx.main_queue.count -= 1;
2047
+ result = true;
2048
+ //db_printf("Found full slot.\n");
2049
+ } else {
2050
+ //db_printf("Looking for partial slot...\n");
2051
+ unsigned int best_id = PART_ENTRY_COUNT;
2052
+ unsigned int best_slot = STASH_SIZE;
2053
+ unsigned int best_count = 0;
2054
+ for(int i=0; i < FN_ID_COUNT; i++){
2055
+ unsigned int slot = _grp_ctx.main_queue.partial_map[i];
2056
+
2057
+ if( (slot != STASH_SIZE) && (_grp_ctx.stash[slot].count > best_count)){
2058
+ best_id = i;
2059
+ best_slot = slot;
2060
+ best_count = _grp_ctx.stash[slot].count;
2061
+ }
2062
+
2063
+ }
2064
+
2065
+ result = (best_slot != STASH_SIZE);
2066
+ if(result){
2067
+ //db_printf("Found partial slot.\n");
2068
+ _grp_ctx.exec_head = best_slot;
2069
+ _grp_ctx.main_queue.partial_map[best_id] = STASH_SIZE;
2070
+ _grp_ctx.main_queue.count -=1;
2071
+ }
2072
+ }
2073
+
2074
+ }
2075
+
2076
+ __syncwarp(active);
2077
+ return result;
2078
+
2079
+ }
2080
+
2081
+
2082
+ #ifdef ASYNC_LOADS
2083
+ __device__ void force_full_async_loads() {
2084
+
2085
+ _grp_ctx.load_barrier.arrive_and_wait();
2086
+ __syncwarp();
2087
+ #if 1
2088
+ if (util::current_leader()) {
2089
+ unsigned char iter = _grp_ctx.load_queue.full_head;
2090
+ if( iter != STASH_SIZE ){
2091
+ unsigned int main_delta = 1;
2092
+ unsigned char next = _grp_ctx.stash[iter].next.adr;
2093
+ while( next != STASH_SIZE ){
2094
+ iter = next;
2095
+ next = _grp_ctx.stash[iter].next.adr;
2096
+ main_delta += 1;
2097
+ }
2098
+ _grp_ctx.stash[iter].next = _grp_ctx.main_queue.full_head;
2099
+ _grp_ctx.main_queue.full_head = _grp_ctx.load_queue.full_head;
2100
+ _grp_ctx.load_queue.full_head = STASH_SIZE;
2101
+ _grp_ctx.main_queue.count += main_delta;
2102
+ _grp_ctx.load_queue.count -= main_delta;
2103
+ }
2104
+ //printf("(%d:f-%d)",blockIdx.x,_grp_ctx.main_queue.count);
2105
+ }
2106
+ #else
2107
+ if( util::current_leader() ){
2108
+ _grp_ctx.main_queue.full_head = _grp_ctx.load_queue.full_head;
2109
+ _grp_ctx.load_queue.full_head = STASH_SIZE;
2110
+ _grp_ctx.main_queue.count += _grp_ctx.load_queue.count;
2111
+ _grp_ctx.load_queue.count = 0;
2112
+ printf("(%d:f-%d)",blockIdx.x,_grp_ctx.main_queue.count);
2113
+ }
2114
+ #endif
2115
+ __syncwarp();
2116
+
2117
+ }
2118
+ __device__ void force_async_loads() {
2119
+
2120
+ force_full_async_loads();
2121
+
2122
+ for(int i=0; i < FN_ID_COUNT; i++){
2123
+ unsigned char idx = _grp_ctx.load_queue.partial_map[i];
2124
+ if( idx != STASH_SIZE){
2125
+ LinkType& link = _grp_ctx.stash[idx];
2126
+ if( threadIdx.x < link.count ){
2127
+ async_call(0,link.id,link.promises[threadIdx.x]);
2128
+ }
2129
+ __syncwarp();
2130
+ if( util::current_leader() ){
2131
+ insert_empty_slot(idx);
2132
+ _grp_ctx.load_queue.partial_map[i] = STASH_SIZE;
2133
+ //printf("(%d:%d)\n",blockIdx.x,idx);
2134
+ }
2135
+ }
2136
+ }
2137
+ __syncwarp();
2138
+ if(util::current_leader()){
2139
+ init(&_grp_ctx.load_barrier,WORK_GROUP_SIZE);
2140
+ _grp_ctx.load_queue.count = 0;
2141
+ }
2142
+
2143
+ }
2144
+ #endif
2145
+
2146
+
2147
+
2148
+ /*
2149
+ // Tries to perform up to one work group worth of work by selecting a link from shared memory (or,
2150
+ // if necessary, fetching a link from main memory), and running the function on the data within
2151
+ // the link, as directed by the function id the link is labeled with. This function returns false
2152
+ // if a halting condition has been reached (either due to lack of work or an event) and true
2153
+ // otherwise.
2154
+ */
2155
+ __device__ void exec_cycle(){
2156
+
2157
+
2158
+
2159
+ clear_exec_head();
2160
+
2161
+ /*
2162
+ // Advance the stash iterator to the next chunk of work that needs to be done.
2163
+ */
2164
+ //*
2165
+
2166
+
2167
+ if( stash_overfilled() ) {
2168
+ //printf("{Stash overfilled}");
2169
+ #ifdef BARRIER_SPILL
2170
+ if( util::current_leader() ){
2171
+ _grp_ctx.spill_barrier.release_full(*this);
2172
+ }
2173
+ #endif
2174
+
2175
+ if(_grp_ctx.link_stash_count < STASH_MARGIN){
2176
+ fill_stash_links(STASH_MARGIN);
2177
+ }
2178
+ //printf("{Spilling for call.}");
2179
+ spill_stash(STASH_HIGH_WATER-1);
2180
+ }
2181
+
2182
+
2183
+ beg_time(1);
2184
+
2185
+ #ifdef ASYNC_LOADS
2186
+ if( (_grp_ctx.main_queue.full_head == STASH_SIZE) && (_grp_ctx.load_queue.full_head != STASH_SIZE) ){
2187
+ force_full_async_loads();
2188
+ }
2189
+ #endif
2190
+
2191
+ const PromiseCount GLOBAL_WORK_THRESHOLD = STASH_SIZE/2;
2192
+ //if ( ( ((_dev_ctx.stack->frames[0].children_residents) & 0xFFFF ) > (gridDim.x*blockIdx.x*2) ) && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
2193
+ if ( ( ((_dev_ctx.stack->frames[0].children_residents.get_right()) ) > GLOBAL_WORK_THRESHOLD ) && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
2194
+ fill_stash(STASH_HIGH_WATER,false);
2195
+ }
2196
+
2197
+ end_time(1);
2198
+
2199
+ beg_time(5);
2200
+
2201
+ if (_grp_ctx.main_queue.full_head == STASH_SIZE){
2202
+
2203
+
2204
+ #ifdef EAGER_FILLING
2205
+ const PromiseCount GLOBAL_WORK_THRESHOLD = STASH_SIZE/2;
2206
+ if( ((_dev_ctx.stack->frames[0].children_residents.get_right()) ) > GLOBAL_WORK_THRESHOLD ) {
2207
+ fill_stash(STASH_HIGH_WATER,false);
2208
+ } else {
2209
+ #endif
2210
+ if ( _grp_ctx.can_make_work && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
2211
+ _grp_ctx.can_make_work = __any_sync(0xFFFFFFFF,PROGRAM_SPEC::make_work(*this));
2212
+ if( util::current_leader() && (! _grp_ctx.busy ) && ( _grp_ctx.main_queue.count != 0 ) ){
2213
+ unsigned int depth_live = atomicAdd(&(_dev_ctx.stack->depth_live),1);
2214
+ _grp_ctx.busy = true;
2215
+ //printf("{made self busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
2216
+ }
2217
+ __syncwarp();
2218
+ }
2219
+ #ifdef EAGER_FILLING
2220
+ }
2221
+ #endif
2222
+ }
2223
+
2224
+ end_time(5);
2225
+
2226
+ #ifdef ASYNC_LOADS
2227
+ if( (_grp_ctx.main_queue.full_head == STASH_SIZE) && (_grp_ctx.load_queue.count > 0) ){
2228
+ force_async_loads();
2229
+ }
2230
+ #endif
2231
+
2232
+ #if 1
2233
+
2234
+ /*
2235
+ if ( _grp_ctx.main_queue.full_head == STASH_SIZE ) {
2236
+ fill_stash(STASH_SIZE-2);
2237
+ }
2238
+ */
2239
+ #else
2240
+ if(_grp_ctx.main_queue.full_head == STASH_SIZE){
2241
+ if( !_grp_ctx.scarce_work ){
2242
+ fill_stash(STASH_SIZE-2);
2243
+ if( util::current_leader() && (_grp_ctx.main_queue.full_head == STASH_SIZE) ){
2244
+ _grp_ctx.scarce_work = true;
2245
+ }
2246
+ }
2247
+ } else {
2248
+ if( util::current_leader() ){
2249
+ _grp_ctx.scarce_work = false;
2250
+ }
2251
+ }
2252
+ #endif
2253
+ // */
2254
+
2255
+ #if 0
2256
+ if( util::current_leader() ){
2257
+ PairPack<PromiseCount> cr = _dev_ctx.stack->frames[0].children_residents;
2258
+ printf("{group %d children_residents=(%d,%d), can_make_work=%d}\n",blockIdx.x,cr.get_left(),cr.get_right(),_grp_ctx.can_make_work);
2259
+ }
2260
+ #endif
2261
+ beg_time(2);
2262
+ if( !advance_stash_iter() ){
2263
+ /*
2264
+ // No more work exists in the stash, so try to fetch it from the stack.
2265
+ */
2266
+ beg_time(10);
2267
+ fill_stash(STASH_HIGH_WATER,true);
2268
+ end_time(10);
2269
+
2270
+ if( _grp_ctx.keep_running && !advance_stash_iter() ){
2271
+ /*
2272
+ // REALLY BAD: The fill_stash function was successful, however
2273
+ // the stash still has no work to perform. In this situation,
2274
+ // we set an error flag and halt.
2275
+ */
2276
+ /*
2277
+ if(util::current_leader()){
2278
+ db_printf("\nBad stuff afoot!\n\n");
2279
+ }
2280
+ set_flags(_grp_ctx,STASH_FAIL_FLAG);
2281
+ _grp_ctx.keep_running = false;
2282
+ */
2283
+ }
2284
+ }
2285
+ end_time(2);
2286
+
2287
+
2288
+ unsigned int active = __activemask();
2289
+ __syncwarp(active);
2290
+
2291
+
2292
+ beg_time(4);
2293
+ if( _grp_ctx.exec_head != STASH_SIZE ){
2294
+ /*
2295
+ // Find which function the current link corresponds to.
2296
+ */
2297
+ OpDisc func_id = _grp_ctx.stash[_grp_ctx.exec_head].id;
2298
+ unsigned int promise_count = _grp_ctx.stash[_grp_ctx.exec_head].count;
2299
+
2300
+ /*
2301
+ // Only execute if there is a promise in the current link corresponding to the thread that
2302
+ // is being executed.
2303
+ */
2304
+ if(util::current_leader()){
2305
+ db_printf("Executing slot %d, which is %d promises of type %d\n",_grp_ctx.exec_head,promise_count,func_id);
2306
+ }
2307
+ if( threadIdx.x < promise_count ){
2308
+ //db_printf("Executing...\n");
2309
+ PromiseUnionType& promise = _grp_ctx.stash[_grp_ctx.exec_head].promises[threadIdx.x];
2310
+ //do_async(func_id,promise);
2311
+ promise.template loose_eval<ProgramType>(*this,func_id);
2312
+ }
2313
+ }
2314
+
2315
+ __syncwarp(active);
2316
+ end_time(4);
2317
+
2318
+
2319
+ }
2320
+
2321
+
2322
+
2323
+ __device__ void cleanup_runtime(){
2324
+
2325
+
2326
+ //unsigned int active = __activemask();
2327
+ //__syncwarp(active);
2328
+ __syncwarp();
2329
+
2330
+
2331
+ if(_thd_ctx.spare_index > 0){
2332
+ dealloc_links(_thd_ctx.spare_links,_thd_ctx.spare_index);
2333
+ }
2334
+
2335
+ if(threadIdx.x == 0){
2336
+
2337
+ #ifdef BARRIER_SPILL
2338
+ _grp_ctx.spill_barrier.release(*this);
2339
+ #endif
2340
+
2341
+ q_printf("CLEANING UP\n");
2342
+ clear_exec_head();
2343
+
2344
+ spill_stash(0);
2345
+ spill_stash_links(0);
2346
+
2347
+ if(_grp_ctx.can_make_work){
2348
+ //printf("{Setting early halt flag.}");
2349
+ set_flags(EARLY_HALT_FLAG);
2350
+ }
2351
+
2352
+ if(_grp_ctx.busy){
2353
+ unsigned int depth_live = atomicSub(&(_dev_ctx.stack->depth_live),1);
2354
+ //printf("{wrap busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
2355
+ }
2356
+ }
2357
+
2358
+ //__syncwarp(active);
2359
+ __syncwarp();
2360
+ __threadfence();
2361
+ //__syncwarp(active);
2362
+ __syncwarp();
2363
+
2364
+ if(threadIdx.x == 0){
2365
+ unsigned int checkout_index = atomicAdd(&(_dev_ctx.stack->checkout),1);
2366
+ __threadfence();
2367
+ if( checkout_index == (gridDim.x-1) ){
2368
+ //printf("{Final}\n");
2369
+ atomicExch(&(_dev_ctx.stack->checkout),0);
2370
+ unsigned int old_flags = atomicAnd(&(_dev_ctx.stack->status_flags),~EARLY_HALT_FLAG);
2371
+ unsigned int depth_live = atomicAdd(&(_dev_ctx.stack->depth_live),0);
2372
+ bool halted_early = ( old_flags & EARLY_HALT_FLAG );
2373
+ bool work_left = ( (depth_live & 0xFFFF0000) != 0 );
2374
+
2375
+ if( (!halted_early) && (!work_left) ){
2376
+ set_flags(COMPLETION_FLAG);
2377
+ }
2378
+
2379
+ //printf("{depth_live is (%d,%d)}",(depth_live&0xFFFF0000)>>16,depth_live&0xFFFF );
2380
+ //unsigned int cr = atomicAdd(&(_dev_ctx.stack->frames[0].children_residents),0);
2381
+ //printf("{Level 0 CR is (%d,%d)}",(cr&0xFFFF0000)>>16,cr&0xFFFF );
2382
+ }
2383
+
2384
+ #ifdef HRM_TIME
2385
+ end_time(0);
2386
+ for(int i=0; i<HRM_TIME; i++){
2387
+ atomicAdd(&_dev_ctx.time_totals[i],_grp_ctx.time_totals[i]);
2388
+ }
2389
+ #endif
2390
+
2391
+ }
2392
+
2393
+ }
2394
+
2395
+
2396
+ public:
2397
+ /*
2398
+ //
2399
+ // This must be run once on the resources used for execution, prior to execution. Given that this
2400
+ // essentially wipes all data from these resources and zeros all values, it is not advised that
2401
+ // this function be used at any other time, except to setup for a re-start or to clear out after
2402
+ // calling the pull_runtime to prevent promise duplication.
2403
+ //
2404
+ */
2405
+ __device__ void init_program(){
2406
+
2407
+ /* Initialize per-thread resources */
2408
+ init_thread();
2409
+
2410
+ const unsigned int threads_per_frame = FRAME_SIZE + 1;
2411
+ const unsigned int total_stack_work = StackType::NULL_LEVEL * threads_per_frame;
2412
+
2413
+ unsigned int worker_count = gridDim.x*blockDim.x;
2414
+
2415
+ /*
2416
+ // If the currently executing thread has device thread index 0, wipe the data in the base
2417
+ // of the stack.
2418
+ */
2419
+ if(_thd_ctx.thread_id == 0){
2420
+ _dev_ctx.stack->status_flags = 0;
2421
+ _dev_ctx.stack->depth_live = 0;
2422
+ _dev_ctx.stack->checkout = 0;
2423
+ }
2424
+
2425
+
2426
+ /*
2427
+ if( _thd_ctx.thread_id == 0 ){
2428
+ printf( "Initializing the stack with\n"
2429
+ "\t- total_stack_work=%d\n"
2430
+ "\t- threads_per_frame=%d\n"
2431
+ "\t- worker_count=%d\n"
2432
+ "\t- stack->frames[0].children_residents=%d\n",
2433
+ total_stack_work,
2434
+ threads_per_frame,
2435
+ worker_count,
2436
+ _dev_ctx.stack->frames[0].children_residents
2437
+ );
2438
+ }
2439
+ */
2440
+
2441
+
2442
+
2443
+ /*
2444
+ // Blank out the frames in the stack. Setting queues to NULL_QUEUE, and zeroing the counts
2445
+ // for resident promises and child promises of each frame.
2446
+ */
2447
+ for(unsigned int index = _thd_ctx.thread_id; index < total_stack_work; index+=worker_count ){
2448
+
2449
+ unsigned int target_level = index / threads_per_frame;
2450
+ unsigned int frame_index = index % threads_per_frame;
2451
+ if( frame_index == FRAME_SIZE ){
2452
+ _dev_ctx.stack->frames[target_level].children_residents.data = 0u;
2453
+ if( _dev_ctx.stack->frames[target_level].children_residents.data != 0u ){
2454
+ printf("BAD A!\n");
2455
+ }
2456
+ } else {
2457
+ _dev_ctx.stack->frames[target_level].pool.queues[frame_index].pair.data = QueueType::null;
2458
+ if( _dev_ctx.stack->frames[target_level].pool.queues[frame_index].pair.data != QueueType::null ) {
2459
+ printf("BAD B!\n");
2460
+ }
2461
+ }
2462
+
2463
+ }
2464
+
2465
+
2466
+ #ifdef LAZY_LINK
2467
+
2468
+ if(_thd_ctx.thread_id == 0){
2469
+ (*_dev_ctx.claim_count) = 0;
2470
+ }
2471
+
2472
+ /*
2473
+ // Initialize the pool, assigning empty queues to each queue slot.
2474
+ */
2475
+ for(unsigned int index = _thd_ctx.thread_id; index < POOL_SIZE; index+=worker_count ){
2476
+
2477
+ _dev_ctx.pool->queues[index].pair.data = QueueType::null;
2478
+ if( _dev_ctx.pool->queues[index].pair.data != QueueType::null ){
2479
+ printf("BAD C!\n");
2480
+ }
2481
+
2482
+ }
2483
+
2484
+ #else
2485
+ /*
2486
+ // Initialize the arena, connecting the contained links into roughly equally sized lists,
2487
+ // zeroing the promise counter in the links and marking the function ID with an invalid
2488
+ // value to make use-before-initialization more obvious during system validation.
2489
+ */
2490
+ unsigned int bump = ((arena_size%POOL_SIZE) != 0) ? 1 : 0;
2491
+ unsigned int arena_init_stride = arena_size/POOL_SIZE + bump;
2492
+ for(unsigned int index = _thd_ctx.thread_id; index < arena_size; index+=worker_count ){
2493
+
2494
+ unsigned int next = index + 1;
2495
+ if( ( (next % arena_init_stride) == 0 ) || (next >= arena_size) ){
2496
+ next = LinkAdrType::null;
2497
+ }
2498
+ _dev_ctx.arena[index].empty(LinkAdrType(next));
2499
+ }
2500
+
2501
+
2502
+ /*
2503
+ // Initialize the pool, giving each queue slot one of the previously created linked lists.
2504
+ */
2505
+ for(unsigned int index = _thd_ctx.thread_id; index < POOL_SIZE; index+=worker_count ){
2506
+
2507
+ unsigned int head = arena_init_stride * index;
2508
+ unsigned int tail = arena_init_stride * (index + 1) - 1;
2509
+ tail = (tail >= arena_size) ? arena_size - 1 : tail;
2510
+ _dev_ctx.pool->queues[index] = QueueType(LinkAdrType(head),LinkAdrType(tail));
2511
+
2512
+ }
2513
+ #endif
2514
+
2515
+
2516
+ }
2517
+
2518
+
2519
+ /*
2520
+ // Unpacks all promise data from the call buffer into the stack of the given context. This
2521
+ // could be useful for backing up program states for debugging or to re-start processing from
2522
+ // a previous state.
2523
+ */
2524
+ __device__ void push_calls(DeviceContext _dev_ctx, LinkType* call_buffer, size_t link_count){
2525
+
2526
+ /* Initialize per-warp resources */
2527
+ __shared__ GroupContext _grp_ctx;
2528
+ init_group(_dev_ctx,_grp_ctx);
2529
+
2530
+ /* Initialize per-thread resources */
2531
+ ThreadContext _thd_ctx;
2532
+ init_local(_thd_ctx);
2533
+
2534
+
2535
+ for(int link_index=blockIdx.x; link_index < link_count; link_index+= gridDim.x){
2536
+ LinkType& the_link = call_buffer[link_index];
2537
+ unsigned int count = the_link.count;
2538
+ unsigned int func_id = the_link.id;
2539
+ if(threadIdx.x < count){
2540
+ db_printf("\nasync_call(id:%d,depth: 0)\n\n",func_id);
2541
+ async_call(func_id,0,the_link.data.data[threadIdx.x]);
2542
+ }
2543
+
2544
+ }
2545
+
2546
+ cleanup_runtime();
2547
+
2548
+ }
2549
+
2550
+
2551
+
2552
+
2553
+ static void check_error(){
2554
+
2555
+ adapt::GPUrtError_t status = adapt::GPUrtGetLastError();
2556
+
2557
+ if(status != adapt::GPUrtSuccess){
2558
+ const char* err_str = adapt::GPUrtGetErrorString(status);
2559
+ printf("ERROR: \"%s\"\n",err_str);
2560
+ }
2561
+
2562
+ }
2563
+
2564
+
2565
+
2566
+
2567
+
2568
+ /*
2569
+ // Places a single function call into the runtime.
2570
+ static void remote_call(Instance &instance, unsigned char func_id, PromiseUnionType promise){
2571
+
2572
+ LinkType* call_buffer;
2573
+ util::host::auto_throw(adapt::GPUrtMalloc( (void**) &call_buffer, sizeof(LinkType) ));
2574
+
2575
+ LinkType host_link;
2576
+ host_link.count = 1;
2577
+ host_link.id = func_id;
2578
+ host_link.next = LinkType::null;
2579
+ host_link.depth = 0;
2580
+ host_link.meta_data.data= 0;
2581
+ host_link.data.data[0] = promise;
2582
+
2583
+ util::host::auto_throw(adapt::GPUrtMemcpy(call_buffer,&host_link,sizeof(LinkType),adapt::GPUrtMemcpyHostToDevice));
2584
+
2585
+
2586
+ push_runtime<<<1,WORK_GROUP_SIZE>>>(instance.to_context(),call_buffer,1);
2587
+
2588
+ check_error();
2589
+
2590
+ util::host::auto_throw(adapt::GPUrtFree(call_buffer));
2591
+
2592
+ }
2593
+ */
2594
+
2595
+
2596
+
2597
+
2598
+
2599
+ /*
2600
+ // Packs all promise data from the runtime stack into the communication buffer (comm_buffer). This
2601
+ // could be useful for backing up program states for debugging or to re-start processing from
2602
+ // a previous state.
2603
+ //
2604
+ // For now, this will not be implemented, as it isn't particularly useful until the system's
2605
+ // correctness has been verified.
2606
+ //
2607
+ */
2608
+ __device__ void pull_promises(Instance &instance){
2609
+
2610
+ /*
2611
+ // [TODO] NOT YET IMPLEMENTED
2612
+ */
2613
+
2614
+ }
2615
+
2616
+
2617
+
2618
+ /*
2619
+ // The workhorse of the program. This function executes until either a halting condition
2620
+ // is encountered or a maximum number of processing cycles has occured. This makes sure
2621
+ // that long-running programs don't time out on the GPU. In practice, cycle_count may have
2622
+ // to be tuned to the average cycle execution time for a given application. This could
2623
+ // potentially be automated using an exponential backoff heuristic.
2624
+ */
2625
+ __device__ void exec(unsigned int cycle_count){
2626
+
2627
+ /* Initialize per-warp resources */
2628
+ init_group();
2629
+
2630
+ /* Initialize per-thread resources */
2631
+ init_thread();
2632
+
2633
+ PROGRAM_SPEC::initialize(*this);
2634
+
2635
+ /*
2636
+ if(util::current_leader()){
2637
+ printf("\n\n\nInitial frame zero resident count is: %d\n\n\n",_dev_ctx.stack->frames[0].children_residents);
2638
+ }
2639
+ */
2640
+
2641
+ /* The execution loop. */
2642
+ #ifdef RACE_COND_PRINT
2643
+ unsigned int cycle_break = cycle_count;
2644
+ #endif
2645
+ for(unsigned int cycle=0u; cycle<cycle_count; cycle++){
2646
+ /* Early halting handled with a break. */
2647
+ exec_cycle();
2648
+ if(!_grp_ctx.keep_running){
2649
+ #ifdef RACE_COND_PRINT
2650
+ cycle_break = cycle+1;
2651
+ #endif
2652
+ break;
2653
+ }
2654
+ }
2655
+
2656
+ PROGRAM_SPEC::finalize(*this);
2657
+
2658
+ /*
2659
+ // Ensure that nothing which should persist between dispatches is lost in the
2660
+ // shared or private memory of the halting program.
2661
+ */
2662
+ cleanup_runtime();
2663
+
2664
+ if(util::current_leader()){
2665
+ rc_printf("SM %d finished after %d cycles with promise delta %d\n",threadIdx.x,cycle_break,_grp_ctx.SM_promise_delta);
2666
+ }
2667
+
2668
+ }
2669
+
2670
+
2671
+
2672
+
2673
+
2674
+ #if 1
2675
+
2676
+
2677
+ __host__ static bool queue_count(Instance runtime, LinkType* host_arena, QueueType queue, LinkAdrType& result){
2678
+
2679
+ //printf("Entered function\n");
2680
+ LinkAdrType head = queue.get_head();
2681
+ LinkAdrType tail = queue.get_tail();
2682
+ LinkAdrType last = LinkAdrType::null;
2683
+ LinkAdrType count = 0;
2684
+
2685
+ //printf("About to check if the head or tail was NULL\n");
2686
+
2687
+ if( head.is_null() ){
2688
+ if( tail.is_null() ) {
2689
+ result = 0;
2690
+ return true;
2691
+ } else {
2692
+ printf("NULL head with a non-NULL tail\n");
2693
+ return false;
2694
+ }
2695
+ } else if ( tail.is_null() ){
2696
+ printf("Non-NULL head with a NULL tail\n");
2697
+ return false;
2698
+ }
2699
+
2700
+ //printf("Just checked if the head or tail was NULL\n");
2701
+ LinkAdrType iter = head;
2702
+ while( ! iter.is_null() ){
2703
+ if( iter.adr > runtime.arena_size ){
2704
+ printf("Queue has bad index pointing to index %d\n",iter.adr);
2705
+ return false;
2706
+ }
2707
+ if(host_arena[iter.adr].meta_data != 0){
2708
+ printf("Link re-visited\n");
2709
+ LinkAdrType loop_point = iter;
2710
+ LinkAdrType visit_count = 0;
2711
+ iter = head;
2712
+ printf("(%d,%d): ",head.adr,tail.adr);
2713
+ LinkAdrType step_count = 0;
2714
+ while(true){
2715
+ if(iter == loop_point){
2716
+ if(visit_count.adr == 0){
2717
+ printf("{%d}->",iter.adr);
2718
+ } else {
2719
+ printf("{%d}\n",iter.adr);
2720
+ break;
2721
+ }
2722
+ visit_count.adr += 1;
2723
+ } else {
2724
+ printf("%d->",iter.adr);
2725
+ }
2726
+ iter = host_arena[iter.adr].next;
2727
+ if( iter.is_null() ){
2728
+ printf("NULL\n",iter.adr);
2729
+ return false;
2730
+ }
2731
+ step_count.adr +=1;
2732
+ if(step_count.adr > 64){
2733
+ printf("...{%d}\n",loop_point.adr);
2734
+ break;
2735
+ }
2736
+ }
2737
+ return false;
2738
+ } else {
2739
+ host_arena[iter.adr].meta_data = 1;
2740
+ }
2741
+ last = iter;
2742
+ iter = host_arena[iter.adr].next;
2743
+ count.adr += 1;
2744
+ }
2745
+
2746
+ if( last.adr != tail.adr ){
2747
+ printf("Final link %d in the queue (%d,%d) not the tail\n",last.adr,head.adr,tail.adr);
2748
+ return false;
2749
+ }
2750
+
2751
+ result = count;
2752
+ return true;
2753
+
2754
+ }
2755
+
2756
+
2757
+
2758
+
2759
+
2760
+
2761
+ /*
2762
+ // Counts the number of links in each queue in the pool and in the stack, storing the counts in
2763
+ // the provided arrays. This funciton returns true if counting was successful and returns false
2764
+ // if an invalid state is detected.
2765
+ */
2766
+ __host__ static bool runtime_overview(Instance runtime){
2767
+
2768
+ bool result = true;
2769
+
2770
+ #ifdef DEBUG_PRINT
2771
+ const bool always_print = true;
2772
+ #else
2773
+ const bool always_print = true;
2774
+ #endif
2775
+
2776
+ LinkAdrType* pool_counts = new LinkAdrType[POOL_SIZE];
2777
+ bool* pool_count_validity = new bool[POOL_SIZE];
2778
+ LinkAdrType* stack_counts = new LinkAdrType[STACK_SIZE*FRAME_SIZE];
2779
+ bool* stack_count_validity = new bool[STACK_SIZE*FRAME_SIZE];
2780
+
2781
+
2782
+ #ifdef LAZY_LINK
2783
+ AdrType claim_count;
2784
+ runtime.claim_count >> claim_count;
2785
+ #endif
2786
+
2787
+ LinkType* host_arena = new LinkType[runtime.arena_size];
2788
+
2789
+ QueueType* host_pool = new QueueType[POOL_SIZE];
2790
+
2791
+ StackType* host_stack = new StackType;
2792
+
2793
+
2794
+ LinkAdrType link_total = 0;
2795
+
2796
+ adapt::GPUrtMemcpy(host_arena,runtime.arena,sizeof(LinkType) *runtime.arena_size,adapt::GPUrtMemcpyDeviceToHost);
2797
+ adapt::GPUrtMemcpy(host_pool ,runtime.pool ,sizeof(QueueType)*POOL_SIZE ,adapt::GPUrtMemcpyDeviceToHost);
2798
+ adapt::GPUrtMemcpy(host_stack,runtime.stack,sizeof(StackType) ,adapt::GPUrtMemcpyDeviceToHost);
2799
+
2800
+
2801
+ for(AdrType i = 0; i < runtime.arena_size; i++){
2802
+ host_arena[i].meta_data = 0;
2803
+ }
2804
+
2805
+
2806
+
2807
+ //printf("Counting through pool links...\n");
2808
+ for(int i=0; i < POOL_SIZE; i++){
2809
+ //printf("Counting pool queue %d\n",i);
2810
+ QueueType queue = host_pool[i];
2811
+ //printf("Read pool queue %d\n",i);
2812
+ pool_count_validity[i] = queue_count(runtime,host_arena,queue,pool_counts[i]);
2813
+ //printf("Just validated pool queue %d\n",i);
2814
+ result = result && pool_count_validity[i];
2815
+ if(pool_count_validity[i]){
2816
+ link_total.adr += pool_counts[i].adr;
2817
+ }
2818
+ }
2819
+
2820
+ //printf("Counting through stack links...\n");
2821
+ for(int i=0; i < STACK_SIZE; i++){
2822
+ for(int j=0; j < FRAME_SIZE; j++){
2823
+ QueueType queue = host_stack->frames[i].pool.queues[j];
2824
+ unsigned int index = i*FRAME_SIZE + j;
2825
+ stack_count_validity[i] = queue_count(runtime,host_arena,queue,stack_counts[index]);
2826
+ result = result && stack_count_validity[i];
2827
+ if(stack_count_validity[i]){
2828
+ link_total.adr += stack_counts[index].adr;
2829
+ }
2830
+ }
2831
+
2832
+ }
2833
+
2834
+
2835
+ if( (!result) || always_print ){
2836
+ printf("POOL:\t[");
2837
+ for(int i=0; i<POOL_SIZE; i++){
2838
+ if(pool_count_validity[i]){
2839
+ printf("\t%d",pool_counts[i].adr);
2840
+ } else {
2841
+ printf("\t????");
2842
+ }
2843
+ }
2844
+ printf("\t]\n");
2845
+
2846
+ unsigned int status_flags = host_stack->status_flags;
2847
+ unsigned int depth = (host_stack->depth_live >> 16) & 0xFFFF;
2848
+ unsigned int live = (host_stack->depth_live) & 0xFFFF;
2849
+
2850
+ #ifdef LAZY_LINK
2851
+ printf("CLAIM_COUNT:\t%d\n",claim_count);
2852
+ #endif
2853
+
2854
+ printf("STACK:\t(status_flags: %#010x\tdepth: %d\tlive: %d)\t{\n",status_flags,depth,live);
2855
+ for(int i=0; i < STACK_SIZE; i++){
2856
+ util::mem::PairPack<PromiseCount> child_res = host_stack->frames[i].children_residents.data;
2857
+ PromiseCount children = child_res.get_left();
2858
+ PromiseCount residents = child_res.get_right();
2859
+ printf("(children: %d\t residents: %d)\t[",children,residents);
2860
+ for(int j=0; j < FRAME_SIZE; j++){
2861
+ unsigned int index = i*FRAME_SIZE + j;
2862
+ if(stack_count_validity[i]){
2863
+ printf("\t%d",stack_counts[index].adr);
2864
+ } else {
2865
+ printf("\t????");
2866
+ }
2867
+ }
2868
+ printf("\t]\n");
2869
+
2870
+ }
2871
+ printf("} LINK TOTAL: %d\n",link_total.adr);
2872
+ }
2873
+
2874
+ delete[] host_arena;
2875
+ delete[] host_pool;
2876
+ delete host_stack;
2877
+
2878
+ delete[] pool_count_validity;
2879
+ delete[] stack_count_validity;
2880
+
2881
+ return result;
2882
+
2883
+ }
2884
+
2885
+ #endif
2886
+
2887
+
2888
+ template<typename TYPE,typename... ARGS>
2889
+ __device__ void async(ARGS... args){
2890
+ async_call_cast<TYPE>(0,args...);
2891
+ }
2892
+
2893
+
2894
+
2895
+
2896
+ template<typename TYPE,typename... ARGS>
2897
+ __device__ void sync(ARGS... args){
2898
+ immediate_call_cast<TYPE>(Promise<TYPE>(args...));
2899
+ }
2900
+
2901
+
2902
+ template<typename TYPE>
2903
+ __device__ float load_fraction()
2904
+ {
2905
+ return NAN;
2906
+ }
2907
+
2908
+ __device__ void halt_early() {
2909
+ set_flags(EARLY_HALT_FLAG);
2910
+ }
2911
+
2912
+
2913
+ };
2914
+
2915
+
2916
+
2917
+
2918
+ #undef MEMBER_SWITCH
2919
+ #undef CONST_SWITCH
2920
+
2921
+ #endif
2922
+
2923
+