harmonize-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harmonize/__init__.py +65 -0
- harmonize/__main__.py +39 -0
- harmonize/adapt.py +105 -0
- harmonize/array.py +588 -0
- harmonize/atomics.py +206 -0
- harmonize/codegen.py +409 -0
- harmonize/config.py +220 -0
- harmonize/cpp/async/future.h +64 -0
- harmonize/cpp/async/mod.h +9 -0
- harmonize/cpp/async/promise.h +747 -0
- harmonize/cpp/harmonize.h +4 -0
- harmonize/cpp/mem/mod.h +14 -0
- harmonize/cpp/mem/work.h +168 -0
- harmonize/cpp/preamble/adapt.h +162 -0
- harmonize/cpp/preamble/mod.h +133 -0
- harmonize/cpp/program/async.h +2923 -0
- harmonize/cpp/program/event.h +560 -0
- harmonize/cpp/program/mod.h +84 -0
- harmonize/cpp/sync/barrier.h +704 -0
- harmonize/cpp/sync/mod.h +9 -0
- harmonize/cpp/util/cli.h +510 -0
- harmonize/cpp/util/host.h +287 -0
- harmonize/cpp/util/iter.h +565 -0
- harmonize/cpp/util/mem.h +1050 -0
- harmonize/cpp/util/mod.h +148 -0
- harmonize/diagnostics.py +487 -0
- harmonize/errors.py +12 -0
- harmonize/logging.py +51 -0
- harmonize/pointer.py +108 -0
- harmonize/prim.py +59 -0
- harmonize/printing.py +104 -0
- harmonize/python/__init__.py +48 -0
- harmonize/runtime.py +1501 -0
- harmonize/templates.py +369 -0
- harmonize/timing.py +10 -0
- harmonize/types.py +16 -0
- harmonize_python-0.1.0.dist-info/METADATA +161 -0
- harmonize_python-0.1.0.dist-info/RECORD +40 -0
- harmonize_python-0.1.0.dist-info/WHEEL +4 -0
- harmonize_python-0.1.0.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,2923 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
#ifndef HARMONIZE_PROGRAM_ASYNC
|
|
5
|
+
#define HARMONIZE_PROGRAM_ASYNC
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
#include "../async/mod.h"
|
|
10
|
+
|
|
11
|
+
#include "../mem/mod.h"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
//! This macro is currently unused and partially implemented. When finished, it will
|
|
15
|
+
//! operate like the `MEMBER_SWITCH` macro, but for internal templates.
|
|
16
|
+
#define TEMPLATE_MEMBER_SWITCH(NAME,DEFAULT_TYPE) \
|
|
17
|
+
template<class PROG,class DEFAULT> static auto NAME##Lookup (int) -> DEFAULT; \
|
|
18
|
+
template<class PROG,class DEFAULT> static auto NAME##Lookup (bool) -> typename Specializer<typename PROG::NAME>::Type; \
|
|
19
|
+
typedef decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_TYPE>(true)) NAME;
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
//! This macro inserts code that detects whether or not an internal type is defined by
|
|
23
|
+
//! the type `PROGRAM_SPEC` (the parameter name used to refer to a program specification)
|
|
24
|
+
//! and either defines an internal type that duplicates the internal type of `PROGRAM_SPEC`
|
|
25
|
+
//! or defines an internal type based off of a default.
|
|
26
|
+
#define MEMBER_SWITCH(NAME,DEFAULT_TYPE) \
|
|
27
|
+
template<class PROG,class DEFAULT> static auto NAME##Lookup (int) -> DEFAULT; \
|
|
28
|
+
template<class PROG,class DEFAULT> static auto NAME##Lookup (bool) -> typename PROG::NAME; \
|
|
29
|
+
typedef decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_TYPE>(true)) NAME;
|
|
30
|
+
|
|
31
|
+
//! This macro behaves the same as `MEMBER_SWITCH`, but for internal static constants,
|
|
32
|
+
//! rather than for types
|
|
33
|
+
#define CONST_SWITCH(TYPE,NAME,DEFAULT_VAL) \
|
|
34
|
+
template<class PROG,TYPE DEFAULT> static auto NAME##Lookup (int) -> std::integral_constant<TYPE, DEFAULT>; \
|
|
35
|
+
template<class PROG,TYPE DEFAULT> static auto NAME##Lookup (bool) -> std::integral_constant<TYPE, PROG::NAME>; \
|
|
36
|
+
static const size_t NAME = decltype(NAME##Lookup<PROGRAM_SPEC,DEFAULT_VAL>(true))::value;
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
////////////////////////////////////////////////////////////////////////////////////
|
|
42
|
+
// //
|
|
43
|
+
// To have a well-defined Harmonize program, we need to define: //
|
|
44
|
+
// //
|
|
45
|
+
// - the set of valid async function identifiers (in the PROMISE_UNION) //
|
|
46
|
+
// - the number of threads within a work group (and hence the number of //
|
|
47
|
+
// promises per link). Only a work group size of 32 is currently supported. //
|
|
48
|
+
// - the type used to index links in their resident arena //
|
|
49
|
+
// - the number of queues present in pools and stack frames, respectively //
|
|
50
|
+
// - the height of the stack (currently, only a flat stack is supported) //
|
|
51
|
+
// - the states that should be tracked per-group and per-thread //
|
|
52
|
+
// //
|
|
53
|
+
////////////////////////////////////////////////////////////////////////////////////
|
|
54
|
+
// //
|
|
55
|
+
// .-->(QUEUES_PER_POOL) .--> (PROMISE_UNION) //
|
|
56
|
+
// (GROUP/THREAD_STATE) / / //
|
|
57
|
+
// A .-----------. / .--> LINK ----> (GROUP_SIZE) //
|
|
58
|
+
// | / X / \ //
|
|
59
|
+
// | / / `-> ARENA------------. \ //
|
|
60
|
+
// PROG ------> POOL --------------. \ \ //
|
|
61
|
+
// | \ \ \ \ //
|
|
62
|
+
// | `--> STACK ----> FRAME -----> QUEUE -------> ADR --> (ADR_TYPE) //
|
|
63
|
+
// | \ \ //
|
|
64
|
+
// | \ `--> (QUEUE_PER_FRAME) //
|
|
65
|
+
// V \ //
|
|
66
|
+
// (STASH_SIZE) `--> (STACK_SIZE) //
|
|
67
|
+
// //
|
|
68
|
+
////////////////////////////////////////////////////////////////////////////////////
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
//! Switches whether or not lazy allocation is enabled for the allocation of a `WorkLink`
|
|
72
|
+
#define LAZY_LINK
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
//! The class that defines an asynchronous program and all of its types.
|
|
78
|
+
template< typename PROGRAM_SPEC >
|
|
79
|
+
class AsyncProgram
|
|
80
|
+
{
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
public:
|
|
84
|
+
|
|
85
|
+
typedef AsyncProgram<PROGRAM_SPEC> ProgramType;
|
|
86
|
+
|
|
87
|
+
//! Templates for use by `TEMPLATE_MEMBER_SWITCH`
|
|
88
|
+
template<typename BASE>
|
|
89
|
+
struct Specializer
|
|
90
|
+
{
|
|
91
|
+
using Type = BASE;
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
template<template<typename>typename BASE>
|
|
95
|
+
struct Specializer<BASE<ProgramType>>
|
|
96
|
+
{
|
|
97
|
+
using Type = BASE<ProgramType>;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
//! Define the type used to address work links
|
|
101
|
+
MEMBER_SWITCH( AdrType,unsigned int)
|
|
102
|
+
//! Define the set of operations
|
|
103
|
+
MEMBER_SWITCH( OpSet, OpUnion<>)
|
|
104
|
+
|
|
105
|
+
template<typename OP_SET, typename ADR_TYPE>
|
|
106
|
+
friend class RemappingBarrier;
|
|
107
|
+
|
|
108
|
+
template<typename... TYPES>
|
|
109
|
+
friend class ArgTuple;
|
|
110
|
+
|
|
111
|
+
//! Define the states stored in global, shared, and private memory
|
|
112
|
+
MEMBER_SWITCH(DeviceState, VoidState)
|
|
113
|
+
MEMBER_SWITCH( GroupState, VoidState)
|
|
114
|
+
MEMBER_SWITCH(ThreadState, VoidState)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
//! Define the type of `PromiseUnion` used by the program.
|
|
118
|
+
typedef PromiseUnion<OpSet> PromiseUnionType;
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
//! Used to look up information about the primary `PromiseUnion` type used
|
|
122
|
+
template<typename TYPE>
|
|
123
|
+
struct Lookup { typedef typename PromiseUnionType::template Lookup<TYPE>::type type; };
|
|
124
|
+
|
|
125
|
+
//! Define internal constants based off of the program specification, or
|
|
126
|
+
//! fall back onto defaults.
|
|
127
|
+
CONST_SWITCH(size_t,STASH_SIZE,16)
|
|
128
|
+
CONST_SWITCH(size_t,FRAME_SIZE,32)
|
|
129
|
+
CONST_SWITCH(size_t, POOL_SIZE,32)
|
|
130
|
+
CONST_SWITCH(size_t,STACK_SIZE, 0)
|
|
131
|
+
CONST_SWITCH(size_t,GROUP_SIZE,32)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
//! Constants used to determine when to spill or fill the stash, and
|
|
135
|
+
//! by how much
|
|
136
|
+
static const size_t STASH_MARGIN = 2;
|
|
137
|
+
static const size_t STASH_HIGH_WATER = STASH_SIZE-STASH_MARGIN;
|
|
138
|
+
|
|
139
|
+
//! The number of async functions present in the program.
|
|
140
|
+
static const unsigned char FN_ID_COUNT = PromiseUnionType::Info::COUNT;
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
//! During system verification/debugging, this will be used as a cutoff to prevent infinite
|
|
144
|
+
//! looping
|
|
145
|
+
static const unsigned int PUSH_QUEUE_RETRY_LIMIT = 32;
|
|
146
|
+
static const unsigned int FILL_STASH_RETRY_LIMIT = 1;
|
|
147
|
+
static const unsigned int FILL_STASH_LINKS_RETRY_LIMIT = 32;
|
|
148
|
+
|
|
149
|
+
static const size_t WORK_GROUP_SIZE = GROUP_SIZE;
|
|
150
|
+
|
|
151
|
+
//! A set of halting condition flags
|
|
152
|
+
static const unsigned int BAD_FUNC_ID_FLAG = 0x00000001;
|
|
153
|
+
static const unsigned int STASH_FAIL_FLAG = 0x00000002;
|
|
154
|
+
static const unsigned int COMPLETION_FLAG = 0x80000000;
|
|
155
|
+
static const unsigned int EARLY_HALT_FLAG = 0x40000000;
|
|
156
|
+
|
|
157
|
+
//! Defining a set of internal short-hand names for the specializaions used by the class
|
|
158
|
+
typedef util::mem::Adr <AdrType> LinkAdrType;
|
|
159
|
+
typedef util::mem::PoolQueue <LinkAdrType> QueueType;
|
|
160
|
+
typedef WorkFrame <QueueType,FRAME_SIZE> FrameType;
|
|
161
|
+
typedef WorkStack <FrameType,STACK_SIZE> StackType;
|
|
162
|
+
typedef WorkPool <QueueType,POOL_SIZE> PoolType;
|
|
163
|
+
|
|
164
|
+
typedef WorkLink <OpSet, LinkAdrType, WORK_GROUP_SIZE> LinkType;
|
|
165
|
+
|
|
166
|
+
typedef WorkArena <LinkAdrType,LinkType> ArenaType;
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
//! The depth of the partial table (1 if stack is flat, 3 otherwise).
|
|
171
|
+
static const unsigned char PART_TABLE_DEPTH = StackType::PART_MULT;
|
|
172
|
+
static const unsigned char PART_ENTRY_COUNT = FN_ID_COUNT*PART_TABLE_DEPTH;
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
static const AdrType SPARE_LINK_COUNT = 2u;
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
//! This struct represents the entire set of data structures that must be stored in thread
|
|
179
|
+
//! memory to track te state of the program defined by the developer as well as the state of
|
|
180
|
+
//! the context which is driving exection.
|
|
181
|
+
struct ThreadContext {
|
|
182
|
+
|
|
183
|
+
unsigned int thread_id;
|
|
184
|
+
unsigned int rand_state;
|
|
185
|
+
unsigned int spare_index;
|
|
186
|
+
LinkAdrType spare_links[SPARE_LINK_COUNT];
|
|
187
|
+
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
//! A non-atomic promise coalescing structure used to track information about full and
|
|
192
|
+
//! partial work links.
|
|
193
|
+
struct RemapQueue {
|
|
194
|
+
unsigned char count;
|
|
195
|
+
unsigned char full_head;
|
|
196
|
+
unsigned char partial_map[PART_ENTRY_COUNT];
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
//! This struct represents the entire set of data structures that must be stored in group
|
|
201
|
+
//! memory to track te state of the program defined by the developer as well as the state of
|
|
202
|
+
//! the context which is driving exection.
|
|
203
|
+
struct GroupContext {
|
|
204
|
+
|
|
205
|
+
size_t level; // Current level being run
|
|
206
|
+
|
|
207
|
+
bool keep_running;
|
|
208
|
+
bool busy;
|
|
209
|
+
bool can_make_work;
|
|
210
|
+
bool scarce_work;
|
|
211
|
+
|
|
212
|
+
unsigned char exec_head; // Indexes the link that is/will be evaluated next
|
|
213
|
+
unsigned char empty_head; // Head of the linked list of empty links
|
|
214
|
+
|
|
215
|
+
RemapQueue main_queue;
|
|
216
|
+
|
|
217
|
+
#ifdef ASYNC_LOADS
|
|
218
|
+
RemapQueue load_queue;
|
|
219
|
+
adapt::GPUrt::barrier<adapt::GPUrt::thread_scope_system> load_barrier;
|
|
220
|
+
#endif
|
|
221
|
+
|
|
222
|
+
unsigned char link_stash_count; // Number of device-space links stored
|
|
223
|
+
|
|
224
|
+
LinkType stash[STASH_SIZE];
|
|
225
|
+
LinkAdrType link_stash[STASH_SIZE];
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
int SM_promise_delta;
|
|
229
|
+
unsigned long long int work_iterator;
|
|
230
|
+
|
|
231
|
+
#ifdef BARRIER_SPILL
|
|
232
|
+
RemappingBarrier<OpSet> spill_barrier;
|
|
233
|
+
#endif
|
|
234
|
+
|
|
235
|
+
#ifdef HRM_TIME
|
|
236
|
+
unsigned long long int time_totals[HRM_TIME];
|
|
237
|
+
#endif
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
//! This struct represents the entire set of data structures that must be stored in device
|
|
244
|
+
//! memory to track the state of the program defined by the developer as well as the state
|
|
245
|
+
//! of the context which is driving execution.
|
|
246
|
+
struct DeviceContext {
|
|
247
|
+
|
|
248
|
+
typedef ProgramType ParentProgramType;
|
|
249
|
+
|
|
250
|
+
#ifdef LAZY_LINK
|
|
251
|
+
AdrType* claim_count;
|
|
252
|
+
#endif
|
|
253
|
+
|
|
254
|
+
#ifdef HRM_TIME
|
|
255
|
+
unsigned long long int* time_totals;
|
|
256
|
+
#endif
|
|
257
|
+
|
|
258
|
+
ArenaType arena;
|
|
259
|
+
PoolType* pool;
|
|
260
|
+
StackType* stack;
|
|
261
|
+
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
protected:
|
|
266
|
+
public:
|
|
267
|
+
|
|
268
|
+
DeviceContext & _dev_ctx;
|
|
269
|
+
GroupContext & _grp_ctx;
|
|
270
|
+
ThreadContext & _thd_ctx;
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
DeviceState & device;
|
|
274
|
+
GroupState & group;
|
|
275
|
+
ThreadState & thread;
|
|
276
|
+
|
|
277
|
+
__device__ AsyncProgram<PROGRAM_SPEC> (
|
|
278
|
+
DeviceContext & d_c,
|
|
279
|
+
GroupContext & g_c,
|
|
280
|
+
ThreadContext & t_c,
|
|
281
|
+
|
|
282
|
+
DeviceState & d,
|
|
283
|
+
GroupState & g,
|
|
284
|
+
ThreadState & t
|
|
285
|
+
)
|
|
286
|
+
: _dev_ctx(d_c)
|
|
287
|
+
, _grp_ctx(g_c)
|
|
288
|
+
, _thd_ctx(t_c)
|
|
289
|
+
, device (d)
|
|
290
|
+
, group (g)
|
|
291
|
+
, thread (t)
|
|
292
|
+
{}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
//! Instances wrap around their program scope's DeviceContext. These differ from a program's
|
|
300
|
+
//! DeviceContext object in that they perform automatic deallocation as soon as they drop
|
|
301
|
+
//! out of scope.
|
|
302
|
+
struct Instance {
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
AdrType arena_size;
|
|
306
|
+
#ifdef LAZY_LINK
|
|
307
|
+
util::host::DevBuf<AdrType> claim_count;
|
|
308
|
+
#endif
|
|
309
|
+
#ifdef HRM_TIME
|
|
310
|
+
util::host::DevBuf<unsigned long long int> time_totals;
|
|
311
|
+
#endif
|
|
312
|
+
util::host::DevBuf<LinkType> arena;
|
|
313
|
+
util::host::DevBuf<PoolType> pool;
|
|
314
|
+
util::host::DevBuf<StackType> stack;
|
|
315
|
+
DeviceState device_state;
|
|
316
|
+
|
|
317
|
+
__host__ Instance (AdrType arsize, DeviceState gs)
|
|
318
|
+
: arena(arsize)
|
|
319
|
+
, pool (1)
|
|
320
|
+
, stack(1)
|
|
321
|
+
, arena_size(arsize)
|
|
322
|
+
, device_state(gs)
|
|
323
|
+
#ifdef LAZY_LINK
|
|
324
|
+
, claim_count(0u)
|
|
325
|
+
#endif
|
|
326
|
+
#ifdef HRM_TIME
|
|
327
|
+
, time_totals((size_t)HRM_TIME)
|
|
328
|
+
#endif
|
|
329
|
+
{
|
|
330
|
+
#ifdef HRM_TIME
|
|
331
|
+
adapt::GPUrtMemset( time_totals, 0, sizeof(unsigned long long int) * HRM_TIME );
|
|
332
|
+
#endif
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
__host__ DeviceContext to_context(){
|
|
336
|
+
|
|
337
|
+
DeviceContext result;
|
|
338
|
+
|
|
339
|
+
#ifdef LAZY_LINK
|
|
340
|
+
result.claim_count = claim_count;
|
|
341
|
+
#endif
|
|
342
|
+
#ifdef HRM_TIME
|
|
343
|
+
result.time_totals = time_totals;
|
|
344
|
+
#endif
|
|
345
|
+
|
|
346
|
+
result.arena.size = arena_size;
|
|
347
|
+
result.arena.links = arena;
|
|
348
|
+
result.pool = pool ;
|
|
349
|
+
result.stack = stack;
|
|
350
|
+
|
|
351
|
+
return result;
|
|
352
|
+
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
#ifdef HRM_TIME
|
|
356
|
+
__host__ void print_times(){
|
|
357
|
+
std::vector<unsigned long long int> times;
|
|
358
|
+
time_totals >> times;
|
|
359
|
+
double total = times[0];
|
|
360
|
+
for(unsigned int i=0; i<times.size(); i++){
|
|
361
|
+
double the_time = times[i];
|
|
362
|
+
double prop = 100.0 * (the_time / total);
|
|
363
|
+
printf("T%d: %llu (~%f%)\n",i,times[i],prop );
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
#endif
|
|
367
|
+
|
|
368
|
+
__host__ bool complete(){
|
|
369
|
+
|
|
370
|
+
unsigned int* flags_ptr = &(((StackType*)stack)->status_flags);
|
|
371
|
+
unsigned int flags = 0;
|
|
372
|
+
util::host::auto_throw(adapt::GPUrtMemcpy(&flags,flags_ptr,sizeof(unsigned int),adapt::GPUrtMemcpyDeviceToHost));
|
|
373
|
+
check_error();
|
|
374
|
+
return ((flags & COMPLETION_FLAG) != 0);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
__host__ void clear_flags(){
|
|
378
|
+
unsigned int flags = 0;
|
|
379
|
+
unsigned int* flags_ptr = &(((StackType*)stack)->status_flags);
|
|
380
|
+
util::host::auto_throw(adapt::GPUrtMemcpy(flags_ptr,&flags,sizeof(unsigned int),adapt::GPUrtMemcpyHostToDevice));
|
|
381
|
+
check_error();
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
protected:
|
|
391
|
+
|
|
392
|
+
//! Returns true only if the stash cannot support the allocation of any more work links
|
|
393
|
+
__device__ bool stash_overfilled(){
|
|
394
|
+
#ifdef ASYNC_LOADS
|
|
395
|
+
return (_grp_ctx.main_queue.count+_grp_ctx.load_queue.count) >= STASH_HIGH_WATER;
|
|
396
|
+
#else
|
|
397
|
+
return _grp_ctx.main_queue.count >= STASH_HIGH_WATER;
|
|
398
|
+
#endif
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
//! Returns an index into the partial map of a group based off of a function id and a depth. If
|
|
402
|
+
//! an invalid depth or function id is used, PART_ENTRY_COUNT is returned.
|
|
403
|
+
__device__ unsigned int partial_map_index(
|
|
404
|
+
OpDisc func_id,
|
|
405
|
+
unsigned int depth,
|
|
406
|
+
unsigned int current_level
|
|
407
|
+
){
|
|
408
|
+
|
|
409
|
+
unsigned int the_id = static_cast<unsigned int>(func_id);
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
if( the_id >= FN_ID_COUNT){
|
|
413
|
+
return PART_ENTRY_COUNT;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
unsigned int result = the_id;
|
|
417
|
+
|
|
418
|
+
if( ! StackType::FLAT ){
|
|
419
|
+
result *= PART_TABLE_DEPTH;
|
|
420
|
+
if( depth == current_level ){
|
|
421
|
+
result += 1;
|
|
422
|
+
} else if ( depth == (current_level+1) ){
|
|
423
|
+
result += 2;
|
|
424
|
+
} else if ( depth != (current_level-1) ){
|
|
425
|
+
result = PART_ENTRY_COUNT;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
return result;
|
|
430
|
+
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
//! Initializes the shared state of a work group, which is stored as a ctx_shared struct. This
|
|
435
|
+
//! is mainly done by initializing handles to the arena, pool, and stack, setting the current
|
|
436
|
+
//! level to null, setting the stash iterator to null, and zeroing the stash.
|
|
437
|
+
__device__ void init_group(){
|
|
438
|
+
|
|
439
|
+
//printf("Initing group\n");
|
|
440
|
+
unsigned int active = __activemask();
|
|
441
|
+
|
|
442
|
+
__syncwarp(active);
|
|
443
|
+
|
|
444
|
+
if(util::current_leader()){
|
|
445
|
+
|
|
446
|
+
if( StackType::FLAT ){
|
|
447
|
+
_grp_ctx.level = 0;
|
|
448
|
+
} else {
|
|
449
|
+
_grp_ctx.level = StackType::NULL_LEVEL;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
_grp_ctx.main_queue.count = 0;
|
|
453
|
+
_grp_ctx.link_stash_count = 0;
|
|
454
|
+
_grp_ctx.keep_running = true;
|
|
455
|
+
_grp_ctx.busy = false;
|
|
456
|
+
_grp_ctx.can_make_work= true;
|
|
457
|
+
_grp_ctx.exec_head = STASH_SIZE;
|
|
458
|
+
_grp_ctx.main_queue.full_head = STASH_SIZE;
|
|
459
|
+
_grp_ctx.empty_head = 0;
|
|
460
|
+
_grp_ctx.work_iterator= 0;
|
|
461
|
+
_grp_ctx.scarce_work = false;
|
|
462
|
+
|
|
463
|
+
#ifdef BARRIER_SPILL
|
|
464
|
+
_grp_ctx.spill_barrier = RemappingBarrier<OpSet>::blank(1);
|
|
465
|
+
#endif
|
|
466
|
+
|
|
467
|
+
for(unsigned int i=0; i<STASH_SIZE; i++){
|
|
468
|
+
_grp_ctx.stash[i].empty(i+1);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
for(unsigned int i=0; i<PART_ENTRY_COUNT; i++){
|
|
472
|
+
_grp_ctx.main_queue.partial_map[i] = STASH_SIZE;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
#ifdef ASYNC_LOADS
|
|
476
|
+
_grp_ctx.load_queue.count = 0;
|
|
477
|
+
_grp_ctx.load_queue.full_head = STASH_SIZE;
|
|
478
|
+
for(unsigned int i=0; i<PART_ENTRY_COUNT; i++){
|
|
479
|
+
_grp_ctx.load_queue.partial_map[i] = STASH_SIZE;
|
|
480
|
+
}
|
|
481
|
+
init(&_grp_ctx.load_barrier,WORK_GROUP_SIZE);
|
|
482
|
+
#endif
|
|
483
|
+
|
|
484
|
+
_grp_ctx.SM_promise_delta = 0;
|
|
485
|
+
|
|
486
|
+
#ifdef HRM_TIME
|
|
487
|
+
for(unsigned int i=0; i<HRM_TIME; i++){
|
|
488
|
+
_grp_ctx.time_totals[i] = 0;
|
|
489
|
+
}
|
|
490
|
+
beg_time(0);
|
|
491
|
+
#endif
|
|
492
|
+
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
__syncwarp(active);
|
|
496
|
+
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
//! Initializes the local state of a thread, which is just the device id of the thread and the
|
|
500
|
+
//! state used by the thread to generate random numbers for stochastic choices needed to manage
|
|
501
|
+
//! the runtime state.
|
|
502
|
+
__device__ void init_thread(){
|
|
503
|
+
|
|
504
|
+
//printf("Initing thread %d\n",threadIdx.x);
|
|
505
|
+
_thd_ctx.thread_id = (blockIdx.x * blockDim.x) + threadIdx.x;
|
|
506
|
+
_thd_ctx.rand_state = _thd_ctx.thread_id;
|
|
507
|
+
_thd_ctx.spare_index = 0;
|
|
508
|
+
for(size_t i=0; i<SPARE_LINK_COUNT; i++){
|
|
509
|
+
_thd_ctx.spare_links[i] = LinkAdrType::null;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
//! Sets the bits in the status_flags field of the stack according to the given flag bits.
|
|
516
|
+
__device__ void set_flags(unsigned int flag_bits){
|
|
517
|
+
|
|
518
|
+
atomicOr(&_dev_ctx.stack->status_flags,flag_bits);
|
|
519
|
+
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
//! Unsets the bits in the status_flags field of the stack according to the given flag bits.
|
|
524
|
+
__device__ void unset_flags(unsigned int flag_bits){
|
|
525
|
+
|
|
526
|
+
atomicAnd(&_dev_ctx.stack->status_flags,~flag_bits);
|
|
527
|
+
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
//! Returns the current highest level in the stack. Given that this program is highly parallel,
|
|
531
|
+
//! this number inherently cannot be trusted. By the time the value is fetched, the stack could
|
|
532
|
+
//! have a different height or the thread that set the height may not have deposited links in the
|
|
533
|
+
//! corresponding level yet.
|
|
534
|
+
__device__ unsigned int highest_level(){
|
|
535
|
+
|
|
536
|
+
return left_half(_dev_ctx.stack->depth_live);
|
|
537
|
+
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
//! Returns a reference to the frame at the requested level in the stack.
|
|
542
|
+
__device__ FrameType& get_frame(unsigned int level){
|
|
543
|
+
|
|
544
|
+
return _dev_ctx.stack->frames[level];
|
|
545
|
+
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
//! Joins two queues such that the right queue is now at the end of the left queue.
|
|
550
|
+
//!
|
|
551
|
+
//! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
|
|
552
|
+
//! atomically. If not, one of the queues manipulated with this function will almost certainly
|
|
553
|
+
//! become malformed at some point.
|
|
554
|
+
__device__ QueueType join_queues(QueueType left_queue, QueueType right_queue){
|
|
555
|
+
|
|
556
|
+
QueueType result;
|
|
557
|
+
|
|
558
|
+
/*
|
|
559
|
+
// If either input queue is null, we can simply return the other queue.
|
|
560
|
+
*/
|
|
561
|
+
if( left_queue.is_null() ){
|
|
562
|
+
result = right_queue;
|
|
563
|
+
} else if ( right_queue.is_null() ){
|
|
564
|
+
result = left_queue;
|
|
565
|
+
} else {
|
|
566
|
+
|
|
567
|
+
LinkAdrType left_tail_adr = left_queue .get_tail();
|
|
568
|
+
LinkAdrType right_head_adr = right_queue.get_head();
|
|
569
|
+
LinkAdrType right_tail_adr = right_queue.get_tail();
|
|
570
|
+
|
|
571
|
+
//! Find last link in the queue referenced by left_queue.
|
|
572
|
+
LinkType& left_tail = _dev_ctx.arena[left_tail_adr];
|
|
573
|
+
|
|
574
|
+
//! Set the index for the tail's successor to the head of the queue referenced by
|
|
575
|
+
//! right_queue.
|
|
576
|
+
left_tail.next = right_head_adr;
|
|
577
|
+
|
|
578
|
+
//! Set the right half of the left_queue handle to index the new tail.
|
|
579
|
+
left_queue.set_tail(right_tail_adr);
|
|
580
|
+
|
|
581
|
+
result = left_queue;
|
|
582
|
+
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
return result;
|
|
586
|
+
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
//! Takes the first link off of the queue and returns the index of the link in the arena. If the
|
|
591
|
+
//! queue is empty, a null address is returned instead.
|
|
592
|
+
//!
|
|
593
|
+
//! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
|
|
594
|
+
//! atomically. If not, one of the queues manipulated with this function will almost certainly
|
|
595
|
+
//! become malformed at some point.
|
|
596
|
+
__device__ LinkAdrType pop_front(QueueType& queue){
|
|
597
|
+
|
|
598
|
+
LinkAdrType result;
|
|
599
|
+
|
|
600
|
+
//! Don't try unless the queue is non-null
|
|
601
|
+
if( queue.is_null() ){
|
|
602
|
+
result.adr = LinkAdrType::null;
|
|
603
|
+
} else {
|
|
604
|
+
result = queue.get_head();
|
|
605
|
+
LinkAdrType next = _dev_ctx.arena[result].next;
|
|
606
|
+
queue.set_head(next);
|
|
607
|
+
if(next.adr == LinkAdrType::null){
|
|
608
|
+
queue.set_tail(next);
|
|
609
|
+
} else if ( queue.get_tail() == result ){
|
|
610
|
+
//printf("ERROR: Final link does not have a null next.\n");
|
|
611
|
+
queue.pair.data = QueueType::null;
|
|
612
|
+
return result;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
return result;
|
|
616
|
+
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
//! Adds the given link to the end of the given queue. This link can NOT be part of another queue,
|
|
621
|
+
//! and its next pointer will be automatically nulled before it is appended. If you need to merge
|
|
622
|
+
//! two queues together, use join_queues.
|
|
623
|
+
//!
|
|
624
|
+
//! WARNING: NOT THREAD SAFE. Only use this on queues that have been claimed from the stack
|
|
625
|
+
//! atomically. If not, one of the queues manipulated with this function will almost certainly
|
|
626
|
+
//! become malformed at some point.
|
|
627
|
+
__device__ void push_back(QueueType& queue, LinkAdrType link_adr){
|
|
628
|
+
|
|
629
|
+
_dev_ctx.arena[link_adr].next = LinkAdrType::null;
|
|
630
|
+
if( queue.is_null() ){
|
|
631
|
+
queue = QueueType(link_adr,link_adr);
|
|
632
|
+
} else {
|
|
633
|
+
LinkAdrType tail = queue.get_tail();
|
|
634
|
+
_dev_ctx.arena[tail].next = link_adr;
|
|
635
|
+
//atomicExch( &(_dev_ctx.arena[tail].next.adr),link_adr.adr);
|
|
636
|
+
queue.set_tail(link_adr);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
//! Attempts to pull a queue from a range of queue slots, trying each slot starting from the given
|
|
644
|
+
//! starting index onto the end of the range and then looping back from the beginning. If, after
|
|
645
|
+
//! trying every slot in the range, no non-null queue was obtained, a QueueType::null value is returned.
|
|
646
|
+
__device__ QueueType pull_queue(QueueType* src, unsigned int start_index, unsigned int range_size, unsigned int& src_index){
|
|
647
|
+
|
|
648
|
+
QueueType result;
|
|
649
|
+
|
|
650
|
+
__threadfence();
|
|
651
|
+
//! First iterate from the starting index to the end of the queue range, attempting to
|
|
652
|
+
//! claim a non-null queue until either there are no more slots to try, or the atomic
|
|
653
|
+
//! swap successfuly retrieves something.
|
|
654
|
+
for(unsigned int i=start_index; i < range_size; i++){
|
|
655
|
+
if( src[i].pair.data != QueueType::null ) {
|
|
656
|
+
result.pair.data = atomicExch(&(src[i].pair.data),QueueType::null);
|
|
657
|
+
if( ! result.is_null() ){
|
|
658
|
+
src_index = i;
|
|
659
|
+
__threadfence();
|
|
660
|
+
return result;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
//! Continue searching from the beginning of the range to just before the beginning of the
|
|
666
|
+
//! previous scan.
|
|
667
|
+
for(unsigned int i=0; i < start_index; i++){
|
|
668
|
+
if( src[i].pair.data != QueueType::null ) {
|
|
669
|
+
result.pair.data = atomicExch(&(src[i].pair.data),QueueType::null);
|
|
670
|
+
if( ! result.is_null() ){
|
|
671
|
+
src_index = i;
|
|
672
|
+
__threadfence();
|
|
673
|
+
return result;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
q_printf("COULD NOT PULL QUEUE\n");
|
|
679
|
+
//! Return QueueType::null if nothing is found
|
|
680
|
+
result.pair.data = QueueType::null;
|
|
681
|
+
return result;
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
//! Repeatedly tries to push a queue to a destination queue slot by atomic exchanges. If a non
|
|
690
|
+
//! null queue is ever returned by the exchange, it attempts to merge with a subsequent exchange.
|
|
691
|
+
//!
|
|
692
|
+
//! If an invalid queue (one half null, the other half non-null) is ever found at the destination,
|
|
693
|
+
//! the value of that invalid queue is returned, the invalid value is replaced into the queue, and
|
|
694
|
+
//! any work gathered from the replacement is assigned to the second argument. Otherwise, the second
|
|
695
|
+
//! argument is set to a null state.
|
|
696
|
+
__device__ QueueType push_queue(QueueType& dest, QueueType& queue){
|
|
697
|
+
|
|
698
|
+
if( queue.is_null() ){
|
|
699
|
+
return queue;
|
|
700
|
+
}
|
|
701
|
+
#ifdef INF_LOOP_SAFE
|
|
702
|
+
while(true)
|
|
703
|
+
#else
|
|
704
|
+
for(int i=0; i<PUSH_QUEUE_RETRY_LIMIT; i++)
|
|
705
|
+
#endif
|
|
706
|
+
{
|
|
707
|
+
__threadfence();
|
|
708
|
+
//printf("%d: Pushing queue (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
|
|
709
|
+
|
|
710
|
+
QueueType swap;
|
|
711
|
+
swap.pair.data = atomicExch(&dest.pair.data,queue.pair.data);
|
|
712
|
+
//! If our swap returns a non-null queue, we are still stuck with a queue that
|
|
713
|
+
//! needs to be offloaded to the stack. In this case, claim the queue from the
|
|
714
|
+
//! slot just swapped with, merge the two, and attempt again to place the queue
|
|
715
|
+
//! back. With this method, swap failures are bounded by the number of pushes to
|
|
716
|
+
//! the queue slot, with at most one failure per push_queue call, but no guarantee
|
|
717
|
+
//! of which attempt from which call will suffer from an incurred failure.
|
|
718
|
+
if( ! swap.is_null() ){
|
|
719
|
+
//printf("%d: We got queue (%d,%d) when trying to push a queue\n",blockIdx.x,swap.get_head().adr,swap.get_tail().adr);
|
|
720
|
+
QueueType other_swap;
|
|
721
|
+
if( swap.get_head().is_null() || swap.get_tail().is_null() ){
|
|
722
|
+
other_swap.pair.data = atomicExch(&dest.pair.data,swap.pair.data);
|
|
723
|
+
queue = other_swap;
|
|
724
|
+
return swap;
|
|
725
|
+
} else {
|
|
726
|
+
other_swap.pair.data = atomicExch(&dest.pair.data,QueueType::null);
|
|
727
|
+
queue = join_queues(other_swap,swap);
|
|
728
|
+
//printf("%d: Merged it to form queue (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
|
|
729
|
+
}
|
|
730
|
+
} else {
|
|
731
|
+
//printf("%d: Finally pushed (%d,%d)\n",blockIdx.x,queue.get_head().adr,queue.get_tail().adr);
|
|
732
|
+
break;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return queue;
|
|
736
|
+
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
//! Claims a link from the link stash. If no link exists in the stash, LinkAdrType::null is returned.
|
|
743
|
+
__device__ LinkAdrType claim_stash_link(){
|
|
744
|
+
|
|
745
|
+
LinkAdrType link = LinkAdrType::null;
|
|
746
|
+
unsigned int count = _grp_ctx.link_stash_count;
|
|
747
|
+
if(count > 0){
|
|
748
|
+
link = _grp_ctx.link_stash[count-1];
|
|
749
|
+
_grp_ctx.link_stash_count = count - 1;
|
|
750
|
+
}
|
|
751
|
+
q_printf("New link stash count: %d\n",_grp_ctx.link_stash_count);
|
|
752
|
+
return link;
|
|
753
|
+
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
//! Inserts an empty slot into the stash. This should only be called if there is enough space in
|
|
759
|
+
//! the link stash.
|
|
760
|
+
__device__ void insert_stash_link(LinkAdrType link){
|
|
761
|
+
|
|
762
|
+
unsigned int count = _grp_ctx.link_stash_count;
|
|
763
|
+
_grp_ctx.link_stash[count] = link;
|
|
764
|
+
_grp_ctx.link_stash_count = count + 1;
|
|
765
|
+
q_printf("New link stash count: %d\n",_grp_ctx.link_stash_count);
|
|
766
|
+
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
//! Claims an empty slot from the stash and returns its index. If no empty slot exists in the stash,
|
|
773
|
+
//! then STASH_SIZE is returned.
|
|
774
|
+
__device__ unsigned int claim_empty_slot(){
|
|
775
|
+
|
|
776
|
+
unsigned int slot = _grp_ctx.empty_head;
|
|
777
|
+
if(slot != STASH_SIZE){
|
|
778
|
+
_grp_ctx.empty_head = _grp_ctx.stash[slot].next.adr;
|
|
779
|
+
db_printf("EMPTY: %d << %d\n",slot,_grp_ctx.empty_head);
|
|
780
|
+
}
|
|
781
|
+
return slot;
|
|
782
|
+
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
//! Inserts an empty slot into the stash. This should only be called if there is enough space in
|
|
787
|
+
//! the link stash.
|
|
788
|
+
__device__ void insert_empty_slot(unsigned int slot){
|
|
789
|
+
|
|
790
|
+
_grp_ctx.stash[slot].next.adr = _grp_ctx.empty_head;
|
|
791
|
+
db_printf("EMPTY: >> %d -> %d\n",slot,_grp_ctx.empty_head);
|
|
792
|
+
_grp_ctx.empty_head = slot;
|
|
793
|
+
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
//! Claims a full slot from the stash and returns its index. If no empty slot exists in the stash,
|
|
798
|
+
//! then STASH_SIZE is returned.
|
|
799
|
+
__device__ unsigned int claim_full_slot(){
|
|
800
|
+
|
|
801
|
+
unsigned int slot = _grp_ctx.main_queue.full_head;
|
|
802
|
+
if(slot != STASH_SIZE){
|
|
803
|
+
_grp_ctx.main_queue.full_head = _grp_ctx.stash[slot].next.adr;
|
|
804
|
+
db_printf("FULL : %d << %d\n",slot,_grp_ctx.main_queue.full_head);
|
|
805
|
+
}
|
|
806
|
+
return slot;
|
|
807
|
+
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
//! Inserts a full slot into the stash. This should only be called if there is enough space in
|
|
812
|
+
//! the link stash.
|
|
813
|
+
__device__ void insert_full_slot(RemapQueue& queue, unsigned int slot){
|
|
814
|
+
|
|
815
|
+
_grp_ctx.stash[slot].next.adr = queue.full_head;
|
|
816
|
+
db_printf("FULL : >> %d -> %d\n",slot,queue.full_head);
|
|
817
|
+
queue.full_head = slot;
|
|
818
|
+
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
__device__ void dealloc_links(LinkAdrType* src, size_t count) {
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
QueueType queue;
|
|
826
|
+
queue.pair.data = QueueType::null;
|
|
827
|
+
|
|
828
|
+
//! Connect all links into a queue
|
|
829
|
+
for(unsigned int i=0; i < count; i++){
|
|
830
|
+
|
|
831
|
+
push_back(queue,src[i]);
|
|
832
|
+
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
//! Push out the queue to the pool
|
|
836
|
+
q_printf("Pushing queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
|
|
837
|
+
unsigned int dest_idx = util::random_uint(_thd_ctx.rand_state) % POOL_SIZE;
|
|
838
|
+
push_queue(_dev_ctx.pool->queues[dest_idx],queue);
|
|
839
|
+
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
//!
|
|
844
|
+
//! A thread-safe way of allocating links from global memory. This function is used
|
|
845
|
+
//! by the warp leader to restock the link stash in between processing promises and
|
|
846
|
+
//! used by individual threads if its current promise needs to use a link while the
|
|
847
|
+
//! link stash is empty.
|
|
848
|
+
//!
|
|
849
|
+
__device__ size_t alloc_links(LinkAdrType* dst, size_t req_count) {
|
|
850
|
+
|
|
851
|
+
size_t alloc_count = 0;
|
|
852
|
+
#ifdef LAZY_LINK
|
|
853
|
+
|
|
854
|
+
#if 1
|
|
855
|
+
if( (alloc_count < req_count) && ((*_dev_ctx.claim_count) < _dev_ctx.arena.size) ){
|
|
856
|
+
AdrType claim_offset = atomicAdd(_dev_ctx.claim_count,req_count);
|
|
857
|
+
unsigned int received = 0;
|
|
858
|
+
if( claim_offset <= (_dev_ctx.arena.size - req_count) ){
|
|
859
|
+
received = req_count;
|
|
860
|
+
} else if ( claim_offset >= _dev_ctx.arena.size ) {
|
|
861
|
+
received = 0;
|
|
862
|
+
} else {
|
|
863
|
+
received = _dev_ctx.arena.size - claim_offset;
|
|
864
|
+
}
|
|
865
|
+
for( unsigned int index=0; index < received; index++){
|
|
866
|
+
//insert_stash_link(LinkAdrType(claim_offset+index));
|
|
867
|
+
dst[alloc_count] = LinkAdrType(claim_offset+index);
|
|
868
|
+
alloc_count++;
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
#else
|
|
872
|
+
for(int i=_grp_ctx.link_stash_count; i < threashold; i++){
|
|
873
|
+
if((*_dev_ctx.claim_count) < _dev_ctx.arena.size ){
|
|
874
|
+
AdrType claim_index = atomicAdd(_dev_ctx.claim_count,1);
|
|
875
|
+
if( claim_index < _dev_ctx.arena.size ){
|
|
876
|
+
//_dev_ctx.arena[claim_index].empty(LinkAdrType::null);
|
|
877
|
+
insert_stash_link(LinkAdrType(claim_index));
|
|
878
|
+
}
|
|
879
|
+
} else {
|
|
880
|
+
break;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
#endif
|
|
884
|
+
|
|
885
|
+
#endif
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
for(int try_itr=0; try_itr < FILL_STASH_LINKS_RETRY_LIMIT; try_itr++){
|
|
889
|
+
|
|
890
|
+
if(alloc_count >= req_count){
|
|
891
|
+
break;
|
|
892
|
+
}
|
|
893
|
+
//! Attempt to pull a queue from the pool. This should be very unlikely to fail unless
|
|
894
|
+
//! almost all links have been exhausted or the pool size is disproportionately small
|
|
895
|
+
//! relative to the number of work groups. In the worst case, this should simply not
|
|
896
|
+
//! al_thd_ctxate any links, and the return value shall report this.
|
|
897
|
+
unsigned int src_index = LinkAdrType::null;
|
|
898
|
+
unsigned int start = util::random_uint(_thd_ctx.rand_state)%POOL_SIZE;
|
|
899
|
+
QueueType queue = pull_queue(_dev_ctx.pool->queues,start,POOL_SIZE,src_index);
|
|
900
|
+
q_printf("Pulled queue (%d,%d) from pool %d\n",queue.get_head().adr,queue.get_tail().adr,src_index);
|
|
901
|
+
|
|
902
|
+
//! Keep popping links from the queue until the full number of links have been added or
|
|
903
|
+
//! the queue runs out of links.
|
|
904
|
+
for(int i=alloc_count; i < req_count; i++){
|
|
905
|
+
LinkAdrType link = pop_front(queue);
|
|
906
|
+
if( ! link.is_null() ){
|
|
907
|
+
//insert_stash_link(link);
|
|
908
|
+
dst[alloc_count] = link;
|
|
909
|
+
alloc_count++;
|
|
910
|
+
q_printf("Inserted link %d into link stash\n",link.adr);
|
|
911
|
+
} else {
|
|
912
|
+
break;
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
push_queue(_dev_ctx.pool->queues[src_index],queue);
|
|
916
|
+
q_printf("Pushed queue (%d,%d) to pool %d\n",queue.get_head().adr,queue.get_tail().adr,src_index);
|
|
917
|
+
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
return alloc_count;
|
|
921
|
+
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
__device__ LinkAdrType alloc_spare_link(){
|
|
925
|
+
if(_thd_ctx.spare_index > 0){
|
|
926
|
+
_thd_ctx.spare_index--;
|
|
927
|
+
LinkAdrType result;
|
|
928
|
+
result = _thd_ctx.spare_links[_thd_ctx.spare_index];
|
|
929
|
+
//printf("Re-used spare alloc (%d)\n",result.adr);
|
|
930
|
+
return result;
|
|
931
|
+
} else {
|
|
932
|
+
LinkAdrType result = LinkAdrType::null;
|
|
933
|
+
alloc_links(&result,1);
|
|
934
|
+
//printf("Fresh spare alloc (%d)\n",result.adr);
|
|
935
|
+
return result;
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
__device__ void dump_spare_link(LinkAdrType link_adr){
|
|
940
|
+
if( link_adr.is_null() ){
|
|
941
|
+
return;
|
|
942
|
+
}
|
|
943
|
+
if(_thd_ctx.spare_index < SPARE_LINK_COUNT){
|
|
944
|
+
//printf("Saving link (%d) for later\n",link_adr.adr);
|
|
945
|
+
_thd_ctx.spare_links[_thd_ctx.spare_index] = link_adr;
|
|
946
|
+
_thd_ctx.spare_index++;
|
|
947
|
+
//printf("New spare index of %d is %d\n",threadIdx.x,_thd_ctx.spare_index);
|
|
948
|
+
} else {
|
|
949
|
+
//printf("Spilling spare link (%d)\n",link_adr.adr);
|
|
950
|
+
dealloc_links(&link_adr,1);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
//! Attempts to fill the link stash to the given threshold with links. This should only ever
|
|
955
|
+
//! be called in a single-threaded manner.
|
|
956
|
+
__device__ void fill_stash_links(unsigned int threashold){
|
|
957
|
+
|
|
958
|
+
unsigned int active = __activemask();
|
|
959
|
+
__syncwarp(active);
|
|
960
|
+
|
|
961
|
+
if( util::current_leader() ){
|
|
962
|
+
|
|
963
|
+
LinkAdrType *dst = _grp_ctx.link_stash+_grp_ctx.link_stash_count;
|
|
964
|
+
size_t count = 0;
|
|
965
|
+
if( threashold < _grp_ctx.link_stash_count ){
|
|
966
|
+
count = 0;
|
|
967
|
+
} else if ( threashold >= STASH_SIZE ){
|
|
968
|
+
count = STASH_SIZE - _grp_ctx.link_stash_count;
|
|
969
|
+
} else {
|
|
970
|
+
count = threashold - _grp_ctx.link_stash_count;
|
|
971
|
+
}
|
|
972
|
+
size_t added = alloc_links(dst,count);
|
|
973
|
+
_grp_ctx.link_stash_count += added;
|
|
974
|
+
|
|
975
|
+
}
|
|
976
|
+
__syncwarp(active);
|
|
977
|
+
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
//! If the number of links in the link stash exceeds the given threshold value, this function frees
|
|
984
|
+
//! enough links to bring the number of links down to the threshold. This should only ever be
|
|
985
|
+
//! called in a single_threaded manner.
|
|
986
|
+
__device__ void spill_stash_links(unsigned int threashold){
|
|
987
|
+
|
|
988
|
+
//! Do not even try if no links can be or need to be removed
|
|
989
|
+
if(threashold >= _grp_ctx.link_stash_count){
|
|
990
|
+
q_printf("Nothing to spill...\n");
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
//! Find where in the link stash to begin removing links
|
|
995
|
+
|
|
996
|
+
QueueType queue;
|
|
997
|
+
queue.pair.data = QueueType::null;
|
|
998
|
+
|
|
999
|
+
//! Connect all links into a queue
|
|
1000
|
+
unsigned int spill_count = _grp_ctx.link_stash_count - threashold;
|
|
1001
|
+
for(unsigned int i=0; i < spill_count; i++){
|
|
1002
|
+
|
|
1003
|
+
LinkAdrType link = claim_stash_link();
|
|
1004
|
+
q_printf("Claimed link %d from link stash\n",link.adr);
|
|
1005
|
+
push_back(queue,link);
|
|
1006
|
+
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
_grp_ctx.link_stash_count = threashold;
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
//! Push out the queue to the pool
|
|
1013
|
+
q_printf("Pushing queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
|
|
1014
|
+
unsigned int dest_idx = util::random_uint(_thd_ctx.rand_state) % POOL_SIZE;
|
|
1015
|
+
push_queue(_dev_ctx.pool->queues[dest_idx],queue);
|
|
1016
|
+
|
|
1017
|
+
q_printf("Pushed queue (%d,%d) to pool\n",queue.get_head().adr,queue.get_tail().adr);
|
|
1018
|
+
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
//! Decrements the child and resident counter of each frame corresponding to a call at level
|
|
1025
|
+
//! start_level in the stack returning to a continuation at level end_level in the stack. To reduce
|
|
1026
|
+
//! overall contention, decrementations are first pooled through a shared atomic operation before
|
|
1027
|
+
//! being applied to the stack.
|
|
1028
|
+
//
|
|
1029
|
+
//! A call without a continuation should use this function with start_level == end_level, which
|
|
1030
|
+
//! simply decrements the resident counter at the call's frame.
|
|
1031
|
+
__device__ void pop_frame_counters(unsigned int start_level, unsigned int end_level){
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
unsigned int depth_dec = 0;
|
|
1035
|
+
PromiseCountPair delta;
|
|
1036
|
+
PromiseCountPair result;
|
|
1037
|
+
|
|
1038
|
+
FrameType& frame = _dev_ctx.stack->frames[start_level];
|
|
1039
|
+
|
|
1040
|
+
//! Decrement the residents counter for the start level
|
|
1041
|
+
delta = util::active_count();
|
|
1042
|
+
if(util::current_leader()){
|
|
1043
|
+
result = atomicSub(&frame.children_residents.data,delta);
|
|
1044
|
+
if(result == 0u){
|
|
1045
|
+
depth_dec += 1;
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
//! Decrement the children counter for the remaining levels
|
|
1050
|
+
for(int d=(start_level-1); d >= end_level; d--){
|
|
1051
|
+
FrameType& frame = _dev_ctx.stack->frames[d];
|
|
1052
|
+
delta = util::active_count();
|
|
1053
|
+
if(util::current_leader()){
|
|
1054
|
+
result = atomicSub(&frame.children_residents.data,delta);
|
|
1055
|
+
if(result == 0u){
|
|
1056
|
+
depth_dec += 1;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
//! Update the stack base once all other counters have been updated.
|
|
1062
|
+
if(util::current_leader()){
|
|
1063
|
+
result = atomicSub(&(_dev_ctx.stack->depth_live),depth_dec);
|
|
1064
|
+
if(result == 0){
|
|
1065
|
+
set_flags(_grp_ctx,COMPLETION_FLAG);
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
//! Repetitively tries to merge the given queue of promises with the queue at the given index in the
|
|
1073
|
+
//! frame at the given level on the stack. This function currently aborts if an error flag is set
|
|
1074
|
+
//! or if too many merge failures occur, however, once its correctness is verified, this function
|
|
1075
|
+
//! will run forever until the merge is successful, as success is essentially guaranteed by
|
|
1076
|
+
//! the nature of the process.
|
|
1077
|
+
__device__ void push_promises(unsigned int level, unsigned int index, QueueType queue, int promise_delta) {
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
LinkAdrType tail = queue.get_tail();
|
|
1081
|
+
LinkAdrType head = queue.get_head();
|
|
1082
|
+
rc_printf("SM %d: push_promises(level:%d,index:%d,queue:(%d,%d),delta:%d)\n",threadIdx.x,level,index,tail.adr,head.adr,promise_delta);
|
|
1083
|
+
//! Do not bother pushing a null queue if there is no delta to report
|
|
1084
|
+
if( ( ! queue.is_null() ) || (promise_delta != 0) ){
|
|
1085
|
+
|
|
1086
|
+
//! Change the resident counter of the destination frame by the number of promises
|
|
1087
|
+
//! that have been added to or removed from the given queue
|
|
1088
|
+
FrameType &dest = get_frame(level);
|
|
1089
|
+
unsigned int old_count;
|
|
1090
|
+
unsigned int new_count;
|
|
1091
|
+
if(promise_delta >= 0) {
|
|
1092
|
+
old_count = atomicAdd(&dest.children_residents.data,(PromiseCountPair) promise_delta);
|
|
1093
|
+
new_count = old_count + (unsigned int) promise_delta;
|
|
1094
|
+
if(old_count > new_count){
|
|
1095
|
+
rc_printf("\n\nOVERFLOW\n\n");
|
|
1096
|
+
}
|
|
1097
|
+
} else {
|
|
1098
|
+
PromiseCountPair neg_delta = -((PromiseCountPair) (-promise_delta));
|
|
1099
|
+
old_count = atomicAdd(&dest.children_residents.data,neg_delta);
|
|
1100
|
+
new_count = old_count + neg_delta;
|
|
1101
|
+
if(old_count < new_count){
|
|
1102
|
+
rc_printf("\n\nUNDERFLOW\n\n");
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
_grp_ctx.SM_promise_delta += promise_delta;
|
|
1108
|
+
rc_printf("SM %d-%d: Old count: %d, New count: %d, Delta: %d\n",blockIdx.x,threadIdx.x,old_count,new_count,promise_delta);
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
rc_printf("SM %d-%d: frame zero resident count is: %d\n",blockIdx.x,threadIdx.x,_dev_ctx.stack->frames[0].children_residents.data);
|
|
1112
|
+
//! If the addition caused a frame to change from empty to non-empty or vice-versa,
|
|
1113
|
+
//! make an appropriate incrementation or decrementation at the stack base.
|
|
1114
|
+
if( (old_count == 0) && (new_count != 0) ){
|
|
1115
|
+
atomicAdd(&(_dev_ctx.stack->depth_live),0x00010000u);
|
|
1116
|
+
} else if( (old_count != 0) && (new_count == 0) ){
|
|
1117
|
+
atomicSub(&(_dev_ctx.stack->depth_live),0x00010000u);
|
|
1118
|
+
} else {
|
|
1119
|
+
rc_printf("SM %d: No change!\n",threadIdx.x);
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
//! Finally, push the queues
|
|
1123
|
+
push_queue(dest.pool.queues[index],queue);
|
|
1124
|
+
rc_printf("SM %d: Pushed queue (%d,%d) to stack at index %d\n",threadIdx.x,queue.get_head().adr,queue.get_tail().adr,index);
|
|
1125
|
+
|
|
1126
|
+
if( (_dev_ctx.stack->frames[0].children_residents.get_right()) == 0 ) {
|
|
1127
|
+
rc_printf("SM %d-%d: After queue pushed to stack, frame zero resident count is: %d\n",blockIdx.x,threadIdx.x,_dev_ctx.stack->frames[0].children_residents.data);
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
}
|
|
1131
|
+
rc_printf("(%d) the delta: %d\n",threadIdx.x,promise_delta);
|
|
1132
|
+
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
//! Attempts to pull a queue of promises from the frame in the stack of the given level, starting the
|
|
1138
|
+
//! pull attempt at the given index in the frame. If no queue could be pulled after attempting a
|
|
1139
|
+
//! pull at each queue in the given frame, a QueueType::null value is returned.
|
|
1140
|
+
__device__ QueueType pull_promises(unsigned int level, unsigned int& source_index) {
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
rc_printf("SM %d: pull_promises(level:%d)\n",threadIdx.x,level);
|
|
1144
|
+
unsigned int src_idx = util::random_uint(_thd_ctx.rand_state) % FRAME_SIZE;
|
|
1145
|
+
|
|
1146
|
+
__threadfence();
|
|
1147
|
+
FrameType &src = get_frame(level);
|
|
1148
|
+
|
|
1149
|
+
QueueType queue = pull_queue(src.pool.queues,src_idx,FRAME_SIZE,source_index);
|
|
1150
|
+
|
|
1151
|
+
if( ! queue.is_null() ){
|
|
1152
|
+
q_printf("SM %d: Pulled queue (%d,%d) from stack at index %d\n",blockIdx.x,queue.get_tail().adr,queue.get_head().adr, src_idx);
|
|
1153
|
+
} else {
|
|
1154
|
+
q_printf("SM %d: Failed to pull queue from stack starting at index %d\n",blockIdx.x, src_idx);
|
|
1155
|
+
}
|
|
1156
|
+
__threadfence();
|
|
1157
|
+
return queue;
|
|
1158
|
+
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
//! Attempts to pull a queue from any frame in the stack, starting from the highest and working
|
|
1163
|
+
//! its way down. If no queue could be pulled, a QueueType::null value is returned.
|
|
1164
|
+
__device__ QueueType pull_promises_any_level(unsigned int& level, unsigned int& source_index){
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
QueueType result;
|
|
1168
|
+
result.data = QueueType::null;
|
|
1169
|
+
unsigned int start_level = highest_level();
|
|
1170
|
+
for(int level_itr = start_level; level_itr>=0; level_itr--){
|
|
1171
|
+
q_printf("Pulling promises at level %d for pull_promises_any_level\n",level_itr);
|
|
1172
|
+
result = pull_promises(level_itr,source_index);
|
|
1173
|
+
if( ! result.is_null() ){
|
|
1174
|
+
level = level_itr;
|
|
1175
|
+
return result;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
result.data = QueueType::null;
|
|
1179
|
+
return result;
|
|
1180
|
+
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
//! Adds the contents of the stash slot at the given index to a link and returns the index of the
|
|
1188
|
+
//! link in the arena. This should only ever be called if there is both a link available to store
|
|
1189
|
+
//! the data and if the index is pointing at a non-empty slot. This also should only ever be
|
|
1190
|
+
//! called in a single-threaded context.
|
|
1191
|
+
__device__ LinkAdrType produce_link(unsigned int slot_index ){
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
__shared__ LinkAdrType result;
|
|
1195
|
+
|
|
1196
|
+
unsigned int active = __activemask();
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
//__syncwarp(active);
|
|
1200
|
+
|
|
1201
|
+
//if(util::current_leader()){
|
|
1202
|
+
LinkAdrType link_index = claim_stash_link();
|
|
1203
|
+
q_printf("Claimed link %d from stash\n",link_index.adr);
|
|
1204
|
+
LinkType& the_link = _dev_ctx.arena[link_index];
|
|
1205
|
+
//_grp_ctx.SM_promise_delta += _grp_ctx.stash[slot_index].count;
|
|
1206
|
+
_grp_ctx.stash[slot_index].next = LinkAdrType::null;
|
|
1207
|
+
the_link = _grp_ctx.stash[slot_index];
|
|
1208
|
+
db_printf("Link has count %d and next %d in main memory",the_link.count, the_link.next);
|
|
1209
|
+
result = link_index;
|
|
1210
|
+
_grp_ctx.main_queue.count -= 1;
|
|
1211
|
+
//}
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
//__syncwarp(active);
|
|
1215
|
+
return result;
|
|
1216
|
+
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
//! Removes all promises in the stash that do not correspond to the given level, or to the levels
|
|
1227
|
+
//! immediately above or below (level+1) and (level-1).
|
|
1228
|
+
__device__ void relevel_stash(unsigned int level){
|
|
1229
|
+
|
|
1230
|
+
if( ! StackType::FLAT ){
|
|
1231
|
+
|
|
1232
|
+
// TODO: Implement for non-flat stack
|
|
1233
|
+
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
//! Dumps all full links not corresponding to the current execution level. Furthermore, should the
|
|
1243
|
+
//! remaining links still put the stash over the given threshold occupancy, links will be further
|
|
1244
|
+
//! removed in the order: full links at the current level, partial links not at the current level,
|
|
1245
|
+
//! partial links at the current level.
|
|
1246
|
+
__device__ void spill_stash(unsigned int threashold){
|
|
1247
|
+
|
|
1248
|
+
unsigned int active =__activemask();
|
|
1249
|
+
__syncwarp(active);
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
#if DEF_STACK_MODE == 0
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
if(util::current_leader() && (_grp_ctx.main_queue.count > threashold)){
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
unsigned int spill_count = _grp_ctx.main_queue.count - threashold;
|
|
1261
|
+
int delta = 0;
|
|
1262
|
+
fill_stash_links(spill_count);
|
|
1263
|
+
|
|
1264
|
+
QueueType queue;
|
|
1265
|
+
queue.pair.data = QueueType::null;
|
|
1266
|
+
unsigned int partial_iter = 0;
|
|
1267
|
+
bool has_full_slots = true;
|
|
1268
|
+
q_printf("{Spilling to %d}",threashold);
|
|
1269
|
+
for(unsigned int i=0; i < spill_count; i++){
|
|
1270
|
+
unsigned int slot = STASH_SIZE;
|
|
1271
|
+
if(has_full_slots){
|
|
1272
|
+
slot = claim_full_slot();
|
|
1273
|
+
if(slot == STASH_SIZE){
|
|
1274
|
+
has_full_slots = false;
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
if(! has_full_slots){
|
|
1278
|
+
for(;partial_iter < FN_ID_COUNT; partial_iter++){
|
|
1279
|
+
db_printf("%d",partial_iter);
|
|
1280
|
+
if(_grp_ctx.main_queue.partial_map[partial_iter] != STASH_SIZE){
|
|
1281
|
+
slot = _grp_ctx.main_queue.partial_map[partial_iter];
|
|
1282
|
+
q_printf("{Spilling partial}");
|
|
1283
|
+
partial_iter++;
|
|
1284
|
+
break;
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
if(slot == STASH_SIZE){
|
|
1289
|
+
break;
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
delta += _grp_ctx.stash[slot].count;
|
|
1293
|
+
q_printf("Slot for production (%d) has %d promises\n",slot,_grp_ctx.stash[slot].count);
|
|
1294
|
+
LinkAdrType link = produce_link(slot);
|
|
1295
|
+
push_back(queue,link);
|
|
1296
|
+
insert_empty_slot(slot);
|
|
1297
|
+
if(_grp_ctx.main_queue.count <= threashold){
|
|
1298
|
+
break;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
unsigned int push_index = util::random_uint(_thd_ctx.rand_state)%FRAME_SIZE;
|
|
1303
|
+
q_printf("Pushing promises in (%d,%d) for spilling\n",queue.get_head().adr,queue.get_tail().adr);
|
|
1304
|
+
push_promises(0,push_index,queue,delta);
|
|
1305
|
+
q_printf("Pushed queue (%d,%d) to stack\n",queue.get_head().adr,queue.get_tail().adr);
|
|
1306
|
+
|
|
1307
|
+
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
#else
|
|
1312
|
+
|
|
1313
|
+
__shared__ unsigned int queues[3];
|
|
1314
|
+
__shared__ unsigned int counts[3];
|
|
1315
|
+
__shared__ unsigned int bucket[4];
|
|
1316
|
+
|
|
1317
|
+
/*
|
|
1318
|
+
// Currently implemented in a single-threaded manner per work group to simplify the initial
|
|
1319
|
+
// correctness checking process. This can later be changed to take advantage of in-group
|
|
1320
|
+
// parallelism.
|
|
1321
|
+
*/
|
|
1322
|
+
if(util::current_leader()){
|
|
1323
|
+
|
|
1324
|
+
/*
|
|
1325
|
+
// Zero the counters and null the queues
|
|
1326
|
+
*/
|
|
1327
|
+
for(unsigned int i=0; i < 3; i++){
|
|
1328
|
+
queues[i] = QueueType::null;
|
|
1329
|
+
counts[i] = 0;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
for(unsigned int i=0; i < 4; i++){
|
|
1333
|
+
bucket[i] = 0;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
/*
|
|
1338
|
+
// Count up each type of link
|
|
1339
|
+
*/
|
|
1340
|
+
for(unsigned int i=0; i < STASH_SIZE; i++){
|
|
1341
|
+
unsigned int depth = _grp_ctx.stash[i].depth;
|
|
1342
|
+
unsigned int size = _grp_ctx.stash[i].size;
|
|
1343
|
+
unsigned int idx = (depth != level) ? 0 : 1;
|
|
1344
|
+
idx += (size >= WARP_COUNT) ? 0 : 2;
|
|
1345
|
+
bucket[idx] += 1;
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/*
|
|
1349
|
+
// Determine how much of which type of link needs to be dumped
|
|
1350
|
+
*/
|
|
1351
|
+
unsigned int dump_total = bucket[0];
|
|
1352
|
+
unsigned int dump_count = (_grp_ctx.main_queue.count > threshold) ? _grp_ctx.main_queue.count - threshold : 0;
|
|
1353
|
+
dump_count = (dump_count <= bucket[0]) : 0 ? dump_count - bucket[0];
|
|
1354
|
+
for(unsigned int i=1; i< 4; i++){
|
|
1355
|
+
unsigned int delta = (bucket[i] <= dump_count) ? bucket[i] : dump_count;
|
|
1356
|
+
dump_count -= delta;
|
|
1357
|
+
bucket[i] = delta;
|
|
1358
|
+
dump_total += delta;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/*
|
|
1362
|
+
// Dump the corresponding number of each type of link
|
|
1363
|
+
*/
|
|
1364
|
+
for(unsigned int i=0; i < _grp_ctx.main_queue.count; i++){
|
|
1365
|
+
unsigned int depth = _grp_ctx.stash[i].depth;
|
|
1366
|
+
unsigned int size = _grp_ctx.stash[i].size;
|
|
1367
|
+
unsigned int bucket_idx = (depth != level) ? 0 : 1;
|
|
1368
|
+
bucket_idx += (size >= WARP_COUNT) ? 0 : 2;
|
|
1369
|
+
if(bucket[bucket_idx] == 0){
|
|
1370
|
+
continue;
|
|
1371
|
+
}
|
|
1372
|
+
LinkAdrType link = _grp_ctx.link_stash[_grp_ctx.link_stash_count];
|
|
1373
|
+
_grp_ctx.link_stash_count -= 1;
|
|
1374
|
+
|
|
1375
|
+
copy_link(_dev_ctx.arena[link], _grp_ctx.stash[i]);
|
|
1376
|
+
|
|
1377
|
+
unsigned int level_index = level+1-depth;
|
|
1378
|
+
counts[level_index] += size;
|
|
1379
|
+
push_back(queues[level_index],link);
|
|
1380
|
+
|
|
1381
|
+
_grp_ctx.stash[i].size = 0;
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
#endif
|
|
1386
|
+
|
|
1387
|
+
__syncwarp(active);
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
//! Spills from stash in advance of inserting more work
|
|
1395
|
+
__device__ void async_call_stash_dump(OpDisc func_id, int depth_delta, unsigned int delta){
|
|
1396
|
+
|
|
1397
|
+
/*
|
|
1398
|
+
// Make room to queue incoming promises, if there isn't enough room already.
|
|
1399
|
+
*/
|
|
1400
|
+
#if 0
|
|
1401
|
+
if(_grp_ctx.link_stash_count < 2){
|
|
1402
|
+
fill_stash_links(2);
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
if(_grp_ctx.main_queue.count >= (STASH_SIZE-2)){
|
|
1406
|
+
spill_stash(STASH_SIZE-3);
|
|
1407
|
+
}
|
|
1408
|
+
#else
|
|
1409
|
+
/*
|
|
1410
|
+
unsigned int depth = (unsigned int) (_grp_ctx.level + depth_delta);
|
|
1411
|
+
unsigned int left_jump = partial_map_index(func_id,depth,_grp_ctx.level);
|
|
1412
|
+
unsigned int space = 0;
|
|
1413
|
+
if( left_jump != PART_ENTRY_COUNT ){
|
|
1414
|
+
unsigned int left_idx = _grp_ctx.main_queue.partial_map[left_jump];
|
|
1415
|
+
if( left_idx != STASH_SIZE ){
|
|
1416
|
+
space = WORK_GROUP_SIZE - _grp_ctx.stash[left_idx].count;
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
*/
|
|
1420
|
+
if( stash_overfilled() ) { //&& (space < delta) ){
|
|
1421
|
+
if(_grp_ctx.link_stash_count < STASH_MARGIN){
|
|
1422
|
+
fill_stash_links(STASH_MARGIN);
|
|
1423
|
+
}
|
|
1424
|
+
//printf("{Spilling for call.}");
|
|
1425
|
+
spill_stash(STASH_HIGH_WATER-1);
|
|
1426
|
+
}
|
|
1427
|
+
#endif
|
|
1428
|
+
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
//! Prepares the stash for the insertion of work
|
|
1433
|
+
__device__ void async_call_stash_prep(RemapQueue& dst, OpDisc func_id, int depth_delta, unsigned int delta,
|
|
1434
|
+
unsigned int &left, unsigned int &left_start, unsigned int &right
|
|
1435
|
+
){
|
|
1436
|
+
|
|
1437
|
+
/*
|
|
1438
|
+
// Locate the destination links in the stash that the promises will be written to. For now,
|
|
1439
|
+
// like many other parts of the code, this will be single-threaded within the work group
|
|
1440
|
+
// to make validation easier but will be optimized for group-level parallelism later.
|
|
1441
|
+
*/
|
|
1442
|
+
if( util::current_leader() ){
|
|
1443
|
+
|
|
1444
|
+
db_printf("{Queueing %d promises of type %d}",delta,func_id);
|
|
1445
|
+
/*
|
|
1446
|
+
// Null out the right index. This index should not be used unless the number of
|
|
1447
|
+
// promises queued spills over beyond the first link being written to (the left one)
|
|
1448
|
+
*/
|
|
1449
|
+
right = STASH_SIZE;
|
|
1450
|
+
|
|
1451
|
+
/*
|
|
1452
|
+
// Find the index of the partial link in the stash corresponding to the id and
|
|
1453
|
+
// depth of the calls being queued (if it exists).
|
|
1454
|
+
*/
|
|
1455
|
+
unsigned int depth = (unsigned int) (_grp_ctx.level + depth_delta);
|
|
1456
|
+
unsigned int left_jump = partial_map_index(func_id,depth,_grp_ctx.level);
|
|
1457
|
+
|
|
1458
|
+
/*
|
|
1459
|
+
// If there is a partially filled link to be filled, assign that to the left index
|
|
1460
|
+
*/
|
|
1461
|
+
if(left_jump != PART_ENTRY_COUNT){
|
|
1462
|
+
db_printf("(%d:A)",blockIdx.x);
|
|
1463
|
+
left = dst.partial_map[left_jump];
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
unsigned int left_count;
|
|
1467
|
+
if(left == STASH_SIZE){
|
|
1468
|
+
left = claim_empty_slot();
|
|
1469
|
+
dst.count += 1;
|
|
1470
|
+
db_printf("(%d:B+1->%d)",blockIdx.x,dst.count);
|
|
1471
|
+
db_printf("Updated stash count: %d\n",dst.count);
|
|
1472
|
+
_grp_ctx.stash[left].id = func_id;
|
|
1473
|
+
dst.partial_map[left_jump] = left;
|
|
1474
|
+
left_count = 0;
|
|
1475
|
+
} else {
|
|
1476
|
+
left_count = _grp_ctx.stash[left].count;
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
if ( (left_count + delta) > WORK_GROUP_SIZE ){
|
|
1480
|
+
right = claim_empty_slot();
|
|
1481
|
+
dst.count += 1;
|
|
1482
|
+
db_printf("(%d:C+1->%d)",blockIdx.x,dst.count);
|
|
1483
|
+
_grp_ctx.stash[right].count = left_count+delta - WORK_GROUP_SIZE;
|
|
1484
|
+
_grp_ctx.stash[right].id = func_id;
|
|
1485
|
+
insert_full_slot(dst,left);
|
|
1486
|
+
dst.partial_map[left_jump] = right;
|
|
1487
|
+
_grp_ctx.stash[left].count = WORK_GROUP_SIZE;
|
|
1488
|
+
} else if ( (left_count + delta) == WORK_GROUP_SIZE ){
|
|
1489
|
+
db_printf("(%d:D)",blockIdx.x);
|
|
1490
|
+
dst.partial_map[left_jump] = STASH_SIZE;
|
|
1491
|
+
insert_full_slot(dst,left);
|
|
1492
|
+
_grp_ctx.stash[left].count = WORK_GROUP_SIZE;
|
|
1493
|
+
} else {
|
|
1494
|
+
db_printf("(%d:E)",blockIdx.x);
|
|
1495
|
+
_grp_ctx.stash[left].count = left_count + delta;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
left_start = left_count;
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
/*
|
|
1507
|
+
// Queues the input promise into a corresponding local link according to the given function id and
|
|
1508
|
+
// at a level corresponding to the current level of he promises being evaluated plus the value of
|
|
1509
|
+
// depth_delta. This scheme ensures that the function being called and the depth of the promises
|
|
1510
|
+
// being created for those calls are consistent across the warp.
|
|
1511
|
+
*/
|
|
1512
|
+
__device__ void async_call(OpDisc func_id, int depth_delta, PromiseUnionType& promise){
|
|
1513
|
+
|
|
1514
|
+
unsigned int active = __activemask();
|
|
1515
|
+
|
|
1516
|
+
/*
|
|
1517
|
+
// Calculate how many promises are being queued as well as the assigned index of the
|
|
1518
|
+
// current thread's promise in the write to the stash.
|
|
1519
|
+
*/
|
|
1520
|
+
unsigned int index = util::warp_inc_scan();
|
|
1521
|
+
unsigned int delta = util::active_count();
|
|
1522
|
+
|
|
1523
|
+
|
|
1524
|
+
#ifdef BARRIER_SPILL
|
|
1525
|
+
if( stash_overfilled() ) { //&& (space < delta) ){
|
|
1526
|
+
_grp_ctx.spill_barrier.union_await<false>(*this,func_id,promise);
|
|
1527
|
+
return;
|
|
1528
|
+
}
|
|
1529
|
+
#else
|
|
1530
|
+
async_call_stash_dump(func_id, depth_delta, delta);
|
|
1531
|
+
#endif
|
|
1532
|
+
|
|
1533
|
+
|
|
1534
|
+
__shared__ unsigned int left, left_start, right;
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
async_call_stash_prep(_grp_ctx.main_queue,func_id,depth_delta,delta,left,left_start,right);
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
/*
|
|
1541
|
+
// Write the promise into the appropriate part of the stash, writing into the left link
|
|
1542
|
+
// when possible and spilling over into the right link when necessary.
|
|
1543
|
+
*/
|
|
1544
|
+
__syncwarp(active);
|
|
1545
|
+
if( (left_start + index) >= WORK_GROUP_SIZE ){
|
|
1546
|
+
//db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1547
|
+
_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].dyn_copy_as(func_id,promise);
|
|
1548
|
+
//_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE] = promise;
|
|
1549
|
+
} else {
|
|
1550
|
+
//db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1551
|
+
_grp_ctx.stash[left].promises[left_start+index].dyn_copy_as(func_id,promise);
|
|
1552
|
+
//_grp_ctx.stash[left].promises[left_start+index] = promise;
|
|
1553
|
+
}
|
|
1554
|
+
__syncwarp(active);
|
|
1555
|
+
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
/*
|
|
1559
|
+
// Like async_call, but allows for one to hand in the underlying type corresponding to a function id directly
|
|
1560
|
+
*/
|
|
1561
|
+
template<typename TYPE, typename... ARGS>
|
|
1562
|
+
__device__ void async_call_cast(int depth_delta, ARGS... args){
|
|
1563
|
+
|
|
1564
|
+
beg_time(7);
|
|
1565
|
+
unsigned int active = __activemask();
|
|
1566
|
+
|
|
1567
|
+
/*
|
|
1568
|
+
// Calculate how many promises are being queued as well as the assigned index of the
|
|
1569
|
+
// current thread's promise in the write to the stash.
|
|
1570
|
+
*/
|
|
1571
|
+
unsigned int index = util::warp_inc_scan();
|
|
1572
|
+
unsigned int delta = util::active_count();
|
|
1573
|
+
|
|
1574
|
+
#ifdef BARRIER_SPILL
|
|
1575
|
+
if( stash_overfilled() ) { //&& (space < delta) ){
|
|
1576
|
+
beg_time(8);
|
|
1577
|
+
_grp_ctx.spill_barrier.await<false>(*this,Promise<TYPE>(args...));
|
|
1578
|
+
end_time(8);
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
#else
|
|
1582
|
+
beg_time(8);
|
|
1583
|
+
async_call_stash_dump(Lookup<TYPE>::type::DISC, depth_delta, delta);
|
|
1584
|
+
end_time(8);
|
|
1585
|
+
#endif
|
|
1586
|
+
|
|
1587
|
+
__shared__ unsigned int left, left_start, right;
|
|
1588
|
+
|
|
1589
|
+
|
|
1590
|
+
#ifdef ASYNC_LOADS
|
|
1591
|
+
RemapQueue& dst_queue = has_load(args...) ? _grp_ctx.load_queue : _grp_ctx.main_queue;
|
|
1592
|
+
#else
|
|
1593
|
+
RemapQueue& dst_queue = _grp_ctx.main_queue;
|
|
1594
|
+
#endif
|
|
1595
|
+
|
|
1596
|
+
beg_time(9);
|
|
1597
|
+
async_call_stash_prep(dst_queue,Lookup<TYPE>::type::DISC,depth_delta,delta,left,left_start,right);
|
|
1598
|
+
end_time(9);
|
|
1599
|
+
|
|
1600
|
+
/*
|
|
1601
|
+
// Write the promise into the appropriate part of the stash, writing into the left link
|
|
1602
|
+
// when possible and spilling over into the right link when necessary.
|
|
1603
|
+
*/
|
|
1604
|
+
__syncwarp(active);
|
|
1605
|
+
if( (left_start + index) >= WORK_GROUP_SIZE ){
|
|
1606
|
+
//db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1607
|
+
#ifdef ASYNC_LOADS
|
|
1608
|
+
_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>().async_load(*this,args...);
|
|
1609
|
+
#else
|
|
1610
|
+
_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>() = Promise<TYPE>(args...);
|
|
1611
|
+
#endif
|
|
1612
|
+
} else {
|
|
1613
|
+
//db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1614
|
+
#ifdef ASYNC_LOADS
|
|
1615
|
+
_grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>().async_load(*this,args...);
|
|
1616
|
+
#else
|
|
1617
|
+
_grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>() = Promise<TYPE>(args...);
|
|
1618
|
+
#endif
|
|
1619
|
+
}
|
|
1620
|
+
__syncwarp(active);
|
|
1621
|
+
end_time(7);
|
|
1622
|
+
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
template<typename TYPE>
|
|
1627
|
+
__device__ void async_call_cast(int depth_delta, Promise<TYPE> promise){
|
|
1628
|
+
beg_time(7);
|
|
1629
|
+
unsigned int active = __activemask();
|
|
1630
|
+
|
|
1631
|
+
/*
|
|
1632
|
+
// Calculate how many promises are being queued as well as the assigned index of the
|
|
1633
|
+
// current thread's promise in the write to the stash.
|
|
1634
|
+
*/
|
|
1635
|
+
unsigned int index = util::warp_inc_scan();
|
|
1636
|
+
unsigned int delta = util::active_count();
|
|
1637
|
+
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
|
|
1641
|
+
#ifdef BARRIER_SPILL
|
|
1642
|
+
if( stash_overfilled() ) { //&& (space < delta) ){
|
|
1643
|
+
beg_time(8);
|
|
1644
|
+
_grp_ctx.spill_barrier.await<false>(*this,promise);
|
|
1645
|
+
end_time(8);
|
|
1646
|
+
return;
|
|
1647
|
+
}
|
|
1648
|
+
#else
|
|
1649
|
+
beg_time(8);
|
|
1650
|
+
async_call_stash_dump(Lookup<TYPE>::type::DISC, depth_delta, delta);
|
|
1651
|
+
end_time(8);
|
|
1652
|
+
#endif
|
|
1653
|
+
|
|
1654
|
+
__shared__ unsigned int left, left_start, right;
|
|
1655
|
+
|
|
1656
|
+
|
|
1657
|
+
beg_time(9);
|
|
1658
|
+
async_call_stash_prep(_grp_ctx.main_queue,Lookup<TYPE>::type::DISC,depth_delta,delta,left,left_start,right);
|
|
1659
|
+
end_time(9);
|
|
1660
|
+
|
|
1661
|
+
/*
|
|
1662
|
+
// Write the promise into the appropriate part of the stash, writing into the left link
|
|
1663
|
+
// when possible and spilling over into the right link when necessary.
|
|
1664
|
+
*/
|
|
1665
|
+
__syncwarp(active);
|
|
1666
|
+
if( (left_start + index) >= WORK_GROUP_SIZE ){
|
|
1667
|
+
//db_printf("Overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1668
|
+
_grp_ctx.stash[right].promises[left_start+index-WORK_GROUP_SIZE].template cast<TYPE>() = promise;
|
|
1669
|
+
} else {
|
|
1670
|
+
//db_printf("Non-overflow: id: %d, left: %d, left_start: %d, index: %d\n",threadIdx.x,left,left_start,index);
|
|
1671
|
+
_grp_ctx.stash[left].promises[left_start+index].template cast<TYPE>() = promise;
|
|
1672
|
+
}
|
|
1673
|
+
__syncwarp(active);
|
|
1674
|
+
end_time(7);
|
|
1675
|
+
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
|
|
1679
|
+
template<typename TYPE>
|
|
1680
|
+
__device__ void immediate_call_cast(Promise<TYPE> param_value){
|
|
1681
|
+
PromiseUnionType promise;
|
|
1682
|
+
promise.template cast<TYPE>() = param_value;
|
|
1683
|
+
promise.template rigid_eval<ProgramType,TYPE>(*this);
|
|
1684
|
+
//promise_eval<ProgramType,FUNC_ID>(param_value);
|
|
1685
|
+
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
|
|
1690
|
+
#define PARACON
|
|
1691
|
+
|
|
1692
|
+
/*
|
|
1693
|
+
// Adds the contents of the link at the given index to the stash and adds the given link to link
|
|
1694
|
+
// stash. Once complete, it returns the number of promises added to the stash by the operation.
|
|
1695
|
+
// This should only ever be called if there is enough space to store the extra work and link.
|
|
1696
|
+
*/
|
|
1697
|
+
__device__ unsigned int consume_link(LinkAdrType link_index ){
|
|
1698
|
+
|
|
1699
|
+
|
|
1700
|
+
#if 0 //def PARACON
|
|
1701
|
+
__shared__ LinkAdrType the_index;
|
|
1702
|
+
__shared__ unsigned int add_count;
|
|
1703
|
+
__shared__ OpDisc func_id;
|
|
1704
|
+
|
|
1705
|
+
unsigned int active = __activemask();
|
|
1706
|
+
|
|
1707
|
+
__syncwarp(active);
|
|
1708
|
+
|
|
1709
|
+
if(util::current_leader()){
|
|
1710
|
+
|
|
1711
|
+
q_printf("Consuming link %d\n",link_index.adr);
|
|
1712
|
+
|
|
1713
|
+
the_index = link_index;
|
|
1714
|
+
add_count = _dev_ctx.arena[link_index].count;
|
|
1715
|
+
func_id = _dev_ctx.arena[link_index].id;
|
|
1716
|
+
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
__syncwarp(active);
|
|
1721
|
+
|
|
1722
|
+
#if 0
|
|
1723
|
+
if(threadIdx.x < add_count){
|
|
1724
|
+
async_call(func_id,0,_dev_ctx.arena[the_index].promises[threadIdx.x]);
|
|
1725
|
+
}
|
|
1726
|
+
#else
|
|
1727
|
+
unsigned int idx = util::warp_inc_scan();
|
|
1728
|
+
unsigned int tot = util::active_count();
|
|
1729
|
+
for(unsigned int i=idx; i<add_count; i+=tot){
|
|
1730
|
+
async_call(func_id,0,_dev_ctx.arena[the_index].promises[i]);
|
|
1731
|
+
}
|
|
1732
|
+
#endif
|
|
1733
|
+
|
|
1734
|
+
|
|
1735
|
+
__syncwarp(active);
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
if(util::current_leader()){
|
|
1739
|
+
insert_stash_link(link_index);
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
return add_count;
|
|
1743
|
+
|
|
1744
|
+
|
|
1745
|
+
#else
|
|
1746
|
+
|
|
1747
|
+
LinkAdrType the_index;
|
|
1748
|
+
unsigned int add_count;
|
|
1749
|
+
OpDisc func_id;
|
|
1750
|
+
|
|
1751
|
+
unsigned int active = __activemask();
|
|
1752
|
+
unsigned int acount = util::active_count();
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
|
|
1756
|
+
the_index = link_index;
|
|
1757
|
+
add_count = _dev_ctx.arena[link_index].count;
|
|
1758
|
+
func_id = _dev_ctx.arena[link_index].id;
|
|
1759
|
+
|
|
1760
|
+
//_grp_ctx.SM_promise_delta -= add_count;
|
|
1761
|
+
|
|
1762
|
+
db_printf("active count: %d, add count: %d\n",acount,add_count);
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
db_printf("\n\nprior stash count: %d\n\n\n",_grp_ctx.main_queue.count);
|
|
1766
|
+
//*
|
|
1767
|
+
for(unsigned int i=0; i< add_count; i++){
|
|
1768
|
+
//PromiseUnionType promise = _dev_ctx.arena[the_index].promises[i];
|
|
1769
|
+
//async_call(func_id,0,promise);
|
|
1770
|
+
async_call(func_id,0, _dev_ctx.arena[the_index].promises[i] );
|
|
1771
|
+
}
|
|
1772
|
+
// */
|
|
1773
|
+
//PromiseType promise = _dev_ctx.arena[the_index].data.data[0];
|
|
1774
|
+
//async_call(func_id,0,promise);
|
|
1775
|
+
|
|
1776
|
+
db_printf("\n\nafter stash count: %d\n\n\n",_grp_ctx.main_queue.count);
|
|
1777
|
+
|
|
1778
|
+
|
|
1779
|
+
insert_stash_link(link_index);
|
|
1780
|
+
|
|
1781
|
+
return add_count;
|
|
1782
|
+
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
#endif
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
|
|
1794
|
+
|
|
1795
|
+
/*
|
|
1796
|
+
// Tries to transfer links from the stack into the stash of the work group until the stash
|
|
1797
|
+
// is filled to the given threashold. If a halting condition is reached, this function will set
|
|
1798
|
+
// the keep_running value in the shared context to false.
|
|
1799
|
+
*/
|
|
1800
|
+
__device__ void fill_stash(unsigned int threashold, bool halt_on_fail){
|
|
1801
|
+
|
|
1802
|
+
unsigned int active =__activemask();
|
|
1803
|
+
__syncwarp(active);
|
|
1804
|
+
|
|
1805
|
+
|
|
1806
|
+
#ifdef PARACON
|
|
1807
|
+
__shared__ unsigned int link_count;
|
|
1808
|
+
__shared__ LinkAdrType links[STASH_SIZE];
|
|
1809
|
+
#endif
|
|
1810
|
+
|
|
1811
|
+
/*
|
|
1812
|
+
// Currently implemented in a single-threaded manner per work group to simplify the initial
|
|
1813
|
+
// correctness checking process. This can later be changed to take advantage of in-group
|
|
1814
|
+
// parallelism.
|
|
1815
|
+
*/
|
|
1816
|
+
if(util::current_leader()){
|
|
1817
|
+
|
|
1818
|
+
//db_printf("Filling stash...\n");
|
|
1819
|
+
|
|
1820
|
+
unsigned int taken = 0;
|
|
1821
|
+
|
|
1822
|
+
beg_time(12);
|
|
1823
|
+
threashold = (threashold > STASH_SIZE) ? STASH_SIZE : threashold;
|
|
1824
|
+
//printf("{Filling to %d @ %d}",threashold,blockIdx.x);
|
|
1825
|
+
|
|
1826
|
+
unsigned int gather_count = (threashold < _grp_ctx.main_queue.count) ? 0 : threashold - _grp_ctx.main_queue.count;
|
|
1827
|
+
if( (STASH_SIZE - _grp_ctx.link_stash_count) < gather_count){
|
|
1828
|
+
unsigned int spill_thresh = STASH_SIZE - gather_count;
|
|
1829
|
+
spill_stash_links(spill_thresh);
|
|
1830
|
+
}
|
|
1831
|
+
end_time(12);
|
|
1832
|
+
|
|
1833
|
+
|
|
1834
|
+
#ifdef PARACON
|
|
1835
|
+
unsigned int _thd_ctx_link_count = 0;
|
|
1836
|
+
#endif
|
|
1837
|
+
|
|
1838
|
+
#ifdef RACE_COND_PRINT
|
|
1839
|
+
unsigned int p_depth_live = _dev_ctx.stack->depth_live;
|
|
1840
|
+
rc_printf("SM %d: depth_live is (%d,%d)\n",threadIdx.x,(p_depth_live&0xFFFF0000)>>16,p_depth_live&0xFFFF);
|
|
1841
|
+
#endif
|
|
1842
|
+
|
|
1843
|
+
for(unsigned int i = 0; i < FILL_STASH_RETRY_LIMIT; i++){
|
|
1844
|
+
|
|
1845
|
+
/* If the stack is empty or a flag is set, return false */
|
|
1846
|
+
unsigned int depth_live = _dev_ctx.stack->depth_live;
|
|
1847
|
+
if( (depth_live == 0u) || ( _dev_ctx.stack->status_flags != 0u) ){
|
|
1848
|
+
if( halt_on_fail || ( _dev_ctx.stack->status_flags != 0u) ) {
|
|
1849
|
+
_grp_ctx.keep_running = false;
|
|
1850
|
+
}
|
|
1851
|
+
break;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
|
|
1855
|
+
unsigned int src_index;
|
|
1856
|
+
QueueType queue;
|
|
1857
|
+
|
|
1858
|
+
beg_time(3);
|
|
1859
|
+
#if DEF_STACK_MODE == 0
|
|
1860
|
+
|
|
1861
|
+
db_printf("STACK MODE ZERO\n");
|
|
1862
|
+
q_printf("%dth try pulling promises for fill\n",i+1);
|
|
1863
|
+
if( get_frame(_grp_ctx.level).children_residents.data != 0 ){
|
|
1864
|
+
queue = pull_promises(_grp_ctx.level,src_index);
|
|
1865
|
+
} else {
|
|
1866
|
+
queue.pair.data = QueueType::null;
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
#else
|
|
1870
|
+
/*
|
|
1871
|
+
// Determine whether or not to pull from the current level in the stack
|
|
1872
|
+
*/
|
|
1873
|
+
unsigned int depth = left_half(depth_live);
|
|
1874
|
+
bool pull_any = (depth < _grp_ctx.level);
|
|
1875
|
+
FrameType ¤t_frame = get_frame(depth);
|
|
1876
|
+
if(!pull_any){
|
|
1877
|
+
pull_any = ((current_frame.children_residents.get_right()) == 0);
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
/*
|
|
1882
|
+
// Retrieve a queue from the stack.
|
|
1883
|
+
*/
|
|
1884
|
+
|
|
1885
|
+
if(pull_any){
|
|
1886
|
+
unsigned int new_level;
|
|
1887
|
+
queue = pull_promises_any_level(new_level,src_index);
|
|
1888
|
+
relevel_stash(new_level);
|
|
1889
|
+
} else {
|
|
1890
|
+
queue = pull_promises(_grp_ctx.level,src_index);
|
|
1891
|
+
}
|
|
1892
|
+
#endif
|
|
1893
|
+
end_time(3);
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
beg_time(11);
|
|
1897
|
+
#ifdef PARACON
|
|
1898
|
+
db_printf("About to pop promises\n");
|
|
1899
|
+
while( ( ! queue.is_null() )
|
|
1900
|
+
&& (_thd_ctx_link_count < gather_count)
|
|
1901
|
+
&& (_grp_ctx.link_stash_count < STASH_SIZE)
|
|
1902
|
+
){
|
|
1903
|
+
beg_time(13);
|
|
1904
|
+
LinkAdrType link = pop_front(queue);
|
|
1905
|
+
end_time(13);
|
|
1906
|
+
if( ! link.is_null() ){
|
|
1907
|
+
beg_time(14);
|
|
1908
|
+
db_printf("Popping front %d\n",link);
|
|
1909
|
+
links[_thd_ctx_link_count] = link;
|
|
1910
|
+
taken += _dev_ctx.arena[link].count;
|
|
1911
|
+
_thd_ctx_link_count++;
|
|
1912
|
+
end_time(14);
|
|
1913
|
+
} else {
|
|
1914
|
+
break;
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
#else
|
|
1918
|
+
db_printf("About to pop promises\n");
|
|
1919
|
+
while( ( ! queue.is_null() )
|
|
1920
|
+
&& (_grp_ctx.main_queue.count < threashold)
|
|
1921
|
+
&& (_grp_ctx.link_stash_count < STASH_SIZE)
|
|
1922
|
+
){
|
|
1923
|
+
beg_time(13);
|
|
1924
|
+
LinkAdrType link = pop_front(queue);
|
|
1925
|
+
end_time(13);
|
|
1926
|
+
|
|
1927
|
+
q_printf("Popping front %d. Q is now (%d,%d)\n",link.adr,queue.get_head().adr,queue.get_tail().adr);
|
|
1928
|
+
|
|
1929
|
+
if( ! link.is_null() ){
|
|
1930
|
+
beg_time(14);
|
|
1931
|
+
taken += consume_link(link);
|
|
1932
|
+
end_time(14);
|
|
1933
|
+
} else {
|
|
1934
|
+
break;
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
#endif
|
|
1938
|
+
end_time(11);
|
|
1939
|
+
|
|
1940
|
+
db_printf("Popped promises\n");
|
|
1941
|
+
if(taken != 0){
|
|
1942
|
+
if(!_grp_ctx.busy){
|
|
1943
|
+
atomicAdd(&(_dev_ctx.stack->depth_live),1);
|
|
1944
|
+
_grp_ctx.busy = true;
|
|
1945
|
+
//printf("{got busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
|
|
1946
|
+
rc_printf("SM %d: Incremented depth value\n",threadIdx.x);
|
|
1947
|
+
}
|
|
1948
|
+
rc_printf("Pushing promises for filling\n");
|
|
1949
|
+
push_promises(_grp_ctx.level,src_index,queue,-taken);
|
|
1950
|
+
break;
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
#ifdef PARACON
|
|
1954
|
+
if( _thd_ctx_link_count >= gather_count ){
|
|
1955
|
+
break;
|
|
1956
|
+
}
|
|
1957
|
+
#else
|
|
1958
|
+
if( _grp_ctx.main_queue.count >= threashold ){
|
|
1959
|
+
break;
|
|
1960
|
+
}
|
|
1961
|
+
#endif
|
|
1962
|
+
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
|
|
1969
|
+
#ifdef PARACON
|
|
1970
|
+
if(_grp_ctx.busy && (_grp_ctx.main_queue.count == 0) && (taken == 0) ){
|
|
1971
|
+
#else
|
|
1972
|
+
if(_grp_ctx.busy && (_grp_ctx.main_queue.count == 0)){
|
|
1973
|
+
#endif
|
|
1974
|
+
unsigned int depth_live = atomicSub(&(_dev_ctx.stack->depth_live),1);
|
|
1975
|
+
//printf("{unbusy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
|
|
1976
|
+
rc_printf("SM %d: Decremented depth value\n",threadIdx.x);
|
|
1977
|
+
_grp_ctx.busy = false;
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
|
|
1981
|
+
#ifdef PARACON
|
|
1982
|
+
link_count = _thd_ctx_link_count;
|
|
1983
|
+
#endif
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
__syncwarp(active);
|
|
1989
|
+
|
|
1990
|
+
|
|
1991
|
+
|
|
1992
|
+
#ifdef PARACON
|
|
1993
|
+
|
|
1994
|
+
__threadfence();
|
|
1995
|
+
beg_time(15);
|
|
1996
|
+
if(util::current_leader()){
|
|
1997
|
+
for(int i=0; i<link_count;i++){
|
|
1998
|
+
consume_link(links[i]);
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
end_time(15);
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
__syncwarp(active);
|
|
2005
|
+
__threadfence();
|
|
2006
|
+
#endif
|
|
2007
|
+
|
|
2008
|
+
|
|
2009
|
+
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
__device__ void clear_exec_head(){
|
|
2016
|
+
|
|
2017
|
+
|
|
2018
|
+
if( util::current_leader() && (_grp_ctx.exec_head != STASH_SIZE) ){
|
|
2019
|
+
insert_empty_slot(_grp_ctx.exec_head);
|
|
2020
|
+
_grp_ctx.exec_head = STASH_SIZE;
|
|
2021
|
+
}
|
|
2022
|
+
__syncwarp();
|
|
2023
|
+
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
|
|
2029
|
+
/*
|
|
2030
|
+
// Selects the next link in the stash. This selection process could become more sophisticated
|
|
2031
|
+
// in later version to account for the average branching factor of each async function. For now,
|
|
2032
|
+
// it selects the fullest slot of the current level if it can. If no slots with promises for the
|
|
2033
|
+
// current level exist in the stash, the function returns false.
|
|
2034
|
+
*/
|
|
2035
|
+
__device__ bool advance_stash_iter(){
|
|
2036
|
+
|
|
2037
|
+
__shared__ bool result;
|
|
2038
|
+
unsigned int active =__activemask();
|
|
2039
|
+
__syncwarp(active);
|
|
2040
|
+
|
|
2041
|
+
|
|
2042
|
+
if(util::current_leader()){
|
|
2043
|
+
|
|
2044
|
+
if(_grp_ctx.main_queue.full_head != STASH_SIZE){
|
|
2045
|
+
_grp_ctx.exec_head = claim_full_slot();
|
|
2046
|
+
_grp_ctx.main_queue.count -= 1;
|
|
2047
|
+
result = true;
|
|
2048
|
+
//db_printf("Found full slot.\n");
|
|
2049
|
+
} else {
|
|
2050
|
+
//db_printf("Looking for partial slot...\n");
|
|
2051
|
+
unsigned int best_id = PART_ENTRY_COUNT;
|
|
2052
|
+
unsigned int best_slot = STASH_SIZE;
|
|
2053
|
+
unsigned int best_count = 0;
|
|
2054
|
+
for(int i=0; i < FN_ID_COUNT; i++){
|
|
2055
|
+
unsigned int slot = _grp_ctx.main_queue.partial_map[i];
|
|
2056
|
+
|
|
2057
|
+
if( (slot != STASH_SIZE) && (_grp_ctx.stash[slot].count > best_count)){
|
|
2058
|
+
best_id = i;
|
|
2059
|
+
best_slot = slot;
|
|
2060
|
+
best_count = _grp_ctx.stash[slot].count;
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
}
|
|
2064
|
+
|
|
2065
|
+
result = (best_slot != STASH_SIZE);
|
|
2066
|
+
if(result){
|
|
2067
|
+
//db_printf("Found partial slot.\n");
|
|
2068
|
+
_grp_ctx.exec_head = best_slot;
|
|
2069
|
+
_grp_ctx.main_queue.partial_map[best_id] = STASH_SIZE;
|
|
2070
|
+
_grp_ctx.main_queue.count -=1;
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
__syncwarp(active);
|
|
2077
|
+
return result;
|
|
2078
|
+
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
|
|
2082
|
+
#ifdef ASYNC_LOADS
|
|
2083
|
+
__device__ void force_full_async_loads() {
|
|
2084
|
+
|
|
2085
|
+
_grp_ctx.load_barrier.arrive_and_wait();
|
|
2086
|
+
__syncwarp();
|
|
2087
|
+
#if 1
|
|
2088
|
+
if (util::current_leader()) {
|
|
2089
|
+
unsigned char iter = _grp_ctx.load_queue.full_head;
|
|
2090
|
+
if( iter != STASH_SIZE ){
|
|
2091
|
+
unsigned int main_delta = 1;
|
|
2092
|
+
unsigned char next = _grp_ctx.stash[iter].next.adr;
|
|
2093
|
+
while( next != STASH_SIZE ){
|
|
2094
|
+
iter = next;
|
|
2095
|
+
next = _grp_ctx.stash[iter].next.adr;
|
|
2096
|
+
main_delta += 1;
|
|
2097
|
+
}
|
|
2098
|
+
_grp_ctx.stash[iter].next = _grp_ctx.main_queue.full_head;
|
|
2099
|
+
_grp_ctx.main_queue.full_head = _grp_ctx.load_queue.full_head;
|
|
2100
|
+
_grp_ctx.load_queue.full_head = STASH_SIZE;
|
|
2101
|
+
_grp_ctx.main_queue.count += main_delta;
|
|
2102
|
+
_grp_ctx.load_queue.count -= main_delta;
|
|
2103
|
+
}
|
|
2104
|
+
//printf("(%d:f-%d)",blockIdx.x,_grp_ctx.main_queue.count);
|
|
2105
|
+
}
|
|
2106
|
+
#else
|
|
2107
|
+
if( util::current_leader() ){
|
|
2108
|
+
_grp_ctx.main_queue.full_head = _grp_ctx.load_queue.full_head;
|
|
2109
|
+
_grp_ctx.load_queue.full_head = STASH_SIZE;
|
|
2110
|
+
_grp_ctx.main_queue.count += _grp_ctx.load_queue.count;
|
|
2111
|
+
_grp_ctx.load_queue.count = 0;
|
|
2112
|
+
printf("(%d:f-%d)",blockIdx.x,_grp_ctx.main_queue.count);
|
|
2113
|
+
}
|
|
2114
|
+
#endif
|
|
2115
|
+
__syncwarp();
|
|
2116
|
+
|
|
2117
|
+
}
|
|
2118
|
+
__device__ void force_async_loads() {
|
|
2119
|
+
|
|
2120
|
+
force_full_async_loads();
|
|
2121
|
+
|
|
2122
|
+
for(int i=0; i < FN_ID_COUNT; i++){
|
|
2123
|
+
unsigned char idx = _grp_ctx.load_queue.partial_map[i];
|
|
2124
|
+
if( idx != STASH_SIZE){
|
|
2125
|
+
LinkType& link = _grp_ctx.stash[idx];
|
|
2126
|
+
if( threadIdx.x < link.count ){
|
|
2127
|
+
async_call(0,link.id,link.promises[threadIdx.x]);
|
|
2128
|
+
}
|
|
2129
|
+
__syncwarp();
|
|
2130
|
+
if( util::current_leader() ){
|
|
2131
|
+
insert_empty_slot(idx);
|
|
2132
|
+
_grp_ctx.load_queue.partial_map[i] = STASH_SIZE;
|
|
2133
|
+
//printf("(%d:%d)\n",blockIdx.x,idx);
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
__syncwarp();
|
|
2138
|
+
if(util::current_leader()){
|
|
2139
|
+
init(&_grp_ctx.load_barrier,WORK_GROUP_SIZE);
|
|
2140
|
+
_grp_ctx.load_queue.count = 0;
|
|
2141
|
+
}
|
|
2142
|
+
|
|
2143
|
+
}
|
|
2144
|
+
#endif
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
|
|
2148
|
+
/*
|
|
2149
|
+
// Tries to perform up to one work group worth of work by selecting a link from shared memory (or,
|
|
2150
|
+
// if necessary, fetching a link from main memory), and running the function on the data within
|
|
2151
|
+
// the link, as directed by the function id the link is labeled with. This function returns false
|
|
2152
|
+
// if a halting condition has been reached (either due to lack of work or an event) and true
|
|
2153
|
+
// otherwise.
|
|
2154
|
+
*/
|
|
2155
|
+
__device__ void exec_cycle(){
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
|
|
2159
|
+
clear_exec_head();
|
|
2160
|
+
|
|
2161
|
+
/*
|
|
2162
|
+
// Advance the stash iterator to the next chunk of work that needs to be done.
|
|
2163
|
+
*/
|
|
2164
|
+
//*
|
|
2165
|
+
|
|
2166
|
+
|
|
2167
|
+
if( stash_overfilled() ) {
|
|
2168
|
+
//printf("{Stash overfilled}");
|
|
2169
|
+
#ifdef BARRIER_SPILL
|
|
2170
|
+
if( util::current_leader() ){
|
|
2171
|
+
_grp_ctx.spill_barrier.release_full(*this);
|
|
2172
|
+
}
|
|
2173
|
+
#endif
|
|
2174
|
+
|
|
2175
|
+
if(_grp_ctx.link_stash_count < STASH_MARGIN){
|
|
2176
|
+
fill_stash_links(STASH_MARGIN);
|
|
2177
|
+
}
|
|
2178
|
+
//printf("{Spilling for call.}");
|
|
2179
|
+
spill_stash(STASH_HIGH_WATER-1);
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
|
|
2183
|
+
beg_time(1);
|
|
2184
|
+
|
|
2185
|
+
#ifdef ASYNC_LOADS
|
|
2186
|
+
if( (_grp_ctx.main_queue.full_head == STASH_SIZE) && (_grp_ctx.load_queue.full_head != STASH_SIZE) ){
|
|
2187
|
+
force_full_async_loads();
|
|
2188
|
+
}
|
|
2189
|
+
#endif
|
|
2190
|
+
|
|
2191
|
+
const PromiseCount GLOBAL_WORK_THRESHOLD = STASH_SIZE/2;
|
|
2192
|
+
//if ( ( ((_dev_ctx.stack->frames[0].children_residents) & 0xFFFF ) > (gridDim.x*blockIdx.x*2) ) && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
|
|
2193
|
+
if ( ( ((_dev_ctx.stack->frames[0].children_residents.get_right()) ) > GLOBAL_WORK_THRESHOLD ) && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
|
|
2194
|
+
fill_stash(STASH_HIGH_WATER,false);
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
end_time(1);
|
|
2198
|
+
|
|
2199
|
+
beg_time(5);
|
|
2200
|
+
|
|
2201
|
+
if (_grp_ctx.main_queue.full_head == STASH_SIZE){
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
#ifdef EAGER_FILLING
|
|
2205
|
+
const PromiseCount GLOBAL_WORK_THRESHOLD = STASH_SIZE/2;
|
|
2206
|
+
if( ((_dev_ctx.stack->frames[0].children_residents.get_right()) ) > GLOBAL_WORK_THRESHOLD ) {
|
|
2207
|
+
fill_stash(STASH_HIGH_WATER,false);
|
|
2208
|
+
} else {
|
|
2209
|
+
#endif
|
|
2210
|
+
if ( _grp_ctx.can_make_work && (_grp_ctx.main_queue.full_head == STASH_SIZE) ) {
|
|
2211
|
+
_grp_ctx.can_make_work = __any_sync(0xFFFFFFFF,PROGRAM_SPEC::make_work(*this));
|
|
2212
|
+
if( util::current_leader() && (! _grp_ctx.busy ) && ( _grp_ctx.main_queue.count != 0 ) ){
|
|
2213
|
+
unsigned int depth_live = atomicAdd(&(_dev_ctx.stack->depth_live),1);
|
|
2214
|
+
_grp_ctx.busy = true;
|
|
2215
|
+
//printf("{made self busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
|
|
2216
|
+
}
|
|
2217
|
+
__syncwarp();
|
|
2218
|
+
}
|
|
2219
|
+
#ifdef EAGER_FILLING
|
|
2220
|
+
}
|
|
2221
|
+
#endif
|
|
2222
|
+
}
|
|
2223
|
+
|
|
2224
|
+
end_time(5);
|
|
2225
|
+
|
|
2226
|
+
#ifdef ASYNC_LOADS
|
|
2227
|
+
if( (_grp_ctx.main_queue.full_head == STASH_SIZE) && (_grp_ctx.load_queue.count > 0) ){
|
|
2228
|
+
force_async_loads();
|
|
2229
|
+
}
|
|
2230
|
+
#endif
|
|
2231
|
+
|
|
2232
|
+
#if 1
|
|
2233
|
+
|
|
2234
|
+
/*
|
|
2235
|
+
if ( _grp_ctx.main_queue.full_head == STASH_SIZE ) {
|
|
2236
|
+
fill_stash(STASH_SIZE-2);
|
|
2237
|
+
}
|
|
2238
|
+
*/
|
|
2239
|
+
#else
|
|
2240
|
+
if(_grp_ctx.main_queue.full_head == STASH_SIZE){
|
|
2241
|
+
if( !_grp_ctx.scarce_work ){
|
|
2242
|
+
fill_stash(STASH_SIZE-2);
|
|
2243
|
+
if( util::current_leader() && (_grp_ctx.main_queue.full_head == STASH_SIZE) ){
|
|
2244
|
+
_grp_ctx.scarce_work = true;
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
} else {
|
|
2248
|
+
if( util::current_leader() ){
|
|
2249
|
+
_grp_ctx.scarce_work = false;
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
#endif
|
|
2253
|
+
// */
|
|
2254
|
+
|
|
2255
|
+
#if 0
|
|
2256
|
+
if( util::current_leader() ){
|
|
2257
|
+
PairPack<PromiseCount> cr = _dev_ctx.stack->frames[0].children_residents;
|
|
2258
|
+
printf("{group %d children_residents=(%d,%d), can_make_work=%d}\n",blockIdx.x,cr.get_left(),cr.get_right(),_grp_ctx.can_make_work);
|
|
2259
|
+
}
|
|
2260
|
+
#endif
|
|
2261
|
+
beg_time(2);
|
|
2262
|
+
if( !advance_stash_iter() ){
|
|
2263
|
+
/*
|
|
2264
|
+
// No more work exists in the stash, so try to fetch it from the stack.
|
|
2265
|
+
*/
|
|
2266
|
+
beg_time(10);
|
|
2267
|
+
fill_stash(STASH_HIGH_WATER,true);
|
|
2268
|
+
end_time(10);
|
|
2269
|
+
|
|
2270
|
+
if( _grp_ctx.keep_running && !advance_stash_iter() ){
|
|
2271
|
+
/*
|
|
2272
|
+
// REALLY BAD: The fill_stash function was successful, however
|
|
2273
|
+
// the stash still has no work to perform. In this situation,
|
|
2274
|
+
// we set an error flag and halt.
|
|
2275
|
+
*/
|
|
2276
|
+
/*
|
|
2277
|
+
if(util::current_leader()){
|
|
2278
|
+
db_printf("\nBad stuff afoot!\n\n");
|
|
2279
|
+
}
|
|
2280
|
+
set_flags(_grp_ctx,STASH_FAIL_FLAG);
|
|
2281
|
+
_grp_ctx.keep_running = false;
|
|
2282
|
+
*/
|
|
2283
|
+
}
|
|
2284
|
+
}
|
|
2285
|
+
end_time(2);
|
|
2286
|
+
|
|
2287
|
+
|
|
2288
|
+
unsigned int active = __activemask();
|
|
2289
|
+
__syncwarp(active);
|
|
2290
|
+
|
|
2291
|
+
|
|
2292
|
+
beg_time(4);
|
|
2293
|
+
if( _grp_ctx.exec_head != STASH_SIZE ){
|
|
2294
|
+
/*
|
|
2295
|
+
// Find which function the current link corresponds to.
|
|
2296
|
+
*/
|
|
2297
|
+
OpDisc func_id = _grp_ctx.stash[_grp_ctx.exec_head].id;
|
|
2298
|
+
unsigned int promise_count = _grp_ctx.stash[_grp_ctx.exec_head].count;
|
|
2299
|
+
|
|
2300
|
+
/*
|
|
2301
|
+
// Only execute if there is a promise in the current link corresponding to the thread that
|
|
2302
|
+
// is being executed.
|
|
2303
|
+
*/
|
|
2304
|
+
if(util::current_leader()){
|
|
2305
|
+
db_printf("Executing slot %d, which is %d promises of type %d\n",_grp_ctx.exec_head,promise_count,func_id);
|
|
2306
|
+
}
|
|
2307
|
+
if( threadIdx.x < promise_count ){
|
|
2308
|
+
//db_printf("Executing...\n");
|
|
2309
|
+
PromiseUnionType& promise = _grp_ctx.stash[_grp_ctx.exec_head].promises[threadIdx.x];
|
|
2310
|
+
//do_async(func_id,promise);
|
|
2311
|
+
promise.template loose_eval<ProgramType>(*this,func_id);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
__syncwarp(active);
|
|
2316
|
+
end_time(4);
|
|
2317
|
+
|
|
2318
|
+
|
|
2319
|
+
}
|
|
2320
|
+
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
__device__ void cleanup_runtime(){
|
|
2324
|
+
|
|
2325
|
+
|
|
2326
|
+
//unsigned int active = __activemask();
|
|
2327
|
+
//__syncwarp(active);
|
|
2328
|
+
__syncwarp();
|
|
2329
|
+
|
|
2330
|
+
|
|
2331
|
+
if(_thd_ctx.spare_index > 0){
|
|
2332
|
+
dealloc_links(_thd_ctx.spare_links,_thd_ctx.spare_index);
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
if(threadIdx.x == 0){
|
|
2336
|
+
|
|
2337
|
+
#ifdef BARRIER_SPILL
|
|
2338
|
+
_grp_ctx.spill_barrier.release(*this);
|
|
2339
|
+
#endif
|
|
2340
|
+
|
|
2341
|
+
q_printf("CLEANING UP\n");
|
|
2342
|
+
clear_exec_head();
|
|
2343
|
+
|
|
2344
|
+
spill_stash(0);
|
|
2345
|
+
spill_stash_links(0);
|
|
2346
|
+
|
|
2347
|
+
if(_grp_ctx.can_make_work){
|
|
2348
|
+
//printf("{Setting early halt flag.}");
|
|
2349
|
+
set_flags(EARLY_HALT_FLAG);
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
if(_grp_ctx.busy){
|
|
2353
|
+
unsigned int depth_live = atomicSub(&(_dev_ctx.stack->depth_live),1);
|
|
2354
|
+
//printf("{wrap busy %d depth_live=(%d,%d)}",blockIdx.x,(depth_live & 0xFFFF0000)>>16u, depth_live & 0xFFFF);
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
//__syncwarp(active);
|
|
2359
|
+
__syncwarp();
|
|
2360
|
+
__threadfence();
|
|
2361
|
+
//__syncwarp(active);
|
|
2362
|
+
__syncwarp();
|
|
2363
|
+
|
|
2364
|
+
if(threadIdx.x == 0){
|
|
2365
|
+
unsigned int checkout_index = atomicAdd(&(_dev_ctx.stack->checkout),1);
|
|
2366
|
+
__threadfence();
|
|
2367
|
+
if( checkout_index == (gridDim.x-1) ){
|
|
2368
|
+
//printf("{Final}\n");
|
|
2369
|
+
atomicExch(&(_dev_ctx.stack->checkout),0);
|
|
2370
|
+
unsigned int old_flags = atomicAnd(&(_dev_ctx.stack->status_flags),~EARLY_HALT_FLAG);
|
|
2371
|
+
unsigned int depth_live = atomicAdd(&(_dev_ctx.stack->depth_live),0);
|
|
2372
|
+
bool halted_early = ( old_flags & EARLY_HALT_FLAG );
|
|
2373
|
+
bool work_left = ( (depth_live & 0xFFFF0000) != 0 );
|
|
2374
|
+
|
|
2375
|
+
if( (!halted_early) && (!work_left) ){
|
|
2376
|
+
set_flags(COMPLETION_FLAG);
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2379
|
+
//printf("{depth_live is (%d,%d)}",(depth_live&0xFFFF0000)>>16,depth_live&0xFFFF );
|
|
2380
|
+
//unsigned int cr = atomicAdd(&(_dev_ctx.stack->frames[0].children_residents),0);
|
|
2381
|
+
//printf("{Level 0 CR is (%d,%d)}",(cr&0xFFFF0000)>>16,cr&0xFFFF );
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
#ifdef HRM_TIME
|
|
2385
|
+
end_time(0);
|
|
2386
|
+
for(int i=0; i<HRM_TIME; i++){
|
|
2387
|
+
atomicAdd(&_dev_ctx.time_totals[i],_grp_ctx.time_totals[i]);
|
|
2388
|
+
}
|
|
2389
|
+
#endif
|
|
2390
|
+
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
public:
|
|
2397
|
+
/*
|
|
2398
|
+
//
|
|
2399
|
+
// This must be run once on the resources used for execution, prior to execution. Given that this
|
|
2400
|
+
// essentially wipes all data from these resources and zeros all values, it is not advised that
|
|
2401
|
+
// this function be used at any other time, except to setup for a re-start or to clear out after
|
|
2402
|
+
// calling the pull_runtime to prevent promise duplication.
|
|
2403
|
+
//
|
|
2404
|
+
*/
|
|
2405
|
+
__device__ void init_program(){
|
|
2406
|
+
|
|
2407
|
+
/* Initialize per-thread resources */
|
|
2408
|
+
init_thread();
|
|
2409
|
+
|
|
2410
|
+
const unsigned int threads_per_frame = FRAME_SIZE + 1;
|
|
2411
|
+
const unsigned int total_stack_work = StackType::NULL_LEVEL * threads_per_frame;
|
|
2412
|
+
|
|
2413
|
+
unsigned int worker_count = gridDim.x*blockDim.x;
|
|
2414
|
+
|
|
2415
|
+
/*
|
|
2416
|
+
// If the currently executing thread has device thread index 0, wipe the data in the base
|
|
2417
|
+
// of the stack.
|
|
2418
|
+
*/
|
|
2419
|
+
if(_thd_ctx.thread_id == 0){
|
|
2420
|
+
_dev_ctx.stack->status_flags = 0;
|
|
2421
|
+
_dev_ctx.stack->depth_live = 0;
|
|
2422
|
+
_dev_ctx.stack->checkout = 0;
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
|
|
2426
|
+
/*
|
|
2427
|
+
if( _thd_ctx.thread_id == 0 ){
|
|
2428
|
+
printf( "Initializing the stack with\n"
|
|
2429
|
+
"\t- total_stack_work=%d\n"
|
|
2430
|
+
"\t- threads_per_frame=%d\n"
|
|
2431
|
+
"\t- worker_count=%d\n"
|
|
2432
|
+
"\t- stack->frames[0].children_residents=%d\n",
|
|
2433
|
+
total_stack_work,
|
|
2434
|
+
threads_per_frame,
|
|
2435
|
+
worker_count,
|
|
2436
|
+
_dev_ctx.stack->frames[0].children_residents
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
*/
|
|
2440
|
+
|
|
2441
|
+
|
|
2442
|
+
|
|
2443
|
+
/*
|
|
2444
|
+
// Blank out the frames in the stack. Setting queues to NULL_QUEUE, and zeroing the counts
|
|
2445
|
+
// for resident promises and child promises of each frame.
|
|
2446
|
+
*/
|
|
2447
|
+
for(unsigned int index = _thd_ctx.thread_id; index < total_stack_work; index+=worker_count ){
|
|
2448
|
+
|
|
2449
|
+
unsigned int target_level = index / threads_per_frame;
|
|
2450
|
+
unsigned int frame_index = index % threads_per_frame;
|
|
2451
|
+
if( frame_index == FRAME_SIZE ){
|
|
2452
|
+
_dev_ctx.stack->frames[target_level].children_residents.data = 0u;
|
|
2453
|
+
if( _dev_ctx.stack->frames[target_level].children_residents.data != 0u ){
|
|
2454
|
+
printf("BAD A!\n");
|
|
2455
|
+
}
|
|
2456
|
+
} else {
|
|
2457
|
+
_dev_ctx.stack->frames[target_level].pool.queues[frame_index].pair.data = QueueType::null;
|
|
2458
|
+
if( _dev_ctx.stack->frames[target_level].pool.queues[frame_index].pair.data != QueueType::null ) {
|
|
2459
|
+
printf("BAD B!\n");
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
|
|
2466
|
+
#ifdef LAZY_LINK
|
|
2467
|
+
|
|
2468
|
+
if(_thd_ctx.thread_id == 0){
|
|
2469
|
+
(*_dev_ctx.claim_count) = 0;
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
/*
|
|
2473
|
+
// Initialize the pool, assigning empty queues to each queue slot.
|
|
2474
|
+
*/
|
|
2475
|
+
for(unsigned int index = _thd_ctx.thread_id; index < POOL_SIZE; index+=worker_count ){
|
|
2476
|
+
|
|
2477
|
+
_dev_ctx.pool->queues[index].pair.data = QueueType::null;
|
|
2478
|
+
if( _dev_ctx.pool->queues[index].pair.data != QueueType::null ){
|
|
2479
|
+
printf("BAD C!\n");
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
}
|
|
2483
|
+
|
|
2484
|
+
#else
|
|
2485
|
+
/*
|
|
2486
|
+
// Initialize the arena, connecting the contained links into roughly equally sized lists,
|
|
2487
|
+
// zeroing the promise counter in the links and marking the function ID with an invalid
|
|
2488
|
+
// value to make use-before-initialization more obvious during system validation.
|
|
2489
|
+
*/
|
|
2490
|
+
unsigned int bump = ((arena_size%POOL_SIZE) != 0) ? 1 : 0;
|
|
2491
|
+
unsigned int arena_init_stride = arena_size/POOL_SIZE + bump;
|
|
2492
|
+
for(unsigned int index = _thd_ctx.thread_id; index < arena_size; index+=worker_count ){
|
|
2493
|
+
|
|
2494
|
+
unsigned int next = index + 1;
|
|
2495
|
+
if( ( (next % arena_init_stride) == 0 ) || (next >= arena_size) ){
|
|
2496
|
+
next = LinkAdrType::null;
|
|
2497
|
+
}
|
|
2498
|
+
_dev_ctx.arena[index].empty(LinkAdrType(next));
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
|
|
2502
|
+
/*
|
|
2503
|
+
// Initialize the pool, giving each queue slot one of the previously created linked lists.
|
|
2504
|
+
*/
|
|
2505
|
+
for(unsigned int index = _thd_ctx.thread_id; index < POOL_SIZE; index+=worker_count ){
|
|
2506
|
+
|
|
2507
|
+
unsigned int head = arena_init_stride * index;
|
|
2508
|
+
unsigned int tail = arena_init_stride * (index + 1) - 1;
|
|
2509
|
+
tail = (tail >= arena_size) ? arena_size - 1 : tail;
|
|
2510
|
+
_dev_ctx.pool->queues[index] = QueueType(LinkAdrType(head),LinkAdrType(tail));
|
|
2511
|
+
|
|
2512
|
+
}
|
|
2513
|
+
#endif
|
|
2514
|
+
|
|
2515
|
+
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
|
|
2519
|
+
/*
|
|
2520
|
+
// Unpacks all promise data from the call buffer into the stack of the given context. This
|
|
2521
|
+
// could be useful for backing up program states for debugging or to re-start processing from
|
|
2522
|
+
// a previous state.
|
|
2523
|
+
*/
|
|
2524
|
+
__device__ void push_calls(DeviceContext _dev_ctx, LinkType* call_buffer, size_t link_count){
|
|
2525
|
+
|
|
2526
|
+
/* Initialize per-warp resources */
|
|
2527
|
+
__shared__ GroupContext _grp_ctx;
|
|
2528
|
+
init_group(_dev_ctx,_grp_ctx);
|
|
2529
|
+
|
|
2530
|
+
/* Initialize per-thread resources */
|
|
2531
|
+
ThreadContext _thd_ctx;
|
|
2532
|
+
init_local(_thd_ctx);
|
|
2533
|
+
|
|
2534
|
+
|
|
2535
|
+
for(int link_index=blockIdx.x; link_index < link_count; link_index+= gridDim.x){
|
|
2536
|
+
LinkType& the_link = call_buffer[link_index];
|
|
2537
|
+
unsigned int count = the_link.count;
|
|
2538
|
+
unsigned int func_id = the_link.id;
|
|
2539
|
+
if(threadIdx.x < count){
|
|
2540
|
+
db_printf("\nasync_call(id:%d,depth: 0)\n\n",func_id);
|
|
2541
|
+
async_call(func_id,0,the_link.data.data[threadIdx.x]);
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2544
|
+
}
|
|
2545
|
+
|
|
2546
|
+
cleanup_runtime();
|
|
2547
|
+
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
|
|
2551
|
+
|
|
2552
|
+
|
|
2553
|
+
static void check_error(){
|
|
2554
|
+
|
|
2555
|
+
adapt::GPUrtError_t status = adapt::GPUrtGetLastError();
|
|
2556
|
+
|
|
2557
|
+
if(status != adapt::GPUrtSuccess){
|
|
2558
|
+
const char* err_str = adapt::GPUrtGetErrorString(status);
|
|
2559
|
+
printf("ERROR: \"%s\"\n",err_str);
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
}
|
|
2563
|
+
|
|
2564
|
+
|
|
2565
|
+
|
|
2566
|
+
|
|
2567
|
+
|
|
2568
|
+
/*
|
|
2569
|
+
// Places a single function call into the runtime.
|
|
2570
|
+
static void remote_call(Instance &instance, unsigned char func_id, PromiseUnionType promise){
|
|
2571
|
+
|
|
2572
|
+
LinkType* call_buffer;
|
|
2573
|
+
util::host::auto_throw(adapt::GPUrtMalloc( (void**) &call_buffer, sizeof(LinkType) ));
|
|
2574
|
+
|
|
2575
|
+
LinkType host_link;
|
|
2576
|
+
host_link.count = 1;
|
|
2577
|
+
host_link.id = func_id;
|
|
2578
|
+
host_link.next = LinkType::null;
|
|
2579
|
+
host_link.depth = 0;
|
|
2580
|
+
host_link.meta_data.data= 0;
|
|
2581
|
+
host_link.data.data[0] = promise;
|
|
2582
|
+
|
|
2583
|
+
util::host::auto_throw(adapt::GPUrtMemcpy(call_buffer,&host_link,sizeof(LinkType),adapt::GPUrtMemcpyHostToDevice));
|
|
2584
|
+
|
|
2585
|
+
|
|
2586
|
+
push_runtime<<<1,WORK_GROUP_SIZE>>>(instance.to_context(),call_buffer,1);
|
|
2587
|
+
|
|
2588
|
+
check_error();
|
|
2589
|
+
|
|
2590
|
+
util::host::auto_throw(adapt::GPUrtFree(call_buffer));
|
|
2591
|
+
|
|
2592
|
+
}
|
|
2593
|
+
*/
|
|
2594
|
+
|
|
2595
|
+
|
|
2596
|
+
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
/*
|
|
2600
|
+
// Packs all promise data from the runtime stack into the communication buffer (comm_buffer). This
|
|
2601
|
+
// could be useful for backing up program states for debugging or to re-start processing from
|
|
2602
|
+
// a previous state.
|
|
2603
|
+
//
|
|
2604
|
+
// For now, this will not be implemented, as it isn't particularly useful until the system's
|
|
2605
|
+
// correctness has been verified.
|
|
2606
|
+
//
|
|
2607
|
+
*/
|
|
2608
|
+
__device__ void pull_promises(Instance &instance){
|
|
2609
|
+
|
|
2610
|
+
/*
|
|
2611
|
+
// [TODO] NOT YET IMPLEMENTED
|
|
2612
|
+
*/
|
|
2613
|
+
|
|
2614
|
+
}
|
|
2615
|
+
|
|
2616
|
+
|
|
2617
|
+
|
|
2618
|
+
/*
|
|
2619
|
+
// The workhorse of the program. This function executes until either a halting condition
|
|
2620
|
+
// is encountered or a maximum number of processing cycles has occured. This makes sure
|
|
2621
|
+
// that long-running programs don't time out on the GPU. In practice, cycle_count may have
|
|
2622
|
+
// to be tuned to the average cycle execution time for a given application. This could
|
|
2623
|
+
// potentially be automated using an exponential backoff heuristic.
|
|
2624
|
+
*/
|
|
2625
|
+
__device__ void exec(unsigned int cycle_count){
|
|
2626
|
+
|
|
2627
|
+
/* Initialize per-warp resources */
|
|
2628
|
+
init_group();
|
|
2629
|
+
|
|
2630
|
+
/* Initialize per-thread resources */
|
|
2631
|
+
init_thread();
|
|
2632
|
+
|
|
2633
|
+
PROGRAM_SPEC::initialize(*this);
|
|
2634
|
+
|
|
2635
|
+
/*
|
|
2636
|
+
if(util::current_leader()){
|
|
2637
|
+
printf("\n\n\nInitial frame zero resident count is: %d\n\n\n",_dev_ctx.stack->frames[0].children_residents);
|
|
2638
|
+
}
|
|
2639
|
+
*/
|
|
2640
|
+
|
|
2641
|
+
/* The execution loop. */
|
|
2642
|
+
#ifdef RACE_COND_PRINT
|
|
2643
|
+
unsigned int cycle_break = cycle_count;
|
|
2644
|
+
#endif
|
|
2645
|
+
for(unsigned int cycle=0u; cycle<cycle_count; cycle++){
|
|
2646
|
+
/* Early halting handled with a break. */
|
|
2647
|
+
exec_cycle();
|
|
2648
|
+
if(!_grp_ctx.keep_running){
|
|
2649
|
+
#ifdef RACE_COND_PRINT
|
|
2650
|
+
cycle_break = cycle+1;
|
|
2651
|
+
#endif
|
|
2652
|
+
break;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
PROGRAM_SPEC::finalize(*this);
|
|
2657
|
+
|
|
2658
|
+
/*
|
|
2659
|
+
// Ensure that nothing which should persist between dispatches is lost in the
|
|
2660
|
+
// shared or private memory of the halting program.
|
|
2661
|
+
*/
|
|
2662
|
+
cleanup_runtime();
|
|
2663
|
+
|
|
2664
|
+
if(util::current_leader()){
|
|
2665
|
+
rc_printf("SM %d finished after %d cycles with promise delta %d\n",threadIdx.x,cycle_break,_grp_ctx.SM_promise_delta);
|
|
2666
|
+
}
|
|
2667
|
+
|
|
2668
|
+
}
|
|
2669
|
+
|
|
2670
|
+
|
|
2671
|
+
|
|
2672
|
+
|
|
2673
|
+
|
|
2674
|
+
#if 1
|
|
2675
|
+
|
|
2676
|
+
|
|
2677
|
+
__host__ static bool queue_count(Instance runtime, LinkType* host_arena, QueueType queue, LinkAdrType& result){
|
|
2678
|
+
|
|
2679
|
+
//printf("Entered function\n");
|
|
2680
|
+
LinkAdrType head = queue.get_head();
|
|
2681
|
+
LinkAdrType tail = queue.get_tail();
|
|
2682
|
+
LinkAdrType last = LinkAdrType::null;
|
|
2683
|
+
LinkAdrType count = 0;
|
|
2684
|
+
|
|
2685
|
+
//printf("About to check if the head or tail was NULL\n");
|
|
2686
|
+
|
|
2687
|
+
if( head.is_null() ){
|
|
2688
|
+
if( tail.is_null() ) {
|
|
2689
|
+
result = 0;
|
|
2690
|
+
return true;
|
|
2691
|
+
} else {
|
|
2692
|
+
printf("NULL head with a non-NULL tail\n");
|
|
2693
|
+
return false;
|
|
2694
|
+
}
|
|
2695
|
+
} else if ( tail.is_null() ){
|
|
2696
|
+
printf("Non-NULL head with a NULL tail\n");
|
|
2697
|
+
return false;
|
|
2698
|
+
}
|
|
2699
|
+
|
|
2700
|
+
//printf("Just checked if the head or tail was NULL\n");
|
|
2701
|
+
LinkAdrType iter = head;
|
|
2702
|
+
while( ! iter.is_null() ){
|
|
2703
|
+
if( iter.adr > runtime.arena_size ){
|
|
2704
|
+
printf("Queue has bad index pointing to index %d\n",iter.adr);
|
|
2705
|
+
return false;
|
|
2706
|
+
}
|
|
2707
|
+
if(host_arena[iter.adr].meta_data != 0){
|
|
2708
|
+
printf("Link re-visited\n");
|
|
2709
|
+
LinkAdrType loop_point = iter;
|
|
2710
|
+
LinkAdrType visit_count = 0;
|
|
2711
|
+
iter = head;
|
|
2712
|
+
printf("(%d,%d): ",head.adr,tail.adr);
|
|
2713
|
+
LinkAdrType step_count = 0;
|
|
2714
|
+
while(true){
|
|
2715
|
+
if(iter == loop_point){
|
|
2716
|
+
if(visit_count.adr == 0){
|
|
2717
|
+
printf("{%d}->",iter.adr);
|
|
2718
|
+
} else {
|
|
2719
|
+
printf("{%d}\n",iter.adr);
|
|
2720
|
+
break;
|
|
2721
|
+
}
|
|
2722
|
+
visit_count.adr += 1;
|
|
2723
|
+
} else {
|
|
2724
|
+
printf("%d->",iter.adr);
|
|
2725
|
+
}
|
|
2726
|
+
iter = host_arena[iter.adr].next;
|
|
2727
|
+
if( iter.is_null() ){
|
|
2728
|
+
printf("NULL\n",iter.adr);
|
|
2729
|
+
return false;
|
|
2730
|
+
}
|
|
2731
|
+
step_count.adr +=1;
|
|
2732
|
+
if(step_count.adr > 64){
|
|
2733
|
+
printf("...{%d}\n",loop_point.adr);
|
|
2734
|
+
break;
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
return false;
|
|
2738
|
+
} else {
|
|
2739
|
+
host_arena[iter.adr].meta_data = 1;
|
|
2740
|
+
}
|
|
2741
|
+
last = iter;
|
|
2742
|
+
iter = host_arena[iter.adr].next;
|
|
2743
|
+
count.adr += 1;
|
|
2744
|
+
}
|
|
2745
|
+
|
|
2746
|
+
if( last.adr != tail.adr ){
|
|
2747
|
+
printf("Final link %d in the queue (%d,%d) not the tail\n",last.adr,head.adr,tail.adr);
|
|
2748
|
+
return false;
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
result = count;
|
|
2752
|
+
return true;
|
|
2753
|
+
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
|
|
2757
|
+
|
|
2758
|
+
|
|
2759
|
+
|
|
2760
|
+
|
|
2761
|
+
/*
|
|
2762
|
+
// Counts the number of links in each queue in the pool and in the stack, storing the counts in
|
|
2763
|
+
// the provided arrays. This funciton returns true if counting was successful and returns false
|
|
2764
|
+
// if an invalid state is detected.
|
|
2765
|
+
*/
|
|
2766
|
+
__host__ static bool runtime_overview(Instance runtime){
|
|
2767
|
+
|
|
2768
|
+
bool result = true;
|
|
2769
|
+
|
|
2770
|
+
#ifdef DEBUG_PRINT
|
|
2771
|
+
const bool always_print = true;
|
|
2772
|
+
#else
|
|
2773
|
+
const bool always_print = true;
|
|
2774
|
+
#endif
|
|
2775
|
+
|
|
2776
|
+
LinkAdrType* pool_counts = new LinkAdrType[POOL_SIZE];
|
|
2777
|
+
bool* pool_count_validity = new bool[POOL_SIZE];
|
|
2778
|
+
LinkAdrType* stack_counts = new LinkAdrType[STACK_SIZE*FRAME_SIZE];
|
|
2779
|
+
bool* stack_count_validity = new bool[STACK_SIZE*FRAME_SIZE];
|
|
2780
|
+
|
|
2781
|
+
|
|
2782
|
+
#ifdef LAZY_LINK
|
|
2783
|
+
AdrType claim_count;
|
|
2784
|
+
runtime.claim_count >> claim_count;
|
|
2785
|
+
#endif
|
|
2786
|
+
|
|
2787
|
+
LinkType* host_arena = new LinkType[runtime.arena_size];
|
|
2788
|
+
|
|
2789
|
+
QueueType* host_pool = new QueueType[POOL_SIZE];
|
|
2790
|
+
|
|
2791
|
+
StackType* host_stack = new StackType;
|
|
2792
|
+
|
|
2793
|
+
|
|
2794
|
+
LinkAdrType link_total = 0;
|
|
2795
|
+
|
|
2796
|
+
adapt::GPUrtMemcpy(host_arena,runtime.arena,sizeof(LinkType) *runtime.arena_size,adapt::GPUrtMemcpyDeviceToHost);
|
|
2797
|
+
adapt::GPUrtMemcpy(host_pool ,runtime.pool ,sizeof(QueueType)*POOL_SIZE ,adapt::GPUrtMemcpyDeviceToHost);
|
|
2798
|
+
adapt::GPUrtMemcpy(host_stack,runtime.stack,sizeof(StackType) ,adapt::GPUrtMemcpyDeviceToHost);
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
for(AdrType i = 0; i < runtime.arena_size; i++){
|
|
2802
|
+
host_arena[i].meta_data = 0;
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
|
|
2806
|
+
|
|
2807
|
+
//printf("Counting through pool links...\n");
|
|
2808
|
+
for(int i=0; i < POOL_SIZE; i++){
|
|
2809
|
+
//printf("Counting pool queue %d\n",i);
|
|
2810
|
+
QueueType queue = host_pool[i];
|
|
2811
|
+
//printf("Read pool queue %d\n",i);
|
|
2812
|
+
pool_count_validity[i] = queue_count(runtime,host_arena,queue,pool_counts[i]);
|
|
2813
|
+
//printf("Just validated pool queue %d\n",i);
|
|
2814
|
+
result = result && pool_count_validity[i];
|
|
2815
|
+
if(pool_count_validity[i]){
|
|
2816
|
+
link_total.adr += pool_counts[i].adr;
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
//printf("Counting through stack links...\n");
|
|
2821
|
+
for(int i=0; i < STACK_SIZE; i++){
|
|
2822
|
+
for(int j=0; j < FRAME_SIZE; j++){
|
|
2823
|
+
QueueType queue = host_stack->frames[i].pool.queues[j];
|
|
2824
|
+
unsigned int index = i*FRAME_SIZE + j;
|
|
2825
|
+
stack_count_validity[i] = queue_count(runtime,host_arena,queue,stack_counts[index]);
|
|
2826
|
+
result = result && stack_count_validity[i];
|
|
2827
|
+
if(stack_count_validity[i]){
|
|
2828
|
+
link_total.adr += stack_counts[index].adr;
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
}
|
|
2833
|
+
|
|
2834
|
+
|
|
2835
|
+
if( (!result) || always_print ){
|
|
2836
|
+
printf("POOL:\t[");
|
|
2837
|
+
for(int i=0; i<POOL_SIZE; i++){
|
|
2838
|
+
if(pool_count_validity[i]){
|
|
2839
|
+
printf("\t%d",pool_counts[i].adr);
|
|
2840
|
+
} else {
|
|
2841
|
+
printf("\t????");
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
printf("\t]\n");
|
|
2845
|
+
|
|
2846
|
+
unsigned int status_flags = host_stack->status_flags;
|
|
2847
|
+
unsigned int depth = (host_stack->depth_live >> 16) & 0xFFFF;
|
|
2848
|
+
unsigned int live = (host_stack->depth_live) & 0xFFFF;
|
|
2849
|
+
|
|
2850
|
+
#ifdef LAZY_LINK
|
|
2851
|
+
printf("CLAIM_COUNT:\t%d\n",claim_count);
|
|
2852
|
+
#endif
|
|
2853
|
+
|
|
2854
|
+
printf("STACK:\t(status_flags: %#010x\tdepth: %d\tlive: %d)\t{\n",status_flags,depth,live);
|
|
2855
|
+
for(int i=0; i < STACK_SIZE; i++){
|
|
2856
|
+
util::mem::PairPack<PromiseCount> child_res = host_stack->frames[i].children_residents.data;
|
|
2857
|
+
PromiseCount children = child_res.get_left();
|
|
2858
|
+
PromiseCount residents = child_res.get_right();
|
|
2859
|
+
printf("(children: %d\t residents: %d)\t[",children,residents);
|
|
2860
|
+
for(int j=0; j < FRAME_SIZE; j++){
|
|
2861
|
+
unsigned int index = i*FRAME_SIZE + j;
|
|
2862
|
+
if(stack_count_validity[i]){
|
|
2863
|
+
printf("\t%d",stack_counts[index].adr);
|
|
2864
|
+
} else {
|
|
2865
|
+
printf("\t????");
|
|
2866
|
+
}
|
|
2867
|
+
}
|
|
2868
|
+
printf("\t]\n");
|
|
2869
|
+
|
|
2870
|
+
}
|
|
2871
|
+
printf("} LINK TOTAL: %d\n",link_total.adr);
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2874
|
+
delete[] host_arena;
|
|
2875
|
+
delete[] host_pool;
|
|
2876
|
+
delete host_stack;
|
|
2877
|
+
|
|
2878
|
+
delete[] pool_count_validity;
|
|
2879
|
+
delete[] stack_count_validity;
|
|
2880
|
+
|
|
2881
|
+
return result;
|
|
2882
|
+
|
|
2883
|
+
}
|
|
2884
|
+
|
|
2885
|
+
#endif
|
|
2886
|
+
|
|
2887
|
+
|
|
2888
|
+
template<typename TYPE,typename... ARGS>
|
|
2889
|
+
__device__ void async(ARGS... args){
|
|
2890
|
+
async_call_cast<TYPE>(0,args...);
|
|
2891
|
+
}
|
|
2892
|
+
|
|
2893
|
+
|
|
2894
|
+
|
|
2895
|
+
|
|
2896
|
+
template<typename TYPE,typename... ARGS>
|
|
2897
|
+
__device__ void sync(ARGS... args){
|
|
2898
|
+
immediate_call_cast<TYPE>(Promise<TYPE>(args...));
|
|
2899
|
+
}
|
|
2900
|
+
|
|
2901
|
+
|
|
2902
|
+
template<typename TYPE>
|
|
2903
|
+
__device__ float load_fraction()
|
|
2904
|
+
{
|
|
2905
|
+
return NAN;
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
__device__ void halt_early() {
|
|
2909
|
+
set_flags(EARLY_HALT_FLAG);
|
|
2910
|
+
}
|
|
2911
|
+
|
|
2912
|
+
|
|
2913
|
+
};
|
|
2914
|
+
|
|
2915
|
+
|
|
2916
|
+
|
|
2917
|
+
|
|
2918
|
+
#undef MEMBER_SWITCH
|
|
2919
|
+
#undef CONST_SWITCH
|
|
2920
|
+
|
|
2921
|
+
#endif
|
|
2922
|
+
|
|
2923
|
+
|