harmonize-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harmonize/__init__.py +65 -0
- harmonize/__main__.py +39 -0
- harmonize/adapt.py +105 -0
- harmonize/array.py +588 -0
- harmonize/atomics.py +206 -0
- harmonize/codegen.py +409 -0
- harmonize/config.py +220 -0
- harmonize/cpp/async/future.h +64 -0
- harmonize/cpp/async/mod.h +9 -0
- harmonize/cpp/async/promise.h +747 -0
- harmonize/cpp/harmonize.h +4 -0
- harmonize/cpp/mem/mod.h +14 -0
- harmonize/cpp/mem/work.h +168 -0
- harmonize/cpp/preamble/adapt.h +162 -0
- harmonize/cpp/preamble/mod.h +133 -0
- harmonize/cpp/program/async.h +2923 -0
- harmonize/cpp/program/event.h +560 -0
- harmonize/cpp/program/mod.h +84 -0
- harmonize/cpp/sync/barrier.h +704 -0
- harmonize/cpp/sync/mod.h +9 -0
- harmonize/cpp/util/cli.h +510 -0
- harmonize/cpp/util/host.h +287 -0
- harmonize/cpp/util/iter.h +565 -0
- harmonize/cpp/util/mem.h +1050 -0
- harmonize/cpp/util/mod.h +148 -0
- harmonize/diagnostics.py +487 -0
- harmonize/errors.py +12 -0
- harmonize/logging.py +51 -0
- harmonize/pointer.py +108 -0
- harmonize/prim.py +59 -0
- harmonize/printing.py +104 -0
- harmonize/python/__init__.py +48 -0
- harmonize/runtime.py +1501 -0
- harmonize/templates.py +369 -0
- harmonize/timing.py +10 -0
- harmonize/types.py +16 -0
- harmonize_python-0.1.0.dist-info/METADATA +161 -0
- harmonize_python-0.1.0.dist-info/RECORD +40 -0
- harmonize_python-0.1.0.dist-info/WHEEL +4 -0
- harmonize_python-0.1.0.dist-info/licenses/LICENSE +29 -0
harmonize/cpp/util/mem.h
ADDED
|
@@ -0,0 +1,1050 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
#ifndef HARMONIZE_UTIL_MEM
|
|
5
|
+
#define HARMONIZE_UTIL_MEM
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
namespace util {
|
|
9
|
+
|
|
10
|
+
namespace mem {
|
|
11
|
+
|
|
12
|
+
template<typename T>
|
|
13
|
+
struct PairEquivalent;
|
|
14
|
+
|
|
15
|
+
template<>
|
|
16
|
+
struct PairEquivalent<unsigned short>
|
|
17
|
+
{
|
|
18
|
+
typedef unsigned int Type;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
template<>
|
|
23
|
+
struct PairEquivalent<unsigned int>
|
|
24
|
+
{
|
|
25
|
+
typedef unsigned long long int Type;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
template<typename T>
|
|
30
|
+
struct PairPack
|
|
31
|
+
{
|
|
32
|
+
|
|
33
|
+
typedef PairPack<T> Self;
|
|
34
|
+
|
|
35
|
+
typedef typename PairEquivalent<T>::Type PairType;
|
|
36
|
+
|
|
37
|
+
static const PairType RIGHT_MASK = std::numeric_limits<T>::max();
|
|
38
|
+
static const size_t HALF_WIDTH = std::numeric_limits<T>::digits;
|
|
39
|
+
static const PairType LEFT_MASK = RIGHT_MASK << (PairType) HALF_WIDTH;
|
|
40
|
+
|
|
41
|
+
PairType data;
|
|
42
|
+
|
|
43
|
+
__host__ __device__ T get_left() {
|
|
44
|
+
return (data >> HALF_WIDTH) & RIGHT_MASK;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
__host__ __device__ void set_left(T val) {
|
|
48
|
+
data &= RIGHT_MASK;
|
|
49
|
+
data |= ((PairType) val) << (PairType) HALF_WIDTH;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
__host__ __device__ T get_right(){
|
|
53
|
+
return data & RIGHT_MASK;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
__host__ __device__ void set_right(T val){
|
|
57
|
+
data &= LEFT_MASK;
|
|
58
|
+
data |= val;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
PairPack<T> () = default;
|
|
62
|
+
|
|
63
|
+
__host__ __device__ PairPack<T> (T left, T right){
|
|
64
|
+
data = left;
|
|
65
|
+
data <<= HALF_WIDTH;
|
|
66
|
+
data |= right;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
template <typename ADR_INNER_TYPE>
|
|
78
|
+
struct Adr
|
|
79
|
+
{
|
|
80
|
+
|
|
81
|
+
typedef ADR_INNER_TYPE AdrType;
|
|
82
|
+
|
|
83
|
+
static const AdrType null = std::numeric_limits<AdrType>::max();
|
|
84
|
+
|
|
85
|
+
AdrType adr;
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
Adr<AdrType>() = default;
|
|
89
|
+
|
|
90
|
+
__host__ __device__ Adr<AdrType>(AdrType adr_val) : adr(adr_val) {}
|
|
91
|
+
|
|
92
|
+
__host__ __device__ bool operator==(const Adr<AdrType>& link_adr){
|
|
93
|
+
return (adr == link_adr.adr);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
__host__ __device__ bool is_null(){
|
|
98
|
+
return (adr == null);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
template <typename ADR_TYPE>
|
|
109
|
+
struct PoolQueue;
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
template <typename ADR_TYPE>
|
|
113
|
+
struct PoolQueue <Adr<ADR_TYPE>>
|
|
114
|
+
{
|
|
115
|
+
|
|
116
|
+
typedef ADR_TYPE AdrType;
|
|
117
|
+
typedef Adr<AdrType> LinkAdrType;
|
|
118
|
+
typedef PoolQueue<LinkAdrType> Self;
|
|
119
|
+
|
|
120
|
+
typedef typename PairEquivalent<AdrType>::Type QueueType;
|
|
121
|
+
PairPack<AdrType> pair;
|
|
122
|
+
|
|
123
|
+
static const QueueType null = std::numeric_limits<QueueType>::max();
|
|
124
|
+
|
|
125
|
+
__host__ __device__ LinkAdrType get_head(){
|
|
126
|
+
return pair.get_left();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/*
|
|
130
|
+
// This function extracts the tail of the given queue
|
|
131
|
+
*/
|
|
132
|
+
__host__ __device__ LinkAdrType get_tail(){
|
|
133
|
+
return pair.get_right();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/*
|
|
137
|
+
// Enable default constructor
|
|
138
|
+
*/
|
|
139
|
+
PoolQueue<Adr<ADR_TYPE>>() = default;
|
|
140
|
+
|
|
141
|
+
/*
|
|
142
|
+
// This function concatenates two link addresses into a queue
|
|
143
|
+
*/
|
|
144
|
+
__host__ __device__ PoolQueue<Adr<ADR_TYPE>>
|
|
145
|
+
(LinkAdrType head, LinkAdrType tail)
|
|
146
|
+
{
|
|
147
|
+
pair = PairPack<AdrType>(head.adr,tail.adr);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/*
|
|
151
|
+
// This function sets the head of the queue to the given link address
|
|
152
|
+
*/
|
|
153
|
+
__host__ __device__ void set_head(LinkAdrType head){
|
|
154
|
+
pair.set_left(head.adr);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/*
|
|
158
|
+
// This function sets the tail of the queue to the given link address
|
|
159
|
+
*/
|
|
160
|
+
__host__ __device__ void set_tail(LinkAdrType tail){
|
|
161
|
+
pair.set_right(tail.adr);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/*
|
|
165
|
+
// Returns true if and only if queue is empty
|
|
166
|
+
*/
|
|
167
|
+
__host__ __device__ bool is_null(){
|
|
168
|
+
return (pair.data == Self::null);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
__host__ __device__ static const Self null_queue() {
|
|
173
|
+
Self result;
|
|
174
|
+
result.pair.data = Self::null;
|
|
175
|
+
return result;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
template<typename T, typename AdrType>
|
|
188
|
+
__host__ void mempool_check(T* arena, size_t arena_size, PoolQueue<AdrType>* pool, size_t pool_size){
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
#define LAZY_MEM
|
|
203
|
+
//#define SMART_MEM
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
template<typename T, typename INDEX> struct MemPool;
|
|
207
|
+
|
|
208
|
+
template<typename T, typename INDEX>
|
|
209
|
+
__global__ void mempool_init(MemPool<T,INDEX> mempool);
|
|
210
|
+
|
|
211
|
+
template<typename T, typename INDEX>
|
|
212
|
+
struct MemPool {
|
|
213
|
+
|
|
214
|
+
typedef T DataType;
|
|
215
|
+
typedef INDEX Index;
|
|
216
|
+
typedef Adr<Index> AdrType;
|
|
217
|
+
typedef PoolQueue<AdrType> QueueType;
|
|
218
|
+
|
|
219
|
+
static const unsigned int RETRY_COUNT = 32;
|
|
220
|
+
|
|
221
|
+
union Link {
|
|
222
|
+
DataType data;
|
|
223
|
+
AdrType next;
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
#ifdef LAZY_MEM
|
|
227
|
+
AdrType claim_count;
|
|
228
|
+
#endif
|
|
229
|
+
|
|
230
|
+
Link* arena;
|
|
231
|
+
AdrType arena_size;
|
|
232
|
+
|
|
233
|
+
QueueType* pool;
|
|
234
|
+
AdrType pool_size;
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
__host__ void host_init()
|
|
238
|
+
{
|
|
239
|
+
#ifdef LAZY_MEM
|
|
240
|
+
claim_count = 0;
|
|
241
|
+
#endif
|
|
242
|
+
|
|
243
|
+
if( arena_size.adr != 0 ){
|
|
244
|
+
arena = host::hardMalloc<Link> ( arena_size.adr );
|
|
245
|
+
}
|
|
246
|
+
if( pool_size.adr != 0 ){
|
|
247
|
+
pool = host::hardMalloc<QueueType>( pool_size .adr );
|
|
248
|
+
}
|
|
249
|
+
if( (arena_size.adr != 0) || (pool_size.adr != 0 ) ){
|
|
250
|
+
mempool_init<<<256,32>>>(*this);
|
|
251
|
+
util::host::auto_throw(adapt::GPUrtDeviceSynchronize());
|
|
252
|
+
}
|
|
253
|
+
//next_print();
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
__host__ void host_free()
|
|
257
|
+
{
|
|
258
|
+
if( arena_size.adr != 0 ){
|
|
259
|
+
host::auto_throw( adapt::GPUrtFree( arena ) );
|
|
260
|
+
}
|
|
261
|
+
if( pool_size.adr != 0 ){
|
|
262
|
+
host::auto_throw( adapt::GPUrtFree( pool ) );
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
__host__ void next_print(){
|
|
268
|
+
Link* host_copy = (Link*)malloc(sizeof(Link)*arena_size.adr);
|
|
269
|
+
adapt::GPUrtMemcpy(host_copy,arena,sizeof(Link)*arena_size.adr,adapt::GPUrtMemcpyDeviceToHost);
|
|
270
|
+
for(int i=0; i<arena_size.adr; i++){
|
|
271
|
+
printf("%d,",host_copy[i].next.adr);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
__host__ MemPool<DataType,Index>( Index as, Index ps )
|
|
277
|
+
: arena_size(as)
|
|
278
|
+
, pool_size (ps)
|
|
279
|
+
{}
|
|
280
|
+
|
|
281
|
+
__host__ MemPool<DataType,Index>()
|
|
282
|
+
: arena_size(0)
|
|
283
|
+
, pool_size (0)
|
|
284
|
+
{}
|
|
285
|
+
|
|
286
|
+
__device__ DataType& operator[] (Index index){
|
|
287
|
+
return arena[index].data;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
__device__ Index pop_front(QueueType& queue){
|
|
292
|
+
AdrType result;
|
|
293
|
+
if( queue.is_null() ){
|
|
294
|
+
result.adr = AdrType::null;
|
|
295
|
+
} else {
|
|
296
|
+
result = queue.get_head();
|
|
297
|
+
AdrType next = arena[result.adr].next;
|
|
298
|
+
queue.set_head(next);
|
|
299
|
+
if( next.is_null() ){
|
|
300
|
+
queue.set_tail(next);
|
|
301
|
+
} else if ( queue.get_tail() == result ){
|
|
302
|
+
printf("ERROR: Final link does not have a null next.\n");
|
|
303
|
+
queue.pair.data = QueueType::null;
|
|
304
|
+
return result.adr;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return result.adr;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
__device__ QueueType join(QueueType dst, QueueType src){
|
|
312
|
+
if( dst.is_null() ){
|
|
313
|
+
return src;
|
|
314
|
+
} else if ( src.is_null() ) {
|
|
315
|
+
return dst;
|
|
316
|
+
} else {
|
|
317
|
+
AdrType left_tail_adr = dst.get_tail();
|
|
318
|
+
AdrType right_head_adr = src.get_head();
|
|
319
|
+
AdrType right_tail_adr = src.get_tail();
|
|
320
|
+
|
|
321
|
+
/*
|
|
322
|
+
// Find last link in the dst queue and set succcessor to head of src queue.
|
|
323
|
+
*/
|
|
324
|
+
arena[left_tail_adr.adr].next = right_head_adr;
|
|
325
|
+
|
|
326
|
+
/* Set the right half of the left_queue handle to index the new tail. */
|
|
327
|
+
dst.set_tail(right_tail_adr);
|
|
328
|
+
|
|
329
|
+
return dst;
|
|
330
|
+
}
|
|
331
|
+
//arena[dst.get_tail().adr].next = src.get_head();
|
|
332
|
+
//dst.set_tail(src.get_tail());
|
|
333
|
+
//return dst;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
__device__ QueueType pull_queue(Index& pull_idx){
|
|
338
|
+
Index start_idx = pull_idx;
|
|
339
|
+
//printf("Pulling from %d",start_idx);
|
|
340
|
+
bool done = false;
|
|
341
|
+
QueueType queue = QueueType::null_queue();
|
|
342
|
+
__threadfence();
|
|
343
|
+
for(Index i=start_idx; i<pool_size.adr; i++){
|
|
344
|
+
queue.pair.data = atomicExch(&(pool[i].pair.data),QueueType::null);
|
|
345
|
+
if( ! queue.is_null() ){
|
|
346
|
+
done = true;
|
|
347
|
+
pull_idx = i;
|
|
348
|
+
//printf("{pulled(%d,%d) from %d}",queue.get_head().adr,queue.get_tail().adr,pull_idx);
|
|
349
|
+
return queue;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
if( !done ){
|
|
353
|
+
for(Index i=0; i<start_idx; i++){
|
|
354
|
+
queue.pair.data = atomicExch(&(pool[i].pair.data),QueueType::null);
|
|
355
|
+
if( ! queue.is_null() ){
|
|
356
|
+
done = true;
|
|
357
|
+
pull_idx = i;
|
|
358
|
+
//printf("{pulled(%d,%d) from %d}",queue.get_head().adr,queue.get_tail().adr,pull_idx);
|
|
359
|
+
return queue;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
//printf("{pulled(%d,%d)}",queue.get_head().adr,queue.get_tail().adr);
|
|
364
|
+
return queue;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
__device__ void push_queue(QueueType queue, Index push_idx){
|
|
370
|
+
if( queue.is_null() ){
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
//printf("Pushing (%d,%d) into %d",queue.get_head().adr,queue.get_tail().adr,push_idx);
|
|
374
|
+
unsigned int try_count = 0;
|
|
375
|
+
while(true){
|
|
376
|
+
__threadfence();
|
|
377
|
+
QueueType swap;
|
|
378
|
+
swap.pair.data = atomicExch(&(pool[push_idx].pair.data),queue.pair.data);
|
|
379
|
+
if( swap.is_null() ){
|
|
380
|
+
//printf("{%d tries}",try_count);
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
try_count++;
|
|
384
|
+
queue.pair.data = atomicExch(&(pool[push_idx].pair.data),QueueType::null);
|
|
385
|
+
queue = join(swap,queue);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
__device__ Index pull_span( Index* dst, Index count, Index stride, unsigned int& rand_state ){
|
|
391
|
+
|
|
392
|
+
Index result = 0;
|
|
393
|
+
for(unsigned int t=0; t<RETRY_COUNT; t++){
|
|
394
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
395
|
+
QueueType queue = pull_queue(queue_index);
|
|
396
|
+
if( ! queue.is_null() ){
|
|
397
|
+
AdrType adr = pop_front(queue);
|
|
398
|
+
while( (adr.adr != AdrType::null) && (result < count) ){
|
|
399
|
+
dst[result*stride] = adr.adr;
|
|
400
|
+
adr = pop_front(queue);
|
|
401
|
+
result++;
|
|
402
|
+
}
|
|
403
|
+
push_queue(queue,queue_index);
|
|
404
|
+
}
|
|
405
|
+
if( result >= count ){
|
|
406
|
+
return result;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return result;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
__device__ void push_span( Index* src, Index count, Index stride, unsigned int& rand_state ){
|
|
415
|
+
Index first = AdrType::null;
|
|
416
|
+
Index last = AdrType::null;
|
|
417
|
+
for( Index i=0; i<count; i++){
|
|
418
|
+
Index& slot = src[stride*i];
|
|
419
|
+
if( slot != AdrType::null ){
|
|
420
|
+
if( first == AdrType::null ){
|
|
421
|
+
first = slot;
|
|
422
|
+
} else {
|
|
423
|
+
arena[last].next = slot;
|
|
424
|
+
}
|
|
425
|
+
last = slot;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if( last != AdrType::null ){
|
|
429
|
+
arena[last].next = AdrType::null;
|
|
430
|
+
}
|
|
431
|
+
QueueType queue = QueueType(first,last);
|
|
432
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
433
|
+
push_queue(queue,queue_index);
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
__device__ Index pull_span_atomic( Index* dst, Index count, Index stride, unsigned int& rand_state ){
|
|
440
|
+
|
|
441
|
+
Index result = 0;
|
|
442
|
+
for(unsigned int t=0; t<RETRY_COUNT; t++){
|
|
443
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
444
|
+
QueueType queue = pull_queue(queue_index);
|
|
445
|
+
if( ! queue.is_null() ){
|
|
446
|
+
for( ; result<count; result++){
|
|
447
|
+
Index old = atomicCAS(&dst[result*stride],AdrType::null,queue.get_head());
|
|
448
|
+
if( old != AdrType::null ){
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
AdrType adr = pop_front(queue);
|
|
452
|
+
if( adr == AdrType::null ){
|
|
453
|
+
break;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
push_queue(queue,queue_index);
|
|
457
|
+
}
|
|
458
|
+
if( result >= count ){
|
|
459
|
+
return result;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
return result;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
__device__ void push_span_atomic( Index* src, Index count, Index stride, unsigned int& rand_state ){
|
|
466
|
+
Index first = AdrType::null;
|
|
467
|
+
Index last = AdrType::null;
|
|
468
|
+
for( Index i=0; i<count; i++){
|
|
469
|
+
Index swp = atomicExch(&src[stride*i],AdrType::null);
|
|
470
|
+
if( swp != AdrType::null ){
|
|
471
|
+
if( first == AdrType::null ){
|
|
472
|
+
first = swp;
|
|
473
|
+
} else {
|
|
474
|
+
arena[last].next = swp;
|
|
475
|
+
}
|
|
476
|
+
last = swp;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
if( last != AdrType::null ){
|
|
480
|
+
arena[last].next = AdrType::null;
|
|
481
|
+
}
|
|
482
|
+
QueueType queue = QueueType(first,last);
|
|
483
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
484
|
+
push_queue(queue,queue_index);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
#ifdef LAZY_MEM
|
|
489
|
+
__device__ Index lazy_alloc_index(unsigned int& rand_state){
|
|
490
|
+
|
|
491
|
+
#ifdef SMART_MEM
|
|
492
|
+
__shared__ Index claim_index;
|
|
493
|
+
unsigned int active = __activemask();
|
|
494
|
+
__syncwarp(active);
|
|
495
|
+
unsigned int count = active_count();
|
|
496
|
+
unsigned int offset = warp_inc_scan();
|
|
497
|
+
if(current_leader()){
|
|
498
|
+
if( claim_count.adr < arena_size.adr ){
|
|
499
|
+
claim_index = atomicAdd(&(claim_count.adr),count);
|
|
500
|
+
} else {
|
|
501
|
+
claim_index = arena_size.adr;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
__syncwarp(active);
|
|
505
|
+
if( (claim_index+offset) < arena_size.adr ){
|
|
506
|
+
return claim_index + offset;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
#else
|
|
510
|
+
if( claim_count.adr < arena_size.adr ){
|
|
511
|
+
Index claim_index = atomicAdd(&(claim_count.adr),1);
|
|
512
|
+
if( claim_index < arena_size.adr ){
|
|
513
|
+
return claim_index;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
#endif
|
|
517
|
+
return AdrType::null;
|
|
518
|
+
}
|
|
519
|
+
#endif
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
__device__ Index alloc_index(unsigned int& rand_state){
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
#ifdef LAZY_MEM
|
|
527
|
+
Index result = lazy_alloc_index(rand_state);
|
|
528
|
+
if( result != AdrType::null ){
|
|
529
|
+
return result;
|
|
530
|
+
}
|
|
531
|
+
#endif
|
|
532
|
+
|
|
533
|
+
for( unsigned int t=0; t<RETRY_COUNT; t++){
|
|
534
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
535
|
+
QueueType queue = pull_queue(queue_index);
|
|
536
|
+
if( ! queue.is_null() ){
|
|
537
|
+
AdrType adr = pop_front(queue);
|
|
538
|
+
push_queue(queue,queue_index);
|
|
539
|
+
return adr.adr;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
printf("{Alloc fail}");
|
|
543
|
+
return AdrType::null;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
__device__ DataType* alloc(unsigned int& rand_state){
|
|
548
|
+
Index result_index = alloc_index(rand_state);
|
|
549
|
+
if( result_index != Adr<Index>::null ){
|
|
550
|
+
return &(arena[result_index].data);
|
|
551
|
+
}
|
|
552
|
+
return NULL;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
__device__ void free(Index index, unsigned int& rand_state){
|
|
557
|
+
if( index == AdrType::null ){
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
Index queue_index = random_uint(rand_state) % pool_size.adr;
|
|
561
|
+
arena[index].next = AdrType::null;
|
|
562
|
+
__threadfence();
|
|
563
|
+
QueueType queue = QueueType(AdrType(index),AdrType(index));
|
|
564
|
+
push_queue(queue,queue_index);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
__device__ void free(DataType* address, unsigned int& rand_state){
|
|
570
|
+
if( address == NULL ) {
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
Index index = address - arena;
|
|
574
|
+
if( index == AdrType::null ){
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
free(index,rand_state);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
__device__ void serial_init(){
|
|
583
|
+
parallel_init(0,1);
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
__device__ void parallel_init(Index thread_index, Index thread_count){
|
|
589
|
+
|
|
590
|
+
typedef MemPool<T,INDEX> PoolType;
|
|
591
|
+
typedef INDEX Index;
|
|
592
|
+
|
|
593
|
+
#ifdef LAZY_MEM
|
|
594
|
+
claim_count = 0;
|
|
595
|
+
|
|
596
|
+
for(Index i=thread_index; i<pool_size.adr; i+=thread_count){
|
|
597
|
+
pool [i] = QueueType(AdrType::null,AdrType::null);
|
|
598
|
+
//printf("{%d:(%d,%d)}",i,pool[i].get_head().adr,pool[i].get_tail().adr);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
#else
|
|
602
|
+
|
|
603
|
+
Index span = arena_size.adr / pool_size.adr;
|
|
604
|
+
|
|
605
|
+
Index limit = arena_size.adr;
|
|
606
|
+
for(Index i=thread_index; i<limit; i+=thread_count){
|
|
607
|
+
if( ( (i%span) == (span-1) ) || ( i == (limit-1) ) ){
|
|
608
|
+
arena[i].next = AdrType::null;
|
|
609
|
+
} else {
|
|
610
|
+
arena[i].next = i+1;
|
|
611
|
+
}
|
|
612
|
+
//printf("(%d:%d)",i,arena[i].next.adr);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
for(Index i=thread_index; i<pool_size.adr; i+=thread_count){
|
|
616
|
+
Index last;
|
|
617
|
+
if( ((i+1)*span-1) >= arena_size.adr ) {
|
|
618
|
+
last = arena_size.adr - 1;
|
|
619
|
+
} else {
|
|
620
|
+
last = (i+1)*span-1;
|
|
621
|
+
}
|
|
622
|
+
pool [i] = QueueType(AdrType(i*span),AdrType(last));
|
|
623
|
+
//printf("{%d:(%d,%d)}",i,pool[i].get_head().adr,pool[i].get_tail().adr);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
#endif
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
template<typename T, typename INDEX>
|
|
634
|
+
__global__ void mempool_init(MemPool<T,INDEX> mempool){
|
|
635
|
+
|
|
636
|
+
typedef INDEX Index;
|
|
637
|
+
|
|
638
|
+
Index thread_count = blockDim.x * gridDim.x;
|
|
639
|
+
Index thread_index = blockDim.x * blockIdx.x + threadIdx.x;
|
|
640
|
+
|
|
641
|
+
mempool.parallel_init(thread_index,thread_count);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
template<typename T, size_t SIZE, size_t WARP_SIZE=32>
|
|
648
|
+
struct MemCache {
|
|
649
|
+
|
|
650
|
+
typedef typename T::DataType DataType;
|
|
651
|
+
typedef typename T::Index Index;
|
|
652
|
+
typedef Adr<Index> AdrType;
|
|
653
|
+
typedef MemPool<DataType,Index> PoolType;
|
|
654
|
+
|
|
655
|
+
#ifdef LAZY_MEM
|
|
656
|
+
bool try_lazy;
|
|
657
|
+
#endif
|
|
658
|
+
|
|
659
|
+
PoolType* parent;
|
|
660
|
+
Index indexes[SIZE*WARP_SIZE];
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
#if __CUDA_ARCH__ < 600
|
|
664
|
+
#define atomicExch_block atomicExch
|
|
665
|
+
#define atomicCAS_block atomicCAS
|
|
666
|
+
#endif
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
__device__ Index& get_index( unsigned int offset ){
|
|
670
|
+
unsigned int real_offset = (WARP_SIZE*offset + threadIdx.x + (offset/SIZE)) % (SIZE*WARP_SIZE);
|
|
671
|
+
return indexes[real_offset];
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
__device__ void initialize (PoolType& p) {
|
|
676
|
+
if( current_leader() ){
|
|
677
|
+
#ifdef LAZY_MEM
|
|
678
|
+
try_lazy = true;
|
|
679
|
+
#endif
|
|
680
|
+
parent = &p;
|
|
681
|
+
}
|
|
682
|
+
for ( unsigned int i=0; i<SIZE; i++ ) {
|
|
683
|
+
get_index(i) = AdrType::null;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
__device__ void finalize (unsigned int &rand_state) {
|
|
688
|
+
for ( unsigned int i=0; i<SIZE; i++ ) {
|
|
689
|
+
Index& slot = get_index(i);
|
|
690
|
+
if( slot != AdrType::null ){
|
|
691
|
+
parent->free(slot,rand_state);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
__device__ Index alloc_index(unsigned int& rand_state){
|
|
699
|
+
|
|
700
|
+
#if 0
|
|
701
|
+
if( try_lazy ){
|
|
702
|
+
Index result = parent->lazy_alloc_index(rand_state);
|
|
703
|
+
if( result != AdrType::null ){
|
|
704
|
+
return result;
|
|
705
|
+
} else {
|
|
706
|
+
try_lazy = false;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
#endif
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
#if 0
|
|
713
|
+
for ( unsigned int i=threadIdx.x; i<SIZE; i++ ) {
|
|
714
|
+
Index swap = atomicExch_block(&(indexes[i]),AdrType::null);
|
|
715
|
+
if( swap != AdrType::null ){
|
|
716
|
+
return swap;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
for ( unsigned int i=0; i<threadIdx.x; i++ ) {
|
|
720
|
+
Index swap = atomicExch_block(&(indexes[i]),AdrType::null);
|
|
721
|
+
if( swap != AdrType::null ){
|
|
722
|
+
return swap;
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
#else
|
|
726
|
+
for ( unsigned int i=0; i<(SIZE*WARP_SIZE); i++ ) {
|
|
727
|
+
Index& slot = get_index(i);
|
|
728
|
+
Index swap = atomicExch_block(&slot,AdrType::null);
|
|
729
|
+
if( swap != AdrType::null ){
|
|
730
|
+
return swap;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
#endif
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
return parent->alloc_index(rand_state);
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
__device__ DataType* alloc(unsigned int& rand_state){
|
|
741
|
+
Index result_index = alloc_index(rand_state);
|
|
742
|
+
if ( result_index != Adr<Index>::null ) {
|
|
743
|
+
return (parent->arena[result_index].data);
|
|
744
|
+
}
|
|
745
|
+
return NULL;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
__device__ void free(Index index, unsigned int& rand_state){
|
|
750
|
+
#if 0
|
|
751
|
+
for ( unsigned int i=threadIdx.x; i<SIZE; i++ ) {
|
|
752
|
+
Index swap = atomicCAS_block(&(indexes[i]),AdrType::null,index);
|
|
753
|
+
if( swap == AdrType::null ){
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
for ( unsigned int i=0; i<threadIdx.x; i++ ) {
|
|
758
|
+
Index swap = atomicCAS_block(&(indexes[i]),AdrType::null,index);
|
|
759
|
+
if( swap == AdrType::null ){
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
#else
|
|
764
|
+
for ( unsigned int i=0; i<(SIZE*WARP_SIZE); i++ ) {
|
|
765
|
+
Index& slot = get_index(i);
|
|
766
|
+
Index swap = atomicCAS_block(&slot,AdrType::null,index);
|
|
767
|
+
if( swap == AdrType::null ){
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
#endif
|
|
772
|
+
parent->free(index,rand_state);
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
__device__ void free(DataType* address, unsigned int& rand_state){
|
|
777
|
+
if( address == NULL ){
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
Index index = address - (parent->arena);
|
|
781
|
+
free(index,rand_state);
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
#if __CUDA_ARCH__ < 600
|
|
785
|
+
#undef atomicExch_block
|
|
786
|
+
#undef atomicCAS_block
|
|
787
|
+
#endif
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
};
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
template<typename T, size_t SIZE, size_t WARP_SIZE=32>
|
|
796
|
+
struct SimpleMemCache {
|
|
797
|
+
|
|
798
|
+
typedef typename T::DataType DataType;
|
|
799
|
+
typedef typename T::Index Index;
|
|
800
|
+
typedef Adr<Index> AdrType;
|
|
801
|
+
typedef MemPool<DataType,Index> PoolType;
|
|
802
|
+
|
|
803
|
+
#ifdef LAZY_MEM
|
|
804
|
+
bool try_lazy;
|
|
805
|
+
#endif
|
|
806
|
+
|
|
807
|
+
PoolType* parent;
|
|
808
|
+
unsigned int counts[WARP_SIZE];
|
|
809
|
+
Index indexes[SIZE*WARP_SIZE];
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
#if __CUDA_ARCH__ < 600
|
|
813
|
+
#define atomicExch_block atomicExch
|
|
814
|
+
#define atomicCAS_block atomicCAS
|
|
815
|
+
#endif
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
__device__ Index& get_index( unsigned int offset ){
|
|
819
|
+
unsigned int real_offset = (WARP_SIZE*offset + threadIdx.x + (offset/SIZE)) % (SIZE*WARP_SIZE);
|
|
820
|
+
return indexes[real_offset];
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
__device__ void initialize (PoolType& p) {
|
|
825
|
+
if( current_leader() ){
|
|
826
|
+
parent = &p;
|
|
827
|
+
}
|
|
828
|
+
counts[threadIdx.x] = 0;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
__device__ void finalize (unsigned int &rand_state) {
|
|
832
|
+
unsigned int count = counts[threadIdx.x];
|
|
833
|
+
for ( unsigned int i=0; i<count; i++ ) {
|
|
834
|
+
Index& slot = get_index(i);
|
|
835
|
+
parent->free(slot,rand_state);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
__device__ void fill_up(unsigned int& rand_state) {
|
|
841
|
+
unsigned int count = counts[threadIdx.x];
|
|
842
|
+
if( SIZE/2 > count ) {
|
|
843
|
+
parent->pull_span( &get_index(count), (SIZE/2)-count, WARP_SIZE, rand_state);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
__device__ Index alloc_index(unsigned int& rand_state) {
|
|
849
|
+
|
|
850
|
+
unsigned int& count = counts[threadIdx.x];
|
|
851
|
+
if( count > 0 ){
|
|
852
|
+
count -= 1;
|
|
853
|
+
Index& slot = get_index(count);
|
|
854
|
+
return slot;
|
|
855
|
+
} else {
|
|
856
|
+
return parent->alloc_index(rand_state);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
__device__ DataType* alloc(unsigned int& rand_state){
|
|
862
|
+
Index result_index = alloc_index(rand_state);
|
|
863
|
+
if ( result_index != Adr<Index>::null ) {
|
|
864
|
+
return (parent->arena[result_index].data);
|
|
865
|
+
}
|
|
866
|
+
return NULL;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
__device__ void free(Index index, unsigned int& rand_state){
|
|
871
|
+
|
|
872
|
+
unsigned int& count = counts[threadIdx.x];
|
|
873
|
+
if( count < SIZE ){
|
|
874
|
+
get_index(count) = index;
|
|
875
|
+
count += 1;
|
|
876
|
+
} else {
|
|
877
|
+
parent->free(index,rand_state);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
__device__ void free(DataType* address, unsigned int& rand_state){
|
|
884
|
+
if( address == NULL ){
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
Index index = address - (parent->arena);
|
|
888
|
+
free(index,rand_state);
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
#if __CUDA_ARCH__ < 600
|
|
892
|
+
#undef atomicExch_block
|
|
893
|
+
#undef atomicCAS_block
|
|
894
|
+
#endif
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
};
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
template<typename T, typename INDEX, typename MASK = unsigned int>
|
|
905
|
+
struct MemChunk {
|
|
906
|
+
|
|
907
|
+
typedef INDEX Index;
|
|
908
|
+
typedef MASK Mask;
|
|
909
|
+
typedef MemPool<MemChunk<T,Index,Mask>,Index> PoolType;
|
|
910
|
+
|
|
911
|
+
static const Index SIZE = std::numeric_limits<Mask>::digits;
|
|
912
|
+
static const Mask FULL = std::numeric_limits<Mask>::max();
|
|
913
|
+
static const MASK HIGH_BIT = 1<<(SIZE-1);
|
|
914
|
+
static const Index OFFSET_MASK = ~(SIZE-1);
|
|
915
|
+
|
|
916
|
+
Mask fill_mask;
|
|
917
|
+
T data[SIZE];
|
|
918
|
+
|
|
919
|
+
__device__ Index offset(PoolType& parent){
|
|
920
|
+
return (this - parent.arena);
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
__device__ bool full(){
|
|
924
|
+
return (fill_mask == FULL);
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
__device__ bool empty(){
|
|
928
|
+
return (fill_mask == 0);
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
__device__ bool fill_count(){
|
|
932
|
+
return __popc(fill_mask);
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
__device__ MemChunk<T,Index,Mask>()
|
|
936
|
+
: fill_mask(0)
|
|
937
|
+
{}
|
|
938
|
+
|
|
939
|
+
__device__ Index alloc(PoolType& parent, Mask& rand_state) {
|
|
940
|
+
Mask try_mask = random_uint(rand_state);
|
|
941
|
+
while( ! full() ){
|
|
942
|
+
Mask scan_mask = try_mask & ~fill_mask;
|
|
943
|
+
scan_mask = (scan_mask == 0) ? ~fill_mask : scan_mask;
|
|
944
|
+
Index lead = leading_zeros(scan_mask);
|
|
945
|
+
Mask select = HIGH_BIT >> lead;
|
|
946
|
+
if( select == 0 ){
|
|
947
|
+
break;
|
|
948
|
+
}
|
|
949
|
+
Mask prior = atomicOr(fill_mask,select);
|
|
950
|
+
if( (prior & select) == 0 ){
|
|
951
|
+
return ( SIZE * offset(parent) ) + ( SIZE - 1 - lead );
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
return Adr<Index>::null;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
__device__ void free(Index index, PoolType& parent, Mask& rand_state) {
|
|
958
|
+
if( (index & OFFSET_MASK) == offset() ){
|
|
959
|
+
Index inner_index = index & ~OFFSET_MASK;
|
|
960
|
+
Mask select = 1 << inner_index;
|
|
961
|
+
atomicAnd(fill_mask, ~select);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
};
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
/*
|
|
969
|
+
template<typename T, typename INDEX, typename MASK, INDEX SIZE_VAL>
|
|
970
|
+
struct MemPoolBank {
|
|
971
|
+
|
|
972
|
+
typedef INDEX Index;
|
|
973
|
+
typedef MASK MaskType;
|
|
974
|
+
typedef MemChunk<T,Index,MaskType> ChunkType;
|
|
975
|
+
typedef MemPool<ChunkType,Index> PoolType;
|
|
976
|
+
|
|
977
|
+
static const Index SIZE = SIZE_VAL;
|
|
978
|
+
|
|
979
|
+
unsigned int rand_state;
|
|
980
|
+
Index full_count;
|
|
981
|
+
|
|
982
|
+
PoolType& parent;
|
|
983
|
+
Index chunks[SIZE];
|
|
984
|
+
|
|
985
|
+
__device__ Index containing_chunk(Index target){
|
|
986
|
+
Index result = ChunkType::offset_mask & index;
|
|
987
|
+
if( result >= parent.arena_size ){
|
|
988
|
+
return Adr<Index>::null;
|
|
989
|
+
}
|
|
990
|
+
return result;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
__device__ MemPoolBank<T,Index,MaskType,SIZE_VAL>
|
|
994
|
+
(PoolType& parent_ref, unsigned int seed, bool& err)
|
|
995
|
+
: parent(parent_ref)
|
|
996
|
+
, full_count(0)
|
|
997
|
+
, rand_state(seed)
|
|
998
|
+
{
|
|
999
|
+
for(int i=0; i<SIZE; i++){
|
|
1000
|
+
chunks[i] = Adr<Index>::null;
|
|
1001
|
+
}
|
|
1002
|
+
err = parent.pull_span(chunks,SIZE,seed);
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
__device__ Index alloc() {
|
|
1007
|
+
for(int i=0; i<SIZE; i++){
|
|
1008
|
+
if( chunks[i] == Adr<Index>::null ){
|
|
1009
|
+
continue;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
__device__ Index hard_neighbor_alloc(Index target, MaskType& rand_state){
|
|
1016
|
+
return parent.arena[ChunkType::offset_mask & index].alloc(parent,rand_state);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
__device__ Index neighbor_alloc(Index target, MaskType& rand_state){
|
|
1020
|
+
Index result = hard_neighbor_alloc(target,rand_state);
|
|
1021
|
+
if( result == Adr<Index>::null){
|
|
1022
|
+
return alloc(rand_state);
|
|
1023
|
+
} else {
|
|
1024
|
+
return result;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
__device__ Index coalesce(Index original) {
|
|
1030
|
+
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
__device__ void free(Index index) {
|
|
1034
|
+
Index chunk_index = ChunkType::offset_mask & index;
|
|
1035
|
+
if( index < parent.arena_size ){
|
|
1036
|
+
parent.arena[chunk_index].free(parent,index);
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
};
|
|
1041
|
+
*/
|
|
1042
|
+
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
#endif
|
|
1049
|
+
|
|
1050
|
+
|