harmonize-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1050 @@
1
+
2
+
3
+
4
+ #ifndef HARMONIZE_UTIL_MEM
5
+ #define HARMONIZE_UTIL_MEM
6
+
7
+
8
+ namespace util {
9
+
10
+ namespace mem {
11
+
12
+ template<typename T>
13
+ struct PairEquivalent;
14
+
15
+ template<>
16
+ struct PairEquivalent<unsigned short>
17
+ {
18
+ typedef unsigned int Type;
19
+ };
20
+
21
+
22
+ template<>
23
+ struct PairEquivalent<unsigned int>
24
+ {
25
+ typedef unsigned long long int Type;
26
+ };
27
+
28
+
29
+ template<typename T>
30
+ struct PairPack
31
+ {
32
+
33
+ typedef PairPack<T> Self;
34
+
35
+ typedef typename PairEquivalent<T>::Type PairType;
36
+
37
+ static const PairType RIGHT_MASK = std::numeric_limits<T>::max();
38
+ static const size_t HALF_WIDTH = std::numeric_limits<T>::digits;
39
+ static const PairType LEFT_MASK = RIGHT_MASK << (PairType) HALF_WIDTH;
40
+
41
+ PairType data;
42
+
43
+ __host__ __device__ T get_left() {
44
+ return (data >> HALF_WIDTH) & RIGHT_MASK;
45
+ }
46
+
47
+ __host__ __device__ void set_left(T val) {
48
+ data &= RIGHT_MASK;
49
+ data |= ((PairType) val) << (PairType) HALF_WIDTH;
50
+ }
51
+
52
+ __host__ __device__ T get_right(){
53
+ return data & RIGHT_MASK;
54
+ }
55
+
56
+ __host__ __device__ void set_right(T val){
57
+ data &= LEFT_MASK;
58
+ data |= val;
59
+ }
60
+
61
+ PairPack<T> () = default;
62
+
63
+ __host__ __device__ PairPack<T> (T left, T right){
64
+ data = left;
65
+ data <<= HALF_WIDTH;
66
+ data |= right;
67
+ }
68
+
69
+ };
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+ template <typename ADR_INNER_TYPE>
78
+ struct Adr
79
+ {
80
+
81
+ typedef ADR_INNER_TYPE AdrType;
82
+
83
+ static const AdrType null = std::numeric_limits<AdrType>::max();
84
+
85
+ AdrType adr;
86
+
87
+
88
+ Adr<AdrType>() = default;
89
+
90
+ __host__ __device__ Adr<AdrType>(AdrType adr_val) : adr(adr_val) {}
91
+
92
+ __host__ __device__ bool operator==(const Adr<AdrType>& link_adr){
93
+ return (adr == link_adr.adr);
94
+ }
95
+
96
+
97
+ __host__ __device__ bool is_null(){
98
+ return (adr == null);
99
+ }
100
+
101
+
102
+ };
103
+
104
+
105
+
106
+
107
+
108
+ template <typename ADR_TYPE>
109
+ struct PoolQueue;
110
+
111
+
112
+ template <typename ADR_TYPE>
113
+ struct PoolQueue <Adr<ADR_TYPE>>
114
+ {
115
+
116
+ typedef ADR_TYPE AdrType;
117
+ typedef Adr<AdrType> LinkAdrType;
118
+ typedef PoolQueue<LinkAdrType> Self;
119
+
120
+ typedef typename PairEquivalent<AdrType>::Type QueueType;
121
+ PairPack<AdrType> pair;
122
+
123
+ static const QueueType null = std::numeric_limits<QueueType>::max();
124
+
125
+ __host__ __device__ LinkAdrType get_head(){
126
+ return pair.get_left();
127
+ }
128
+
129
+ /*
130
+ // This function extracts the tail of the given queue
131
+ */
132
+ __host__ __device__ LinkAdrType get_tail(){
133
+ return pair.get_right();
134
+ }
135
+
136
+ /*
137
+ // Enable default constructor
138
+ */
139
+ PoolQueue<Adr<ADR_TYPE>>() = default;
140
+
141
+ /*
142
+ // This function concatenates two link addresses into a queue
143
+ */
144
+ __host__ __device__ PoolQueue<Adr<ADR_TYPE>>
145
+ (LinkAdrType head, LinkAdrType tail)
146
+ {
147
+ pair = PairPack<AdrType>(head.adr,tail.adr);
148
+ }
149
+
150
+ /*
151
+ // This function sets the head of the queue to the given link address
152
+ */
153
+ __host__ __device__ void set_head(LinkAdrType head){
154
+ pair.set_left(head.adr);
155
+ }
156
+
157
+ /*
158
+ // This function sets the tail of the queue to the given link address
159
+ */
160
+ __host__ __device__ void set_tail(LinkAdrType tail){
161
+ pair.set_right(tail.adr);
162
+ }
163
+
164
+ /*
165
+ // Returns true if and only if queue is empty
166
+ */
167
+ __host__ __device__ bool is_null(){
168
+ return (pair.data == Self::null);
169
+ }
170
+
171
+
172
+ __host__ __device__ static const Self null_queue() {
173
+ Self result;
174
+ result.pair.data = Self::null;
175
+ return result;
176
+ }
177
+
178
+ };
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+ template<typename T, typename AdrType>
188
+ __host__ void mempool_check(T* arena, size_t arena_size, PoolQueue<AdrType>* pool, size_t pool_size){
189
+
190
+
191
+
192
+
193
+ }
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+ #define LAZY_MEM
203
+ //#define SMART_MEM
204
+
205
+
206
+ template<typename T, typename INDEX> struct MemPool;
207
+
208
+ template<typename T, typename INDEX>
209
+ __global__ void mempool_init(MemPool<T,INDEX> mempool);
210
+
211
+ template<typename T, typename INDEX>
212
+ struct MemPool {
213
+
214
+ typedef T DataType;
215
+ typedef INDEX Index;
216
+ typedef Adr<Index> AdrType;
217
+ typedef PoolQueue<AdrType> QueueType;
218
+
219
+ static const unsigned int RETRY_COUNT = 32;
220
+
221
+ union Link {
222
+ DataType data;
223
+ AdrType next;
224
+ };
225
+
226
+ #ifdef LAZY_MEM
227
+ AdrType claim_count;
228
+ #endif
229
+
230
+ Link* arena;
231
+ AdrType arena_size;
232
+
233
+ QueueType* pool;
234
+ AdrType pool_size;
235
+
236
+
237
+ __host__ void host_init()
238
+ {
239
+ #ifdef LAZY_MEM
240
+ claim_count = 0;
241
+ #endif
242
+
243
+ if( arena_size.adr != 0 ){
244
+ arena = host::hardMalloc<Link> ( arena_size.adr );
245
+ }
246
+ if( pool_size.adr != 0 ){
247
+ pool = host::hardMalloc<QueueType>( pool_size .adr );
248
+ }
249
+ if( (arena_size.adr != 0) || (pool_size.adr != 0 ) ){
250
+ mempool_init<<<256,32>>>(*this);
251
+ util::host::auto_throw(adapt::GPUrtDeviceSynchronize());
252
+ }
253
+ //next_print();
254
+ }
255
+
256
+ __host__ void host_free()
257
+ {
258
+ if( arena_size.adr != 0 ){
259
+ host::auto_throw( adapt::GPUrtFree( arena ) );
260
+ }
261
+ if( pool_size.adr != 0 ){
262
+ host::auto_throw( adapt::GPUrtFree( pool ) );
263
+ }
264
+ }
265
+
266
+
267
+ __host__ void next_print(){
268
+ Link* host_copy = (Link*)malloc(sizeof(Link)*arena_size.adr);
269
+ adapt::GPUrtMemcpy(host_copy,arena,sizeof(Link)*arena_size.adr,adapt::GPUrtMemcpyDeviceToHost);
270
+ for(int i=0; i<arena_size.adr; i++){
271
+ printf("%d,",host_copy[i].next.adr);
272
+ }
273
+ }
274
+
275
+
276
+ __host__ MemPool<DataType,Index>( Index as, Index ps )
277
+ : arena_size(as)
278
+ , pool_size (ps)
279
+ {}
280
+
281
+ __host__ MemPool<DataType,Index>()
282
+ : arena_size(0)
283
+ , pool_size (0)
284
+ {}
285
+
286
+ __device__ DataType& operator[] (Index index){
287
+ return arena[index].data;
288
+ }
289
+
290
+
291
+ __device__ Index pop_front(QueueType& queue){
292
+ AdrType result;
293
+ if( queue.is_null() ){
294
+ result.adr = AdrType::null;
295
+ } else {
296
+ result = queue.get_head();
297
+ AdrType next = arena[result.adr].next;
298
+ queue.set_head(next);
299
+ if( next.is_null() ){
300
+ queue.set_tail(next);
301
+ } else if ( queue.get_tail() == result ){
302
+ printf("ERROR: Final link does not have a null next.\n");
303
+ queue.pair.data = QueueType::null;
304
+ return result.adr;
305
+ }
306
+ }
307
+ return result.adr;
308
+ }
309
+
310
+
311
+ __device__ QueueType join(QueueType dst, QueueType src){
312
+ if( dst.is_null() ){
313
+ return src;
314
+ } else if ( src.is_null() ) {
315
+ return dst;
316
+ } else {
317
+ AdrType left_tail_adr = dst.get_tail();
318
+ AdrType right_head_adr = src.get_head();
319
+ AdrType right_tail_adr = src.get_tail();
320
+
321
+ /*
322
+ // Find last link in the dst queue and set succcessor to head of src queue.
323
+ */
324
+ arena[left_tail_adr.adr].next = right_head_adr;
325
+
326
+ /* Set the right half of the left_queue handle to index the new tail. */
327
+ dst.set_tail(right_tail_adr);
328
+
329
+ return dst;
330
+ }
331
+ //arena[dst.get_tail().adr].next = src.get_head();
332
+ //dst.set_tail(src.get_tail());
333
+ //return dst;
334
+ }
335
+
336
+
337
+ __device__ QueueType pull_queue(Index& pull_idx){
338
+ Index start_idx = pull_idx;
339
+ //printf("Pulling from %d",start_idx);
340
+ bool done = false;
341
+ QueueType queue = QueueType::null_queue();
342
+ __threadfence();
343
+ for(Index i=start_idx; i<pool_size.adr; i++){
344
+ queue.pair.data = atomicExch(&(pool[i].pair.data),QueueType::null);
345
+ if( ! queue.is_null() ){
346
+ done = true;
347
+ pull_idx = i;
348
+ //printf("{pulled(%d,%d) from %d}",queue.get_head().adr,queue.get_tail().adr,pull_idx);
349
+ return queue;
350
+ }
351
+ }
352
+ if( !done ){
353
+ for(Index i=0; i<start_idx; i++){
354
+ queue.pair.data = atomicExch(&(pool[i].pair.data),QueueType::null);
355
+ if( ! queue.is_null() ){
356
+ done = true;
357
+ pull_idx = i;
358
+ //printf("{pulled(%d,%d) from %d}",queue.get_head().adr,queue.get_tail().adr,pull_idx);
359
+ return queue;
360
+ }
361
+ }
362
+ }
363
+ //printf("{pulled(%d,%d)}",queue.get_head().adr,queue.get_tail().adr);
364
+ return queue;
365
+ }
366
+
367
+
368
+
369
+ __device__ void push_queue(QueueType queue, Index push_idx){
370
+ if( queue.is_null() ){
371
+ return;
372
+ }
373
+ //printf("Pushing (%d,%d) into %d",queue.get_head().adr,queue.get_tail().adr,push_idx);
374
+ unsigned int try_count = 0;
375
+ while(true){
376
+ __threadfence();
377
+ QueueType swap;
378
+ swap.pair.data = atomicExch(&(pool[push_idx].pair.data),queue.pair.data);
379
+ if( swap.is_null() ){
380
+ //printf("{%d tries}",try_count);
381
+ return;
382
+ }
383
+ try_count++;
384
+ queue.pair.data = atomicExch(&(pool[push_idx].pair.data),QueueType::null);
385
+ queue = join(swap,queue);
386
+ }
387
+ }
388
+
389
+
390
+ __device__ Index pull_span( Index* dst, Index count, Index stride, unsigned int& rand_state ){
391
+
392
+ Index result = 0;
393
+ for(unsigned int t=0; t<RETRY_COUNT; t++){
394
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
395
+ QueueType queue = pull_queue(queue_index);
396
+ if( ! queue.is_null() ){
397
+ AdrType adr = pop_front(queue);
398
+ while( (adr.adr != AdrType::null) && (result < count) ){
399
+ dst[result*stride] = adr.adr;
400
+ adr = pop_front(queue);
401
+ result++;
402
+ }
403
+ push_queue(queue,queue_index);
404
+ }
405
+ if( result >= count ){
406
+ return result;
407
+ }
408
+ }
409
+ return result;
410
+ }
411
+
412
+
413
+
414
+ __device__ void push_span( Index* src, Index count, Index stride, unsigned int& rand_state ){
415
+ Index first = AdrType::null;
416
+ Index last = AdrType::null;
417
+ for( Index i=0; i<count; i++){
418
+ Index& slot = src[stride*i];
419
+ if( slot != AdrType::null ){
420
+ if( first == AdrType::null ){
421
+ first = slot;
422
+ } else {
423
+ arena[last].next = slot;
424
+ }
425
+ last = slot;
426
+ }
427
+ }
428
+ if( last != AdrType::null ){
429
+ arena[last].next = AdrType::null;
430
+ }
431
+ QueueType queue = QueueType(first,last);
432
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
433
+ push_queue(queue,queue_index);
434
+ }
435
+
436
+
437
+
438
+
439
+ __device__ Index pull_span_atomic( Index* dst, Index count, Index stride, unsigned int& rand_state ){
440
+
441
+ Index result = 0;
442
+ for(unsigned int t=0; t<RETRY_COUNT; t++){
443
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
444
+ QueueType queue = pull_queue(queue_index);
445
+ if( ! queue.is_null() ){
446
+ for( ; result<count; result++){
447
+ Index old = atomicCAS(&dst[result*stride],AdrType::null,queue.get_head());
448
+ if( old != AdrType::null ){
449
+ continue;
450
+ }
451
+ AdrType adr = pop_front(queue);
452
+ if( adr == AdrType::null ){
453
+ break;
454
+ }
455
+ }
456
+ push_queue(queue,queue_index);
457
+ }
458
+ if( result >= count ){
459
+ return result;
460
+ }
461
+ }
462
+ return result;
463
+ }
464
+
465
+ __device__ void push_span_atomic( Index* src, Index count, Index stride, unsigned int& rand_state ){
466
+ Index first = AdrType::null;
467
+ Index last = AdrType::null;
468
+ for( Index i=0; i<count; i++){
469
+ Index swp = atomicExch(&src[stride*i],AdrType::null);
470
+ if( swp != AdrType::null ){
471
+ if( first == AdrType::null ){
472
+ first = swp;
473
+ } else {
474
+ arena[last].next = swp;
475
+ }
476
+ last = swp;
477
+ }
478
+ }
479
+ if( last != AdrType::null ){
480
+ arena[last].next = AdrType::null;
481
+ }
482
+ QueueType queue = QueueType(first,last);
483
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
484
+ push_queue(queue,queue_index);
485
+ }
486
+
487
+
488
+ #ifdef LAZY_MEM
489
+ __device__ Index lazy_alloc_index(unsigned int& rand_state){
490
+
491
+ #ifdef SMART_MEM
492
+ __shared__ Index claim_index;
493
+ unsigned int active = __activemask();
494
+ __syncwarp(active);
495
+ unsigned int count = active_count();
496
+ unsigned int offset = warp_inc_scan();
497
+ if(current_leader()){
498
+ if( claim_count.adr < arena_size.adr ){
499
+ claim_index = atomicAdd(&(claim_count.adr),count);
500
+ } else {
501
+ claim_index = arena_size.adr;
502
+ }
503
+ }
504
+ __syncwarp(active);
505
+ if( (claim_index+offset) < arena_size.adr ){
506
+ return claim_index + offset;
507
+ }
508
+
509
+ #else
510
+ if( claim_count.adr < arena_size.adr ){
511
+ Index claim_index = atomicAdd(&(claim_count.adr),1);
512
+ if( claim_index < arena_size.adr ){
513
+ return claim_index;
514
+ }
515
+ }
516
+ #endif
517
+ return AdrType::null;
518
+ }
519
+ #endif
520
+
521
+
522
+
523
+ __device__ Index alloc_index(unsigned int& rand_state){
524
+
525
+
526
+ #ifdef LAZY_MEM
527
+ Index result = lazy_alloc_index(rand_state);
528
+ if( result != AdrType::null ){
529
+ return result;
530
+ }
531
+ #endif
532
+
533
+ for( unsigned int t=0; t<RETRY_COUNT; t++){
534
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
535
+ QueueType queue = pull_queue(queue_index);
536
+ if( ! queue.is_null() ){
537
+ AdrType adr = pop_front(queue);
538
+ push_queue(queue,queue_index);
539
+ return adr.adr;
540
+ }
541
+ }
542
+ printf("{Alloc fail}");
543
+ return AdrType::null;
544
+ }
545
+
546
+
547
+ __device__ DataType* alloc(unsigned int& rand_state){
548
+ Index result_index = alloc_index(rand_state);
549
+ if( result_index != Adr<Index>::null ){
550
+ return &(arena[result_index].data);
551
+ }
552
+ return NULL;
553
+ }
554
+
555
+
556
+ __device__ void free(Index index, unsigned int& rand_state){
557
+ if( index == AdrType::null ){
558
+ return;
559
+ }
560
+ Index queue_index = random_uint(rand_state) % pool_size.adr;
561
+ arena[index].next = AdrType::null;
562
+ __threadfence();
563
+ QueueType queue = QueueType(AdrType(index),AdrType(index));
564
+ push_queue(queue,queue_index);
565
+ }
566
+
567
+
568
+
569
+ __device__ void free(DataType* address, unsigned int& rand_state){
570
+ if( address == NULL ) {
571
+ return;
572
+ }
573
+ Index index = address - arena;
574
+ if( index == AdrType::null ){
575
+ return;
576
+ }
577
+ free(index,rand_state);
578
+ }
579
+
580
+
581
+
582
+ __device__ void serial_init(){
583
+ parallel_init(0,1);
584
+ }
585
+
586
+
587
+
588
+ __device__ void parallel_init(Index thread_index, Index thread_count){
589
+
590
+ typedef MemPool<T,INDEX> PoolType;
591
+ typedef INDEX Index;
592
+
593
+ #ifdef LAZY_MEM
594
+ claim_count = 0;
595
+
596
+ for(Index i=thread_index; i<pool_size.adr; i+=thread_count){
597
+ pool [i] = QueueType(AdrType::null,AdrType::null);
598
+ //printf("{%d:(%d,%d)}",i,pool[i].get_head().adr,pool[i].get_tail().adr);
599
+ }
600
+
601
+ #else
602
+
603
+ Index span = arena_size.adr / pool_size.adr;
604
+
605
+ Index limit = arena_size.adr;
606
+ for(Index i=thread_index; i<limit; i+=thread_count){
607
+ if( ( (i%span) == (span-1) ) || ( i == (limit-1) ) ){
608
+ arena[i].next = AdrType::null;
609
+ } else {
610
+ arena[i].next = i+1;
611
+ }
612
+ //printf("(%d:%d)",i,arena[i].next.adr);
613
+ }
614
+
615
+ for(Index i=thread_index; i<pool_size.adr; i+=thread_count){
616
+ Index last;
617
+ if( ((i+1)*span-1) >= arena_size.adr ) {
618
+ last = arena_size.adr - 1;
619
+ } else {
620
+ last = (i+1)*span-1;
621
+ }
622
+ pool [i] = QueueType(AdrType(i*span),AdrType(last));
623
+ //printf("{%d:(%d,%d)}",i,pool[i].get_head().adr,pool[i].get_tail().adr);
624
+ }
625
+
626
+ #endif
627
+ }
628
+
629
+ };
630
+
631
+
632
+
633
+ template<typename T, typename INDEX>
634
+ __global__ void mempool_init(MemPool<T,INDEX> mempool){
635
+
636
+ typedef INDEX Index;
637
+
638
+ Index thread_count = blockDim.x * gridDim.x;
639
+ Index thread_index = blockDim.x * blockIdx.x + threadIdx.x;
640
+
641
+ mempool.parallel_init(thread_index,thread_count);
642
+ }
643
+
644
+
645
+
646
+
647
+ template<typename T, size_t SIZE, size_t WARP_SIZE=32>
648
+ struct MemCache {
649
+
650
+ typedef typename T::DataType DataType;
651
+ typedef typename T::Index Index;
652
+ typedef Adr<Index> AdrType;
653
+ typedef MemPool<DataType,Index> PoolType;
654
+
655
+ #ifdef LAZY_MEM
656
+ bool try_lazy;
657
+ #endif
658
+
659
+ PoolType* parent;
660
+ Index indexes[SIZE*WARP_SIZE];
661
+
662
+
663
+ #if __CUDA_ARCH__ < 600
664
+ #define atomicExch_block atomicExch
665
+ #define atomicCAS_block atomicCAS
666
+ #endif
667
+
668
+
669
+ __device__ Index& get_index( unsigned int offset ){
670
+ unsigned int real_offset = (WARP_SIZE*offset + threadIdx.x + (offset/SIZE)) % (SIZE*WARP_SIZE);
671
+ return indexes[real_offset];
672
+ }
673
+
674
+
675
+ __device__ void initialize (PoolType& p) {
676
+ if( current_leader() ){
677
+ #ifdef LAZY_MEM
678
+ try_lazy = true;
679
+ #endif
680
+ parent = &p;
681
+ }
682
+ for ( unsigned int i=0; i<SIZE; i++ ) {
683
+ get_index(i) = AdrType::null;
684
+ }
685
+ }
686
+
687
+ __device__ void finalize (unsigned int &rand_state) {
688
+ for ( unsigned int i=0; i<SIZE; i++ ) {
689
+ Index& slot = get_index(i);
690
+ if( slot != AdrType::null ){
691
+ parent->free(slot,rand_state);
692
+ }
693
+ }
694
+ }
695
+
696
+
697
+
698
+ __device__ Index alloc_index(unsigned int& rand_state){
699
+
700
+ #if 0
701
+ if( try_lazy ){
702
+ Index result = parent->lazy_alloc_index(rand_state);
703
+ if( result != AdrType::null ){
704
+ return result;
705
+ } else {
706
+ try_lazy = false;
707
+ }
708
+ }
709
+ #endif
710
+
711
+
712
+ #if 0
713
+ for ( unsigned int i=threadIdx.x; i<SIZE; i++ ) {
714
+ Index swap = atomicExch_block(&(indexes[i]),AdrType::null);
715
+ if( swap != AdrType::null ){
716
+ return swap;
717
+ }
718
+ }
719
+ for ( unsigned int i=0; i<threadIdx.x; i++ ) {
720
+ Index swap = atomicExch_block(&(indexes[i]),AdrType::null);
721
+ if( swap != AdrType::null ){
722
+ return swap;
723
+ }
724
+ }
725
+ #else
726
+ for ( unsigned int i=0; i<(SIZE*WARP_SIZE); i++ ) {
727
+ Index& slot = get_index(i);
728
+ Index swap = atomicExch_block(&slot,AdrType::null);
729
+ if( swap != AdrType::null ){
730
+ return swap;
731
+ }
732
+ }
733
+ #endif
734
+
735
+
736
+ return parent->alloc_index(rand_state);
737
+ }
738
+
739
+
740
+ __device__ DataType* alloc(unsigned int& rand_state){
741
+ Index result_index = alloc_index(rand_state);
742
+ if ( result_index != Adr<Index>::null ) {
743
+ return (parent->arena[result_index].data);
744
+ }
745
+ return NULL;
746
+ }
747
+
748
+
749
+ __device__ void free(Index index, unsigned int& rand_state){
750
+ #if 0
751
+ for ( unsigned int i=threadIdx.x; i<SIZE; i++ ) {
752
+ Index swap = atomicCAS_block(&(indexes[i]),AdrType::null,index);
753
+ if( swap == AdrType::null ){
754
+ return;
755
+ }
756
+ }
757
+ for ( unsigned int i=0; i<threadIdx.x; i++ ) {
758
+ Index swap = atomicCAS_block(&(indexes[i]),AdrType::null,index);
759
+ if( swap == AdrType::null ){
760
+ return;
761
+ }
762
+ }
763
+ #else
764
+ for ( unsigned int i=0; i<(SIZE*WARP_SIZE); i++ ) {
765
+ Index& slot = get_index(i);
766
+ Index swap = atomicCAS_block(&slot,AdrType::null,index);
767
+ if( swap == AdrType::null ){
768
+ return;
769
+ }
770
+ }
771
+ #endif
772
+ parent->free(index,rand_state);
773
+ }
774
+
775
+
776
+ __device__ void free(DataType* address, unsigned int& rand_state){
777
+ if( address == NULL ){
778
+ return;
779
+ }
780
+ Index index = address - (parent->arena);
781
+ free(index,rand_state);
782
+ }
783
+
784
+ #if __CUDA_ARCH__ < 600
785
+ #undef atomicExch_block
786
+ #undef atomicCAS_block
787
+ #endif
788
+
789
+
790
+ };
791
+
792
+
793
+
794
+
795
+ template<typename T, size_t SIZE, size_t WARP_SIZE=32>
796
+ struct SimpleMemCache {
797
+
798
+ typedef typename T::DataType DataType;
799
+ typedef typename T::Index Index;
800
+ typedef Adr<Index> AdrType;
801
+ typedef MemPool<DataType,Index> PoolType;
802
+
803
+ #ifdef LAZY_MEM
804
+ bool try_lazy;
805
+ #endif
806
+
807
+ PoolType* parent;
808
+ unsigned int counts[WARP_SIZE];
809
+ Index indexes[SIZE*WARP_SIZE];
810
+
811
+
812
+ #if __CUDA_ARCH__ < 600
813
+ #define atomicExch_block atomicExch
814
+ #define atomicCAS_block atomicCAS
815
+ #endif
816
+
817
+
818
+ __device__ Index& get_index( unsigned int offset ){
819
+ unsigned int real_offset = (WARP_SIZE*offset + threadIdx.x + (offset/SIZE)) % (SIZE*WARP_SIZE);
820
+ return indexes[real_offset];
821
+ }
822
+
823
+
824
+ __device__ void initialize (PoolType& p) {
825
+ if( current_leader() ){
826
+ parent = &p;
827
+ }
828
+ counts[threadIdx.x] = 0;
829
+ }
830
+
831
+ __device__ void finalize (unsigned int &rand_state) {
832
+ unsigned int count = counts[threadIdx.x];
833
+ for ( unsigned int i=0; i<count; i++ ) {
834
+ Index& slot = get_index(i);
835
+ parent->free(slot,rand_state);
836
+ }
837
+ }
838
+
839
+
840
+ __device__ void fill_up(unsigned int& rand_state) {
841
+ unsigned int count = counts[threadIdx.x];
842
+ if( SIZE/2 > count ) {
843
+ parent->pull_span( &get_index(count), (SIZE/2)-count, WARP_SIZE, rand_state);
844
+ }
845
+ }
846
+
847
+
848
+ __device__ Index alloc_index(unsigned int& rand_state) {
849
+
850
+ unsigned int& count = counts[threadIdx.x];
851
+ if( count > 0 ){
852
+ count -= 1;
853
+ Index& slot = get_index(count);
854
+ return slot;
855
+ } else {
856
+ return parent->alloc_index(rand_state);
857
+ }
858
+ }
859
+
860
+
861
+ __device__ DataType* alloc(unsigned int& rand_state){
862
+ Index result_index = alloc_index(rand_state);
863
+ if ( result_index != Adr<Index>::null ) {
864
+ return (parent->arena[result_index].data);
865
+ }
866
+ return NULL;
867
+ }
868
+
869
+
870
+ __device__ void free(Index index, unsigned int& rand_state){
871
+
872
+ unsigned int& count = counts[threadIdx.x];
873
+ if( count < SIZE ){
874
+ get_index(count) = index;
875
+ count += 1;
876
+ } else {
877
+ parent->free(index,rand_state);
878
+ }
879
+
880
+ }
881
+
882
+
883
+ __device__ void free(DataType* address, unsigned int& rand_state){
884
+ if( address == NULL ){
885
+ return;
886
+ }
887
+ Index index = address - (parent->arena);
888
+ free(index,rand_state);
889
+ }
890
+
891
+ #if __CUDA_ARCH__ < 600
892
+ #undef atomicExch_block
893
+ #undef atomicCAS_block
894
+ #endif
895
+
896
+
897
+ };
898
+
899
+
900
+
901
+
902
+
903
+
904
+ template<typename T, typename INDEX, typename MASK = unsigned int>
905
+ struct MemChunk {
906
+
907
+ typedef INDEX Index;
908
+ typedef MASK Mask;
909
+ typedef MemPool<MemChunk<T,Index,Mask>,Index> PoolType;
910
+
911
+ static const Index SIZE = std::numeric_limits<Mask>::digits;
912
+ static const Mask FULL = std::numeric_limits<Mask>::max();
913
+ static const MASK HIGH_BIT = 1<<(SIZE-1);
914
+ static const Index OFFSET_MASK = ~(SIZE-1);
915
+
916
+ Mask fill_mask;
917
+ T data[SIZE];
918
+
919
+ __device__ Index offset(PoolType& parent){
920
+ return (this - parent.arena);
921
+ }
922
+
923
+ __device__ bool full(){
924
+ return (fill_mask == FULL);
925
+ }
926
+
927
+ __device__ bool empty(){
928
+ return (fill_mask == 0);
929
+ }
930
+
931
+ __device__ bool fill_count(){
932
+ return __popc(fill_mask);
933
+ }
934
+
935
+ __device__ MemChunk<T,Index,Mask>()
936
+ : fill_mask(0)
937
+ {}
938
+
939
+ __device__ Index alloc(PoolType& parent, Mask& rand_state) {
940
+ Mask try_mask = random_uint(rand_state);
941
+ while( ! full() ){
942
+ Mask scan_mask = try_mask & ~fill_mask;
943
+ scan_mask = (scan_mask == 0) ? ~fill_mask : scan_mask;
944
+ Index lead = leading_zeros(scan_mask);
945
+ Mask select = HIGH_BIT >> lead;
946
+ if( select == 0 ){
947
+ break;
948
+ }
949
+ Mask prior = atomicOr(fill_mask,select);
950
+ if( (prior & select) == 0 ){
951
+ return ( SIZE * offset(parent) ) + ( SIZE - 1 - lead );
952
+ }
953
+ }
954
+ return Adr<Index>::null;
955
+ }
956
+
957
+ __device__ void free(Index index, PoolType& parent, Mask& rand_state) {
958
+ if( (index & OFFSET_MASK) == offset() ){
959
+ Index inner_index = index & ~OFFSET_MASK;
960
+ Mask select = 1 << inner_index;
961
+ atomicAnd(fill_mask, ~select);
962
+ }
963
+ }
964
+
965
+ };
966
+
967
+
968
+ /*
969
+ template<typename T, typename INDEX, typename MASK, INDEX SIZE_VAL>
970
+ struct MemPoolBank {
971
+
972
+ typedef INDEX Index;
973
+ typedef MASK MaskType;
974
+ typedef MemChunk<T,Index,MaskType> ChunkType;
975
+ typedef MemPool<ChunkType,Index> PoolType;
976
+
977
+ static const Index SIZE = SIZE_VAL;
978
+
979
+ unsigned int rand_state;
980
+ Index full_count;
981
+
982
+ PoolType& parent;
983
+ Index chunks[SIZE];
984
+
985
+ __device__ Index containing_chunk(Index target){
986
+ Index result = ChunkType::offset_mask & index;
987
+ if( result >= parent.arena_size ){
988
+ return Adr<Index>::null;
989
+ }
990
+ return result;
991
+ }
992
+
993
+ __device__ MemPoolBank<T,Index,MaskType,SIZE_VAL>
994
+ (PoolType& parent_ref, unsigned int seed, bool& err)
995
+ : parent(parent_ref)
996
+ , full_count(0)
997
+ , rand_state(seed)
998
+ {
999
+ for(int i=0; i<SIZE; i++){
1000
+ chunks[i] = Adr<Index>::null;
1001
+ }
1002
+ err = parent.pull_span(chunks,SIZE,seed);
1003
+ }
1004
+
1005
+
1006
+ __device__ Index alloc() {
1007
+ for(int i=0; i<SIZE; i++){
1008
+ if( chunks[i] == Adr<Index>::null ){
1009
+ continue;
1010
+ }
1011
+
1012
+ }
1013
+ }
1014
+
1015
+ __device__ Index hard_neighbor_alloc(Index target, MaskType& rand_state){
1016
+ return parent.arena[ChunkType::offset_mask & index].alloc(parent,rand_state);
1017
+ }
1018
+
1019
+ __device__ Index neighbor_alloc(Index target, MaskType& rand_state){
1020
+ Index result = hard_neighbor_alloc(target,rand_state);
1021
+ if( result == Adr<Index>::null){
1022
+ return alloc(rand_state);
1023
+ } else {
1024
+ return result;
1025
+ }
1026
+ }
1027
+
1028
+
1029
+ __device__ Index coalesce(Index original) {
1030
+
1031
+ }
1032
+
1033
+ __device__ void free(Index index) {
1034
+ Index chunk_index = ChunkType::offset_mask & index;
1035
+ if( index < parent.arena_size ){
1036
+ parent.arena[chunk_index].free(parent,index);
1037
+ }
1038
+ }
1039
+
1040
+ };
1041
+ */
1042
+
1043
+ }
1044
+
1045
+ }
1046
+
1047
+
1048
+ #endif
1049
+
1050
+