00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029 #ifndef __TBB_machine_H
00030 #define __TBB_machine_H
00031
00124 #include "tbb_stddef.h"
00125
00126 namespace tbb {
00127 namespace internal {
00128
00130
00131
00132
00133
00134
00135 template <typename T, std::size_t S>
00136 struct machine_load_store;
00137
00138 template <typename T, std::size_t S>
00139 struct machine_load_store_relaxed;
00140
00141 template <typename T, std::size_t S>
00142 struct machine_load_store_seq_cst;
00143
00144
00146
00147 template<size_t S> struct atomic_selector;
00148
00149 template<> struct atomic_selector<1> {
00150 typedef int8_t word;
00151 inline static word fetch_store ( volatile void* location, word value );
00152 };
00153
00154 template<> struct atomic_selector<2> {
00155 typedef int16_t word;
00156 inline static word fetch_store ( volatile void* location, word value );
00157 };
00158
00159 template<> struct atomic_selector<4> {
00160 #if _MSC_VER && !_WIN64
00161
00162 typedef intptr_t word;
00163 #else
00164 typedef int32_t word;
00165 #endif
00166 inline static word fetch_store ( volatile void* location, word value );
00167 };
00168
00169 template<> struct atomic_selector<8> {
00170 typedef int64_t word;
00171 inline static word fetch_store ( volatile void* location, word value );
00172 };
00173
00174 }}
00175
00176 #define __TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(M) \
00177 inline void __TBB_machine_generic_store8##M(volatile void *ptr, int64_t value) { \
00178 for(;;) { \
00179 int64_t result = *(int64_t *)ptr; \
00180 if( __TBB_machine_cmpswp8##M(ptr,value,result)==result ) break; \
00181 } \
00182 } \
00183
00184 #define __TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(M) \
00185 inline int64_t __TBB_machine_generic_load8##M(const volatile void *ptr) { \
00186 \
00187 \
00188 const int64_t anyvalue = 2305843009213693951LL; \
00189 return __TBB_machine_cmpswp8##M(const_cast<volatile void *>(ptr),anyvalue,anyvalue); \
00190 } \
00191
00192 #if _WIN32||_WIN64
00193
00194 #ifdef _MANAGED
00195 #pragma managed(push, off)
00196 #endif
00197
00198 #if __MINGW64__ || __MINGW32__
00199 extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void );
00200 #define __TBB_Yield() SwitchToThread()
00201 #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT)
00202 #include "machine/gcc_generic.h"
00203 #elif __MINGW64__
00204 #include "machine/linux_intel64.h"
00205 #elif __MINGW32__
00206 #include "machine/linux_ia32.h"
00207 #endif
00208 #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
00209 #include "machine/icc_generic.h"
00210 #elif defined(_M_IX86)
00211 #include "machine/windows_ia32.h"
00212 #elif defined(_M_X64)
00213 #include "machine/windows_intel64.h"
00214 #elif _XBOX
00215 #include "machine/xbox360_ppc.h"
00216 #endif
00217
00218 #ifdef _MANAGED
00219 #pragma managed(pop)
00220 #endif
00221
00222 #elif __TBB_DEFINE_MIC
00223
00224 #include "machine/mic_common.h"
00225
00226 #include "machine/linux_intel64.h"
00227
00228 #elif __linux__ || __FreeBSD__ || __NetBSD__
00229
00230 #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT)
00231 #include "machine/gcc_generic.h"
00232 #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
00233 #include "machine/icc_generic.h"
00234 #elif __i386__
00235 #include "machine/linux_ia32.h"
00236 #elif __x86_64__
00237 #include "machine/linux_intel64.h"
00238 #elif __ia64__
00239 #include "machine/linux_ia64.h"
00240 #elif __powerpc__
00241 #include "machine/mac_ppc.h"
00242 #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT
00243 #include "machine/gcc_generic.h"
00244 #endif
00245 #include "machine/linux_common.h"
00246
00247 #elif __APPLE__
00248
00249 #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT)
00250 #include "machine/icc_generic.h"
00251 #elif __i386__
00252 #include "machine/linux_ia32.h"
00253 #elif __x86_64__
00254 #include "machine/linux_intel64.h"
00255 #elif __POWERPC__
00256 #include "machine/mac_ppc.h"
00257 #endif
00258 #include "machine/macos_common.h"
00259
00260 #elif _AIX
00261
00262 #include "machine/ibm_aix51.h"
00263
00264 #elif __sun || __SUNPRO_CC
00265
00266 #define __asm__ asm
00267 #define __volatile__ volatile
00268
00269 #if __i386 || __i386__
00270 #include "machine/linux_ia32.h"
00271 #elif __x86_64__
00272 #include "machine/linux_intel64.h"
00273 #elif __sparc
00274 #include "machine/sunos_sparc.h"
00275 #endif
00276 #include <sched.h>
00277
00278 #define __TBB_Yield() sched_yield()
00279
00280 #endif
00281
00282 #ifndef __TBB_64BIT_ATOMICS
00283 #define __TBB_64BIT_ATOMICS 1
00284 #endif
00285
00286
00287
00288
00289 #if __TBB_USE_FENCED_ATOMICS
00290 #define __TBB_machine_cmpswp1 __TBB_machine_cmpswp1full_fence
00291 #define __TBB_machine_cmpswp2 __TBB_machine_cmpswp2full_fence
00292 #define __TBB_machine_cmpswp4 __TBB_machine_cmpswp4full_fence
00293 #define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8full_fence
00294
00295 #if __TBB_WORDSIZE==8
00296 #define __TBB_machine_fetchadd8 __TBB_machine_fetchadd8full_fence
00297 #define __TBB_machine_fetchstore8 __TBB_machine_fetchstore8full_fence
00298 #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd8release(P,V)
00299 #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd8acquire(P,1)
00300 #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd8release(P,(-1))
00301 #else
00302 #define __TBB_machine_fetchadd4 __TBB_machine_fetchadd4full_fence
00303 #define __TBB_machine_fetchstore4 __TBB_machine_fetchstore4full_fence
00304 #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd4release(P,V)
00305 #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd4acquire(P,1)
00306 #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd4release(P,(-1))
00307 #endif
00308 #else
00309 #define __TBB_FetchAndAddWrelease(P,V) __TBB_FetchAndAddW(P,V)
00310 #define __TBB_FetchAndIncrementWacquire(P) __TBB_FetchAndAddW(P,1)
00311 #define __TBB_FetchAndDecrementWrelease(P) __TBB_FetchAndAddW(P,(-1))
00312 #endif
00313
00314 #if __TBB_WORDSIZE==4
00315 #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp4(P,V,C)
00316 #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd4(P,V)
00317 #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore4(P,V)
00318 #elif __TBB_WORDSIZE==8
00319 #if __TBB_USE_GENERIC_DWORD_LOAD_STORE || __TBB_USE_GENERIC_DWORD_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_STORE
00320 #error These macros should only be used on 32-bit platforms.
00321 #endif
00322
00323 #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp8(P,V,C)
00324 #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd8(P,V)
00325 #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore8(P,V)
00326 #else
00327 #error Unsupported machine word size.
00328 #endif
00329
00330 #ifndef __TBB_Pause
00331 inline void __TBB_Pause(int32_t) {
00332 __TBB_Yield();
00333 }
00334 #endif
00335
00336 namespace tbb {
00337
00339 inline void atomic_fence () { __TBB_full_memory_fence(); }
00340
00341 namespace internal {
00342
00344
00345 class atomic_backoff : no_copy {
00347
00349 static const int32_t LOOPS_BEFORE_YIELD = 16;
00350 int32_t count;
00351 public:
00352 atomic_backoff() : count(1) {}
00353
00355 void pause() {
00356 if( count<=LOOPS_BEFORE_YIELD ) {
00357 __TBB_Pause(count);
00358
00359 count*=2;
00360 } else {
00361
00362 __TBB_Yield();
00363 }
00364 }
00365
00366
00367 bool bounded_pause() {
00368 if( count<=LOOPS_BEFORE_YIELD ) {
00369 __TBB_Pause(count);
00370
00371 count*=2;
00372 return true;
00373 } else {
00374 return false;
00375 }
00376 }
00377
00378 void reset() {
00379 count = 1;
00380 }
00381 };
00382
00384
00385 template<typename T, typename U>
00386 void spin_wait_while_eq( const volatile T& location, U value ) {
00387 atomic_backoff backoff;
00388 while( location==value ) backoff.pause();
00389 }
00390
00392
00393 template<typename T, typename U>
00394 void spin_wait_until_eq( const volatile T& location, const U value ) {
00395 atomic_backoff backoff;
00396 while( location!=value ) backoff.pause();
00397 }
00398
00399
00400
00401
00402
00403
00404
00405
00406
00407
00408
00409
00410 #if (__TBB_BIG_ENDIAN!=-1)
00411 #if ( __TBB_USE_GENERIC_PART_WORD_CAS)
00412 #error generic implementation of part-word CAS was explicitly disabled for this configuration
00413 #endif
00414 template<typename T>
00415 inline T __TBB_MaskedCompareAndSwap (volatile T * const ptr, const T value, const T comparand ) {
00416 struct endianness{ static bool is_big_endian(){
00417 #ifndef __TBB_BIG_ENDIAN
00418 const uint32_t probe = 0x03020100;
00419 return (((const char*)(&probe))[0]==0x03);
00420 #elif (__TBB_BIG_ENDIAN==0) || (__TBB_BIG_ENDIAN==1)
00421 return __TBB_BIG_ENDIAN;
00422 #else
00423 #error unexpected value of __TBB_BIG_ENDIAN
00424 #endif
00425 }};
00426
00427 const uint32_t byte_offset = (uint32_t) ((uintptr_t)ptr & 0x3);
00428 volatile uint32_t * const aligned_ptr = (uint32_t*)((uintptr_t)ptr - byte_offset );
00429
00430
00431 const uint32_t bits_to_shift = 8*(endianness::is_big_endian() ? (4 - sizeof(T) - (byte_offset)) : byte_offset);
00432 const uint32_t mask = (((uint32_t)1<<(sizeof(T)*8)) - 1 )<<bits_to_shift;
00433 const uint32_t shifted_comparand = ((uint32_t)comparand << bits_to_shift)&mask;
00434 const uint32_t shifted_value = ((uint32_t)value << bits_to_shift)&mask;
00435
00436 for(atomic_backoff b;;b.pause()) {
00437 const uint32_t surroundings = *aligned_ptr & ~mask ;
00438 const uint32_t big_comparand = surroundings | shifted_comparand ;
00439 const uint32_t big_value = surroundings | shifted_value ;
00440
00441
00442 const uint32_t big_result = (uint32_t)__TBB_machine_cmpswp4( aligned_ptr, big_value, big_comparand );
00443 if( big_result == big_comparand
00444 || ((big_result ^ big_comparand) & mask) != 0)
00445 {
00446 return T((big_result & mask) >> bits_to_shift);
00447 }
00448 else continue;
00449 }
00450 }
00451 #endif
00452 template<size_t S, typename T>
00453 inline T __TBB_CompareAndSwapGeneric (volatile void *ptr, T value, T comparand );
00454
00455 template<>
00456 inline uint8_t __TBB_CompareAndSwapGeneric <1,uint8_t> (volatile void *ptr, uint8_t value, uint8_t comparand ) {
00457 #if __TBB_USE_GENERIC_PART_WORD_CAS
00458 return __TBB_MaskedCompareAndSwap<uint8_t>((volatile uint8_t *)ptr,value,comparand);
00459 #else
00460 return __TBB_machine_cmpswp1(ptr,value,comparand);
00461 #endif
00462 }
00463
00464 template<>
00465 inline uint16_t __TBB_CompareAndSwapGeneric <2,uint16_t> (volatile void *ptr, uint16_t value, uint16_t comparand ) {
00466 #if __TBB_USE_GENERIC_PART_WORD_CAS
00467 return __TBB_MaskedCompareAndSwap<uint16_t>((volatile uint16_t *)ptr,value,comparand);
00468 #else
00469 return __TBB_machine_cmpswp2(ptr,value,comparand);
00470 #endif
00471 }
00472
00473 template<>
00474 inline uint32_t __TBB_CompareAndSwapGeneric <4,uint32_t> (volatile void *ptr, uint32_t value, uint32_t comparand ) {
00475
00476 return (uint32_t)__TBB_machine_cmpswp4(ptr,value,comparand);
00477 }
00478
00479 #if __TBB_64BIT_ATOMICS
00480 template<>
00481 inline uint64_t __TBB_CompareAndSwapGeneric <8,uint64_t> (volatile void *ptr, uint64_t value, uint64_t comparand ) {
00482 return __TBB_machine_cmpswp8(ptr,value,comparand);
00483 }
00484 #endif
00485
00486 template<size_t S, typename T>
00487 inline T __TBB_FetchAndAddGeneric (volatile void *ptr, T addend) {
00488 atomic_backoff b;
00489 T result;
00490 for(;;) {
00491 result = *reinterpret_cast<volatile T *>(ptr);
00492
00493 if( __TBB_CompareAndSwapGeneric<S,T> ( ptr, result+addend, result )==result )
00494 break;
00495 b.pause();
00496 }
00497 return result;
00498 }
00499
00500 template<size_t S, typename T>
00501 inline T __TBB_FetchAndStoreGeneric (volatile void *ptr, T value) {
00502 atomic_backoff b;
00503 T result;
00504 for(;;) {
00505 result = *reinterpret_cast<volatile T *>(ptr);
00506
00507 if( __TBB_CompareAndSwapGeneric<S,T> ( ptr, value, result )==result )
00508 break;
00509 b.pause();
00510 }
00511 return result;
00512 }
00513
00514 #if __TBB_USE_GENERIC_PART_WORD_CAS
00515 #define __TBB_machine_cmpswp1 tbb::internal::__TBB_CompareAndSwapGeneric<1,uint8_t>
00516 #define __TBB_machine_cmpswp2 tbb::internal::__TBB_CompareAndSwapGeneric<2,uint16_t>
00517 #endif
00518
00519 #if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_PART_WORD_FETCH_ADD
00520 #define __TBB_machine_fetchadd1 tbb::internal::__TBB_FetchAndAddGeneric<1,uint8_t>
00521 #define __TBB_machine_fetchadd2 tbb::internal::__TBB_FetchAndAddGeneric<2,uint16_t>
00522 #endif
00523
00524 #if __TBB_USE_GENERIC_FETCH_ADD
00525 #define __TBB_machine_fetchadd4 tbb::internal::__TBB_FetchAndAddGeneric<4,uint32_t>
00526 #endif
00527
00528 #if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_ADD
00529 #define __TBB_machine_fetchadd8 tbb::internal::__TBB_FetchAndAddGeneric<8,uint64_t>
00530 #endif
00531
00532 #if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_PART_WORD_FETCH_STORE
00533 #define __TBB_machine_fetchstore1 tbb::internal::__TBB_FetchAndStoreGeneric<1,uint8_t>
00534 #define __TBB_machine_fetchstore2 tbb::internal::__TBB_FetchAndStoreGeneric<2,uint16_t>
00535 #endif
00536
00537 #if __TBB_USE_GENERIC_FETCH_STORE
00538 #define __TBB_machine_fetchstore4 tbb::internal::__TBB_FetchAndStoreGeneric<4,uint32_t>
00539 #endif
00540
00541 #if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_DWORD_FETCH_STORE
00542 #define __TBB_machine_fetchstore8 tbb::internal::__TBB_FetchAndStoreGeneric<8,uint64_t>
00543 #endif
00544
00545 #if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE
00546 #define __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(S) \
00547 atomic_selector<S>::word atomic_selector<S>::fetch_store ( volatile void* location, word value ) { \
00548 return __TBB_machine_fetchstore##S( location, value ); \
00549 }
00550
00551 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(1)
00552 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(2)
00553 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(4)
00554 __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(8)
00555
00556 #undef __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE
00557 #endif
00558
00559 #if __TBB_USE_GENERIC_DWORD_LOAD_STORE
00560
00561 #if ! __TBB_USE_FENCED_ATOMICS
00562
00563
00564 #define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8
00565 #endif
00566 __TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence)
00567 __TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence)
00568
00569 #if ! __TBB_USE_FENCED_ATOMICS
00570 #undef __TBB_machine_cmpswp8full_fence
00571 #endif
00572
00573 #define __TBB_machine_store8 tbb::internal::__TBB_machine_generic_store8full_fence
00574 #define __TBB_machine_load8 tbb::internal::__TBB_machine_generic_load8full_fence
00575 #endif
00576
00577 #if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE
00578
00584 template <typename T, size_t S>
00585 struct machine_load_store {
00586 static T load_with_acquire ( const volatile T& location ) {
00587 T to_return = location;
00588 __TBB_acquire_consistency_helper();
00589 return to_return;
00590 }
00591 static void store_with_release ( volatile T &location, T value ) {
00592 __TBB_release_consistency_helper();
00593 location = value;
00594 }
00595 };
00596
00597
00598 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
00599 template <typename T>
00600 struct machine_load_store<T,8> {
00601 static T load_with_acquire ( const volatile T& location ) {
00602 return (T)__TBB_machine_load8( (const volatile void*)&location );
00603 }
00604 static void store_with_release ( volatile T& location, T value ) {
00605 __TBB_machine_store8( (volatile void*)&location, (int64_t)value );
00606 }
00607 };
00608 #endif
00609 #endif
00610
00611 #if __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE
00612 template <typename T, size_t S>
00613 struct machine_load_store_seq_cst {
00614 static T load ( const volatile T& location ) {
00615 __TBB_full_memory_fence();
00616 return machine_load_store<T,S>::load_with_acquire( location );
00617 }
00618 #if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE
00619 static void store ( volatile T &location, T value ) {
00620 atomic_selector<S>::fetch_store( (volatile void*)&location, (typename atomic_selector<S>::word)value );
00621 }
00622 #else
00623 static void store ( volatile T &location, T value ) {
00624 machine_load_store<T,S>::store_with_release( location, value );
00625 __TBB_full_memory_fence();
00626 }
00627 #endif
00628 };
00629
00630 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
00631
00633 template <typename T>
00634 struct machine_load_store_seq_cst<T,8> {
00635 static T load ( const volatile T& location ) {
00636
00637
00638 const int64_t anyvalue = 2305843009213693951LL;
00639 return __TBB_machine_cmpswp8( (volatile void*)const_cast<volatile T*>(&location), anyvalue, anyvalue );
00640 }
00641 static void store ( volatile T &location, T value ) {
00642 int64_t result = (volatile int64_t&)location;
00643 while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result )
00644 result = (volatile int64_t&)location;
00645 }
00646 };
00647 #endif
00648 #endif
00649
00650 #if __TBB_USE_GENERIC_RELAXED_LOAD_STORE
00651
00655 template <typename T, size_t S>
00656 struct machine_load_store_relaxed {
00657 static inline T load ( const volatile T& location ) {
00658 return location;
00659 }
00660 static inline void store ( volatile T& location, T value ) {
00661 location = value;
00662 }
00663 };
00664
00665 #if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS
00666 template <typename T>
00667 struct machine_load_store_relaxed<T,8> {
00668 static inline T load ( const volatile T& location ) {
00669 return (T)__TBB_machine_load8( (const volatile void*)&location );
00670 }
00671 static inline void store ( volatile T& location, T value ) {
00672 __TBB_machine_store8( (volatile void*)&location, (int64_t)value );
00673 }
00674 };
00675 #endif
00676 #endif
00677
00678 #undef __TBB_WORDSIZE //this macro is forbidden to use outside of atomic machinery
00679
00680 template<typename T>
00681 inline T __TBB_load_with_acquire(const volatile T &location) {
00682 return machine_load_store<T,sizeof(T)>::load_with_acquire( location );
00683 }
00684 template<typename T, typename V>
00685 inline void __TBB_store_with_release(volatile T& location, V value) {
00686 machine_load_store<T,sizeof(T)>::store_with_release( location, T(value) );
00687 }
00689 inline void __TBB_store_with_release(volatile size_t& location, size_t value) {
00690 machine_load_store<size_t,sizeof(size_t)>::store_with_release( location, value );
00691 }
00692
00693 template<typename T>
00694 inline T __TBB_load_full_fence(const volatile T &location) {
00695 return machine_load_store_seq_cst<T,sizeof(T)>::load( location );
00696 }
00697 template<typename T, typename V>
00698 inline void __TBB_store_full_fence(volatile T& location, V value) {
00699 machine_load_store_seq_cst<T,sizeof(T)>::store( location, T(value) );
00700 }
00702 inline void __TBB_store_full_fence(volatile size_t& location, size_t value) {
00703 machine_load_store_seq_cst<size_t,sizeof(size_t)>::store( location, value );
00704 }
00705
00706 template<typename T>
00707 inline T __TBB_load_relaxed (const volatile T& location) {
00708 return machine_load_store_relaxed<T,sizeof(T)>::load( const_cast<T&>(location) );
00709 }
00710 template<typename T, typename V>
00711 inline void __TBB_store_relaxed ( volatile T& location, V value ) {
00712 machine_load_store_relaxed<T,sizeof(T)>::store( const_cast<T&>(location), T(value) );
00713 }
00715 inline void __TBB_store_relaxed ( volatile size_t& location, size_t value ) {
00716 machine_load_store_relaxed<size_t,sizeof(size_t)>::store( const_cast<size_t&>(location), value );
00717 }
00718
00719
00720
00721
00722
00723
00724
00725
00726 #ifndef __TBB_TypeWithAlignmentAtLeastAsStrict
00727
00728 #if __TBB_ATTRIBUTE_ALIGNED_PRESENT
00729
00730 #define __TBB_DefineTypeWithAlignment(PowerOf2) \
00731 struct __TBB_machine_type_with_alignment_##PowerOf2 { \
00732 uint32_t member[PowerOf2/sizeof(uint32_t)]; \
00733 } __attribute__((aligned(PowerOf2)));
00734 #define __TBB_alignof(T) __alignof__(T)
00735
00736 #elif __TBB_DECLSPEC_ALIGN_PRESENT
00737
00738 #define __TBB_DefineTypeWithAlignment(PowerOf2) \
00739 __declspec(align(PowerOf2)) \
00740 struct __TBB_machine_type_with_alignment_##PowerOf2 { \
00741 uint32_t member[PowerOf2/sizeof(uint32_t)]; \
00742 };
00743 #define __TBB_alignof(T) __alignof(T)
00744
00745 #else
00746 #error Must define __TBB_TypeWithAlignmentAtLeastAsStrict(T)
00747 #endif
00748
00749
00750
00751 __TBB_DefineTypeWithAlignment(16)
00752 __TBB_DefineTypeWithAlignment(32)
00753 __TBB_DefineTypeWithAlignment(64)
00754
00755 typedef __TBB_machine_type_with_alignment_64 __TBB_machine_type_with_strictest_alignment;
00756
00757
00758 template<size_t N> struct type_with_alignment;
00759
00760
00761 template<> struct type_with_alignment<1> { char member; };
00762 template<> struct type_with_alignment<2> { uint16_t member; };
00763 template<> struct type_with_alignment<4> { uint32_t member; };
00764 template<> struct type_with_alignment<8> { uint64_t member; };
00765 template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; };
00766 template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; };
00767 template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; };
00768
00769 #if __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN
00771
00773 template<size_t Size, typename T>
00774 struct work_around_alignment_bug {
00775 static const size_t alignment = __TBB_alignof(T);
00776 };
00777 #define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<tbb::internal::work_around_alignment_bug<sizeof(T),T>::alignment>
00778 #else
00779 #define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<__TBB_alignof(T)>
00780 #endif
00781
00782 #endif
00783
00784
00785 template<typename T>
00786 struct reverse {
00787 static const T byte_table[256];
00788 };
00789
00790
00791 template<typename T>
00792 const T reverse<T>::byte_table[256] = {
00793 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
00794 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
00795 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
00796 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
00797 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
00798 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
00799 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
00800 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
00801 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
00802 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
00803 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
00804 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
00805 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
00806 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
00807 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
00808 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF
00809 };
00810
00811 }
00812 }
00813
00814
00815 using tbb::internal::__TBB_load_with_acquire;
00816 using tbb::internal::__TBB_store_with_release;
00817
00818
00819 #define __TBB_load_acquire __TBB_load_with_acquire
00820 #define __TBB_store_release __TBB_store_with_release
00821
00822 #ifndef __TBB_Log2
00823 inline intptr_t __TBB_Log2( uintptr_t x ) {
00824 if( x==0 ) return -1;
00825 intptr_t result = 0;
00826 uintptr_t tmp;
00827
00828 if( sizeof(x)>4 && (tmp = ((uint64_t)x)>>32)) { x=tmp; result += 32; }
00829 if( (tmp = x>>16) ) { x=tmp; result += 16; }
00830 if( (tmp = x>>8) ) { x=tmp; result += 8; }
00831 if( (tmp = x>>4) ) { x=tmp; result += 4; }
00832 if( (tmp = x>>2) ) { x=tmp; result += 2; }
00833 return (x&2)? result+1: result;
00834 }
00835 #endif
00836
00837 #ifndef __TBB_AtomicOR
00838 inline void __TBB_AtomicOR( volatile void *operand, uintptr_t addend ) {
00839 tbb::internal::atomic_backoff b;
00840 for(;;) {
00841 uintptr_t tmp = *(volatile uintptr_t *)operand;
00842 uintptr_t result = __TBB_CompareAndSwapW(operand, tmp|addend, tmp);
00843 if( result==tmp ) break;
00844 b.pause();
00845 }
00846 }
00847 #endif
00848
00849 #ifndef __TBB_AtomicAND
00850 inline void __TBB_AtomicAND( volatile void *operand, uintptr_t addend ) {
00851 tbb::internal::atomic_backoff b;
00852 for(;;) {
00853 uintptr_t tmp = *(volatile uintptr_t *)operand;
00854 uintptr_t result = __TBB_CompareAndSwapW(operand, tmp&addend, tmp);
00855 if( result==tmp ) break;
00856 b.pause();
00857 }
00858 }
00859 #endif
00860
00861 #ifndef __TBB_Flag
00862 typedef unsigned char __TBB_Flag;
00863 #endif
00864 typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag;
00865
00866 #ifndef __TBB_TryLockByte
00867 inline bool __TBB_TryLockByte( __TBB_atomic_flag &flag ) {
00868 return __TBB_machine_cmpswp1(&flag,1,0)==0;
00869 }
00870 #endif
00871
00872 #ifndef __TBB_LockByte
00873 inline __TBB_Flag __TBB_LockByte( __TBB_atomic_flag& flag ) {
00874 if ( !__TBB_TryLockByte(flag) ) {
00875 tbb::internal::atomic_backoff b;
00876 do {
00877 b.pause();
00878 } while ( !__TBB_TryLockByte(flag) );
00879 }
00880 return 0;
00881 }
00882 #endif
00883
00884 #ifndef __TBB_UnlockByte
00885 #define __TBB_UnlockByte __TBB_store_with_release
00886 #endif
00887
00888 #ifndef __TBB_ReverseByte
00889 inline unsigned char __TBB_ReverseByte(unsigned char src) {
00890 return tbb::internal::reverse<unsigned char>::byte_table[src];
00891 }
00892 #endif
00893
00894 template<typename T>
00895 T __TBB_ReverseBits(T src) {
00896 T dst;
00897 unsigned char *original = (unsigned char *) &src;
00898 unsigned char *reversed = (unsigned char *) &dst;
00899
00900 for( int i = sizeof(T)-1; i >= 0; i-- )
00901 reversed[i] = __TBB_ReverseByte( original[sizeof(T)-i-1] );
00902
00903 return dst;
00904 }
00905
00906 #endif