00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021 #ifndef __TBB_concurrent_hash_map_H
00022 #define __TBB_concurrent_hash_map_H
00023
00024 #include <stdexcept>
00025 #include <iterator>
00026 #include <utility>
00027 #include <cstring>
00028 #include <string>
00029 #include "tbb_stddef.h"
00030 #include "cache_aligned_allocator.h"
00031 #include "tbb_allocator.h"
00032 #include "spin_rw_mutex.h"
00033 #include "atomic.h"
00034 #include "aligned_space.h"
00035 #if TBB_USE_PERFORMANCE_WARNINGS
00036 #include <typeinfo>
00037 #endif
00038
00039 namespace tbb {
00040
00041 template<typename T> struct tbb_hash_compare;
00042 template<typename Key, typename T, typename HashCompare = tbb_hash_compare<Key>, typename A = tbb_allocator<std::pair<Key, T> > >
00043 class concurrent_hash_map;
00044
00046 namespace internal {
00048 void* __TBB_EXPORTED_FUNC itt_load_pointer_with_acquire_v3( const void* src );
00050 void __TBB_EXPORTED_FUNC itt_store_pointer_with_release_v3( void* dst, void* src );
00051
00053 typedef size_t hashcode_t;
00055 class hash_map_base {
00056 public:
00058 typedef size_t size_type;
00060 typedef size_t hashcode_t;
00062 typedef size_t segment_index_t;
00064 struct node_base : no_copy {
00066 typedef spin_rw_mutex mutex_t;
00068 typedef mutex_t::scoped_lock scoped_t;
00070 node_base *next;
00071 mutex_t mutex;
00072 };
00074 # define __TBB_rehash_req reinterpret_cast<node_base*>(1)
00076 # define __TBB_empty_rehashed reinterpret_cast<node_base*>(0)
00078 struct bucket : no_copy {
00080 typedef spin_rw_mutex mutex_t;
00082 typedef mutex_t::scoped_lock scoped_t;
00083 mutex_t mutex;
00084 node_base *node_list;
00085 };
00087 static size_type const embedded_block = 1;
00089 static size_type const embedded_buckets = 1<<embedded_block;
00091 static size_type const first_block = 8;
00093 static size_type const pointers_per_table = sizeof(segment_index_t) * 8;
00095 typedef bucket *segment_ptr_t;
00097 typedef segment_ptr_t segments_table_t[pointers_per_table];
00099 atomic<hashcode_t> my_mask;
00101 segments_table_t my_table;
00103 atomic<size_type> my_size;
00105 bucket my_embedded_segment[embedded_buckets];
00106
00108 hash_map_base() {
00109 std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t)
00110 + sizeof(my_size) + sizeof(my_mask)
00111 + embedded_buckets*sizeof(bucket) );
00112 for( size_type i = 0; i < embedded_block; i++ )
00113 my_table[i] = my_embedded_segment + segment_base(i);
00114 my_mask = embedded_buckets - 1;
00115 __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
00116 }
00117
00119 static segment_index_t segment_index_of( size_type index ) {
00120 return segment_index_t( __TBB_Log2( index|1 ) );
00121 }
00122
00124 static segment_index_t segment_base( segment_index_t k ) {
00125 return (segment_index_t(1)<<k & ~segment_index_t(1));
00126 }
00127
00129 static size_type segment_size( segment_index_t k ) {
00130 return size_type(1)<<k;
00131 }
00132
00134 static bool is_valid( void *ptr ) {
00135 return ptr > reinterpret_cast<void*>(1);
00136 }
00137
00139 static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
00140 if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
00141 else for(size_type i = 0; i < sz; i++, ptr++) {
00142 *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
00143 ptr->node_list = __TBB_rehash_req;
00144 }
00145 }
00146
00148 static void add_to_bucket( bucket *b, node_base *n ) {
00149 n->next = b->node_list;
00150 b->node_list = n;
00151 }
00152
00154 struct enable_segment_failsafe {
00155 segment_ptr_t *my_segment_ptr;
00156 enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
00157 ~enable_segment_failsafe() {
00158 if( my_segment_ptr ) *my_segment_ptr = 0;
00159 }
00160 };
00161
00163 void enable_segment( segment_index_t k, bool is_initial = false ) {
00164 __TBB_ASSERT( k, "Zero segment must be embedded" );
00165 enable_segment_failsafe watchdog( my_table, k );
00166 cache_aligned_allocator<bucket> alloc;
00167 size_type sz;
00168 __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
00169 if( k >= first_block ) {
00170 sz = segment_size( k );
00171 segment_ptr_t ptr = alloc.allocate( sz );
00172 init_buckets( ptr, sz, is_initial );
00173 #if TBB_USE_THREADING_TOOLS
00174 itt_store_pointer_with_release_v3( my_table + k, ptr );
00175 #else
00176 my_table[k] = ptr;
00177 #endif
00178 sz <<= 1;
00179 } else {
00180 __TBB_ASSERT( k == embedded_block, "Wrong segment index" );
00181 sz = segment_size( first_block );
00182 segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets );
00183 init_buckets( ptr, sz - embedded_buckets, is_initial );
00184 ptr -= segment_base(embedded_block);
00185 for(segment_index_t i = embedded_block; i < first_block; i++)
00186 #if TBB_USE_THREADING_TOOLS
00187 itt_store_pointer_with_release_v3( my_table + i, ptr + segment_base(i) );
00188 #else
00189 my_table[i] = ptr + segment_base(i);
00190 #endif
00191 }
00192 #if TBB_USE_THREADING_TOOLS
00193 itt_store_pointer_with_release_v3( &my_mask, (void*)(sz-1) );
00194 #else
00195 my_mask = sz - 1;
00196 #endif
00197 watchdog.my_segment_ptr = 0;
00198 }
00199
00201 bucket *get_bucket( hashcode_t h ) const throw() {
00202 segment_index_t s = segment_index_of( h );
00203 h -= segment_base(s);
00204 segment_ptr_t seg = my_table[s];
00205 __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" );
00206 return &seg[h];
00207 }
00208
00210
00211 inline bool check_mask_race( const hashcode_t h, hashcode_t &m ) const {
00212 hashcode_t m_now, m_old = m;
00213 #if TBB_USE_THREADING_TOOLS
00214 m_now = (hashcode_t) itt_load_pointer_with_acquire_v3( &my_mask );
00215 #else
00216 m_now = my_mask;
00217 #endif
00218 if( m_old != m_now )
00219 return check_rehashing_collision( h, m_old, m = m_now );
00220 return false;
00221 }
00222
00224 bool check_rehashing_collision( const hashcode_t h, hashcode_t m_old, hashcode_t m ) const {
00225 __TBB_ASSERT(m_old != m, NULL);
00226 if( (h & m_old) != (h & m) ) {
00227
00228
00229 for( ++m_old; !(h & m_old); m_old <<= 1 );
00230 m_old = (m_old<<1) - 1;
00231 __TBB_ASSERT((m_old&(m_old+1))==0 && m_old <= m, NULL);
00232
00233 if( __TBB_load_with_acquire(get_bucket( h & m_old )->node_list) != __TBB_rehash_req )
00234 return true;
00235 }
00236 return false;
00237 }
00238
00240 segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) {
00241 size_type sz = ++my_size;
00242 add_to_bucket( b, n );
00243
00244 if( sz >= mask ) {
00245 segment_index_t new_seg = segment_index_of( mask+1 );
00246 __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated");
00247 if( !my_table[new_seg] && __TBB_CompareAndSwapW(&my_table[new_seg], 1, 0) == 0 )
00248 return new_seg;
00249 }
00250 return 0;
00251 }
00252
00254 void reserve(size_type buckets) {
00255 if( !buckets-- ) return;
00256 bool is_initial = !my_size;
00257 for( size_type m = my_mask; buckets > m; m = my_mask )
00258 enable_segment( segment_index_of( m+1 ), is_initial );
00259 }
00261 void internal_swap(hash_map_base &table) {
00262 std::swap(this->my_mask, table.my_mask);
00263 std::swap(this->my_size, table.my_size);
00264 for(size_type i = 0; i < embedded_buckets; i++)
00265 std::swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list);
00266 for(size_type i = embedded_block; i < pointers_per_table; i++)
00267 std::swap(this->my_table[i], table.my_table[i]);
00268 }
00269 };
00270
00271 template<typename Iterator>
00272 class hash_map_range;
00273
00275
00277 template<typename Container, typename Value>
00278 class hash_map_iterator
00279 : public std::iterator<std::forward_iterator_tag,Value>
00280 {
00281 typedef Container map_type;
00282 typedef typename Container::node node;
00283 typedef hash_map_base::node_base node_base;
00284 typedef hash_map_base::bucket bucket;
00285
00286 template<typename C, typename T, typename U>
00287 friend bool operator==( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00288
00289 template<typename C, typename T, typename U>
00290 friend bool operator!=( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00291
00292 template<typename C, typename T, typename U>
00293 friend ptrdiff_t operator-( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00294
00295 template<typename C, typename U>
00296 friend class internal::hash_map_iterator;
00297
00298 template<typename I>
00299 friend class internal::hash_map_range;
00300
00301 void advance_to_next_bucket() {
00302 size_t k = my_index+1;
00303 while( my_bucket && k <= my_map->my_mask ) {
00304
00305 if( k& (k-2) )
00306 ++my_bucket;
00307 else my_bucket = my_map->get_bucket( k );
00308 my_node = static_cast<node*>( my_bucket->node_list );
00309 if( hash_map_base::is_valid(my_node) ) {
00310 my_index = k; return;
00311 }
00312 ++k;
00313 }
00314 my_bucket = 0; my_node = 0; my_index = k;
00315 }
00316 #if !defined(_MSC_VER) || defined(__INTEL_COMPILER)
00317 template<typename Key, typename T, typename HashCompare, typename A>
00318 friend class tbb::concurrent_hash_map;
00319 #else
00320 public:
00321 #endif
00323 const Container *my_map;
00324
00326 size_t my_index;
00327
00329 const bucket *my_bucket;
00330
00332 node *my_node;
00333
00334 hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n );
00335
00336 public:
00338 hash_map_iterator() {}
00339 hash_map_iterator( const hash_map_iterator<Container,typename Container::value_type> &other ) :
00340 my_map(other.my_map),
00341 my_index(other.my_index),
00342 my_bucket(other.my_bucket),
00343 my_node(other.my_node)
00344 {}
00345 Value& operator*() const {
00346 __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" );
00347 return my_node->item;
00348 }
00349 Value* operator->() const {return &operator*();}
00350 hash_map_iterator& operator++();
00351
00353 Value* operator++(int) {
00354 Value* result = &operator*();
00355 operator++();
00356 return result;
00357 }
00358 };
00359
00360 template<typename Container, typename Value>
00361 hash_map_iterator<Container,Value>::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) :
00362 my_map(&map),
00363 my_index(index),
00364 my_bucket(b),
00365 my_node( static_cast<node*>(n) )
00366 {
00367 if( b && !hash_map_base::is_valid(n) )
00368 advance_to_next_bucket();
00369 }
00370
00371 template<typename Container, typename Value>
00372 hash_map_iterator<Container,Value>& hash_map_iterator<Container,Value>::operator++() {
00373 my_node = static_cast<node*>( my_node->next );
00374 if( !my_node ) advance_to_next_bucket();
00375 return *this;
00376 }
00377
00378 template<typename Container, typename T, typename U>
00379 bool operator==( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00380 return i.my_node == j.my_node && i.my_map == j.my_map;
00381 }
00382
00383 template<typename Container, typename T, typename U>
00384 bool operator!=( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00385 return i.my_node != j.my_node || i.my_map != j.my_map;
00386 }
00387
00389
00390 template<typename Iterator>
00391 class hash_map_range {
00392 typedef typename Iterator::map_type map_type;
00393 Iterator my_begin;
00394 Iterator my_end;
00395 mutable Iterator my_midpoint;
00396 size_t my_grainsize;
00398 void set_midpoint() const;
00399 template<typename U> friend class hash_map_range;
00400 public:
00402 typedef std::size_t size_type;
00403 typedef typename Iterator::value_type value_type;
00404 typedef typename Iterator::reference reference;
00405 typedef typename Iterator::difference_type difference_type;
00406 typedef Iterator iterator;
00407
00409 bool empty() const {return my_begin==my_end;}
00410
00412 bool is_divisible() const {
00413 return my_midpoint!=my_end;
00414 }
00416 hash_map_range( hash_map_range& r, split ) :
00417 my_end(r.my_end),
00418 my_grainsize(r.my_grainsize)
00419 {
00420 r.my_end = my_begin = r.my_midpoint;
00421 __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" );
00422 __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" );
00423 set_midpoint();
00424 r.set_midpoint();
00425 }
00427 template<typename U>
00428 hash_map_range( hash_map_range<U>& r) :
00429 my_begin(r.my_begin),
00430 my_end(r.my_end),
00431 my_midpoint(r.my_midpoint),
00432 my_grainsize(r.my_grainsize)
00433 {}
00434 #if TBB_DEPRECATED
00436 hash_map_range( const Iterator& begin_, const Iterator& end_, size_type grainsize = 1 ) :
00437 my_begin(begin_),
00438 my_end(end_),
00439 my_grainsize(grainsize)
00440 {
00441 if(!my_end.my_index && !my_end.my_bucket)
00442 my_end.my_index = my_end.my_map->my_mask + 1;
00443 set_midpoint();
00444 __TBB_ASSERT( grainsize>0, "grainsize must be positive" );
00445 }
00446 #endif
00448 hash_map_range( const map_type &map, size_type grainsize = 1 ) :
00449 my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ),
00450 my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ),
00451 my_grainsize( grainsize )
00452 {
00453 __TBB_ASSERT( grainsize>0, "grainsize must be positive" );
00454 set_midpoint();
00455 }
00456 const Iterator& begin() const {return my_begin;}
00457 const Iterator& end() const {return my_end;}
00459 size_type grainsize() const {return my_grainsize;}
00460 };
00461
00462 template<typename Iterator>
00463 void hash_map_range<Iterator>::set_midpoint() const {
00464
00465 size_t m = my_end.my_index-my_begin.my_index;
00466 if( m > my_grainsize ) {
00467 m = my_begin.my_index + m/2u;
00468 hash_map_base::bucket *b = my_begin.my_map->get_bucket(m);
00469 my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list);
00470 } else {
00471 my_midpoint = my_end;
00472 }
00473 __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index,
00474 "my_begin is after my_midpoint" );
00475 __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index,
00476 "my_midpoint is after my_end" );
00477 __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end,
00478 "[my_begin, my_midpoint) range should not be empty" );
00479 }
00480 }
00482
00484 static const size_t hash_multiplier = sizeof(size_t)==4? 2654435769U : 11400714819323198485ULL;
00486 template<typename T>
00487 inline static size_t tbb_hasher( const T& t ) {
00488 return static_cast<size_t>( t ) * hash_multiplier;
00489 }
00490 template<typename P>
00491 inline static size_t tbb_hasher( P* ptr ) {
00492 size_t const h = reinterpret_cast<size_t>( ptr );
00493 return (h >> 3) ^ h;
00494 }
00495 template<typename E, typename S, typename A>
00496 inline static size_t tbb_hasher( const std::basic_string<E,S,A>& s ) {
00497 size_t h = 0;
00498 for( const E* c = s.c_str(); *c; c++ )
00499 h = static_cast<size_t>(*c) ^ (h * hash_multiplier);
00500 return h;
00501 }
00502 template<typename F, typename S>
00503 inline static size_t tbb_hasher( const std::pair<F,S>& p ) {
00504 return tbb_hasher(p.first) ^ tbb_hasher(p.second);
00505 }
00506
00508 template<typename T>
00509 struct tbb_hash_compare {
00510 static size_t hash( const T& t ) { return tbb_hasher(t); }
00511 static bool equal( const T& a, const T& b ) { return a == b; }
00512 };
00513
00515
00543 template<typename Key, typename T, typename HashCompare, typename Allocator>
00544 class concurrent_hash_map : protected internal::hash_map_base {
00545 template<typename Container, typename Value>
00546 friend class internal::hash_map_iterator;
00547
00548 template<typename I>
00549 friend class internal::hash_map_range;
00550
00551 public:
00552 typedef Key key_type;
00553 typedef T mapped_type;
00554 typedef std::pair<const Key,T> value_type;
00555 typedef internal::hash_map_base::size_type size_type;
00556 typedef ptrdiff_t difference_type;
00557 typedef value_type *pointer;
00558 typedef const value_type *const_pointer;
00559 typedef value_type &reference;
00560 typedef const value_type &const_reference;
00561 typedef internal::hash_map_iterator<concurrent_hash_map,value_type> iterator;
00562 typedef internal::hash_map_iterator<concurrent_hash_map,const value_type> const_iterator;
00563 typedef internal::hash_map_range<iterator> range_type;
00564 typedef internal::hash_map_range<const_iterator> const_range_type;
00565 typedef Allocator allocator_type;
00566
00567 protected:
00568 friend class const_accessor;
00569 struct node;
00570 typedef typename Allocator::template rebind<node>::other node_allocator_type;
00571 node_allocator_type my_allocator;
00572 HashCompare my_hash_compare;
00573
00574 struct node : public node_base {
00575 value_type item;
00576 node( const Key &key ) : item(key, T()) {}
00577 node( const Key &key, const T &t ) : item(key, t) {}
00578
00579 void *operator new( size_t , node_allocator_type &a ) {
00580 void *ptr = a.allocate(1);
00581 if(!ptr) throw std::bad_alloc();
00582 return ptr;
00583 }
00584
00585 void operator delete( void *ptr, node_allocator_type &a ) {return a.deallocate(static_cast<node*>(ptr),1); }
00586 };
00587
00588 void delete_node( node_base *n ) {
00589 my_allocator.destroy( static_cast<node*>(n) );
00590 my_allocator.deallocate( static_cast<node*>(n), 1);
00591 }
00592
00593 node *search_bucket( const key_type &key, bucket *b ) const {
00594 node *n = static_cast<node*>( b->node_list );
00595 while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) )
00596 n = static_cast<node*>( n->next );
00597 __TBB_ASSERT(n != __TBB_rehash_req, "Search can be executed only for rehashed bucket");
00598 return n;
00599 }
00600
00602 class bucket_accessor : public bucket::scoped_t {
00603 bool my_is_writer;
00604 bucket *my_b;
00605 public:
00606 bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); }
00608 inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) {
00609 my_b = base->get_bucket( h );
00610 #if TBB_USE_THREADING_TOOLS
00611 if( itt_load_pointer_with_acquire_v3(&my_b->node_list) == __TBB_rehash_req
00612 #else
00613 if( __TBB_load_with_acquire(my_b->node_list) == __TBB_rehash_req
00614 #endif
00615 && try_acquire( my_b->mutex, true ) )
00616 {
00617 if( my_b->node_list == __TBB_rehash_req ) base->rehash_bucket( my_b, h );
00618 my_is_writer = true;
00619 }
00620 else bucket::scoped_t::acquire( my_b->mutex, my_is_writer = writer );
00621 __TBB_ASSERT( my_b->node_list != __TBB_rehash_req, NULL);
00622 }
00624 bool is_writer() { return my_is_writer; }
00626 bucket *operator() () { return my_b; }
00627
00628 bool upgrade_to_writer() { my_is_writer = true; return bucket::scoped_t::upgrade_to_writer(); }
00629 };
00630
00631
00632 void rehash_bucket( bucket *b_new, const hashcode_t h ) {
00633 __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)");
00634 __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
00635 __TBB_store_with_release(b_new->node_list, __TBB_empty_rehashed);
00636 hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1;
00637
00638 bucket_accessor b_old( this, h & mask );
00639
00640 mask = (mask<<1) | 1;
00641 __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL );
00642 restart:
00643 for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) {
00644 hashcode_t c = my_hash_compare.hash( static_cast<node*>(n)->item.first );
00645 if( (c & mask) == h ) {
00646 if( !b_old.is_writer() )
00647 if( !b_old.upgrade_to_writer() ) {
00648 goto restart;
00649 }
00650 *p = n->next;
00651 add_to_bucket( b_new, n );
00652 } else p = &n->next;
00653 }
00654 }
00655
00656 public:
00657
00658 class accessor;
00660 class const_accessor {
00661 friend class concurrent_hash_map<Key,T,HashCompare,Allocator>;
00662 friend class accessor;
00663 void operator=( const accessor & ) const;
00664 const_accessor( const accessor & );
00665 public:
00667 typedef const typename concurrent_hash_map::value_type value_type;
00668
00670 bool empty() const {return !my_node;}
00671
00673 void release() {
00674 if( my_node ) {
00675 my_lock.release();
00676 my_node = 0;
00677 }
00678 }
00679
00681 const_reference operator*() const {
00682 __TBB_ASSERT( my_node, "attempt to dereference empty accessor" );
00683 return my_node->item;
00684 }
00685
00687 const_pointer operator->() const {
00688 return &operator*();
00689 }
00690
00692 const_accessor() : my_node(NULL) {}
00693
00695 ~const_accessor() {
00696 my_node = NULL;
00697 }
00698 private:
00699 node *my_node;
00700 typename node::scoped_t my_lock;
00701 hashcode_t my_hash;
00702 };
00703
00705 class accessor: public const_accessor {
00706 public:
00708 typedef typename concurrent_hash_map::value_type value_type;
00709
00711 reference operator*() const {
00712 __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" );
00713 return this->my_node->item;
00714 }
00715
00717 pointer operator->() const {
00718 return &operator*();
00719 }
00720 };
00721
00723 concurrent_hash_map(const allocator_type &a = allocator_type())
00724 : my_allocator(a)
00725 {}
00726
00728 concurrent_hash_map( const concurrent_hash_map& table, const allocator_type &a = allocator_type())
00729 : my_allocator(a)
00730 {
00731 internal_copy(table);
00732 }
00733
00735 template<typename I>
00736 concurrent_hash_map(I first, I last, const allocator_type &a = allocator_type())
00737 : my_allocator(a)
00738 {
00739 reserve( std::distance(first, last) );
00740 internal_copy(first, last);
00741 }
00742
00744 concurrent_hash_map& operator=( const concurrent_hash_map& table ) {
00745 if( this!=&table ) {
00746 clear();
00747 internal_copy(table);
00748 }
00749 return *this;
00750 }
00751
00752
00754 void clear();
00755
00757 ~concurrent_hash_map() { clear(); }
00758
00759
00760
00761
00762 range_type range( size_type grainsize=1 ) {
00763 return range_type( *this, grainsize );
00764 }
00765 const_range_type range( size_type grainsize=1 ) const {
00766 return const_range_type( *this, grainsize );
00767 }
00768
00769
00770
00771
00772 iterator begin() {return iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00773 iterator end() {return iterator(*this,0,0,0);}
00774 const_iterator begin() const {return const_iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00775 const_iterator end() const {return const_iterator(*this,0,0,0);}
00776 std::pair<iterator, iterator> equal_range( const Key& key ) { return internal_equal_range(key, end()); }
00777 std::pair<const_iterator, const_iterator> equal_range( const Key& key ) const { return internal_equal_range(key, end()); }
00778
00780 size_type size() const { return my_size; }
00781
00783 bool empty() const { return my_size == 0; }
00784
00786 size_type max_size() const {return (~size_type(0))/sizeof(node);}
00787
00789 allocator_type get_allocator() const { return this->my_allocator; }
00790
00792 void swap(concurrent_hash_map &table);
00793
00794
00795
00796
00797
00799 size_type count( const Key &key ) const {
00800 return const_cast<concurrent_hash_map*>(this)->lookup(false, key, NULL, NULL, false );
00801 }
00802
00804
00805 bool find( const_accessor &result, const Key &key ) const {
00806 result.release();
00807 return const_cast<concurrent_hash_map*>(this)->lookup(false, key, NULL, &result, false );
00808 }
00809
00811
00812 bool find( accessor &result, const Key &key ) {
00813 result.release();
00814 return lookup(false, key, NULL, &result, true );
00815 }
00816
00818
00819 bool insert( const_accessor &result, const Key &key ) {
00820 result.release();
00821 return lookup(true, key, NULL, &result, false );
00822 }
00823
00825
00826 bool insert( accessor &result, const Key &key ) {
00827 result.release();
00828 return lookup(true, key, NULL, &result, true );
00829 }
00830
00832
00833 bool insert( const_accessor &result, const value_type &value ) {
00834 result.release();
00835 return lookup(true, value.first, &value.second, &result, false );
00836 }
00837
00839
00840 bool insert( accessor &result, const value_type &value ) {
00841 result.release();
00842 return lookup(true, value.first, &value.second, &result, true );
00843 }
00844
00846
00847 bool insert( const value_type &value ) {
00848 return lookup(true, value.first, &value.second, NULL, false );
00849 }
00850
00852 template<typename I>
00853 void insert(I first, I last) {
00854 for(; first != last; ++first)
00855 insert( *first );
00856 }
00857
00859
00860 bool erase( const Key& key );
00861
00863
00864 bool erase( const_accessor& item_accessor ) {
00865 return exclude( item_accessor, true );
00866 }
00867
00869
00870 bool erase( accessor& item_accessor ) {
00871 return exclude( item_accessor, false );
00872 }
00873
00874 protected:
00876 bool lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write );
00877
00879 bool exclude( const_accessor &item_accessor, bool readonly );
00880
00882 template<typename I>
00883 std::pair<I, I> internal_equal_range( const Key& key, I end ) const;
00884
00886 void internal_copy( const concurrent_hash_map& source );
00887
00888 template<typename I>
00889 void internal_copy(I first, I last);
00890
00892 const_pointer find( const Key& key ) const {
00893 hashcode_t h = my_hash_compare.hash( key );
00894 hashcode_t m = my_mask;
00895 restart:
00896 __TBB_ASSERT((m&(m+1))==0, NULL);
00897 bucket *b = get_bucket( h & m );
00898 if( b->node_list == __TBB_rehash_req ) {
00899 bucket::scoped_t lock;
00900 if( lock.try_acquire( b->mutex, true ) && b->node_list == __TBB_rehash_req )
00901 const_cast<concurrent_hash_map*>(this)->rehash_bucket( b, h & m );
00902 else internal::spin_wait_while_eq( b->node_list, __TBB_rehash_req );
00903 }
00904 node *n = search_bucket( key, b );
00905 if( check_mask_race( h, m ) )
00906 goto restart;
00907 if( n )
00908 return &n->item;
00909 return 0;
00910 }
00911 };
00912
00913 #if _MSC_VER && !defined(__INTEL_COMPILER)
00914
00915 #pragma warning( push )
00916 #pragma warning( disable: 4127 )
00917 #endif
00918
00919 template<typename Key, typename T, typename HashCompare, typename A>
00920 bool concurrent_hash_map<Key,T,HashCompare,A>::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write ) {
00921 __TBB_ASSERT( !result || !result->my_node, NULL );
00922 segment_index_t grow_segment;
00923 bool return_value;
00924 node *n, *tmp_n = 0;
00925 hashcode_t const h = my_hash_compare.hash( key );
00926 #if TBB_USE_THREADING_TOOLS
00927 hashcode_t m = (hashcode_t) itt_load_pointer_with_acquire_v3( &my_mask );
00928 #else
00929 hashcode_t m = my_mask;
00930 #endif
00931 restart:
00932 {
00933 __TBB_ASSERT((m&(m+1))==0, NULL);
00934 return_value = false;
00935
00936 bucket_accessor b( this, h & m );
00937
00938
00939 n = search_bucket( key, b() );
00940 if( op_insert ) {
00941
00942 if( !is_valid(n) ) {
00943 if( !tmp_n ) {
00944 if(t) tmp_n = new( my_allocator ) node(key, *t);
00945 else tmp_n = new( my_allocator ) node(key);
00946 }
00947 if( !b.is_writer() && !b.upgrade_to_writer() ) {
00948
00949 n = search_bucket( key, b() );
00950 if( is_valid(n) ) {
00951 b.downgrade_to_reader();
00952 goto exists;
00953 }
00954 }
00955 if( check_mask_race(h, m) )
00956 goto restart;
00957
00958 grow_segment = insert_new_node( b(), n = tmp_n, m );
00959 tmp_n = 0;
00960 return_value = true;
00961 } else {
00962 exists: grow_segment = 0;
00963 }
00964 } else {
00965 if( !n ) {
00966 if( check_mask_race( h, m ) )
00967 goto restart;
00968 return false;
00969 }
00970 return_value = true;
00971 grow_segment = 0;
00972 }
00973 if( !result ) goto check_growth;
00974
00975
00976 if( !result->my_lock.try_acquire( n->mutex, write ) ) {
00977
00978 internal::atomic_backoff trials;
00979 do {
00980 if( !trials.bounded_pause() ) {
00981
00982 b.release();
00983 __TBB_Yield();
00984 m = my_mask;
00985 goto restart;
00986 }
00987 } while( !result->my_lock.try_acquire( n->mutex, write ) );
00988 }
00989 }
00990 result->my_node = n;
00991 result->my_hash = h;
00992 check_growth:
00993
00994 if( grow_segment )
00995 enable_segment( grow_segment );
00996 if( tmp_n )
00997 delete_node( tmp_n );
00998 return return_value;
00999 }
01000
01001 template<typename Key, typename T, typename HashCompare, typename A>
01002 template<typename I>
01003 std::pair<I, I> concurrent_hash_map<Key,T,HashCompare,A>::internal_equal_range( const Key& key, I end ) const {
01004 hashcode_t h = my_hash_compare.hash( key );
01005 hashcode_t m = my_mask;
01006 __TBB_ASSERT((m&(m+1))==0, NULL);
01007 h &= m;
01008 bucket *b = get_bucket( h );
01009 while( b->node_list == __TBB_rehash_req ) {
01010 m = ( 1u<<__TBB_Log2( h ) ) - 1;
01011 b = get_bucket( h &= m );
01012 }
01013 node *n = search_bucket( key, b );
01014 if( !n )
01015 return std::make_pair(end, end);
01016 iterator lower(*this, h, b, n), upper(lower);
01017 return std::make_pair(lower, ++upper);
01018 }
01019
01020 template<typename Key, typename T, typename HashCompare, typename A>
01021 bool concurrent_hash_map<Key,T,HashCompare,A>::exclude( const_accessor &item_accessor, bool readonly ) {
01022 __TBB_ASSERT( item_accessor.my_node, NULL );
01023 node_base *const n = item_accessor.my_node;
01024 item_accessor.my_node = NULL;
01025 hashcode_t const h = item_accessor.my_hash;
01026 hashcode_t m = my_mask;
01027 do {
01028
01029 bucket_accessor b( this, h & m, true );
01030 node_base **p = &b()->node_list;
01031 while( *p && *p != n )
01032 p = &(*p)->next;
01033 if( !*p ) {
01034 if( check_mask_race( h, m ) )
01035 continue;
01036 item_accessor.my_lock.release();
01037 return false;
01038 }
01039 __TBB_ASSERT( *p == n, NULL );
01040 *p = n->next;
01041 my_size--;
01042 break;
01043 } while(true);
01044 if( readonly )
01045 item_accessor.my_lock.upgrade_to_writer();
01046 item_accessor.my_lock.release();
01047 delete_node( n );
01048 return true;
01049 }
01050
01051 template<typename Key, typename T, typename HashCompare, typename A>
01052 bool concurrent_hash_map<Key,T,HashCompare,A>::erase( const Key &key ) {
01053 node_base *n;
01054 hashcode_t const h = my_hash_compare.hash( key );
01055 hashcode_t m = my_mask;
01056 {
01057 restart:
01058
01059 bucket_accessor b( this, h & m );
01060 search:
01061 node_base **p = &b()->node_list;
01062 n = *p;
01063 while( is_valid(n) && !my_hash_compare.equal(key, static_cast<node*>(n)->item.first ) ) {
01064 p = &n->next;
01065 n = *p;
01066 }
01067 if( !n ) {
01068 if( check_mask_race( h, m ) )
01069 goto restart;
01070 return false;
01071 }
01072 else if( !b.is_writer() && !b.upgrade_to_writer() ) {
01073 if( check_mask_race( h, m ) )
01074 goto restart;
01075 goto search;
01076 }
01077 *p = n->next;
01078 my_size--;
01079 }
01080 {
01081 typename node::scoped_t item_locker( n->mutex, true );
01082 }
01083
01084 delete_node( n );
01085 return true;
01086 }
01087
01088 template<typename Key, typename T, typename HashCompare, typename A>
01089 void concurrent_hash_map<Key,T,HashCompare,A>::swap(concurrent_hash_map<Key,T,HashCompare,A> &table) {
01090 std::swap(this->my_allocator, table.my_allocator);
01091 std::swap(this->my_hash_compare, table.my_hash_compare);
01092 internal_swap(table);
01093 }
01094
01095 template<typename Key, typename T, typename HashCompare, typename A>
01096 void concurrent_hash_map<Key,T,HashCompare,A>::clear() {
01097 hashcode_t m = my_mask;
01098 __TBB_ASSERT((m&(m+1))==0, NULL);
01099 #if TBB_USE_DEBUG || TBB_USE_PERFORMANCE_WARNINGS
01100 #if TBB_USE_PERFORMANCE_WARNINGS
01101 int size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0;
01102 static bool reported = false;
01103 #endif
01104
01105 for( segment_index_t b = 0; b <= m; b++ ) {
01106 node_base *n = get_bucket(b)->node_list;
01107 #if TBB_USE_PERFORMANCE_WARNINGS
01108 if( n == __TBB_empty_rehashed ) empty_buckets++;
01109 else if( n == __TBB_rehash_req ) buckets--;
01110 else if( n->next ) overpopulated_buckets++;
01111 #endif
01112 for(; is_valid(n); n = n->next ) {
01113 hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first );
01114 h &= m;
01115 __TBB_ASSERT( h == b || get_bucket(h)->node_list == __TBB_rehash_req, "Rehashing is not finished until serial stage due to concurrent or terminated operation" );
01116 }
01117 }
01118 #if TBB_USE_PERFORMANCE_WARNINGS
01119 if( buckets > size) empty_buckets -= buckets - size;
01120 else overpopulated_buckets -= size - buckets;
01121 if( !reported && buckets >= 512 && ( 2*empty_buckets >= size || 2*overpopulated_buckets > size ) ) {
01122 internal::runtime_warning(
01123 "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d",
01124 typeid(*this).name(), size, empty_buckets, overpopulated_buckets );
01125 reported = true;
01126 }
01127 #endif
01128 #endif//TBB_USE_DEBUG || TBB_USE_PERFORMANCE_WARNINGS
01129 my_size = 0;
01130 segment_index_t s = segment_index_of( m );
01131 __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" );
01132 cache_aligned_allocator<bucket> alloc;
01133 do {
01134 __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" );
01135 segment_ptr_t buckets = my_table[s];
01136 size_type sz = segment_size( s ? s : 1 );
01137 for( segment_index_t i = 0; i < sz; i++ )
01138 for( node_base *n = buckets[i].node_list; is_valid(n); n = buckets[i].node_list ) {
01139 buckets[i].node_list = n->next;
01140 delete_node( n );
01141 }
01142 if( s >= first_block)
01143 alloc.deallocate( buckets, sz );
01144 else if( s == embedded_block && embedded_block != first_block )
01145 alloc.deallocate( buckets, segment_size(first_block)-embedded_buckets );
01146 if( s >= embedded_block ) my_table[s] = 0;
01147 } while(s-- > 0);
01148 my_mask = embedded_buckets - 1;
01149 }
01150
01151 template<typename Key, typename T, typename HashCompare, typename A>
01152 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy( const concurrent_hash_map& source ) {
01153 reserve( source.my_size );
01154 if( my_mask == source.my_mask ) {
01155 for( const_iterator it = source.begin(), end = source.end(); it != end; ++it ) {
01156 bucket *b = get_bucket( it.my_index );
01157 __TBB_ASSERT( b->node_list != __TBB_rehash_req, "Invalid bucket in destination table");
01158 node *n = new( my_allocator ) node(it->first, it->second);
01159 add_to_bucket( b, n );
01160 ++my_size;
01161 }
01162 } else internal_copy( source.begin(), source.end() );
01163 }
01164
01165 template<typename Key, typename T, typename HashCompare, typename A>
01166 template<typename I>
01167 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy(I first, I last) {
01168 hashcode_t m = my_mask;
01169 for(; first != last; ++first) {
01170 hashcode_t h = my_hash_compare.hash( first->first );
01171 bucket *b = get_bucket( h & m );
01172 __TBB_ASSERT( b->node_list != __TBB_rehash_req, "Invalid bucket in destination table");
01173 node *n = new( my_allocator ) node(first->first, first->second);
01174 add_to_bucket( b, n );
01175 ++my_size;
01176 }
01177 }
01178
01179 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01180 inline bool operator==(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b) {
01181 if(a.size() != b.size()) return false;
01182 typename concurrent_hash_map<Key, T, HashCompare, A1>::const_iterator i(a.begin()), i_end(a.end());
01183 typename concurrent_hash_map<Key, T, HashCompare, A2>::const_iterator j, j_end(b.end());
01184 for(; i != i_end; ++i) {
01185 j = b.equal_range(i->first).first;
01186 if( j == j_end || !(i->second == j->second) ) return false;
01187 }
01188 return true;
01189 }
01190
01191 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01192 inline bool operator!=(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b)
01193 { return !(a == b); }
01194
01195 template<typename Key, typename T, typename HashCompare, typename A>
01196 inline void swap(concurrent_hash_map<Key, T, HashCompare, A> &a, concurrent_hash_map<Key, T, HashCompare, A> &b)
01197 { a.swap( b ); }
01198
01199 #if _MSC_VER && !defined(__INTEL_COMPILER)
01200 #pragma warning( pop )
01201 #endif // warning 4127 is back
01202
01203 }
01204
01205 #endif