00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029 #ifndef __TBB_concurrent_hash_map_H
00030 #define __TBB_concurrent_hash_map_H
00031
00032 #include "tbb_stddef.h"
00033
00034 #if !TBB_USE_EXCEPTIONS && _MSC_VER
00035
00036 #pragma warning (push)
00037 #pragma warning (disable: 4530)
00038 #endif
00039
00040 #include <iterator>
00041 #include <utility>
00042 #include <cstring>
00043
00044 #if !TBB_USE_EXCEPTIONS && _MSC_VER
00045 #pragma warning (pop)
00046 #endif
00047
00048 #include "cache_aligned_allocator.h"
00049 #include "tbb_allocator.h"
00050 #include "spin_rw_mutex.h"
00051 #include "atomic.h"
00052 #include "aligned_space.h"
00053 #include "tbb_exception.h"
00054 #include "tbb_profiling.h"
00055 #include "internal/_concurrent_unordered_impl.h"
00056 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
00057 #include <typeinfo>
00058 #endif
00059 #if __TBB_STATISTICS
00060 #include <stdio.h>
00061 #endif
00062
00063 namespace tbb {
00064
00066 template<typename Key>
00067 struct tbb_hash_compare {
00068 static size_t hash( const Key& a ) { return tbb_hasher(a); }
00069 static bool equal( const Key& a, const Key& b ) { return a == b; }
00070 };
00071
00072 namespace interface5 {
00073
00074 template<typename Key, typename T, typename HashCompare = tbb_hash_compare<Key>, typename A = tbb_allocator<std::pair<Key, T> > >
00075 class concurrent_hash_map;
00076
00078 namespace internal {
00079
00080
00082 typedef size_t hashcode_t;
00084 struct hash_map_node_base : tbb::internal::no_copy {
00086 typedef spin_rw_mutex mutex_t;
00088 typedef mutex_t::scoped_lock scoped_t;
00090 hash_map_node_base *next;
00091 mutex_t mutex;
00092 };
00094 static hash_map_node_base *const rehash_req = reinterpret_cast<hash_map_node_base*>(size_t(3));
00096 static hash_map_node_base *const empty_rehashed = reinterpret_cast<hash_map_node_base*>(size_t(0));
00098 class hash_map_base {
00099 public:
00101 typedef size_t size_type;
00103 typedef size_t hashcode_t;
00105 typedef size_t segment_index_t;
00107 typedef hash_map_node_base node_base;
00109 struct bucket : tbb::internal::no_copy {
00111 typedef spin_rw_mutex mutex_t;
00113 typedef mutex_t::scoped_lock scoped_t;
00114 mutex_t mutex;
00115 node_base *node_list;
00116 };
00118 static size_type const embedded_block = 1;
00120 static size_type const embedded_buckets = 1<<embedded_block;
00122 static size_type const first_block = 8;
00124 static size_type const pointers_per_table = sizeof(segment_index_t) * 8;
00126 typedef bucket *segment_ptr_t;
00128 typedef segment_ptr_t segments_table_t[pointers_per_table];
00130 atomic<hashcode_t> my_mask;
00132 segments_table_t my_table;
00134 atomic<size_type> my_size;
00136 bucket my_embedded_segment[embedded_buckets];
00137 #if __TBB_STATISTICS
00138 atomic<unsigned> my_info_resizes;
00139 mutable atomic<unsigned> my_info_restarts;
00140 atomic<unsigned> my_info_rehashes;
00141 #endif
00143 hash_map_base() {
00144 std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t)
00145 + sizeof(my_size) + sizeof(my_mask)
00146 + embedded_buckets*sizeof(bucket) );
00147 for( size_type i = 0; i < embedded_block; i++ )
00148 my_table[i] = my_embedded_segment + segment_base(i);
00149 my_mask = embedded_buckets - 1;
00150 __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
00151 #if __TBB_STATISTICS
00152 my_info_resizes = 0;
00153 my_info_restarts = 0;
00154 my_info_rehashes = 0;
00155 #endif
00156 }
00157
00159 static segment_index_t segment_index_of( size_type index ) {
00160 return segment_index_t( __TBB_Log2( index|1 ) );
00161 }
00162
00164 static segment_index_t segment_base( segment_index_t k ) {
00165 return (segment_index_t(1)<<k & ~segment_index_t(1));
00166 }
00167
00169 static size_type segment_size( segment_index_t k ) {
00170 return size_type(1)<<k;
00171 }
00172
00174 static bool is_valid( void *ptr ) {
00175 return reinterpret_cast<uintptr_t>(ptr) > uintptr_t(63);
00176 }
00177
00179 static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
00180 if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
00181 else for(size_type i = 0; i < sz; i++, ptr++) {
00182 *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
00183 ptr->node_list = rehash_req;
00184 }
00185 }
00186
00188 static void add_to_bucket( bucket *b, node_base *n ) {
00189 __TBB_ASSERT(b->node_list != rehash_req, NULL);
00190 n->next = b->node_list;
00191 b->node_list = n;
00192 }
00193
00195 struct enable_segment_failsafe : tbb::internal::no_copy {
00196 segment_ptr_t *my_segment_ptr;
00197 enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
00198 ~enable_segment_failsafe() {
00199 if( my_segment_ptr ) *my_segment_ptr = 0;
00200 }
00201 };
00202
00204 void enable_segment( segment_index_t k, bool is_initial = false ) {
00205 __TBB_ASSERT( k, "Zero segment must be embedded" );
00206 enable_segment_failsafe watchdog( my_table, k );
00207 cache_aligned_allocator<bucket> alloc;
00208 size_type sz;
00209 __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
00210 if( k >= first_block ) {
00211 sz = segment_size( k );
00212 segment_ptr_t ptr = alloc.allocate( sz );
00213 init_buckets( ptr, sz, is_initial );
00214 itt_hide_store_word( my_table[k], ptr );
00215 sz <<= 1;
00216 } else {
00217 __TBB_ASSERT( k == embedded_block, "Wrong segment index" );
00218 sz = segment_size( first_block );
00219 segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets );
00220 init_buckets( ptr, sz - embedded_buckets, is_initial );
00221 ptr -= segment_base(embedded_block);
00222 for(segment_index_t i = embedded_block; i < first_block; i++)
00223 itt_hide_store_word( my_table[i], ptr + segment_base(i) );
00224 }
00225 itt_store_word_with_release( my_mask, sz-1 );
00226 watchdog.my_segment_ptr = 0;
00227 }
00228
00230 bucket *get_bucket( hashcode_t h ) const throw() {
00231 segment_index_t s = segment_index_of( h );
00232 h -= segment_base(s);
00233 segment_ptr_t seg = my_table[s];
00234 __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" );
00235 return &seg[h];
00236 }
00237
00238
00239 void mark_rehashed_levels( hashcode_t h ) throw () {
00240 segment_index_t s = segment_index_of( h );
00241 while( segment_ptr_t seg = my_table[++s] )
00242 if( seg[h].node_list == rehash_req ) {
00243 seg[h].node_list = empty_rehashed;
00244 mark_rehashed_levels( h + ((hashcode_t)1<<s) );
00245 }
00246 }
00247
00249
00250 inline bool check_mask_race( const hashcode_t h, hashcode_t &m ) const {
00251 hashcode_t m_now, m_old = m;
00252 m_now = (hashcode_t) itt_load_word_with_acquire( my_mask );
00253 if( m_old != m_now )
00254 return check_rehashing_collision( h, m_old, m = m_now );
00255 return false;
00256 }
00257
00259 bool check_rehashing_collision( const hashcode_t h, hashcode_t m_old, hashcode_t m ) const {
00260 __TBB_ASSERT(m_old != m, NULL);
00261 if( (h & m_old) != (h & m) ) {
00262
00263
00264 for( ++m_old; !(h & m_old); m_old <<= 1 )
00265 ;
00266 m_old = (m_old<<1) - 1;
00267 __TBB_ASSERT((m_old&(m_old+1))==0 && m_old <= m, NULL);
00268
00269 if( itt_load_word_with_acquire(get_bucket(h & m_old)->node_list) != rehash_req )
00270 {
00271 #if __TBB_STATISTICS
00272 my_info_restarts++;
00273 #endif
00274 return true;
00275 }
00276 }
00277 return false;
00278 }
00279
00281 segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) {
00282 size_type sz = ++my_size;
00283 add_to_bucket( b, n );
00284
00285 if( sz >= mask ) {
00286 segment_index_t new_seg = __TBB_Log2( mask+1 );
00287 __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated");
00288 if( !itt_hide_load_word(my_table[new_seg])
00289 && __TBB_CompareAndSwapW(&my_table[new_seg], 2, 0) == 0 )
00290 return new_seg;
00291 }
00292 return 0;
00293 }
00294
00296 void reserve(size_type buckets) {
00297 if( !buckets-- ) return;
00298 bool is_initial = !my_size;
00299 for( size_type m = my_mask; buckets > m; m = my_mask )
00300 enable_segment( segment_index_of( m+1 ), is_initial );
00301 }
00303 void internal_swap(hash_map_base &table) {
00304 std::swap(this->my_mask, table.my_mask);
00305 std::swap(this->my_size, table.my_size);
00306 for(size_type i = 0; i < embedded_buckets; i++)
00307 std::swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list);
00308 for(size_type i = embedded_block; i < pointers_per_table; i++)
00309 std::swap(this->my_table[i], table.my_table[i]);
00310 }
00311 };
00312
00313 template<typename Iterator>
00314 class hash_map_range;
00315
00317
00319 template<typename Container, typename Value>
00320 class hash_map_iterator
00321 : public std::iterator<std::forward_iterator_tag,Value>
00322 {
00323 typedef Container map_type;
00324 typedef typename Container::node node;
00325 typedef hash_map_base::node_base node_base;
00326 typedef hash_map_base::bucket bucket;
00327
00328 template<typename C, typename T, typename U>
00329 friend bool operator==( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00330
00331 template<typename C, typename T, typename U>
00332 friend bool operator!=( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00333
00334 template<typename C, typename T, typename U>
00335 friend ptrdiff_t operator-( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00336
00337 template<typename C, typename U>
00338 friend class hash_map_iterator;
00339
00340 template<typename I>
00341 friend class hash_map_range;
00342
00343 void advance_to_next_bucket() {
00344 size_t k = my_index+1;
00345 while( my_bucket && k <= my_map->my_mask ) {
00346
00347 if( k& (k-2) )
00348 ++my_bucket;
00349 else my_bucket = my_map->get_bucket( k );
00350 my_node = static_cast<node*>( my_bucket->node_list );
00351 if( hash_map_base::is_valid(my_node) ) {
00352 my_index = k; return;
00353 }
00354 ++k;
00355 }
00356 my_bucket = 0; my_node = 0; my_index = k;
00357 }
00358 #if !defined(_MSC_VER) || defined(__INTEL_COMPILER)
00359 template<typename Key, typename T, typename HashCompare, typename A>
00360 friend class interface5::concurrent_hash_map;
00361 #else
00362 public:
00363 #endif
00365 const Container *my_map;
00366
00368 size_t my_index;
00369
00371 const bucket *my_bucket;
00372
00374 node *my_node;
00375
00376 hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n );
00377
00378 public:
00380 hash_map_iterator() {}
00381 hash_map_iterator( const hash_map_iterator<Container,typename Container::value_type> &other ) :
00382 my_map(other.my_map),
00383 my_index(other.my_index),
00384 my_bucket(other.my_bucket),
00385 my_node(other.my_node)
00386 {}
00387 Value& operator*() const {
00388 __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" );
00389 return my_node->item;
00390 }
00391 Value* operator->() const {return &operator*();}
00392 hash_map_iterator& operator++();
00393
00395 hash_map_iterator operator++(int) {
00396 hash_map_iterator old(*this);
00397 operator++();
00398 return old;
00399 }
00400 };
00401
00402 template<typename Container, typename Value>
00403 hash_map_iterator<Container,Value>::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) :
00404 my_map(&map),
00405 my_index(index),
00406 my_bucket(b),
00407 my_node( static_cast<node*>(n) )
00408 {
00409 if( b && !hash_map_base::is_valid(n) )
00410 advance_to_next_bucket();
00411 }
00412
00413 template<typename Container, typename Value>
00414 hash_map_iterator<Container,Value>& hash_map_iterator<Container,Value>::operator++() {
00415 my_node = static_cast<node*>( my_node->next );
00416 if( !my_node ) advance_to_next_bucket();
00417 return *this;
00418 }
00419
00420 template<typename Container, typename T, typename U>
00421 bool operator==( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00422 return i.my_node == j.my_node && i.my_map == j.my_map;
00423 }
00424
00425 template<typename Container, typename T, typename U>
00426 bool operator!=( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00427 return i.my_node != j.my_node || i.my_map != j.my_map;
00428 }
00429
00431
00432 template<typename Iterator>
00433 class hash_map_range {
00434 typedef typename Iterator::map_type map_type;
00435 Iterator my_begin;
00436 Iterator my_end;
00437 mutable Iterator my_midpoint;
00438 size_t my_grainsize;
00440 void set_midpoint() const;
00441 template<typename U> friend class hash_map_range;
00442 public:
00444 typedef std::size_t size_type;
00445 typedef typename Iterator::value_type value_type;
00446 typedef typename Iterator::reference reference;
00447 typedef typename Iterator::difference_type difference_type;
00448 typedef Iterator iterator;
00449
00451 bool empty() const {return my_begin==my_end;}
00452
00454 bool is_divisible() const {
00455 return my_midpoint!=my_end;
00456 }
00458 hash_map_range( hash_map_range& r, split ) :
00459 my_end(r.my_end),
00460 my_grainsize(r.my_grainsize)
00461 {
00462 r.my_end = my_begin = r.my_midpoint;
00463 __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" );
00464 __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" );
00465 set_midpoint();
00466 r.set_midpoint();
00467 }
00469 template<typename U>
00470 hash_map_range( hash_map_range<U>& r) :
00471 my_begin(r.my_begin),
00472 my_end(r.my_end),
00473 my_midpoint(r.my_midpoint),
00474 my_grainsize(r.my_grainsize)
00475 {}
00476 #if TBB_DEPRECATED
00478 hash_map_range( const Iterator& begin_, const Iterator& end_, size_type grainsize_ = 1 ) :
00479 my_begin(begin_),
00480 my_end(end_),
00481 my_grainsize(grainsize_)
00482 {
00483 if(!my_end.my_index && !my_end.my_bucket)
00484 my_end.my_index = my_end.my_map->my_mask + 1;
00485 set_midpoint();
00486 __TBB_ASSERT( grainsize_>0, "grainsize must be positive" );
00487 }
00488 #endif
00490 hash_map_range( const map_type &map, size_type grainsize_ = 1 ) :
00491 my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ),
00492 my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ),
00493 my_grainsize( grainsize_ )
00494 {
00495 __TBB_ASSERT( grainsize_>0, "grainsize must be positive" );
00496 set_midpoint();
00497 }
00498 const Iterator& begin() const {return my_begin;}
00499 const Iterator& end() const {return my_end;}
00501 size_type grainsize() const {return my_grainsize;}
00502 };
00503
00504 template<typename Iterator>
00505 void hash_map_range<Iterator>::set_midpoint() const {
00506
00507 size_t m = my_end.my_index-my_begin.my_index;
00508 if( m > my_grainsize ) {
00509 m = my_begin.my_index + m/2u;
00510 hash_map_base::bucket *b = my_begin.my_map->get_bucket(m);
00511 my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list);
00512 } else {
00513 my_midpoint = my_end;
00514 }
00515 __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index,
00516 "my_begin is after my_midpoint" );
00517 __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index,
00518 "my_midpoint is after my_end" );
00519 __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end,
00520 "[my_begin, my_midpoint) range should not be empty" );
00521 }
00522
00523 }
00525
00527
00556 template<typename Key, typename T, typename HashCompare, typename Allocator>
00557 class concurrent_hash_map : protected internal::hash_map_base {
00558 template<typename Container, typename Value>
00559 friend class internal::hash_map_iterator;
00560
00561 template<typename I>
00562 friend class internal::hash_map_range;
00563
00564 public:
00565 typedef Key key_type;
00566 typedef T mapped_type;
00567 typedef std::pair<const Key,T> value_type;
00568 typedef hash_map_base::size_type size_type;
00569 typedef ptrdiff_t difference_type;
00570 typedef value_type *pointer;
00571 typedef const value_type *const_pointer;
00572 typedef value_type &reference;
00573 typedef const value_type &const_reference;
00574 typedef internal::hash_map_iterator<concurrent_hash_map,value_type> iterator;
00575 typedef internal::hash_map_iterator<concurrent_hash_map,const value_type> const_iterator;
00576 typedef internal::hash_map_range<iterator> range_type;
00577 typedef internal::hash_map_range<const_iterator> const_range_type;
00578 typedef Allocator allocator_type;
00579
00580 protected:
00581 friend class const_accessor;
00582 struct node;
00583 typedef typename Allocator::template rebind<node>::other node_allocator_type;
00584 node_allocator_type my_allocator;
00585 HashCompare my_hash_compare;
00586
00587 struct node : public node_base {
00588 value_type item;
00589 node( const Key &key ) : item(key, T()) {}
00590 node( const Key &key, const T &t ) : item(key, t) {}
00591
00592 void *operator new( size_t , node_allocator_type &a ) {
00593 void *ptr = a.allocate(1);
00594 if(!ptr)
00595 tbb::internal::throw_exception(tbb::internal::eid_bad_alloc);
00596 return ptr;
00597 }
00598
00599 void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast<node*>(ptr),1); }
00600 };
00601
00602 void delete_node( node_base *n ) {
00603 my_allocator.destroy( static_cast<node*>(n) );
00604 my_allocator.deallocate( static_cast<node*>(n), 1);
00605 }
00606
00607 node *search_bucket( const key_type &key, bucket *b ) const {
00608 node *n = static_cast<node*>( b->node_list );
00609 while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) )
00610 n = static_cast<node*>( n->next );
00611 __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket");
00612 return n;
00613 }
00614
00616 class bucket_accessor : public bucket::scoped_t {
00617 bucket *my_b;
00618 public:
00619 bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); }
00621 inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) {
00622 my_b = base->get_bucket( h );
00623
00624 if( itt_load_word_with_acquire(my_b->node_list) == internal::rehash_req
00625 && try_acquire( my_b->mutex, true ) )
00626 {
00627 if( my_b->node_list == internal::rehash_req ) base->rehash_bucket( my_b, h );
00628 }
00629 else bucket::scoped_t::acquire( my_b->mutex, writer );
00630 __TBB_ASSERT( my_b->node_list != internal::rehash_req, NULL);
00631 }
00633 bool is_writer() { return bucket::scoped_t::is_writer; }
00635 bucket *operator() () { return my_b; }
00636 };
00637
00638
00639 void rehash_bucket( bucket *b_new, const hashcode_t h ) {
00640 __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)");
00641 __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
00642 __TBB_store_with_release(b_new->node_list, internal::empty_rehashed);
00643 hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1;
00644 #if __TBB_STATISTICS
00645 my_info_rehashes++;
00646 #endif
00647
00648 bucket_accessor b_old( this, h & mask );
00649
00650 mask = (mask<<1) | 1;
00651 __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL );
00652 restart:
00653 for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) {
00654 hashcode_t c = my_hash_compare.hash( static_cast<node*>(n)->item.first );
00655 #if TBB_USE_ASSERT
00656 hashcode_t bmask = h & (mask>>1);
00657 bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1;
00658 __TBB_ASSERT( (c & bmask) == (h & bmask), "hash() function changed for key in table" );
00659 #endif
00660 if( (c & mask) == h ) {
00661 if( !b_old.is_writer() )
00662 if( !b_old.upgrade_to_writer() ) {
00663 goto restart;
00664 }
00665 *p = n->next;
00666 add_to_bucket( b_new, n );
00667 } else p = &n->next;
00668 }
00669 }
00670
00671 public:
00672
00673 class accessor;
00675 class const_accessor : private node::scoped_t {
00676 friend class concurrent_hash_map<Key,T,HashCompare,Allocator>;
00677 friend class accessor;
00678 public:
00680 typedef const typename concurrent_hash_map::value_type value_type;
00681
00683 bool empty() const {return !my_node;}
00684
00686 void release() {
00687 if( my_node ) {
00688 node::scoped_t::release();
00689 my_node = 0;
00690 }
00691 }
00692
00694 const_reference operator*() const {
00695 __TBB_ASSERT( my_node, "attempt to dereference empty accessor" );
00696 return my_node->item;
00697 }
00698
00700 const_pointer operator->() const {
00701 return &operator*();
00702 }
00703
00705 const_accessor() : my_node(NULL) {}
00706
00708 ~const_accessor() {
00709 my_node = NULL;
00710 }
00711 protected:
00712 bool is_writer() { return node::scoped_t::is_writer; }
00713 node *my_node;
00714 hashcode_t my_hash;
00715 };
00716
00718 class accessor: public const_accessor {
00719 public:
00721 typedef typename concurrent_hash_map::value_type value_type;
00722
00724 reference operator*() const {
00725 __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" );
00726 return this->my_node->item;
00727 }
00728
00730 pointer operator->() const {
00731 return &operator*();
00732 }
00733 };
00734
00736 concurrent_hash_map(const allocator_type &a = allocator_type())
00737 : internal::hash_map_base(), my_allocator(a)
00738 {}
00739
00741 concurrent_hash_map(size_type n, const allocator_type &a = allocator_type())
00742 : my_allocator(a)
00743 {
00744 reserve( n );
00745 }
00746
00748 concurrent_hash_map( const concurrent_hash_map& table, const allocator_type &a = allocator_type())
00749 : internal::hash_map_base(), my_allocator(a)
00750 {
00751 internal_copy(table);
00752 }
00753
00755 template<typename I>
00756 concurrent_hash_map(I first, I last, const allocator_type &a = allocator_type())
00757 : my_allocator(a)
00758 {
00759 reserve( std::distance(first, last) );
00760 internal_copy(first, last);
00761 }
00762
00764 concurrent_hash_map& operator=( const concurrent_hash_map& table ) {
00765 if( this!=&table ) {
00766 clear();
00767 internal_copy(table);
00768 }
00769 return *this;
00770 }
00771
00772
00774
00776 void rehash(size_type n = 0);
00777
00779 void clear();
00780
00782 ~concurrent_hash_map() { clear(); }
00783
00784
00785
00786
00787 range_type range( size_type grainsize=1 ) {
00788 return range_type( *this, grainsize );
00789 }
00790 const_range_type range( size_type grainsize=1 ) const {
00791 return const_range_type( *this, grainsize );
00792 }
00793
00794
00795
00796
00797 iterator begin() {return iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00798 iterator end() {return iterator(*this,0,0,0);}
00799 const_iterator begin() const {return const_iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00800 const_iterator end() const {return const_iterator(*this,0,0,0);}
00801 std::pair<iterator, iterator> equal_range( const Key& key ) { return internal_equal_range(key, end()); }
00802 std::pair<const_iterator, const_iterator> equal_range( const Key& key ) const { return internal_equal_range(key, end()); }
00803
00805 size_type size() const { return my_size; }
00806
00808 bool empty() const { return my_size == 0; }
00809
00811 size_type max_size() const {return (~size_type(0))/sizeof(node);}
00812
00814 size_type bucket_count() const { return my_mask+1; }
00815
00817 allocator_type get_allocator() const { return this->my_allocator; }
00818
00820 void swap(concurrent_hash_map &table);
00821
00822
00823
00824
00825
00827 size_type count( const Key &key ) const {
00828 return const_cast<concurrent_hash_map*>(this)->lookup(false, key, NULL, NULL, false );
00829 }
00830
00832
00833 bool find( const_accessor &result, const Key &key ) const {
00834 result.release();
00835 return const_cast<concurrent_hash_map*>(this)->lookup(false, key, NULL, &result, false );
00836 }
00837
00839
00840 bool find( accessor &result, const Key &key ) {
00841 result.release();
00842 return lookup(false, key, NULL, &result, true );
00843 }
00844
00846
00847 bool insert( const_accessor &result, const Key &key ) {
00848 result.release();
00849 return lookup(true, key, NULL, &result, false );
00850 }
00851
00853
00854 bool insert( accessor &result, const Key &key ) {
00855 result.release();
00856 return lookup(true, key, NULL, &result, true );
00857 }
00858
00860
00861 bool insert( const_accessor &result, const value_type &value ) {
00862 result.release();
00863 return lookup(true, value.first, &value.second, &result, false );
00864 }
00865
00867
00868 bool insert( accessor &result, const value_type &value ) {
00869 result.release();
00870 return lookup(true, value.first, &value.second, &result, true );
00871 }
00872
00874
00875 bool insert( const value_type &value ) {
00876 return lookup(true, value.first, &value.second, NULL, false );
00877 }
00878
00880 template<typename I>
00881 void insert(I first, I last) {
00882 for(; first != last; ++first)
00883 insert( *first );
00884 }
00885
00887
00888 bool erase( const Key& key );
00889
00891
00892 bool erase( const_accessor& item_accessor ) {
00893 return exclude( item_accessor );
00894 }
00895
00897
00898 bool erase( accessor& item_accessor ) {
00899 return exclude( item_accessor );
00900 }
00901
00902 protected:
00904 bool lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write );
00905
00907 bool exclude( const_accessor &item_accessor );
00908
00910 template<typename I>
00911 std::pair<I, I> internal_equal_range( const Key& key, I end ) const;
00912
00914 void internal_copy( const concurrent_hash_map& source );
00915
00916 template<typename I>
00917 void internal_copy(I first, I last);
00918
00920
00922 const_pointer internal_fast_find( const Key& key ) const {
00923 hashcode_t h = my_hash_compare.hash( key );
00924 hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
00925 node *n;
00926 restart:
00927 __TBB_ASSERT((m&(m+1))==0, NULL);
00928 bucket *b = get_bucket( h & m );
00929
00930 if( itt_load_word_with_acquire(b->node_list) == internal::rehash_req )
00931 {
00932 bucket::scoped_t lock;
00933 if( lock.try_acquire( b->mutex, true ) ) {
00934 if( b->node_list == internal::rehash_req)
00935 const_cast<concurrent_hash_map*>(this)->rehash_bucket( b, h & m );
00936 }
00937 else lock.acquire( b->mutex, false );
00938 __TBB_ASSERT(b->node_list!=internal::rehash_req,NULL);
00939 }
00940 n = search_bucket( key, b );
00941 if( n )
00942 return &n->item;
00943 else if( check_mask_race( h, m ) )
00944 goto restart;
00945 return 0;
00946 }
00947 };
00948
00949 #if _MSC_VER && !defined(__INTEL_COMPILER)
00950
00951 #pragma warning( push )
00952 #pragma warning( disable: 4127 )
00953 #endif
00954
00955 template<typename Key, typename T, typename HashCompare, typename A>
00956 bool concurrent_hash_map<Key,T,HashCompare,A>::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write ) {
00957 __TBB_ASSERT( !result || !result->my_node, NULL );
00958 bool return_value;
00959 hashcode_t const h = my_hash_compare.hash( key );
00960 hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
00961 segment_index_t grow_segment = 0;
00962 node *n, *tmp_n = 0;
00963 restart:
00964 {
00965 __TBB_ASSERT((m&(m+1))==0, NULL);
00966 return_value = false;
00967
00968 bucket_accessor b( this, h & m );
00969
00970
00971 n = search_bucket( key, b() );
00972 if( op_insert ) {
00973
00974 if( !n ) {
00975 if( !tmp_n ) {
00976 if(t) tmp_n = new( my_allocator ) node(key, *t);
00977 else tmp_n = new( my_allocator ) node(key);
00978 }
00979 if( !b.is_writer() && !b.upgrade_to_writer() ) {
00980
00981 n = search_bucket( key, b() );
00982 if( is_valid(n) ) {
00983 b.downgrade_to_reader();
00984 goto exists;
00985 }
00986 }
00987 if( check_mask_race(h, m) )
00988 goto restart;
00989
00990 grow_segment = insert_new_node( b(), n = tmp_n, m );
00991 tmp_n = 0;
00992 return_value = true;
00993 }
00994 } else {
00995 if( !n ) {
00996 if( check_mask_race( h, m ) )
00997 goto restart;
00998 return false;
00999 }
01000 return_value = true;
01001 }
01002 exists:
01003 if( !result ) goto check_growth;
01004
01005
01006 if( !result->try_acquire( n->mutex, write ) ) {
01007
01008 tbb::internal::atomic_backoff trials;
01009 do {
01010 if( !trials.bounded_pause() ) {
01011
01012 b.release();
01013 __TBB_ASSERT( !op_insert || !return_value, "Can't acquire new item in locked bucket?" );
01014 __TBB_Yield();
01015 m = (hashcode_t) itt_load_word_with_acquire( my_mask );
01016 goto restart;
01017 }
01018 } while( !result->try_acquire( n->mutex, write ) );
01019 }
01020 }
01021 result->my_node = n;
01022 result->my_hash = h;
01023 check_growth:
01024
01025 if( grow_segment ) {
01026 #if __TBB_STATISTICS
01027 my_info_resizes++;
01028 #endif
01029 enable_segment( grow_segment );
01030 }
01031 if( tmp_n )
01032 delete_node( tmp_n );
01033 return return_value;
01034 }
01035
01036 template<typename Key, typename T, typename HashCompare, typename A>
01037 template<typename I>
01038 std::pair<I, I> concurrent_hash_map<Key,T,HashCompare,A>::internal_equal_range( const Key& key, I end_ ) const {
01039 hashcode_t h = my_hash_compare.hash( key );
01040 hashcode_t m = my_mask;
01041 __TBB_ASSERT((m&(m+1))==0, NULL);
01042 h &= m;
01043 bucket *b = get_bucket( h );
01044 while( b->node_list == internal::rehash_req ) {
01045 m = ( 1u<<__TBB_Log2( h ) ) - 1;
01046 b = get_bucket( h &= m );
01047 }
01048 node *n = search_bucket( key, b );
01049 if( !n )
01050 return std::make_pair(end_, end_);
01051 iterator lower(*this, h, b, n), upper(lower);
01052 return std::make_pair(lower, ++upper);
01053 }
01054
01055 template<typename Key, typename T, typename HashCompare, typename A>
01056 bool concurrent_hash_map<Key,T,HashCompare,A>::exclude( const_accessor &item_accessor ) {
01057 __TBB_ASSERT( item_accessor.my_node, NULL );
01058 node_base *const n = item_accessor.my_node;
01059 hashcode_t const h = item_accessor.my_hash;
01060 hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
01061 do {
01062
01063 bucket_accessor b( this, h & m, true );
01064 node_base **p = &b()->node_list;
01065 while( *p && *p != n )
01066 p = &(*p)->next;
01067 if( !*p ) {
01068 if( check_mask_race( h, m ) )
01069 continue;
01070 item_accessor.release();
01071 return false;
01072 }
01073 __TBB_ASSERT( *p == n, NULL );
01074 *p = n->next;
01075 my_size--;
01076 break;
01077 } while(true);
01078 if( !item_accessor.is_writer() )
01079 item_accessor.upgrade_to_writer();
01080 item_accessor.release();
01081 delete_node( n );
01082 return true;
01083 }
01084
01085 template<typename Key, typename T, typename HashCompare, typename A>
01086 bool concurrent_hash_map<Key,T,HashCompare,A>::erase( const Key &key ) {
01087 node_base *n;
01088 hashcode_t const h = my_hash_compare.hash( key );
01089 hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask );
01090 restart:
01091 {
01092
01093 bucket_accessor b( this, h & m );
01094 search:
01095 node_base **p = &b()->node_list;
01096 n = *p;
01097 while( is_valid(n) && !my_hash_compare.equal(key, static_cast<node*>(n)->item.first ) ) {
01098 p = &n->next;
01099 n = *p;
01100 }
01101 if( !n ) {
01102 if( check_mask_race( h, m ) )
01103 goto restart;
01104 return false;
01105 }
01106 else if( !b.is_writer() && !b.upgrade_to_writer() ) {
01107 if( check_mask_race( h, m ) )
01108 goto restart;
01109 goto search;
01110 }
01111 *p = n->next;
01112 my_size--;
01113 }
01114 {
01115 typename node::scoped_t item_locker( n->mutex, true );
01116 }
01117
01118 delete_node( n );
01119 return true;
01120 }
01121
01122 template<typename Key, typename T, typename HashCompare, typename A>
01123 void concurrent_hash_map<Key,T,HashCompare,A>::swap(concurrent_hash_map<Key,T,HashCompare,A> &table) {
01124 std::swap(this->my_allocator, table.my_allocator);
01125 std::swap(this->my_hash_compare, table.my_hash_compare);
01126 internal_swap(table);
01127 }
01128
01129 template<typename Key, typename T, typename HashCompare, typename A>
01130 void concurrent_hash_map<Key,T,HashCompare,A>::rehash(size_type sz) {
01131 reserve( sz );
01132 hashcode_t mask = my_mask;
01133 hashcode_t b = (mask+1)>>1;
01134 __TBB_ASSERT((b&(b-1))==0, NULL);
01135 bucket *bp = get_bucket( b );
01136 for(; b <= mask; b++, bp++ ) {
01137 node_base *n = bp->node_list;
01138 __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" );
01139 __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" );
01140 if( n == internal::rehash_req ) {
01141 hashcode_t h = b; bucket *b_old = bp;
01142 do {
01143 __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
01144 hashcode_t m = ( 1u<<__TBB_Log2( h ) ) - 1;
01145 b_old = get_bucket( h &= m );
01146 } while( b_old->node_list == internal::rehash_req );
01147
01148 mark_rehashed_levels( h );
01149 for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) {
01150 hashcode_t c = my_hash_compare.hash( static_cast<node*>(q)->item.first );
01151 if( (c & mask) != h ) {
01152 *p = q->next;
01153 bucket *b_new = get_bucket( c & mask );
01154 __TBB_ASSERT( b_new->node_list != internal::rehash_req, "hash() function changed for key in table or internal error" );
01155 add_to_bucket( b_new, q );
01156 } else p = &q->next;
01157 }
01158 }
01159 }
01160 #if TBB_USE_PERFORMANCE_WARNINGS
01161 int current_size = int(my_size), buckets = int(mask)+1, empty_buckets = 0, overpopulated_buckets = 0;
01162 static bool reported = false;
01163 #endif
01164 #if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS
01165 for( b = 0; b <= mask; b++ ) {
01166 if( b & (b-2) ) ++bp;
01167 else bp = get_bucket( b );
01168 node_base *n = bp->node_list;
01169 __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" );
01170 __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed, "Broken internal structure" );
01171 #if TBB_USE_PERFORMANCE_WARNINGS
01172 if( n == internal::empty_rehashed ) empty_buckets++;
01173 else if( n->next ) overpopulated_buckets++;
01174 #endif
01175 #if TBB_USE_ASSERT
01176 for( ; is_valid(n); n = n->next ) {
01177 hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first ) & mask;
01178 __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" );
01179 }
01180 #endif
01181 }
01182 #endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS
01183 #if TBB_USE_PERFORMANCE_WARNINGS
01184 if( buckets > current_size) empty_buckets -= buckets - current_size;
01185 else overpopulated_buckets -= current_size - buckets;
01186 if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) {
01187 tbb::internal::runtime_warning(
01188 "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d",
01189 typeid(*this).name(), current_size, empty_buckets, overpopulated_buckets );
01190 reported = true;
01191 }
01192 #endif
01193 }
01194
01195 template<typename Key, typename T, typename HashCompare, typename A>
01196 void concurrent_hash_map<Key,T,HashCompare,A>::clear() {
01197 hashcode_t m = my_mask;
01198 __TBB_ASSERT((m&(m+1))==0, NULL);
01199 #if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
01200 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
01201 int current_size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0;
01202 static bool reported = false;
01203 #endif
01204 bucket *bp = 0;
01205
01206 for( segment_index_t b = 0; b <= m; b++ ) {
01207 if( b & (b-2) ) ++bp;
01208 else bp = get_bucket( b );
01209 node_base *n = bp->node_list;
01210 __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" );
01211 __TBB_ASSERT( *reinterpret_cast<intptr_t*>(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during clear() execution" );
01212 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
01213 if( n == internal::empty_rehashed ) empty_buckets++;
01214 else if( n == internal::rehash_req ) buckets--;
01215 else if( n->next ) overpopulated_buckets++;
01216 #endif
01217 #if __TBB_EXTRA_DEBUG
01218 for(; is_valid(n); n = n->next ) {
01219 hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first );
01220 h &= m;
01221 __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" );
01222 }
01223 #endif
01224 }
01225 #if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
01226 #if __TBB_STATISTICS
01227 printf( "items=%d buckets: capacity=%d rehashed=%d empty=%d overpopulated=%d"
01228 " concurrent: resizes=%u rehashes=%u restarts=%u\n",
01229 current_size, int(m+1), buckets, empty_buckets, overpopulated_buckets,
01230 unsigned(my_info_resizes), unsigned(my_info_rehashes), unsigned(my_info_restarts) );
01231 my_info_resizes = 0;
01232 my_info_restarts = 0;
01233 my_info_rehashes = 0;
01234 #endif
01235 if( buckets > current_size) empty_buckets -= buckets - current_size;
01236 else overpopulated_buckets -= current_size - buckets;
01237 if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) {
01238 tbb::internal::runtime_warning(
01239 "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d",
01240 typeid(*this).name(), current_size, empty_buckets, overpopulated_buckets );
01241 reported = true;
01242 }
01243 #endif
01244 #endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS
01245 my_size = 0;
01246 segment_index_t s = segment_index_of( m );
01247 __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" );
01248 cache_aligned_allocator<bucket> alloc;
01249 do {
01250 __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" );
01251 segment_ptr_t buckets_ptr = my_table[s];
01252 size_type sz = segment_size( s ? s : 1 );
01253 for( segment_index_t i = 0; i < sz; i++ )
01254 for( node_base *n = buckets_ptr[i].node_list; is_valid(n); n = buckets_ptr[i].node_list ) {
01255 buckets_ptr[i].node_list = n->next;
01256 delete_node( n );
01257 }
01258 if( s >= first_block)
01259 alloc.deallocate( buckets_ptr, sz );
01260 else if( s == embedded_block && embedded_block != first_block )
01261 alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets );
01262 if( s >= embedded_block ) my_table[s] = 0;
01263 } while(s-- > 0);
01264 my_mask = embedded_buckets - 1;
01265 }
01266
01267 template<typename Key, typename T, typename HashCompare, typename A>
01268 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy( const concurrent_hash_map& source ) {
01269 reserve( source.my_size );
01270 hashcode_t mask = source.my_mask;
01271 if( my_mask == mask ) {
01272 bucket *dst = 0, *src = 0;
01273 bool rehash_required = false;
01274 for( hashcode_t k = 0; k <= mask; k++ ) {
01275 if( k & (k-2) ) ++dst,src++;
01276 else { dst = get_bucket( k ); src = source.get_bucket( k ); }
01277 __TBB_ASSERT( dst->node_list != internal::rehash_req, "Invalid bucket in destination table");
01278 node *n = static_cast<node*>( src->node_list );
01279 if( n == internal::rehash_req ) {
01280 rehash_required = true;
01281 dst->node_list = internal::rehash_req;
01282 } else for(; n; n = static_cast<node*>( n->next ) ) {
01283 add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) );
01284 ++my_size;
01285 }
01286 }
01287 if( rehash_required ) rehash();
01288 } else internal_copy( source.begin(), source.end() );
01289 }
01290
01291 template<typename Key, typename T, typename HashCompare, typename A>
01292 template<typename I>
01293 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy(I first, I last) {
01294 hashcode_t m = my_mask;
01295 for(; first != last; ++first) {
01296 hashcode_t h = my_hash_compare.hash( first->first );
01297 bucket *b = get_bucket( h & m );
01298 __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table");
01299 node *n = new( my_allocator ) node(first->first, first->second);
01300 add_to_bucket( b, n );
01301 ++my_size;
01302 }
01303 }
01304
01305 }
01306
01307 using interface5::concurrent_hash_map;
01308
01309
01310 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01311 inline bool operator==(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b) {
01312 if(a.size() != b.size()) return false;
01313 typename concurrent_hash_map<Key, T, HashCompare, A1>::const_iterator i(a.begin()), i_end(a.end());
01314 typename concurrent_hash_map<Key, T, HashCompare, A2>::const_iterator j, j_end(b.end());
01315 for(; i != i_end; ++i) {
01316 j = b.equal_range(i->first).first;
01317 if( j == j_end || !(i->second == j->second) ) return false;
01318 }
01319 return true;
01320 }
01321
01322 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01323 inline bool operator!=(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b)
01324 { return !(a == b); }
01325
01326 template<typename Key, typename T, typename HashCompare, typename A>
01327 inline void swap(concurrent_hash_map<Key, T, HashCompare, A> &a, concurrent_hash_map<Key, T, HashCompare, A> &b)
01328 { a.swap( b ); }
01329
01330 #if _MSC_VER && !defined(__INTEL_COMPILER)
01331 #pragma warning( pop )
01332 #endif // warning 4127 is back
01333
01334 }
01335
01336 #endif