concurrent_hash_map.h

00001 /*
00002     Copyright 2005-2009 Intel Corporation.  All Rights Reserved.
00003 
00004     The source code contained or described herein and all documents related
00005     to the source code ("Material") are owned by Intel Corporation or its
00006     suppliers or licensors.  Title to the Material remains with Intel
00007     Corporation or its suppliers and licensors.  The Material is protected
00008     by worldwide copyright laws and treaty provisions.  No part of the
00009     Material may be used, copied, reproduced, modified, published, uploaded,
00010     posted, transmitted, distributed, or disclosed in any way without
00011     Intel's prior express written permission.
00012 
00013     No license under any patent, copyright, trade secret or other
00014     intellectual property right is granted to or conferred upon you by
00015     disclosure or delivery of the Materials, either expressly, by
00016     implication, inducement, estoppel or otherwise.  Any license under such
00017     intellectual property rights must be express and approved by Intel in
00018     writing.
00019 */
00020 
00021 #ifndef __TBB_concurrent_hash_map_H
00022 #define __TBB_concurrent_hash_map_H
00023 
00024 #include <stdexcept>
00025 #include <iterator>
00026 #include <utility>      // Need std::pair
00027 #include <cstring>      // Need std::memset
00028 #include <string>
00029 #include "tbb_stddef.h"
00030 #include "cache_aligned_allocator.h"
00031 #include "tbb_allocator.h"
00032 #include "spin_rw_mutex.h"
00033 #include "atomic.h"
00034 #include "aligned_space.h"
00035 #if TBB_USE_PERFORMANCE_WARNINGS
00036 #include <typeinfo>
00037 #endif
00038 
00039 namespace tbb {
00040 
00041 template<typename T> struct tbb_hash_compare;
00042 template<typename Key, typename T, typename HashCompare = tbb_hash_compare<Key>, typename A = tbb_allocator<std::pair<Key, T> > >
00043 class concurrent_hash_map;
00044 
00046 namespace internal {
00048     void* __TBB_EXPORTED_FUNC itt_load_pointer_with_acquire_v3( const void* src );
00050     void __TBB_EXPORTED_FUNC itt_store_pointer_with_release_v3( void* dst, void* src );
00051 
00053     typedef size_t hashcode_t;
00055     class hash_map_base {
00056     public:
00058         typedef size_t size_type;
00060         typedef size_t hashcode_t;
00062         typedef size_t segment_index_t;
00064         struct node_base : no_copy {
00066             typedef spin_rw_mutex mutex_t;
00068             typedef mutex_t::scoped_lock scoped_t;
00070             node_base *next;
00071             mutex_t mutex;
00072         };
00074 #       define __TBB_rehash_req reinterpret_cast<node_base*>(1)
00076 #       define __TBB_empty_rehashed reinterpret_cast<node_base*>(0)
00078         struct bucket : no_copy {
00080             typedef spin_rw_mutex mutex_t;
00082             typedef mutex_t::scoped_lock scoped_t;
00083             mutex_t mutex;
00084             node_base *node_list;
00085         };
00087         static size_type const embedded_block = 1;
00089         static size_type const embedded_buckets = 1<<embedded_block;
00091         static size_type const first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096
00093         static size_type const pointers_per_table = sizeof(segment_index_t) * 8; // one segment per bit
00095         typedef bucket *segment_ptr_t;
00097         typedef segment_ptr_t segments_table_t[pointers_per_table];
00099         atomic<hashcode_t> my_mask;
00101         segments_table_t my_table;
00103         atomic<size_type> my_size; // It must be in separate cache line from my_mask due to performance effects
00105         bucket my_embedded_segment[embedded_buckets];
00106 
00108         hash_map_base() {
00109             std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128   or 64*8=512
00110                 + sizeof(my_size) + sizeof(my_mask)  // 4+4 or 8+8
00111                 + embedded_buckets*sizeof(bucket) ); // n*8 or n*16
00112             for( size_type i = 0; i < embedded_block; i++ ) // fill the table
00113                 my_table[i] = my_embedded_segment + segment_base(i);
00114             my_mask = embedded_buckets - 1;
00115             __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks");
00116         }
00117 
00119         static segment_index_t segment_index_of( size_type index ) {
00120             return segment_index_t( __TBB_Log2( index|1 ) );
00121         }
00122 
00124         static segment_index_t segment_base( segment_index_t k ) {
00125             return (segment_index_t(1)<<k & ~segment_index_t(1));
00126         }
00127 
00129         static size_type segment_size( segment_index_t k ) {
00130             return size_type(1)<<k; // fake value for k==0
00131         }
00132         
00134         static bool is_valid( void *ptr ) {
00135             return ptr > reinterpret_cast<void*>(1);
00136         }
00137 
00139         static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) {
00140             if( is_initial ) std::memset(ptr, 0, sz*sizeof(bucket) );
00141             else for(size_type i = 0; i < sz; i++, ptr++) {
00142                     *reinterpret_cast<intptr_t*>(&ptr->mutex) = 0;
00143                     ptr->node_list = __TBB_rehash_req;
00144                 }
00145         }
00146         
00148         static void add_to_bucket( bucket *b, node_base *n ) {
00149             n->next = b->node_list;
00150             b->node_list = n; // its under lock and flag is set
00151         }
00152 
00154         struct enable_segment_failsafe {
00155             segment_ptr_t *my_segment_ptr;
00156             enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {}
00157             ~enable_segment_failsafe() {
00158                 if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress
00159             }
00160         };
00161 
00163         void enable_segment( segment_index_t k, bool is_initial = false ) {
00164             __TBB_ASSERT( k, "Zero segment must be embedded" );
00165             enable_segment_failsafe watchdog( my_table, k );
00166             cache_aligned_allocator<bucket> alloc;
00167             size_type sz;
00168             __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment");
00169             if( k >= first_block ) {
00170                 sz = segment_size( k );
00171                 segment_ptr_t ptr = alloc.allocate( sz );
00172                 init_buckets( ptr, sz, is_initial );
00173 #if TBB_USE_THREADING_TOOLS
00174                 itt_store_pointer_with_release_v3( my_table + k, ptr );
00175 #else
00176                 my_table[k] = ptr;// my_mask has release fence
00177 #endif
00178                 sz <<= 1;// double it to get entire capacity of the container
00179             } else { // the first block
00180                 __TBB_ASSERT( k == embedded_block, "Wrong segment index" );
00181                 sz = segment_size( first_block );
00182                 segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets );
00183                 init_buckets( ptr, sz - embedded_buckets, is_initial );
00184                 ptr -= segment_base(embedded_block);
00185                 for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets
00186 #if TBB_USE_THREADING_TOOLS
00187                     itt_store_pointer_with_release_v3( my_table + i, ptr + segment_base(i) );
00188 #else
00189                     my_table[i] = ptr + segment_base(i);
00190 #endif
00191             }
00192 #if TBB_USE_THREADING_TOOLS
00193             itt_store_pointer_with_release_v3( &my_mask, (void*)(sz-1) );
00194 #else
00195             my_mask = sz - 1;
00196 #endif
00197             watchdog.my_segment_ptr = 0;
00198         }
00199 
00201         bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere?
00202             segment_index_t s = segment_index_of( h );
00203             h -= segment_base(s);
00204             segment_ptr_t seg = my_table[s];
00205             __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" );
00206             return &seg[h];
00207         }
00208 
00210         // Splitting into two functions should help inlining
00211         inline bool check_mask_race( const hashcode_t h, hashcode_t &m ) const {
00212             hashcode_t m_now, m_old = m;
00213 #if TBB_USE_THREADING_TOOLS
00214             m_now = (hashcode_t) itt_load_pointer_with_acquire_v3( &my_mask );
00215 #else
00216             m_now = my_mask;
00217 #endif
00218             if( m_old != m_now )
00219                 return check_rehashing_collision( h, m_old, m = m_now );
00220             return false;
00221         }
00222 
00224         bool check_rehashing_collision( const hashcode_t h, hashcode_t m_old, hashcode_t m ) const {
00225             __TBB_ASSERT(m_old != m, NULL); // TODO?: m arg could be optimized out by passing h = h&m
00226             if( (h & m_old) != (h & m) ) { // mask changed for this hashcode, rare event
00227                 // condition above proves that 'h' has some other bits set beside 'm_old'
00228                 // find next applicable mask after m_old    //TODO: look at bsl instruction
00229                 for( ++m_old; !(h & m_old); m_old <<= 1 ); // at maximum few rounds depending on the first block size
00230                 m_old = (m_old<<1) - 1; // get full mask from a bit
00231                 __TBB_ASSERT((m_old&(m_old+1))==0 && m_old <= m, NULL);
00232                 // check whether it is rehashing/ed
00233                 if( __TBB_load_with_acquire(get_bucket( h & m_old )->node_list) != __TBB_rehash_req )
00234                     return true;
00235             }
00236             return false;
00237         }
00238 
00240         segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) {
00241             size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted
00242             add_to_bucket( b, n );
00243             // check load factor
00244             if( sz >= mask ) { // TODO: add custom load_factor 
00245                 segment_index_t new_seg = segment_index_of( mask+1 );
00246                 __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated");
00247                 if( !my_table[new_seg] && __TBB_CompareAndSwapW(&my_table[new_seg], 1, 0) == 0 )
00248                     return new_seg; // The value must be processed
00249             }
00250             return 0;
00251         }
00252 
00254         void reserve(size_type buckets) {
00255             if( !buckets-- ) return;
00256             bool is_initial = !my_size;
00257             for( size_type m = my_mask; buckets > m; m = my_mask )
00258                 enable_segment( segment_index_of( m+1 ), is_initial );
00259         }
00261         void internal_swap(hash_map_base &table) {
00262             std::swap(this->my_mask, table.my_mask);
00263             std::swap(this->my_size, table.my_size);
00264             for(size_type i = 0; i < embedded_buckets; i++)
00265                 std::swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list);
00266             for(size_type i = embedded_block; i < pointers_per_table; i++)
00267                 std::swap(this->my_table[i], table.my_table[i]);
00268         }
00269     };
00270 
00271     template<typename Iterator>
00272     class hash_map_range;
00273 
00275 
00277     template<typename Container, typename Value>
00278     class hash_map_iterator
00279         : public std::iterator<std::forward_iterator_tag,Value>
00280     {
00281         typedef Container map_type;
00282         typedef typename Container::node node;
00283         typedef hash_map_base::node_base node_base;
00284         typedef hash_map_base::bucket bucket;
00285 
00286         template<typename C, typename T, typename U>
00287         friend bool operator==( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00288 
00289         template<typename C, typename T, typename U>
00290         friend bool operator!=( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00291 
00292         template<typename C, typename T, typename U>
00293         friend ptrdiff_t operator-( const hash_map_iterator<C,T>& i, const hash_map_iterator<C,U>& j );
00294     
00295         template<typename C, typename U>
00296         friend class internal::hash_map_iterator;
00297 
00298         template<typename I>
00299         friend class internal::hash_map_range;
00300 
00301         void advance_to_next_bucket() { // TODO?: refactor to iterator_base class
00302             size_t k = my_index+1;
00303             while( my_bucket && k <= my_map->my_mask ) {
00304                 // Following test uses 2's-complement wizardry
00305                 if( k& (k-2) ) // not the begining of a segment
00306                     ++my_bucket;
00307                 else my_bucket = my_map->get_bucket( k );
00308                 my_node = static_cast<node*>( my_bucket->node_list );
00309                 if( hash_map_base::is_valid(my_node) ) {
00310                     my_index = k; return;
00311                 }
00312                 ++k;
00313             }
00314             my_bucket = 0; my_node = 0; my_index = k; // the end
00315         }
00316 #if !defined(_MSC_VER) || defined(__INTEL_COMPILER)
00317         template<typename Key, typename T, typename HashCompare, typename A>
00318         friend class tbb::concurrent_hash_map;
00319 #else
00320     public: // workaround
00321 #endif
00323         const Container *my_map;
00324 
00326         size_t my_index;
00327 
00329         const bucket *my_bucket;
00330 
00332         node *my_node;
00333 
00334         hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n );
00335 
00336     public:
00338         hash_map_iterator() {}
00339         hash_map_iterator( const hash_map_iterator<Container,typename Container::value_type> &other ) :
00340             my_map(other.my_map),
00341             my_index(other.my_index),
00342             my_bucket(other.my_bucket),
00343             my_node(other.my_node)
00344         {}
00345         Value& operator*() const {
00346             __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" );
00347             return my_node->item;
00348         }
00349         Value* operator->() const {return &operator*();}
00350         hash_map_iterator& operator++();
00351         
00353         Value* operator++(int) {
00354             Value* result = &operator*();
00355             operator++();
00356             return result;
00357         }
00358     };
00359 
00360     template<typename Container, typename Value>
00361     hash_map_iterator<Container,Value>::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) :
00362         my_map(&map),
00363         my_index(index),
00364         my_bucket(b),
00365         my_node( static_cast<node*>(n) )
00366     {
00367         if( b && !hash_map_base::is_valid(n) )
00368             advance_to_next_bucket();
00369     }
00370 
00371     template<typename Container, typename Value>
00372     hash_map_iterator<Container,Value>& hash_map_iterator<Container,Value>::operator++() {
00373         my_node = static_cast<node*>( my_node->next );
00374         if( !my_node ) advance_to_next_bucket();
00375         return *this;
00376     }
00377 
00378     template<typename Container, typename T, typename U>
00379     bool operator==( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00380         return i.my_node == j.my_node && i.my_map == j.my_map;
00381     }
00382 
00383     template<typename Container, typename T, typename U>
00384     bool operator!=( const hash_map_iterator<Container,T>& i, const hash_map_iterator<Container,U>& j ) {
00385         return i.my_node != j.my_node || i.my_map != j.my_map;
00386     }
00387 
00389 
00390     template<typename Iterator>
00391     class hash_map_range {
00392         typedef typename Iterator::map_type map_type;
00393         Iterator my_begin;
00394         Iterator my_end;
00395         mutable Iterator my_midpoint;
00396         size_t my_grainsize;
00398         void set_midpoint() const;
00399         template<typename U> friend class hash_map_range;
00400     public:
00402         typedef std::size_t size_type;
00403         typedef typename Iterator::value_type value_type;
00404         typedef typename Iterator::reference reference;
00405         typedef typename Iterator::difference_type difference_type;
00406         typedef Iterator iterator;
00407 
00409         bool empty() const {return my_begin==my_end;}
00410 
00412         bool is_divisible() const {
00413             return my_midpoint!=my_end;
00414         }
00416         hash_map_range( hash_map_range& r, split ) : 
00417             my_end(r.my_end),
00418             my_grainsize(r.my_grainsize)
00419         {
00420             r.my_end = my_begin = r.my_midpoint;
00421             __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" );
00422             __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" );
00423             set_midpoint();
00424             r.set_midpoint();
00425         }
00427         template<typename U>
00428         hash_map_range( hash_map_range<U>& r) : 
00429             my_begin(r.my_begin),
00430             my_end(r.my_end),
00431             my_midpoint(r.my_midpoint),
00432             my_grainsize(r.my_grainsize)
00433         {}
00434 #if TBB_DEPRECATED
00436         hash_map_range( const Iterator& begin_, const Iterator& end_, size_type grainsize = 1 ) : 
00437             my_begin(begin_), 
00438             my_end(end_),
00439             my_grainsize(grainsize)
00440         {
00441             if(!my_end.my_index && !my_end.my_bucket) // end
00442                 my_end.my_index = my_end.my_map->my_mask + 1;
00443             set_midpoint();
00444             __TBB_ASSERT( grainsize>0, "grainsize must be positive" );
00445         }
00446 #endif
00448         hash_map_range( const map_type &map, size_type grainsize = 1 ) : 
00449             my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ),
00450             my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ),
00451             my_grainsize( grainsize )
00452         {
00453             __TBB_ASSERT( grainsize>0, "grainsize must be positive" );
00454             set_midpoint();
00455         }
00456         const Iterator& begin() const {return my_begin;}
00457         const Iterator& end() const {return my_end;}
00459         size_type grainsize() const {return my_grainsize;}
00460     };
00461 
00462     template<typename Iterator>
00463     void hash_map_range<Iterator>::set_midpoint() const {
00464         // Split by groups of nodes
00465         size_t m = my_end.my_index-my_begin.my_index;
00466         if( m > my_grainsize ) {
00467             m = my_begin.my_index + m/2u;
00468             hash_map_base::bucket *b = my_begin.my_map->get_bucket(m);
00469             my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list);
00470         } else {
00471             my_midpoint = my_end;
00472         }
00473         __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index,
00474             "my_begin is after my_midpoint" );
00475         __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index,
00476             "my_midpoint is after my_end" );
00477         __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end,
00478             "[my_begin, my_midpoint) range should not be empty" );
00479     }
00480 } // namespace internal
00482 
00484 static const size_t hash_multiplier = sizeof(size_t)==4? 2654435769U : 11400714819323198485ULL;
00486 template<typename T>
00487 inline static size_t tbb_hasher( const T& t ) {
00488     return static_cast<size_t>( t ) * hash_multiplier;
00489 }
00490 template<typename P>
00491 inline static size_t tbb_hasher( P* ptr ) {
00492     size_t const h = reinterpret_cast<size_t>( ptr );
00493     return (h >> 3) ^ h;
00494 }
00495 template<typename E, typename S, typename A>
00496 inline static size_t tbb_hasher( const std::basic_string<E,S,A>& s ) {
00497     size_t h = 0;
00498     for( const E* c = s.c_str(); *c; c++ )
00499         h = static_cast<size_t>(*c) ^ (h * hash_multiplier);
00500     return h;
00501 }
00502 template<typename F, typename S>
00503 inline static size_t tbb_hasher( const std::pair<F,S>& p ) {
00504     return tbb_hasher(p.first) ^ tbb_hasher(p.second);
00505 }
00506 
00508 template<typename T>
00509 struct tbb_hash_compare {
00510     static size_t hash( const T& t ) { return tbb_hasher(t); }
00511     static bool equal( const T& a, const T& b ) { return a == b; }
00512 };
00513 
00515 
00543 template<typename Key, typename T, typename HashCompare, typename Allocator>
00544 class concurrent_hash_map : protected internal::hash_map_base {
00545     template<typename Container, typename Value>
00546     friend class internal::hash_map_iterator;
00547 
00548     template<typename I>
00549     friend class internal::hash_map_range;
00550 
00551 public:
00552     typedef Key key_type;
00553     typedef T mapped_type;
00554     typedef std::pair<const Key,T> value_type;
00555     typedef internal::hash_map_base::size_type size_type;
00556     typedef ptrdiff_t difference_type;
00557     typedef value_type *pointer;
00558     typedef const value_type *const_pointer;
00559     typedef value_type &reference;
00560     typedef const value_type &const_reference;
00561     typedef internal::hash_map_iterator<concurrent_hash_map,value_type> iterator;
00562     typedef internal::hash_map_iterator<concurrent_hash_map,const value_type> const_iterator;
00563     typedef internal::hash_map_range<iterator> range_type;
00564     typedef internal::hash_map_range<const_iterator> const_range_type;
00565     typedef Allocator allocator_type;
00566 
00567 protected:
00568     friend class const_accessor;
00569     struct node;
00570     typedef typename Allocator::template rebind<node>::other node_allocator_type;
00571     node_allocator_type my_allocator;
00572     HashCompare my_hash_compare;
00573 
00574     struct node : public node_base {
00575         value_type item;
00576         node( const Key &key ) : item(key, T()) {}
00577         node( const Key &key, const T &t ) : item(key, t) {}
00578         // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17
00579         void *operator new( size_t /*size*/, node_allocator_type &a ) {
00580             void *ptr = a.allocate(1);
00581             if(!ptr) throw std::bad_alloc();
00582             return ptr;
00583         }
00584         // match placement-new form above to be called if exception thrown in constructor
00585         void operator delete( void *ptr, node_allocator_type &a ) {return a.deallocate(static_cast<node*>(ptr),1); }
00586     };
00587 
00588     void delete_node( node_base *n ) {
00589         my_allocator.destroy( static_cast<node*>(n) );
00590         my_allocator.deallocate( static_cast<node*>(n), 1);
00591     }
00592 
00593     node *search_bucket( const key_type &key, bucket *b ) const {
00594         node *n = static_cast<node*>( b->node_list );
00595         while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) )
00596             n = static_cast<node*>( n->next );
00597         __TBB_ASSERT(n != __TBB_rehash_req, "Search can be executed only for rehashed bucket");
00598         return n;
00599     }
00600 
00602     class bucket_accessor : public bucket::scoped_t {
00603         bool my_is_writer; // TODO: use it from base type
00604         bucket *my_b;
00605     public:
00606         bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); }
00608         inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) {
00609             my_b = base->get_bucket( h );
00610 #if TBB_USE_THREADING_TOOLS
00611             if( itt_load_pointer_with_acquire_v3(&my_b->node_list) == __TBB_rehash_req
00612 #else
00613             if( __TBB_load_with_acquire(my_b->node_list) == __TBB_rehash_req
00614 #endif
00615                 && try_acquire( my_b->mutex, /*write=*/true ) )
00616             {
00617                 if( my_b->node_list == __TBB_rehash_req ) base->rehash_bucket( my_b, h ); //recursive rehashing
00618                 my_is_writer = true;
00619             }
00620             else bucket::scoped_t::acquire( my_b->mutex, /*write=*/my_is_writer = writer );
00621             __TBB_ASSERT( my_b->node_list != __TBB_rehash_req, NULL);
00622         }
00624         bool is_writer() { return my_is_writer; }
00626         bucket *operator() () { return my_b; }
00627         // TODO: optimize out
00628         bool upgrade_to_writer() { my_is_writer = true; return bucket::scoped_t::upgrade_to_writer(); }
00629     };
00630 
00631     // TODO refactor to hash_base
00632     void rehash_bucket( bucket *b_new, const hashcode_t h ) {
00633         __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)");
00634         __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" );
00635         __TBB_store_with_release(b_new->node_list, __TBB_empty_rehashed); // mark rehashed
00636         hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit
00637 
00638         bucket_accessor b_old( this, h & mask );
00639 
00640         mask = (mask<<1) | 1; // get full mask for new bucket
00641         __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL );
00642     restart:
00643         for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) {
00644             hashcode_t c = my_hash_compare.hash( static_cast<node*>(n)->item.first );
00645             if( (c & mask) == h ) {
00646                 if( !b_old.is_writer() )
00647                     if( !b_old.upgrade_to_writer() ) {
00648                         goto restart; // node ptr can be invalid due to concurrent erase
00649                     }
00650                 *p = n->next; // exclude from b_old
00651                 add_to_bucket( b_new, n );
00652             } else p = &n->next; // iterate to next item
00653         }
00654     }
00655 
00656 public:
00657     
00658     class accessor;
00660     class const_accessor {
00661         friend class concurrent_hash_map<Key,T,HashCompare,Allocator>;
00662         friend class accessor;
00663         void operator=( const accessor & ) const; // Deny access
00664         const_accessor( const accessor & );       // Deny access
00665     public:
00667         typedef const typename concurrent_hash_map::value_type value_type;
00668 
00670         bool empty() const {return !my_node;}
00671 
00673         void release() {
00674             if( my_node ) {
00675                 my_lock.release();
00676                 my_node = 0;
00677             }
00678         }
00679 
00681         const_reference operator*() const {
00682             __TBB_ASSERT( my_node, "attempt to dereference empty accessor" );
00683             return my_node->item;
00684         }
00685 
00687         const_pointer operator->() const {
00688             return &operator*();
00689         }
00690 
00692         const_accessor() : my_node(NULL) {}
00693 
00695         ~const_accessor() {
00696             my_node = NULL; // my_lock.release() is called in scoped_lock destructor
00697         }
00698     private:
00699         node *my_node;
00700         typename node::scoped_t my_lock;
00701         hashcode_t my_hash;
00702     };
00703 
00705     class accessor: public const_accessor {
00706     public:
00708         typedef typename concurrent_hash_map::value_type value_type;
00709 
00711         reference operator*() const {
00712             __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" );
00713             return this->my_node->item;
00714         }
00715 
00717         pointer operator->() const {
00718             return &operator*();
00719         }
00720     };
00721 
00723     concurrent_hash_map(const allocator_type &a = allocator_type())
00724         : my_allocator(a)
00725     {}
00726 
00728     concurrent_hash_map( const concurrent_hash_map& table, const allocator_type &a = allocator_type())
00729         : my_allocator(a)
00730     {
00731         internal_copy(table);
00732     }
00733 
00735     template<typename I>
00736     concurrent_hash_map(I first, I last, const allocator_type &a = allocator_type())
00737         : my_allocator(a)
00738     {
00739         reserve( std::distance(first, last) ); // TODO: load_factor?
00740         internal_copy(first, last);
00741     }
00742 
00744     concurrent_hash_map& operator=( const concurrent_hash_map& table ) {
00745         if( this!=&table ) {
00746             clear();
00747             internal_copy(table);
00748         } 
00749         return *this;
00750     }
00751 
00752 
00754     void clear();
00755 
00757     ~concurrent_hash_map() { clear(); }
00758 
00759     //------------------------------------------------------------------------
00760     // Parallel algorithm support
00761     //------------------------------------------------------------------------
00762     range_type range( size_type grainsize=1 ) {
00763         return range_type( *this, grainsize );
00764     }
00765     const_range_type range( size_type grainsize=1 ) const {
00766         return const_range_type( *this, grainsize );
00767     }
00768 
00769     //------------------------------------------------------------------------
00770     // STL support - not thread-safe methods
00771     //------------------------------------------------------------------------
00772     iterator begin() {return iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00773     iterator end() {return iterator(*this,0,0,0);}
00774     const_iterator begin() const {return const_iterator(*this,0,my_embedded_segment,my_embedded_segment->node_list);}
00775     const_iterator end() const {return const_iterator(*this,0,0,0);}
00776     std::pair<iterator, iterator> equal_range( const Key& key ) { return internal_equal_range(key, end()); }
00777     std::pair<const_iterator, const_iterator> equal_range( const Key& key ) const { return internal_equal_range(key, end()); }
00778     
00780     size_type size() const { return my_size; }
00781 
00783     bool empty() const { return my_size == 0; }
00784 
00786     size_type max_size() const {return (~size_type(0))/sizeof(node);}
00787 
00789     allocator_type get_allocator() const { return this->my_allocator; }
00790 
00792     void swap(concurrent_hash_map &table);
00793 
00794     //------------------------------------------------------------------------
00795     // concurrent map operations
00796     //------------------------------------------------------------------------
00797 
00799     size_type count( const Key &key ) const {
00800         return const_cast<concurrent_hash_map*>(this)->lookup(/*insert*/false, key, NULL, NULL, /*write=*/false );
00801     }
00802 
00804 
00805     bool find( const_accessor &result, const Key &key ) const {
00806         result.release();
00807         return const_cast<concurrent_hash_map*>(this)->lookup(/*insert*/false, key, NULL, &result, /*write=*/false );
00808     }
00809 
00811 
00812     bool find( accessor &result, const Key &key ) {
00813         result.release();
00814         return lookup(/*insert*/false, key, NULL, &result, /*write=*/true );
00815     }
00816         
00818 
00819     bool insert( const_accessor &result, const Key &key ) {
00820         result.release();
00821         return lookup(/*insert*/true, key, NULL, &result, /*write=*/false );
00822     }
00823 
00825 
00826     bool insert( accessor &result, const Key &key ) {
00827         result.release();
00828         return lookup(/*insert*/true, key, NULL, &result, /*write=*/true );
00829     }
00830 
00832 
00833     bool insert( const_accessor &result, const value_type &value ) {
00834         result.release();
00835         return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/false );
00836     }
00837 
00839 
00840     bool insert( accessor &result, const value_type &value ) {
00841         result.release();
00842         return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/true );
00843     }
00844 
00846 
00847     bool insert( const value_type &value ) {
00848         return lookup(/*insert*/true, value.first, &value.second, NULL, /*write=*/false );
00849     }
00850 
00852     template<typename I>
00853     void insert(I first, I last) {
00854         for(; first != last; ++first)
00855             insert( *first );
00856     }
00857 
00859 
00860     bool erase( const Key& key );
00861 
00863 
00864     bool erase( const_accessor& item_accessor ) {
00865         return exclude( item_accessor, /*readonly=*/ true );
00866     }
00867 
00869 
00870     bool erase( accessor& item_accessor ) {
00871         return exclude( item_accessor, /*readonly=*/ false );
00872     }
00873 
00874 protected:
00876     bool lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write );
00877 
00879     bool exclude( const_accessor &item_accessor, bool readonly );
00880 
00882     template<typename I>
00883     std::pair<I, I> internal_equal_range( const Key& key, I end ) const;
00884 
00886     void internal_copy( const concurrent_hash_map& source );
00887 
00888     template<typename I>
00889     void internal_copy(I first, I last);
00890 
00892     const_pointer find( const Key& key ) const {
00893         hashcode_t h = my_hash_compare.hash( key );
00894         hashcode_t m = my_mask;
00895     restart:
00896         __TBB_ASSERT((m&(m+1))==0, NULL);
00897         bucket *b = get_bucket( h & m );
00898         if( b->node_list == __TBB_rehash_req ) {
00899             bucket::scoped_t lock;
00900             if( lock.try_acquire( b->mutex, /*write=*/true ) && b->node_list == __TBB_rehash_req )
00901                 const_cast<concurrent_hash_map*>(this)->rehash_bucket( b, h & m ); //recursive rehashing
00902             else internal::spin_wait_while_eq( b->node_list, __TBB_rehash_req ); //TODO: rework for fast find?
00903         }
00904         node *n = search_bucket( key, b );
00905         if( check_mask_race( h, m ) )
00906             goto restart;
00907         if( n )
00908             return &n->item;
00909         return 0;
00910     }
00911 };
00912 
00913 #if _MSC_VER && !defined(__INTEL_COMPILER)
00914     // Suppress "conditional expression is constant" warning.
00915     #pragma warning( push )
00916     #pragma warning( disable: 4127 )
00917 #endif
00918 
00919 template<typename Key, typename T, typename HashCompare, typename A>
00920 bool concurrent_hash_map<Key,T,HashCompare,A>::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write ) {
00921     __TBB_ASSERT( !result || !result->my_node, NULL );
00922     segment_index_t grow_segment;
00923     bool return_value;
00924     node *n, *tmp_n = 0;
00925     hashcode_t const h = my_hash_compare.hash( key );
00926 #if TBB_USE_THREADING_TOOLS
00927     hashcode_t m = (hashcode_t) itt_load_pointer_with_acquire_v3( &my_mask );
00928 #else
00929     hashcode_t m = my_mask;
00930 #endif
00931     restart:
00932     {//lock scope
00933         __TBB_ASSERT((m&(m+1))==0, NULL);
00934         return_value = false;
00935         // get bucket
00936         bucket_accessor b( this, h & m );
00937 
00938         // find a node
00939         n = search_bucket( key, b() );
00940         if( op_insert ) {
00941             // [opt] insert a key
00942             if( !is_valid(n) ) {
00943                 if( !tmp_n ) {
00944                     if(t) tmp_n = new( my_allocator ) node(key, *t);
00945                     else  tmp_n = new( my_allocator ) node(key);
00946                 }
00947                 if( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion
00948                     // Rerun search_list, in case another thread inserted the item during the upgrade.
00949                     n = search_bucket( key, b() );
00950                     if( is_valid(n) ) { // unfortunately, it did
00951                         b.downgrade_to_reader();
00952                         goto exists;
00953                     }
00954                 }
00955                 if( check_mask_race(h, m) )
00956                     goto restart; // b.release() is done in ~b().
00957                 // insert and set flag to grow the container
00958                 grow_segment = insert_new_node( b(), n = tmp_n, m );
00959                 tmp_n = 0;
00960                 return_value = true;
00961             } else {
00962     exists:     grow_segment = 0;
00963             }
00964         } else { // find or count
00965             if( !n ) {
00966                 if( check_mask_race( h, m ) )
00967                     goto restart; // b.release() is done in ~b(). TODO: replace by continue
00968                 return false;
00969             }
00970             return_value = true;
00971             grow_segment = 0;
00972         }
00973         if( !result ) goto check_growth;
00974         // TODO: the following seems as generic/regular operation
00975         // acquire the item
00976         if( !result->my_lock.try_acquire( n->mutex, write ) ) {
00977             // we are unlucky, prepare for longer wait
00978             internal::atomic_backoff trials;
00979             do {
00980                 if( !trials.bounded_pause() ) {
00981                     // the wait takes really long, restart the operation
00982                     b.release();
00983                     __TBB_Yield();
00984                     m = my_mask;
00985                     goto restart;
00986                 }
00987             } while( !result->my_lock.try_acquire( n->mutex, write ) );
00988         }
00989     }//lock scope
00990     result->my_node = n;
00991     result->my_hash = h;
00992 check_growth:
00993     // [opt] grow the container
00994     if( grow_segment )
00995         enable_segment( grow_segment );
00996     if( tmp_n ) // if op_insert only
00997         delete_node( tmp_n );
00998     return return_value;
00999 }
01000 
01001 template<typename Key, typename T, typename HashCompare, typename A>
01002 template<typename I>
01003 std::pair<I, I> concurrent_hash_map<Key,T,HashCompare,A>::internal_equal_range( const Key& key, I end ) const {
01004     hashcode_t h = my_hash_compare.hash( key );
01005     hashcode_t m = my_mask;
01006     __TBB_ASSERT((m&(m+1))==0, NULL);
01007     h &= m;
01008     bucket *b = get_bucket( h );
01009     while( b->node_list == __TBB_rehash_req ) {
01010         m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit
01011         b = get_bucket( h &= m );
01012     }
01013     node *n = search_bucket( key, b );
01014     if( !n )
01015         return std::make_pair(end, end);
01016     iterator lower(*this, h, b, n), upper(lower);
01017     return std::make_pair(lower, ++upper);
01018 }
01019 
01020 template<typename Key, typename T, typename HashCompare, typename A>
01021 bool concurrent_hash_map<Key,T,HashCompare,A>::exclude( const_accessor &item_accessor, bool readonly ) {
01022     __TBB_ASSERT( item_accessor.my_node, NULL );
01023     node_base *const n = item_accessor.my_node;
01024     item_accessor.my_node = NULL; // we ought release accessor anyway
01025     hashcode_t const h = item_accessor.my_hash;
01026     hashcode_t m = my_mask;
01027     do {
01028         // get bucket
01029         bucket_accessor b( this, h & m, /*writer=*/true );
01030         node_base **p = &b()->node_list;
01031         while( *p && *p != n )
01032             p = &(*p)->next;
01033         if( !*p ) { // someone else was the first
01034             if( check_mask_race( h, m ) )
01035                 continue;
01036             item_accessor.my_lock.release();
01037             return false;
01038         }
01039         __TBB_ASSERT( *p == n, NULL );
01040         *p = n->next; // remove from container
01041         my_size--;
01042         break;
01043     } while(true);
01044     if( readonly ) // need to get exclusive lock
01045         item_accessor.my_lock.upgrade_to_writer(); // return value means nothing here
01046     item_accessor.my_lock.release();
01047     delete_node( n ); // Only one thread can delete it due to write lock on the chain_mutex
01048     return true;
01049 }
01050 
01051 template<typename Key, typename T, typename HashCompare, typename A>
01052 bool concurrent_hash_map<Key,T,HashCompare,A>::erase( const Key &key ) {
01053     node_base *n;
01054     hashcode_t const h = my_hash_compare.hash( key );
01055     hashcode_t m = my_mask;
01056     {//lock scope
01057     restart:
01058         // get bucket
01059         bucket_accessor b( this, h & m );
01060     search:
01061         node_base **p = &b()->node_list;
01062         n = *p;
01063         while( is_valid(n) && !my_hash_compare.equal(key, static_cast<node*>(n)->item.first ) ) {
01064             p = &n->next;
01065             n = *p;
01066         }
01067         if( !n ) { // not found, but mask could be changed
01068             if( check_mask_race( h, m ) )
01069                 goto restart;
01070             return false;
01071         }
01072         else if( !b.is_writer() && !b.upgrade_to_writer() ) {
01073             if( check_mask_race( h, m ) ) // contended upgrade, check mask
01074                 goto restart;
01075             goto search;
01076         }
01077         *p = n->next;
01078         my_size--;
01079     }
01080     {
01081         typename node::scoped_t item_locker( n->mutex, /*write=*/true );
01082     }
01083     // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor!
01084     delete_node( n ); // Only one thread can delete it due to write lock on the bucket
01085     return true;
01086 }
01087 
01088 template<typename Key, typename T, typename HashCompare, typename A>
01089 void concurrent_hash_map<Key,T,HashCompare,A>::swap(concurrent_hash_map<Key,T,HashCompare,A> &table) {
01090     std::swap(this->my_allocator, table.my_allocator);
01091     std::swap(this->my_hash_compare, table.my_hash_compare);
01092     internal_swap(table);
01093 }
01094 
01095 template<typename Key, typename T, typename HashCompare, typename A>
01096 void concurrent_hash_map<Key,T,HashCompare,A>::clear() {
01097     hashcode_t m = my_mask;
01098     __TBB_ASSERT((m&(m+1))==0, NULL);
01099 #if TBB_USE_DEBUG || TBB_USE_PERFORMANCE_WARNINGS
01100 #if TBB_USE_PERFORMANCE_WARNINGS
01101     int size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics
01102     static bool reported = false;
01103 #endif
01104     // check consistency
01105     for( segment_index_t b = 0; b <= m; b++ ) {
01106         node_base *n = get_bucket(b)->node_list;
01107 #if TBB_USE_PERFORMANCE_WARNINGS
01108         if( n == __TBB_empty_rehashed ) empty_buckets++;
01109         else if( n == __TBB_rehash_req ) buckets--;
01110         else if( n->next ) overpopulated_buckets++;
01111 #endif
01112         for(; is_valid(n); n = n->next ) {
01113             hashcode_t h = my_hash_compare.hash( static_cast<node*>(n)->item.first );
01114             h &= m;
01115             __TBB_ASSERT( h == b || get_bucket(h)->node_list == __TBB_rehash_req, "Rehashing is not finished until serial stage due to concurrent or terminated operation" );
01116         }
01117     }
01118 #if TBB_USE_PERFORMANCE_WARNINGS
01119     if( buckets > size) empty_buckets -= buckets - size;
01120     else overpopulated_buckets -= size - buckets; // TODO: load_factor?
01121     if( !reported && buckets >= 512 && ( 2*empty_buckets >= size || 2*overpopulated_buckets > size ) ) {
01122         internal::runtime_warning(
01123             "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d  Empties: %d  Overlaps: %d",
01124             typeid(*this).name(), size, empty_buckets, overpopulated_buckets );
01125         reported = true;
01126     }
01127 #endif
01128 #endif//TBB_USE_DEBUG || TBB_USE_PERFORMANCE_WARNINGS
01129     my_size = 0;
01130     segment_index_t s = segment_index_of( m );
01131     __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" );
01132     cache_aligned_allocator<bucket> alloc;
01133     do {
01134         __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" );
01135         segment_ptr_t buckets = my_table[s];
01136         size_type sz = segment_size( s ? s : 1 );
01137         for( segment_index_t i = 0; i < sz; i++ )
01138             for( node_base *n = buckets[i].node_list; is_valid(n); n = buckets[i].node_list ) {
01139                 buckets[i].node_list = n->next;
01140                 delete_node( n );
01141             }
01142         if( s >= first_block) // the first segment or the next
01143             alloc.deallocate( buckets, sz );
01144         else if( s == embedded_block && embedded_block != first_block )
01145             alloc.deallocate( buckets, segment_size(first_block)-embedded_buckets );
01146         if( s >= embedded_block ) my_table[s] = 0;
01147     } while(s-- > 0);
01148     my_mask = embedded_buckets - 1;
01149 }
01150 
01151 template<typename Key, typename T, typename HashCompare, typename A>
01152 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy( const concurrent_hash_map& source ) {
01153     reserve( source.my_size ); // TODO: load_factor?
01154     if( my_mask == source.my_mask ) { // optimized version
01155         for( const_iterator it = source.begin(), end = source.end(); it != end; ++it ) {
01156             bucket *b = get_bucket( it.my_index );
01157             __TBB_ASSERT( b->node_list != __TBB_rehash_req, "Invalid bucket in destination table");
01158             node *n = new( my_allocator ) node(it->first, it->second);
01159             add_to_bucket( b, n );
01160             ++my_size; // TODO: replace by non-atomic op
01161         }
01162     } else internal_copy( source.begin(), source.end() );
01163 }
01164 
01165 template<typename Key, typename T, typename HashCompare, typename A>
01166 template<typename I>
01167 void concurrent_hash_map<Key,T,HashCompare,A>::internal_copy(I first, I last) {
01168     hashcode_t m = my_mask;
01169     for(; first != last; ++first) {
01170         hashcode_t h = my_hash_compare.hash( first->first );
01171         bucket *b = get_bucket( h & m );
01172         __TBB_ASSERT( b->node_list != __TBB_rehash_req, "Invalid bucket in destination table");
01173         node *n = new( my_allocator ) node(first->first, first->second);
01174         add_to_bucket( b, n );
01175         ++my_size; // TODO: replace by non-atomic op
01176     }
01177 }
01178 
01179 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01180 inline bool operator==(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b) {
01181     if(a.size() != b.size()) return false;
01182     typename concurrent_hash_map<Key, T, HashCompare, A1>::const_iterator i(a.begin()), i_end(a.end());
01183     typename concurrent_hash_map<Key, T, HashCompare, A2>::const_iterator j, j_end(b.end());
01184     for(; i != i_end; ++i) {
01185         j = b.equal_range(i->first).first;
01186         if( j == j_end || !(i->second == j->second) ) return false;
01187     }
01188     return true;
01189 }
01190 
01191 template<typename Key, typename T, typename HashCompare, typename A1, typename A2>
01192 inline bool operator!=(const concurrent_hash_map<Key, T, HashCompare, A1> &a, const concurrent_hash_map<Key, T, HashCompare, A2> &b)
01193 {    return !(a == b); }
01194 
01195 template<typename Key, typename T, typename HashCompare, typename A>
01196 inline void swap(concurrent_hash_map<Key, T, HashCompare, A> &a, concurrent_hash_map<Key, T, HashCompare, A> &b)
01197 {    a.swap( b ); }
01198 
01199 #if _MSC_VER && !defined(__INTEL_COMPILER)
01200     #pragma warning( pop )
01201 #endif // warning 4127 is back
01202 
01203 } // namespace tbb
01204 
01205 #endif /* __TBB_concurrent_hash_map_H */

Copyright © 2005-2009 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.