0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV6.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
28 #ifndef Hypertable_RangeServer_CellStoreV6_h
29 #define Hypertable_RangeServer_CellStoreV6_h
30 
31 #include <map>
32 #include <string>
33 #include <vector>
34 
36 
38 #include "Common/DynamicBuffer.h"
40 #include "Common/BlobHashSet.h"
41 
44 
45 #include "CellStore.h"
46 #include "CellStoreTrailerV6.h"
47 #include "KeyCompressor.h"
48 
49 
53 namespace Hypertable {
55  class Client;
56  class Protocol;
57 }
58 
59 namespace Hypertable {
60 
65  class CellStoreV6 : public CellStore {
66 
67  class IndexBuilder {
68  public:
69  IndexBuilder() : m_bigint(false) { }
70  void add_entry(KeyCompressorPtr &key_compressor, int64_t offset);
73  bool big_int() { return m_bigint; }
74  void chop();
75  void release_fixed_buf() { delete [] m_fixed.release(); }
76  private:
79  bool m_bigint;
80  };
81 
82  public:
83  CellStoreV6(Filesystem *filesys);
84  CellStoreV6(Filesystem *filesys, SchemaPtr &schema);
85  virtual ~CellStoreV6();
86 
87  void create(const char *fname, size_t max_entries,
88  PropertiesPtr &props,
89  const TableIdentifier *table_id=0) override;
90  void add(const Key &key, const ByteString value) override;
91  void finalize(TableIdentifier *table_identifier) override;
92  void open(const String &fname, const String &start_row,
93  const String &end_row, int32_t fd, int64_t file_length,
94  CellStoreTrailer *trailer) override;
95  void rescope(const String &start_row, const String &end_row) override;
96  int64_t get_blocksize() override { return m_trailer.blocksize; }
97  bool may_contain(ScanContext *scan_ctx) override;
98  uint64_t disk_usage() override { return m_disk_usage; }
99  float compression_ratio() override { return m_trailer.compression_ratio; }
100  void split_row_estimate_data(SplitRowDataMapT &split_row_data) override;
101 
112 
113  int64_t get_total_entries() override { return m_trailer.total_entries; }
114  std::string &get_filename() override { return m_filename; }
115  int get_file_id() override { return m_file_id; }
116  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
119  void display_block_info() override;
120  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
121 
122  size_t bloom_filter_size() override {
123  std::lock_guard<std::mutex> lock(m_mutex);
124  return m_bloom_filter ? m_bloom_filter->size() : 0;
125  }
126 
127  int64_t bloom_filter_memory_used() override {
128  std::lock_guard<std::mutex> lock(m_mutex);
130  }
131 
132  int64_t block_index_memory_used() override {
133  std::lock_guard<std::mutex> lock(m_mutex);
135  }
136 
137  uint64_t purge_indexes() override;
138  bool restricted_range() override { return m_restricted_range; }
139  const std::vector<String> &get_replaced_files() override;
140 
141  int32_t get_fd() override {
142  std::lock_guard<std::mutex> lock(m_mutex);
143  return m_fd;
144  }
145 
146  int32_t reopen_fd() override {
147  std::lock_guard<std::mutex> lock(m_mutex);
148  if (m_fd != -1)
149  m_filesys->close(m_fd);
150  m_fd = m_filesys->open(m_filename, 0);
151  return m_fd;
152  }
153 
154  CellStoreTrailer *get_trailer() override { return &m_trailer; }
155 
156  uint16_t block_header_format() override;
157 
158  protected:
159  void create_bloom_filter(bool is_approx = false);
160  void load_bloom_filter();
161  void load_block_index();
162  void load_replaced_files();
163 
165 
168  int32_t m_fd {-1};
169  std::string m_filename;
170  bool m_64bit_index {};
177  int64_t m_offset {};
178  int64_t m_file_length {};
179  int64_t m_disk_usage {};
180  int m_file_id {};
185  size_t m_max_entries {};
187  BloomFilterItems *m_bloom_filter_items {};
188  int64_t m_max_approx_items {};
193  int64_t *m_column_ttl {};
195 
196  // Member that require mutex protection
197 
200 
203 
206  };
207 
210 }
211 
212 #endif // Hypertable_RangeServer_CellStoreV6_h
int64_t m_uncompressed_blocksize
Definition: CellStoreV6.h:183
Abstract base class for cell store trailer.
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV6.h:96
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV6.cc:446
Cell list scanner over a buffer of cells.
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV6.h:114
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV6.cc:143
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV6.h:138
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV6.h:154
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
BloomFilterWithChecksum * m_bloom_filter
Bloom filter.
Definition: CellStoreV6.h:199
const std::vector< String > & get_replaced_files() override
Returns all the cell store files replaced by this CellStore.
Definition: CellStoreV6.cc:305
CellStoreV6(Filesystem *filesys)
Definition: CellStoreV6.cc:69
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV6.h:99
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV6.cc:557
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
std::shared_ptr< KeyCompressor > KeyCompressorPtr
Definition: KeyCompressor.h:45
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV6.h:184
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV6.h:186
Scan context information.
Definition: ScanContext.h:52
KeyCompressorPtr m_key_compressor
Definition: CellStoreV6.h:191
Declarations for BlockCompressionCodec.
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
std::vector< String > Args
Compression codec argument vector.
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV6.cc:274
Declarations for CellStore.
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV6.h:127
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV6.h:113
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV6.h:141
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV6.h:175
std::map< const char *, int64_t, LtCstr, SplitRowDataAlloc > SplitRowDataMapT
Definition: CellList.h:66
A dynamic, resizable memory buffer.
CellStoreBlockIndexArray< uint32_t > m_index_map32
32-bit block index
Definition: CellStoreV6.h:202
IndexBuilder m_index_builder
Definition: CellStoreV6.h:174
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
BloomFilterMode
Enumeration for bloom filter modes.
uint32_t m_outstanding_appends
Definition: CellStoreV6.h:176
std::mutex m_mutex
Definition: CellStore.h:331
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV6.h:187
BlobHashSet BloomFilterItems
Definition: CellStoreV6.h:164
void populate_index_pseudo_table_scanner(CellListScannerBuffer *scanner) override
Populates scanner with key/value pairs generated from CellStore index.
Definition: CellStoreV6.cc:123
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV6.h:132
void display_block_info() override
Displays block information to stdout.
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV6.h:98
A HashSet optimized for blobs.
CellStoreTrailerV6 m_trailer
Definition: CellStoreV6.h:171
size_t size()
Getter for the bloom filter size.
void rescope(const String &start_row, const String &end_row) override
Definition: CellStoreV6.cc:900
Hypertable definitions
Filesystem * m_filesys
Definition: CellStoreV6.h:166
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV6.cc:99
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV6.h:122
DispatchHandler class used to synchronize with response messages.
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV6.cc:862
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
bool may_contain(ScanContext *scan_ctx) override
Bloom filter lookup.
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV6.h:146
BlockCompressionCodec * m_compressor
Definition: CellStoreV6.h:172
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
Provides access to internal components of opaque key.
Definition: Key.h:40
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV6.h:120
DynamicBuffer m_buffer
Definition: CellStoreV6.h:173
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV6.h:115
CellStoreBlockIndexArray< int64_t > m_index_map64
64-bit block index
Definition: CellStoreV6.h:205
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
A Bloom Filter with Checksums.
void split_row_estimate_data(SplitRowDataMapT &split_row_data) override
Populates split_row_data with unique row and count estimates from block index.
Definition: CellStoreV6.cc:108
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_id=0) override
Creates a new cell store.
Definition: CellStoreV6.cc:179
void add_entry(KeyCompressorPtr &key_compressor, int64_t offset)
Definition: CellStoreV6.cc:804
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV6.cc:416
KeyDecompressor * create_key_decompressor() override
Creates a key decompressor suitable for decompressing the keys stored in this cell store...
Definition: CellStoreV6.cc:104
uint16_t block_header_format() override
Declarations for DispatchHandlerSynchronizer.
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
Declarations for CellStoreBlockIndexArray.