0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV7.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
28 #ifndef Hypertable_RangeServer_CellStoreV7_h
29 #define Hypertable_RangeServer_CellStoreV7_h
30 
31 #include "CellStore.h"
33 #include "CellStoreTrailerV7.h"
34 #include "KeyCompressor.h"
35 
38 
40 
41 #include <Common/BlobHashSet.h>
43 #include <Common/DynamicBuffer.h>
44 
45 #include <map>
46 #include <string>
47 #include <vector>
48 
49 namespace Hypertable {
51  class Client;
52  class Protocol;
53 }
54 
55 namespace Hypertable {
56 
61  class CellStoreV7 : public CellStore {
62 
63  class IndexBuilder {
64  public:
65  IndexBuilder() : m_bigint(false) { }
66  void add_entry(KeyCompressorPtr &key_compressor, int64_t offset);
69  bool big_int() { return m_bigint; }
70  void chop();
71  void release_fixed_buf() { delete [] m_fixed.release(); }
72  private:
75  bool m_bigint;
76  };
77 
78  public:
79  CellStoreV7(Filesystem *filesys);
80  CellStoreV7(Filesystem *filesys, SchemaPtr &schema);
81  virtual ~CellStoreV7();
82 
83  void create(const char *fname, size_t max_entries, PropertiesPtr &props,
84  const TableIdentifier *table_id=0) override;
85  void add(const Key &key, const ByteString value) override;
86  void finalize(TableIdentifier *table_identifier) override;
87  void open(const String &fname, const String &start_row,
88  const String &end_row, int32_t fd, int64_t file_length,
89  CellStoreTrailer *trailer) override;
90  void rescope(const String &start_row, const String &end_row) override;
91  int64_t get_blocksize() override { return m_trailer.blocksize; }
92  bool may_contain(ScanContext *scan_ctx) override;
93  uint64_t disk_usage() override { return m_disk_usage; }
94  float compression_ratio() override { return m_trailer.compression_ratio; }
95  void split_row_estimate_data(SplitRowDataMapT &split_row_data) override;
96 
107 
108  int64_t get_total_entries() override { return m_trailer.total_entries; }
109  std::string &get_filename() override { return m_filename; }
110  int get_file_id() override { return m_file_id; }
111  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
114  void display_block_info() override;
115  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
116 
117  size_t bloom_filter_size() override {
118  std::lock_guard<std::mutex> lock(m_mutex);
119  return m_bloom_filter ? m_bloom_filter->size() : 0;
120  }
121 
122  int64_t bloom_filter_memory_used() override {
123  std::lock_guard<std::mutex> lock(m_mutex);
125  }
126 
127  int64_t block_index_memory_used() override {
128  std::lock_guard<std::mutex> lock(m_mutex);
130  }
131 
132  uint64_t purge_indexes() override;
133  bool restricted_range() override { return m_restricted_range; }
134  const std::vector<String> &get_replaced_files() override;
135 
136  int32_t get_fd() override {
137  std::lock_guard<std::mutex> lock(m_mutex);
138  return m_fd;
139  }
140 
141  int32_t reopen_fd() override {
142  std::lock_guard<std::mutex> lock(m_mutex);
143  if (m_fd != -1)
144  m_filesys->close(m_fd);
145  m_fd = m_filesys->open(m_filename, 0);
146  return m_fd;
147  }
148 
149  CellStoreTrailer *get_trailer() override { return &m_trailer; }
150 
151  uint16_t block_header_format() override;
152 
153  protected:
154  void create_bloom_filter(bool is_approx = false);
155  void load_bloom_filter();
156  void load_block_index();
157  void load_replaced_files();
158 
160 
163  int32_t m_fd {-1};
164  std::string m_filename;
165  bool m_64bit_index {};
172  int64_t m_offset {};
173  int64_t m_file_length {};
174  int64_t m_disk_usage {};
175  int m_file_id {};
180  size_t m_max_entries {};
182  BloomFilterItems *m_bloom_filter_items {};
183  int64_t m_max_approx_items {};
188  int64_t *m_column_ttl {};
190 
191  // Member that require mutex protection
192 
195 
198 
201  };
202 
205 } // namespace Hypertable
206 
207 #endif // Hypertable_RangeServer_CellStoreV7_h
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV7.h:108
Abstract base class for cell store trailer.
uint32_t m_outstanding_appends
Definition: CellStoreV7.h:171
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
Cell list scanner over a buffer of cells.
CellStoreTrailerV7 m_trailer
Definition: CellStoreV7.h:166
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV7.h:141
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
CellStoreV7(Filesystem *filesys)
Definition: CellStoreV7.cc:69
void display_block_info() override
Displays block information to stdout.
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV7.h:110
std::shared_ptr< KeyCompressor > KeyCompressorPtr
Definition: KeyCompressor.h:45
uint16_t block_header_format() override
DynamicBuffer m_buffer
Definition: CellStoreV7.h:168
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV7.h:91
BlockCompressionCodec * m_compressor
Definition: CellStoreV7.h:167
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV7.h:115
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV7.h:181
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV7.h:136
BlobHashSet BloomFilterItems
Definition: CellStoreV7.h:159
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV7.cc:99
Scan context information.
Definition: ScanContext.h:52
const std::vector< String > & get_replaced_files() override
Returns all the cell store files replaced by this CellStore.
Definition: CellStoreV7.cc:305
Declarations for BlockCompressionCodec.
KeyDecompressor * create_key_decompressor() override
Creates a key decompressor suitable for decompressing the keys stored in this cell store...
Definition: CellStoreV7.cc:104
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
std::vector< String > Args
Compression codec argument vector.
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
Declarations for CellStore.
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
CellStoreBlockIndexArray< uint32_t > m_index_map32
32-bit block index
Definition: CellStoreV7.h:197
Filesystem * m_filesys
Definition: CellStoreV7.h:161
std::map< const char *, int64_t, LtCstr, SplitRowDataAlloc > SplitRowDataMapT
Definition: CellList.h:66
A dynamic, resizable memory buffer.
int64_t m_uncompressed_blocksize
Definition: CellStoreV7.h:178
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV7.h:127
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_id=0) override
Creates a new cell store.
Definition: CellStoreV7.cc:179
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV7.h:170
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
BloomFilterMode
Enumeration for bloom filter modes.
bool may_contain(ScanContext *scan_ctx) override
Bloom filter lookup.
std::mutex m_mutex
Definition: CellStore.h:331
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV7.h:117
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV7.h:109
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV7.h:133
CellStoreBlockIndexArray< int64_t > m_index_map64
64-bit block index
Definition: CellStoreV7.h:200
A HashSet optimized for blobs.
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV7.cc:557
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV7.h:182
size_t size()
Getter for the bloom filter size.
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV7.cc:446
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV7.h:94
Hypertable definitions
DispatchHandler class used to synchronize with response messages.
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
Represents the trailer for CellStore version 7.
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV7.cc:416
Provides access to internal components of opaque key.
Definition: Key.h:40
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV7.cc:864
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV7.h:179
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV7.h:149
void split_row_estimate_data(SplitRowDataMapT &split_row_data) override
Populates split_row_data with unique row and count estimates from block index.
Definition: CellStoreV7.cc:108
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV7.cc:274
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV7.h:122
Declarations for CellStoreTrailerV7.
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
KeyCompressorPtr m_key_compressor
Definition: CellStoreV7.h:186
void populate_index_pseudo_table_scanner(CellListScannerBuffer *scanner) override
Populates scanner with key/value pairs generated from CellStore index.
Definition: CellStoreV7.cc:123
A Bloom Filter with Checksums.
IndexBuilder m_index_builder
Definition: CellStoreV7.h:169
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV7.h:93
BloomFilterWithChecksum * m_bloom_filter
Bloom filter.
Definition: CellStoreV7.h:194
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV7.cc:143
Declarations for DispatchHandlerSynchronizer.
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
Declarations for CellStoreBlockIndexArray.
void add_entry(KeyCompressorPtr &key_compressor, int64_t offset)
Definition: CellStoreV7.cc:806
void rescope(const String &start_row, const String &end_row) override
Definition: CellStoreV7.cc:902