0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV5.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
22 #ifndef Hypertable_RangeServer_CellStoreV5_h
23 #define Hypertable_RangeServer_CellStoreV5_h
24 
25 #include <map>
26 #include <string>
27 #include <vector>
28 
30 
32 #include "Common/DynamicBuffer.h"
34 #include "Common/BlobHashSet.h"
35 
38 
39 #include "CellStore.h"
40 #include "CellStoreTrailerV5.h"
41 #include "KeyCompressor.h"
42 
43 
47 namespace Hypertable {
49  class Client;
50  class Protocol;
51 }
52 
53 namespace Hypertable {
54 
55  class CellStoreV5 : public CellStore {
56 
57  class IndexBuilder {
58  public:
59  IndexBuilder() : m_bigint(false) { }
60  void add_entry(KeyCompressorPtr &key_compressor, int64_t offset);
63  bool big_int() { return m_bigint; }
64  void chop();
65  void release_fixed_buf() { delete [] m_fixed.release(); }
66  private:
69  bool m_bigint;
70  };
71 
72  public:
73  CellStoreV5(Filesystem *filesys);
74  CellStoreV5(Filesystem *filesys, SchemaPtr &schema);
75  virtual ~CellStoreV5();
76 
77  void create(const char *fname, size_t max_entries,
78  PropertiesPtr &props,
79  const TableIdentifier *table_id=0) override;
80  void add(const Key &key, const ByteString value) override;
81  void finalize(TableIdentifier *table_identifier) override;
82  void open(const String &fname, const String &start_row,
83  const String &end_row, int32_t fd, int64_t file_length,
84  CellStoreTrailer *trailer) override;
85  int64_t get_blocksize() override { return m_trailer.blocksize; }
86  bool may_contain(const void *ptr, size_t len);
87  bool may_contain(const String &key) {
88  return may_contain(key.data(), key.size());
89  }
90  bool may_contain(ScanContext *scan_ctx) override;
91  uint64_t disk_usage() override { return m_disk_usage; }
92  float compression_ratio() override { return m_trailer.compression_ratio; }
93  void split_row_estimate_data(SplitRowDataMapT &split_row_data) override;
94  int64_t get_total_entries() override { return m_trailer.total_entries; }
95  std::string &get_filename() override { return m_filename; }
96  int get_file_id() override { return m_file_id; }
97  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
100  void display_block_info() override;
101  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
102  size_t bloom_filter_size() override { return m_bloom_filter ? m_bloom_filter->size() : 0; }
105  uint64_t purge_indexes() override;
106  bool restricted_range() override { return m_restricted_range; }
107  const std::vector<String> &get_replaced_files() override;
108 
109  int32_t get_fd() override {
110  std::lock_guard<std::mutex> lock(m_mutex);
111  return m_fd;
112  }
113 
114  int32_t reopen_fd() override {
115  std::lock_guard<std::mutex> lock(m_mutex);
116  if (m_fd != -1)
117  m_filesys->close(m_fd);
118  m_fd = m_filesys->open(m_filename, 0);
119  return m_fd;
120  }
121 
122  CellStoreTrailer *get_trailer() override { return &m_trailer; }
123 
124  uint16_t block_header_format() override;
125 
126  protected:
127  void create_bloom_filter(bool is_approx = false);
128  void load_bloom_filter();
129  void load_block_index();
130  void load_replaced_files();
131 
133 
136  int32_t m_fd {-1};
137  std::string m_filename;
140  bool m_64bit_index {};
147  int64_t m_offset {};
148  int64_t m_file_length {};
149  int64_t m_disk_usage {};
150  int m_file_id {};
155  size_t m_max_entries {};
158  BloomFilterItems *m_bloom_filter_items {};
159  int64_t m_max_approx_items {};
164  int64_t *m_column_ttl {};
166  };
167 
168 }
169 
170 #endif // Hypertable_RangeServer_CellStoreV5_h
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV5.h:114
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV5.cc:244
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV5.h:91
Abstract base class for cell store trailer.
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV5.cc:411
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV5.cc:93
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
void add_entry(KeyCompressorPtr &key_compressor, int64_t offset)
Definition: CellStoreV5.cc:768
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV5.h:106
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
CellStoreV5(Filesystem *filesys)
Definition: CellStoreV5.cc:63
std::shared_ptr< KeyCompressor > KeyCompressorPtr
Definition: KeyCompressor.h:45
DynamicBuffer m_buffer
Definition: CellStoreV5.h:143
bool may_contain(const void *ptr, size_t len)
Definition: CellStoreV5.cc:992
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_id=0) override
Creates a new cell store.
Definition: CellStoreV5.cc:149
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV5.cc:525
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV5.h:145
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV5.h:92
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV5.h:95
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV5.h:109
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV5.h:158
Scan context information.
Definition: ScanContext.h:52
Declarations for BlockCompressionCodec.
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
Filesystem * m_filesys
Definition: CellStoreV5.h:134
std::vector< String > Args
Compression codec argument vector.
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV5.h:96
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
CellStoreTrailerV5 m_trailer
Definition: CellStoreV5.h:141
Declarations for CellStore.
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV5.h:122
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
void display_block_info() override
Displays block information to stdout.
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV5.h:85
std::map< const char *, int64_t, LtCstr, SplitRowDataAlloc > SplitRowDataMapT
Definition: CellList.h:66
A dynamic, resizable memory buffer.
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV5.h:156
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV5.h:154
uint32_t m_outstanding_appends
Definition: CellStoreV5.h:146
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
KeyCompressorPtr m_key_compressor
Definition: CellStoreV5.h:162
BloomFilterMode
Enumeration for bloom filter modes.
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV5.cc:116
std::mutex m_mutex
Definition: CellStore.h:331
uint16_t block_header_format() override
CellStoreBlockIndexArray< uint32_t > m_index_map32
Definition: CellStoreV5.h:138
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV5.cc:826
A HashSet optimized for blobs.
size_t size()
Getter for the bloom filter size.
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV5.h:101
BlobHashSet BloomFilterItems
Definition: CellStoreV5.h:132
Hypertable definitions
void split_row_estimate_data(SplitRowDataMapT &split_row_data) override
Populates split_row_data with unique row and count estimates from block index.
Definition: CellStoreV5.cc:102
KeyDecompressor * create_key_decompressor() override
Creates a key decompressor suitable for decompressing the keys stored in this cell store...
Definition: CellStoreV5.cc:98
DispatchHandler class used to synchronize with response messages.
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
BloomFilterWithChecksum * m_bloom_filter
Definition: CellStoreV5.h:157
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
BlockCompressionCodec * m_compressor
Definition: CellStoreV5.h:142
Provides access to internal components of opaque key.
Definition: Key.h:40
const std::vector< String > & get_replaced_files() override
Returns all the cell store files replaced by this CellStore.
Definition: CellStoreV5.cc:275
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV5.h:102
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV5.cc:385
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
IndexBuilder m_index_builder
Definition: CellStoreV5.h:144
CellStoreBlockIndexArray< int64_t > m_index_map64
Definition: CellStoreV5.h:139
A Bloom Filter with Checksums.
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV5.h:104
int64_t m_uncompressed_blocksize
Definition: CellStoreV5.h:153
bool may_contain(const String &key)
Definition: CellStoreV5.h:87
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV5.h:94
Declarations for DispatchHandlerSynchronizer.
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
Declarations for CellStoreBlockIndexArray.
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV5.h:103