0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV2.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
22 #ifndef Hypertable_RangeServer_CellStoreV2_h
23 #define Hypertable_RangeServer_CellStoreV2_h
24 
25 #include <map>
26 #include <string>
27 #include <vector>
28 
30 
32 #include "Common/DynamicBuffer.h"
33 #include "Common/BloomFilter.h"
34 #include "Common/BlobHashSet.h"
35 
38 
39 #include "CellStore.h"
40 #include "CellStoreTrailerV2.h"
41 
42 
46 namespace Hypertable {
48  class Client;
49  class Protocol;
50 }
51 
52 namespace Hypertable {
53 
54  class CellStoreV2 : public CellStore {
55 
56  class IndexBuilder {
57  public:
58  IndexBuilder() : m_bigint(false) { }
59  void add_entry(const SerializedKey key, int64_t offset);
62  bool big_int() { return m_bigint; }
63  void chop();
64  void release_fixed_buf() { delete [] m_fixed.release(); }
65  private:
68  bool m_bigint;
69  };
70 
71  public:
72  CellStoreV2(Filesystem *filesys);
73  virtual ~CellStoreV2();
74 
75  void create(const char *fname, size_t max_entries,
76  PropertiesPtr &props,
77  const TableIdentifier *table_identifier=0) override;
78  void add(const Key &key, const ByteString value) override;
79  void finalize(TableIdentifier *table_identifier) override;
80  void open(const String &fname, const String &start_row,
81  const String &end_row, int32_t fd, int64_t file_length,
82  CellStoreTrailer *trailer) override;
83  int64_t get_blocksize() override { return m_trailer.blocksize; }
84  bool may_contain(const void *ptr, size_t len);
85  bool may_contain(const String &key) {
86  return may_contain(key.data(), key.size());
87  }
88 
89  bool may_contain(ScanContext *scan_ctx) override;
90 
91  uint64_t disk_usage() override {
92  if (m_disk_usage < 0)
93  HT_WARN_OUT << "[Issue 339] Disk usage for " << m_filename << "=" << m_disk_usage
94  << HT_END;
95  return m_disk_usage;
96  }
97  float compression_ratio() override { return m_trailer.compression_ratio; }
98  int64_t get_total_entries() override { return m_trailer.total_entries; }
99  std::string &get_filename() override { return m_filename; }
100  int get_file_id() override { return m_file_id; }
101  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
103  void display_block_info() override;
104  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
105  size_t bloom_filter_size() override { return m_bloom_filter ? m_bloom_filter->size() : 0; }
108  uint64_t purge_indexes() override;
109  bool restricted_range() override { return m_restricted_range; }
110 
111  int32_t get_fd() override {
112  std::lock_guard<std::mutex> lock(m_mutex);
113  return m_fd;
114  }
115 
116  int32_t reopen_fd() override {
117  std::lock_guard<std::mutex> lock(m_mutex);
118  if (m_fd != -1)
119  m_filesys->close(m_fd);
120  m_fd = m_filesys->open(m_filename, 0);
121  return m_fd;
122  }
123 
124  CellStoreTrailer *get_trailer() override { return &m_trailer; }
125 
126  uint16_t block_header_format() override;
127 
128  protected:
129  void create_bloom_filter(bool is_approx = false);
130  void load_bloom_filter();
131  void load_block_index();
132 
134 
136  int32_t m_fd;
137  std::string m_filename;
147  int64_t m_offset;
149  int64_t m_file_length;
150  int64_t m_disk_usage;
157 
160  BloomFilterItems *m_bloom_filter_items;
165  };
166 
167 }
168 
169 #endif // Hypertable_RangeServer_CellStoreV2_h
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV2.cc:282
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV2.cc:623
CellStoreBlockIndexArray< uint32_t > m_index_map32
Definition: CellStoreV2.h:138
Abstract base class for cell store trailer.
DynamicBuffer m_buffer
Definition: CellStoreV2.h:143
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV2.cc:193
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV2.h:106
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV2.h:99
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV2.h:107
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV2.cc:94
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
void display_block_info() override
Displays block information to stdout.
Definition: CellStoreV2.cc:799
CellStoreBlockIndexArray< int64_t > m_index_map64
Definition: CellStoreV2.h:139
A Bloom Filter implementation.
CellStoreV2(Filesystem *filesys)
Definition: CellStoreV2.cc:58
BlockCompressionCodec * m_compressor
Definition: CellStoreV2.h:142
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV2.h:104
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV2.h:105
Scan context information.
Definition: ScanContext.h:52
Declarations for BlockCompressionCodec.
int64_t m_uncompressed_blocksize
Definition: CellStoreV2.h:154
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV2.h:155
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV2.cc:399
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
uint32_t m_outstanding_appends
Definition: CellStoreV2.h:146
std::vector< String > Args
Compression codec argument vector.
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
Declarations for CellStore.
bool may_contain(const void *ptr, size_t len)
Definition: CellStoreV2.cc:783
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV2.h:145
BloomFilter * m_bloom_filter
Definition: CellStoreV2.h:159
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV2.h:83
A dynamic, resizable memory buffer.
size_t size()
Getter for the bloom filter size.
Definition: BloomFilter.h:232
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV2.h:116
BloomFilterMode
Enumeration for bloom filter modes.
std::mutex m_mutex
Definition: CellStore.h:331
#define HT_END
Definition: Logger.h:220
bool may_contain(const String &key)
Definition: CellStoreV2.h:85
IndexBuilder m_index_builder
Definition: CellStoreV2.h:144
A HashSet optimized for blobs.
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV2.h:109
#define HT_WARN_OUT
Definition: Logger.h:291
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV2.h:100
Hypertable definitions
DispatchHandler class used to synchronize with response messages.
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV2.h:158
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV2.h:124
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
uint16_t block_header_format() override
Definition: CellStoreV2.cc:808
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
Provides access to internal components of opaque key.
Definition: Key.h:40
BlobHashSet BloomFilterItems
Definition: CellStoreV2.h:133
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV2.h:91
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV2.h:160
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_identifier=0) override
Creates a new cell store.
Definition: CellStoreV2.cc:110
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
Definition: BloomFilter.h:50
Filesystem * m_filesys
Definition: CellStoreV2.h:135
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV2.h:97
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV2.cc:88
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV2.h:98
Declarations for DispatchHandlerSynchronizer.
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV2.h:111
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
void add_entry(const SerializedKey key, int64_t offset)
Definition: CellStoreV2.cc:565
Declarations for CellStoreBlockIndexArray.
CellStoreTrailerV2 m_trailer
Definition: CellStoreV2.h:141
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV2.cc:308