0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV1.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
22 #ifndef Hypertable_RangeServer_CellStoreV1_h
23 #define Hypertable_RangeServer_CellStoreV1_h
24 
25 #include <map>
26 #include <string>
27 #include <vector>
28 
30 
32 #include "Common/DynamicBuffer.h"
33 #include "Common/BloomFilter.h"
34 #include "Common/BlobHashSet.h"
35 
38 
39 #include "CellStore.h"
40 #include "CellStoreTrailerV1.h"
41 
42 
46 namespace Hypertable {
48  class Client;
49  class Protocol;
50 }
51 
52 namespace Hypertable {
53 
54  class CellStoreV1 : public CellStore {
55 
56  class IndexBuilder {
57  public:
58  IndexBuilder() : m_bigint(false) { }
59  void add_entry(const SerializedKey key, int64_t offset);
62  bool big_int() { return m_bigint; }
63  void chop();
64  void release_fixed_buf() { delete [] m_fixed.release(); }
65  private:
68  bool m_bigint;
69  };
70 
71  public:
72  CellStoreV1(Filesystem *filesys);
73  virtual ~CellStoreV1();
74 
75  void create(const char *fname, size_t max_entries, PropertiesPtr &props,
76  const TableIdentifier *table_id=0) override;
77  void add(const Key &key, const ByteString value) override;
78  void finalize(TableIdentifier *table_identifier) override;
79  void open(const String &fname, const String &start_row,
80  const String &end_row, int32_t fd, int64_t file_length,
81  CellStoreTrailer *trailer) override;
82  int64_t get_blocksize() override { return m_trailer.blocksize; }
83  bool may_contain(const void *ptr, size_t len);
84  bool may_contain(const String &key) {
85  return may_contain(key.data(), key.size());
86  }
87 
88  bool may_contain(ScanContext *scan_ctx) override;
89 
90  uint64_t disk_usage() override {
91  if (m_disk_usage < 0)
92  HT_WARN_OUT << "[Issue 339] Disk usage for " << m_filename << "=" << m_disk_usage
93  << HT_END;
94  return m_disk_usage;
95  }
96  float compression_ratio() override { return m_trailer.compression_ratio; }
97  int64_t get_total_entries() override { return m_trailer.total_entries; }
98  std::string &get_filename() override { return m_filename; }
99  int get_file_id() override { return m_file_id; }
100  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
102  void display_block_info() override;
103  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
104  size_t bloom_filter_size() override { return m_bloom_filter ? m_bloom_filter->size() : 0; }
107  uint64_t purge_indexes() override;
108  bool restricted_range() override { return m_restricted_range; }
109 
110  int32_t get_fd() override {
111  std::lock_guard<std::mutex> lock(m_mutex);
112  return m_fd;
113  }
114 
115  int32_t reopen_fd() override {
116  std::lock_guard<std::mutex> lock(m_mutex);
117  if (m_fd != -1)
118  m_filesys->close(m_fd);
119  m_fd = m_filesys->open(m_filename, 0);
120  return m_fd;
121  }
122 
123  CellStoreTrailer *get_trailer() override { return &m_trailer; }
124 
125  uint16_t block_header_format() override;
126 
127  protected:
128  void create_bloom_filter(bool is_approx = false);
129  void load_bloom_filter();
130  void load_block_index();
131 
133 
135  int32_t m_fd;
136  std::string m_filename;
146  int64_t m_offset;
148  int64_t m_file_length;
149  int64_t m_disk_usage;
156 
159  BloomFilterItems *m_bloom_filter_items;
162  };
163 
164 }
165 
166 #endif // Hypertable_RangeServer_CellStoreV1_h
Abstract base class for cell store trailer.
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
void display_block_info() override
Displays block information to stdout.
Definition: CellStoreV1.cc:768
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV1.cc:93
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV1.cc:281
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV1.h:105
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV1.h:99
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
IndexBuilder m_index_builder
Definition: CellStoreV1.h:143
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV1.h:108
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV1.h:97
A Bloom Filter implementation.
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV1.h:154
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_id=0) override
Creates a new cell store.
Definition: CellStoreV1.cc:109
Scan context information.
Definition: ScanContext.h:52
Declarations for BlockCompressionCodec.
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV1.cc:591
void add_entry(const SerializedKey key, int64_t offset)
Definition: CellStoreV1.cc:533
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV1.h:110
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV1.cc:255
std::vector< String > Args
Compression codec argument vector.
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV1.h:123
uint32_t m_outstanding_appends
Definition: CellStoreV1.h:145
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV1.h:96
Declarations for CellStore.
BlockCompressionCodec * m_compressor
Definition: CellStoreV1.h:141
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV1.h:106
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
CellStoreTrailerV1 m_trailer
Definition: CellStoreV1.h:140
bool may_contain(const void *ptr, size_t len)
Definition: CellStoreV1.cc:752
A dynamic, resizable memory buffer.
CellStoreBlockIndexArray< int64_t > m_index_map64
Definition: CellStoreV1.h:138
size_t size()
Getter for the bloom filter size.
Definition: BloomFilter.h:232
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
BloomFilterMode
Enumeration for bloom filter modes.
std::mutex m_mutex
Definition: CellStore.h:331
int64_t m_uncompressed_blocksize
Definition: CellStoreV1.h:153
#define HT_END
Definition: Logger.h:220
bool may_contain(const String &key)
Definition: CellStoreV1.h:84
A HashSet optimized for blobs.
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV1.h:90
#define HT_WARN_OUT
Definition: Logger.h:291
CellStoreV1(Filesystem *filesys)
Definition: CellStoreV1.cc:58
Hypertable definitions
DispatchHandler class used to synchronize with response messages.
BlobHashSet BloomFilterItems
Definition: CellStoreV1.h:132
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV1.h:157
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
BloomFilter * m_bloom_filter
Definition: CellStoreV1.h:158
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV1.h:82
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
Provides access to internal components of opaque key.
Definition: Key.h:40
uint16_t block_header_format() override
Definition: CellStoreV1.cc:777
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV1.h:115
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV1.cc:173
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV1.h:103
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV1.h:159
DynamicBuffer m_buffer
Definition: CellStoreV1.h:142
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV1.h:104
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
Filesystem * m_filesys
Definition: CellStoreV1.h:134
CellStoreBlockIndexArray< uint32_t > m_index_map32
Definition: CellStoreV1.h:137
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV1.h:98
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
Definition: BloomFilter.h:50
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV1.cc:87
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV1.cc:372
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV1.h:144
Declarations for DispatchHandlerSynchronizer.
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
Declarations for CellStoreBlockIndexArray.