0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV2.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
22 #include "Common/Compat.h"
23 #include <cassert>
24 
25 #include <boost/algorithm/string.hpp>
26 #include <boost/scoped_array.hpp>
27 
28 #include "Common/Config.h"
29 #include "Common/Error.h"
30 #include "Common/Logger.h"
31 #include "Common/System.h"
32 #include "Common/Time.h"
33 
34 #include "AsyncComm/Protocol.h"
35 
38 #include "Hypertable/Lib/Key.h"
39 #include "Hypertable/Lib/Schema.h"
40 
41 #include "CellStoreV2.h"
42 #include "CellStoreTrailerV2.h"
43 #include "CellStoreScanner.h"
44 
45 #include "FileBlockCache.h"
46 #include "Global.h"
47 #include "Config.h"
48 
49 using namespace std;
50 using namespace Hypertable;
51 
52 namespace {
53  const uint32_t MAX_APPENDS_OUTSTANDING = 3;
54  const uint16_t BLOCK_HEADER_FORMAT = 0;
55 }
56 
57 
58 CellStoreV2::CellStoreV2(Filesystem *filesys)
59  : m_filesys(filesys), m_fd(-1), m_filename(), m_64bit_index(false),
60  m_compressor(0), m_buffer(0), m_outstanding_appends(0), m_offset(0),
61  m_last_key(0), m_file_length(0), m_disk_usage(0), m_file_id(0),
62  m_uncompressed_blocksize(0), m_bloom_filter_mode(BLOOM_FILTER_DISABLED),
63  m_bloom_filter(0), m_bloom_filter_items(0),
64  m_filter_false_positive_prob(0.0), m_restricted_range(false) {
66  assert(sizeof(float) == 4);
67 }
68 
69 
71  try {
72  delete m_compressor;
73  delete m_bloom_filter;
74  delete m_bloom_filter_items;
75  if (m_fd != -1)
77  }
78  catch (Exception &e) {
79  HT_ERROR_OUT << e << HT_END;
80  }
81 
84 
85 }
86 
87 
91 }
92 
93 
95  bool need_index = m_restricted_range || scan_ctx->restricted_range || scan_ctx->single_row;
96 
97  if (need_index) {
101  }
102 
103  if (m_64bit_index)
104  return make_shared<CellStoreScanner<CellStoreBlockIndexArray<int64_t>>>(shared_from_this(), scan_ctx, need_index ? &m_index_map64 : 0);
105  return make_shared<CellStoreScanner<CellStoreBlockIndexArray<uint32_t>>>(shared_from_this(), scan_ctx, need_index ? &m_index_map32 : 0);
106 }
107 
108 
109 void
110 CellStoreV2::create(const char *fname, size_t max_entries,
111  PropertiesPtr &props, const TableIdentifier *table_id) {
112  int32_t replication = props->get_i32("replication", int32_t(-1));
113  int64_t blocksize = props->get("blocksize", 0);
114  String compressor = props->get("compressor", String());
115 
116  assert(Config::properties); // requires Config::init* first
117 
118  if (replication == -1 && Config::has("Hypertable.RangeServer.CellStore.DefaultReplication"))
119  replication = Config::get_i32("Hypertable.RangeServer.CellStore.DefaultReplication");
120 
121  if (blocksize == 0)
122  blocksize = Config::get_i32("Hypertable.RangeServer.CellStore"
123  ".DefaultBlockSize");
124  if (compressor.empty())
125  compressor = Config::get_str("Hypertable.RangeServer.CellStore"
126  ".DefaultCompressor");
127  if (!props->has("bloom-filter-mode")) {
128  // probably not called from AccessGroup
129  AccessGroupOptions::parse_bloom_filter(Config::get_str("Hypertable.RangeServer"
130  ".CellStore.DefaultBloomFilter"), props);
131  }
132 
133  m_buffer.reserve(blocksize*4);
134 
135  m_max_entries = max_entries;
136 
137  m_fd = -1;
138  m_offset = 0;
139  m_last_key = 0;
140 
142  m_index_builder.variable_buf().reserve(1024*1024);
143 
144  m_uncompressed_data = 0.0;
145  m_compressed_data = 0.0;
146 
147  m_trailer.clear();
148  m_trailer.blocksize = blocksize;
149  m_uncompressed_blocksize = blocksize;
150 
151  m_filename = fname;
152 
153  m_start_row = "";
155 
157  compressor, m_compressor_args);
158 
162 
164 
165  m_bloom_filter_mode = props->get<BloomFilterMode>("bloom-filter-mode");
166  m_max_approx_items = props->get_i32("max-approx-items");
167 
169  bool has_num_hashes = props->has("num-hashes");
170  bool has_bits_per_item = props->has("bits-per-item");
171 
172  if (has_num_hashes || has_bits_per_item) {
173  if (!(has_num_hashes && has_bits_per_item)) {
174  HT_WARN("Bloom filter option --bits-per-item must be used with "
175  "--num-hashes, defaulting to false probability of 0.01");
177  }
178  else {
179  m_trailer.bloom_filter_hash_count = props->get_i32("num-hashes");
180  m_bloom_bits_per_item = props->get_f64("bits-per-item");
181  }
182  }
183  else
184  m_filter_false_positive_prob = props->get_f64("false-positive");
185  m_bloom_filter_items = new BloomFilterItems(); // aproximator items
186  }
187  HT_DEBUG_OUT <<"bloom-filter-mode="<< m_bloom_filter_mode
188  <<" max-approx-items="<< m_max_approx_items <<" false-positive="
190 }
191 
192 
193 void CellStoreV2::create_bloom_filter(bool is_approx) {
195 
196  HT_DEBUG_OUT << "Creating new BloomFilter for CellStore '"
197  << m_filename <<"' for "<< (is_approx ? "estimated " : "")
198  << m_trailer.filter_items_estimate << " items"<< HT_END;
199  try {
200  if (m_filter_false_positive_prob != 0.0)
203  else
207  }
208  catch(Exception &e) {
209  HT_FATAL_OUT << "Error creating new BloomFilter for CellStore '"
210  << m_filename <<"' for "<< (is_approx ? "estimated " : "")
211  << m_trailer.filter_items_estimate << " items - "<< e << HT_END;
212  }
213 
214  for (const auto &blob : *m_bloom_filter_items)
215  m_bloom_filter->insert(blob.start, blob.size);
216 
217  delete m_bloom_filter_items;
218  m_bloom_filter_items = 0;
219 
220  HT_DEBUG_OUT << "Created new BloomFilter for CellStore '"
221  << m_filename <<"'"<< HT_END;
222 }
223 
224 
225 void CellStoreV2::load_bloom_filter() {
226  size_t len, amount;
227 
229 
230  HT_DEBUG_OUT << "Loading BloomFilter for CellStore '"
232  << " items"<< HT_END;
233  try {
238  }
239  catch(Exception &e) {
240  HT_FATAL_OUT << "Error loading BloomFilter for CellStore '"
242  << " items -"<< e << HT_END;
243  }
244 
246 
247  HT_ASSERT(amount == m_bloom_filter->size());
248 
249  if (amount > 0) {
250 
251  bool second_try = false;
252 
253  while (true) {
254  try {
255  len = m_filesys->pread(m_fd, m_bloom_filter->ptr(), amount,
256  m_trailer.filter_offset, second_try);
257  }
258  catch (Exception &e) {
259  if (!second_try) {
260  second_try=true;
261  continue;
262  }
263  HT_THROW2(e.code(), e, format("Error loading BloomFilter for CellStore '%s'",
264  m_filename.c_str()));
265  }
266  break;
267  }
268 
269  if (len != amount)
270  HT_THROWF(Error::FSBROKER_IO_ERROR, "Problem loading bloomfilter for"
271  "CellStore '%s' : tried to read %lld but only got %lld",
272  m_filename.c_str(), (Lld)amount, (Lld)len);
273  }
274 
277 
278 }
279 
280 
281 
283  uint64_t memory_purged = 0;
284 
286  memory_purged = m_index_stats.bloom_filter_memory;
287  delete m_bloom_filter;
288  m_bloom_filter = 0;
291  }
292 
294  memory_purged += m_index_stats.block_index_memory;
295  if (m_64bit_index)
297  else
301  }
302 
303  return memory_purged;
304 }
305 
306 
307 
308 void CellStoreV2::add(const Key &key, const ByteString value) {
309  EventPtr event_ptr;
310  DynamicBuffer zbuf;
311 
312  if (key.revision > m_trailer.revision)
314 
315  if (key.timestamp != TIMESTAMP_NULL) {
318  else if (key.timestamp > m_trailer.timestamp_max)
320  }
321 
322  if (m_buffer.fill() > (size_t)m_uncompressed_blocksize) {
323  BlockHeaderCellStore header(BLOCK_HEADER_FORMAT, DATA_BLOCK_MAGIC);
324 
326 
327  m_uncompressed_data += (float)m_buffer.fill();
328  m_compressor->deflate(m_buffer, zbuf, header);
329  m_compressed_data += (float)zbuf.fill();
330  m_buffer.clear();
331 
332  uint64_t llval = ((uint64_t)m_trailer.blocksize
333  * (uint64_t)m_uncompressed_data) / (uint64_t)m_compressed_data;
334  m_uncompressed_blocksize = (int64_t)llval;
335 
336  if (m_outstanding_appends >= MAX_APPENDS_OUTSTANDING) {
337  if (!m_sync_handler.wait_for_reply(event_ptr)) {
338  if (event_ptr->type == Event::MESSAGE)
340  "Problem writing to FS file '%s' : %s", m_filename.c_str(),
342  HT_THROWF(event_ptr->error,
343  "Problem writing to FS file '%s'", m_filename.c_str());
344  }
346  }
347 
348  size_t zlen = zbuf.fill();
349  StaticBuffer send_buf(zbuf);
350 
352  catch (Exception &e) {
353  HT_THROW2F(e.code(), e, "Problem writing to FS file '%s'",
354  m_filename.c_str());
355  }
357  m_offset += zlen;
358  }
359 
360  size_t value_len = value.length();
361 
362  m_buffer.ensure(key.length + value_len);
363 
365  m_buffer.add_unchecked(value.ptr, value_len);
366 
369  m_bloom_filter_items->insert(key.row, key.row_len);
370 
372  m_bloom_filter_items->insert(key.row, key.row_len + 2);
373 
375  m_trailer.filter_items_estimate = (size_t)(((double)m_max_entries
376  / (double)m_max_approx_items) * m_bloom_filter_items->size());
377  if (m_trailer.filter_items_estimate == 0) {
378  HT_INFOF("max_entries = %lld, max_approx_items = %lld, bloom_filter_items_size = %lld",
379  (Lld)m_max_entries, (Lld)m_max_approx_items, (Lld)m_bloom_filter_items->size());
381  }
382  create_bloom_filter(true);
383  }
384  }
385  else {
386  assert(!m_bloom_filter_items && m_bloom_filter);
387 
388  m_bloom_filter->insert(key.row);
389 
391  m_bloom_filter->insert(key.row, key.row_len + 2);
392  }
393  }
394 
396 }
397 
398 
399 void CellStoreV2::finalize(TableIdentifier *table_identifier) {
400  EventPtr event_ptr;
401  size_t zlen;
402  DynamicBuffer zbuf(0);
403  SerializedKey key;
404  StaticBuffer send_buf;
405  int64_t index_memory = 0;
406 
407  if (m_buffer.fill() > 0) {
408  BlockHeaderCellStore header(BLOCK_HEADER_FORMAT, DATA_BLOCK_MAGIC);
409 
411 
412  m_uncompressed_data += (float)m_buffer.fill();
413  m_compressor->deflate(m_buffer, zbuf, header);
414  m_compressed_data += (float)zbuf.fill();
415 
416  zlen = zbuf.fill();
417  send_buf = zbuf;
418 
419  if (m_outstanding_appends >= MAX_APPENDS_OUTSTANDING) {
420  if (!m_sync_handler.wait_for_reply(event_ptr))
422  "Problem finalizing CellStore file '%s' : %s",
423  m_filename.c_str(),
424  Protocol::string_format_message(event_ptr).c_str());
426  }
427 
429 
431  m_offset += zlen;
432  }
433 
434  m_buffer.free();
435 
437  if (m_uncompressed_data == 0)
439  else
441 
446 
450  {
451  BlockHeaderCellStore header(BLOCK_HEADER_FORMAT, INDEX_FIXED_BLOCK_MAGIC);
452  m_compressor->deflate(m_index_builder.fixed_buf(), zbuf, header);
453  }
454 
455  zlen = zbuf.fill();
456  send_buf = zbuf;
457 
459 
461  m_offset += zlen;
462 
466  {
467  BlockHeaderCellStore header(BLOCK_HEADER_FORMAT, INDEX_VARIABLE_BLOCK_MAGIC);
470  }
471 
472  zlen = zbuf.fill();
473  send_buf = zbuf;
474 
476 
478  m_offset += zlen;
479 
480  // write filter_offset
482 
483  // if bloom_items haven't been spilled to create a bloom filter yet, do it
486 
487  if (m_bloom_filter_items && m_bloom_filter_items->size() > 0) {
488  m_trailer.filter_items_estimate = m_bloom_filter_items->size();
490  }
491 
492  if (m_bloom_filter) {
497  m_bloom_filter->serialize(send_buf);
501  }
502  }
503 
505 
507  if (m_64bit_index) {
511  index_memory = m_index_map64.memory_used();
513  }
514  else {
518  index_memory = m_index_map32.memory_used();
519  }
520 
521  // deallocate fix index data
523 
524  // Add table information
525  m_trailer.table_id = table_identifier->index();
526  m_trailer.table_generation = table_identifier->generation;
528 
529  // write trailer
530  zbuf.clear();
531  zbuf.reserve(m_trailer.size());
532  m_trailer.serialize(zbuf.ptr);
533  zbuf.ptr += m_trailer.size();
534 
535  zlen = zbuf.fill();
536  send_buf = zbuf;
537 
538  m_filesys->append(m_fd, send_buf);
539 
541  m_offset += zlen;
542 
544  m_filesys->close(m_fd);
545 
548 
550  m_fd = m_filesys->open(m_filename, 0);
551 
553  if (m_disk_usage < 0)
554  HT_WARN_OUT << "[Issue 339] Disk usage for " << m_filename << "=" << m_disk_usage
555  << HT_END;
556  m_index_stats.block_index_memory = sizeof(CellStoreV2) + index_memory;
557 
558  if (m_bloom_filter)
560 
562 }
563 
564 
566  int64_t offset) {
567 
568  // switch to 64-bit offsets if offset being added is >= 2^32
569  if (!m_bigint && offset >= 4294967296LL) {
570  DynamicBuffer tmp_buf(m_fixed.size*2);
571  const uint8_t *src = m_fixed.base;
572  uint8_t *dst = tmp_buf.base;
573  size_t remaining = m_fixed.fill();
574  while (src < m_fixed.ptr)
575  Serialization::encode_i64(&dst, (uint64_t)Serialization::decode_i32(&src, &remaining));
576  delete [] m_fixed.release();
577  m_fixed.base = tmp_buf.base;
578  m_fixed.ptr = dst;
579  m_fixed.size = tmp_buf.size;
580  m_fixed.own = true;
581  tmp_buf.release();
582  m_bigint = true;
583  }
584 
585  // Add key to variable buffer
586  size_t key_len = key.length();
587  m_variable.ensure(key_len);
588  memcpy(m_variable.ptr, key.ptr, key_len);
589  m_variable.ptr += key_len;
590 
591  // Serialize offset into fix index buffer
592  if (m_bigint) {
593  m_fixed.ensure(8);
594  memcpy(m_fixed.ptr, &offset, 8);
595  m_fixed.ptr += 8;
596  }
597  else {
598  m_fixed.ensure(4);
599  memcpy(m_fixed.ptr, &offset, 4);
600  m_fixed.ptr += 4;
601  }
602 }
603 
604 
606  uint8_t *base;
607  size_t len;
608 
609  base = m_fixed.release(&len);
610  m_fixed.reserve(len);
611  m_fixed.add_unchecked(base, len);
612  delete [] base;
613 
614  base = m_variable.release(&len);
615  m_variable.reserve(len);
616  m_variable.add_unchecked(base, len);
617  delete [] base;
618 }
619 
620 
621 
622 void
623 CellStoreV2::open(const String &fname, const String &start_row,
624  const String &end_row, int32_t fd, int64_t file_length,
625  CellStoreTrailer *trailer) {
626  m_filename = fname;
627  m_start_row = start_row;
628  m_end_row = end_row;
629  m_fd = fd;
630  m_file_length = file_length;
631 
633 
634  m_trailer = *static_cast<CellStoreTrailerV2 *>(trailer);
635 
637 
640 
642  m_64bit_index = true;
643 
647  "Bad index offsets in CellStore trailer fix=%lld, var=%lld, "
648  "length=%llu, file='%s'", (Lld)m_trailer.fix_index_offset,
649  (Lld)m_trailer.var_index_offset, (Llu)m_file_length, fname.c_str());
650 
651 }
652 
653 
655  int64_t amount, index_amount;
656  int64_t len = 0;
657  BlockHeaderCellStore header(BLOCK_HEADER_FORMAT);
658  SerializedKey key;
659  bool inflating_fixed=true;
660  bool second_try = false;
661 
663 
664  if (m_compressor == 0)
666 
667  amount = index_amount = m_trailer.filter_offset - m_trailer.fix_index_offset;
668 
669  try_again:
670 
671  try {
672  DynamicBuffer buf(amount);
673 
675  len = m_filesys->pread(m_fd, buf.ptr, amount, m_trailer.fix_index_offset, second_try);
676 
677  if (len != amount)
678  HT_THROWF(Error::FSBROKER_IO_ERROR, "Error loading index for "
679  "CellStore '%s' : tried to read %lld but only got %lld",
680  m_filename.c_str(), (Lld)amount, (Lld)len);
684 
685  inflating_fixed = false;
686 
689 
691  DynamicBuffer vbuf(0, false);
693  vbuf.base = buf.ptr;
694  vbuf.ptr = buf.ptr + amount;
695 
697 
700  }
701  catch (Exception &e) {
702  String msg;
703  if (inflating_fixed) {
704  msg = String("Error inflating FIXED index for cellstore '")
705  + m_filename + "'";
706  HT_ERROR_OUT << msg << ": "<< e << HT_END;
707  }
708  else {
709  msg = "Error inflating VARIABLE index for cellstore '" + m_filename + "'";
710  HT_ERROR_OUT << msg << ": " << e << HT_END;
711  }
712  HT_ERROR_OUT << "pread(fd=" << m_fd << ", len=" << len << ", amount="
713  << index_amount << ")\n" << HT_END;
715  if (second_try)
716  HT_THROW2(e.code(), e, msg);
717  second_try = true;
718  goto try_again;
719  }
720 
722  if (m_64bit_index) {
726  }
727  else {
731  }
732 
734  if (m_disk_usage < 0)
735  HT_WARN_OUT << "[Issue 339] Disk usage for " << m_filename << "=" << m_disk_usage
736  << HT_END;
737 
740 
742 
743 }
744 
745 
747 
749  return true;
750  else if (m_trailer.filter_length == 0) // bloom filter is empty
751  return false;
752  else if (m_bloom_filter == 0)
754 
756 
757  switch (m_bloom_filter_mode) {
758  case BLOOM_FILTER_ROWS:
759  return may_contain(scan_ctx->start_row);
761  if (may_contain(scan_ctx->start_row)) {
762  SchemaPtr &schema = scan_ctx->schema;
763  size_t rowlen = scan_ctx->start_row.length();
764  boost::scoped_array<char> rowcol(new char[rowlen + 2]);
765  memcpy(rowcol.get(), scan_ctx->start_row.c_str(), rowlen + 1);
766 
767  for (auto col : scan_ctx->spec->columns) {
768  uint8_t column_family_id = schema->get_column_family(col)->get_id();
769  rowcol[rowlen + 1] = column_family_id;
770 
771  if (may_contain(rowcol.get(), rowlen + 2))
772  return true;
773  }
774  }
775  return false;
776  default:
777  HT_ASSERT(!"unpossible bloom filter mode!");
778  }
779  return false; // silence stupid compilers
780 }
781 
782 
783 bool CellStoreV2::may_contain(const void *ptr, size_t len) {
784 
786  return true;
787  else if (m_trailer.filter_length == 0) // bloom filter is empty
788  return false;
789  else if (m_bloom_filter == 0)
791 
793  bool may_contain = m_bloom_filter->may_contain(ptr, len);
794  return may_contain;
795 }
796 
797 
798 
802  if (m_64bit_index)
804  else
806 }
807 
809  return BLOCK_HEADER_FORMAT;
810 }
void free()
Frees resources.
size_t get_num_hashes()
Getter for the number of hash functions.
Definition: BloomFilter.h:238
static const char INDEX_FIXED_BLOCK_MAGIC[10]
Definition: CellStore.h:326
#define HT_THROW2F(_code_, _ex_, _fmt_,...)
Definition: Error.h:494
A memory buffer of static size.
Definition: StaticBuffer.h:45
Retrieves system information (hardware, installation directory, etc)
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV2.cc:282
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV2.cc:623
virtual void append(int fd, StaticBuffer &buffer, Flags flags, DispatchHandler *handler)=0
Appends data to a file asynchronously.
Abstract base class for cell store trailer.
CellStoreBlockIndexArray< uint32_t > m_index_map32
Definition: CellStoreV2.h:138
DynamicBuffer m_buffer
Definition: CellStoreV2.h:143
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
int64_t timestamp
Definition: Key.h:134
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV2.cc:193
const char * row
Definition: Key.h:129
virtual void pread(int fd, size_t amount, uint64_t offset, bool verify_checksum, DispatchHandler *handler)=0
Reads data from a file at the specified position asynchronously.
static const char INDEX_VARIABLE_BLOCK_MAGIC[10]
Definition: CellStore.h:327
size_t get_items_actual()
Getter for the actual number of items.
Definition: BloomFilter.h:256
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV2.cc:94
static int32_t response_code(const Event *event)
Returns the response code from an event event generated in response to a request message.
Definition: Protocol.cc:39
PropertiesPtr properties
This singleton map stores all options.
Definition: Config.cc:47
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
uint32_t length
Definition: Key.h:124
bool check_magic(const char *magic)
Compares a given character sequence with the magic field.
Definition: BlockHeader.h:86
void display_block_info() override
Displays block information to stdout.
Definition: CellStoreV2.cc:799
static String string_format_message(const Event *event)
Returns error message decoded standard error MESSAGE generated in response to a request message...
Definition: Protocol.cc:51
CellStoreBlockIndexArray< int64_t > m_index_map64
Definition: CellStoreV2.h:139
String format(const char *fmt,...)
Returns a String using printf like format facilities Vanilla snprintf is about 1.5x faster than this...
Definition: String.cc:37
size_t get_length_bits()
Getter for the number of bits.
Definition: BloomFilter.h:244
std::string m_start_row
Definition: CellList.h:99
virtual size_t size()
Returns the serialized size of the trailer.
long long unsigned int Llu
Shortcut for printf formats.
Definition: String.h:50
CellStoreV2(Filesystem *filesys)
Definition: CellStoreV2.cc:58
Declarations for CellStoreScanner.
static void parse_bloom_filter(const std::string &spec, PropertiesPtr &props)
Parsers a bloom filter specification and sets properties.
std::shared_ptr< Event > EventPtr
Smart pointer to Event.
Definition: Event.h:228
BlockCompressionCodec * m_compressor
Definition: CellStoreV2.h:142
STL namespace.
Scan context information.
Definition: ScanContext.h:52
uint8_t * ptr()
Getter for the bloom filter data.
Definition: BloomFilter.h:226
std::string m_end_row
Definition: CellList.h:100
int64_t m_uncompressed_blocksize
Definition: CellStoreV2.h:154
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV2.h:155
Type
Enumeration for compression type.
uint8_t * ptr
Pointer to the end of the used part of the buffer.
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV2.cc:399
bool wait_for_reply(EventPtr &event)
This method is used by a client to synchronize.
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
uint32_t m_outstanding_appends
Definition: CellStoreV2.h:146
Declarations for Schema.
uint32_t decode_i32(const uint8_t **bufp, size_t *remainp)
Decode a 32-bit integer in little-endian order.
bool has(const String &name)
Check existence of a configuration value.
Definition: Config.h:57
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
#define HT_ASSERT(_e_)
Definition: Logger.h:396
uint32_t row_len
Definition: Key.h:131
bool may_contain(const void *ptr, size_t len)
Definition: CellStoreV2.cc:783
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV2.h:145
BloomFilter * m_bloom_filter
Definition: CellStoreV2.h:159
IndexMemoryStats m_index_stats
Definition: CellStore.h:332
static BlockCompressionCodec::Type parse_block_codec_spec(const std::string &spec, BlockCompressionCodec::Args &args)
Given a block codec config string return its type and put config.
static Hypertable::MemoryTracker * memory_tracker
Definition: Global.h:94
const ScanSpec * spec
Definition: ScanContext.h:55
size_t size()
Getter for the bloom filter size.
Definition: BloomFilter.h:232
static uint64_t access_counter
Definition: Global.h:106
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
uint32_t size
The size of the allocated memory buffer (base)
Logging routines and macros.
BloomFilterMode
Enumeration for bloom filter modes.
Compatibility Macros for C/C++.
BasicBloomFilter BloomFilter
Definition: BloomFilter.h:284
void encode_i64(uint8_t **bufp, uint64_t val)
Encode a 64-bit integer in little-endian order.
#define HT_END
Definition: Logger.h:220
static const int64_t TIMESTAMP_NULL
Definition: KeySpec.h:36
size_t length() const
Retrieves the length of the serialized string.
Definition: ByteString.h:62
static BlockCompressionCodec * create_block_codec(BlockCompressionCodec::Type, const BlockCompressionCodec::Args &args=BlockCompressionCodec::Args())
#define HT_ERROR_OUT
Definition: Logger.h:301
IndexBuilder m_index_builder
Definition: CellStoreV2.h:144
Time related declarations.
bool may_contain(const void *key, size_t len) const
Checks if the data set "may" contain the key.
Definition: BloomFilter.h:188
#define HT_WARN_OUT
Definition: Logger.h:291
bool own
If true then the buffer (base) will be released when going out of scope; if false then the caller has...
const uint8_t * ptr
The pointer to the serialized data.
Definition: ByteString.h:121
Hypertable definitions
SerializedKey serial
Definition: Key.h:123
virtual void clear()
Clears the contents of this trailer;.
long long int Lld
Shortcut for printf formats.
Definition: String.h:53
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV2.h:158
void load(DynamicBuffer &fixed, DynamicBuffer &variable, int64_t end_of_data, const String &start_row="", const String &end_row="")
void clear()
Clears the buffer.
Declarations for BlockHeaderCellStore.
uint16_t block_header_format() override
Definition: CellStoreV2.cc:808
Declarations for Protocol.
#define HT_INFOF(msg,...)
Definition: Logger.h:272
#define HT_THROWF(_code_, _fmt_,...)
Definition: Error.h:490
Provides access to internal components of opaque key.
Definition: Key.h:40
uint8_t * base
Pointer to the allocated memory buffer.
size_t fill() const
Returns the size of the used portion.
Definition: DynamicBuffer.h:70
virtual void deflate(const DynamicBuffer &input, DynamicBuffer &output, BlockHeader &header, size_t reserve=0)=0
Compresses a buffer.
static const char DATA_BLOCK_MAGIC[10]
Definition: CellStore.h:325
Request/response message event.
Definition: Event.h:63
void subtract(int64_t amount)
Subtract to memory used.
Definition: MemoryTracker.h:60
void serialize(StaticBuffer &buf)
Serializes the BloomFilter into a static memory buffer.
Definition: BloomFilter.h:218
This is a generic exception class for Hypertable.
Definition: Error.h:314
int64_t revision
Definition: Key.h:135
BlobHashSet BloomFilterItems
Definition: CellStoreV2.h:133
uint8_t * release(size_t *lenp=0)
Moves ownership of the buffer to the caller.
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
void insert(const void *key, size_t len)
Inserts a new blob into the hash.
Definition: BloomFilter.h:159
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
Configuration settings.
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV2.h:160
#define HT_WARN(msg)
Definition: Logger.h:289
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_identifier=0) override
Creates a new cell store.
Definition: CellStoreV2.cc:110
virtual void inflate(const DynamicBuffer &input, DynamicBuffer &output, BlockHeader &header)=0
Decompresses a buffer.
Error codes, Exception handling, error logging.
#define HT_THROW(_code_, _msg_)
Definition: Error.h:478
static const char * END_ROW_MARKER
Definition: Key.h:49
#define HT_FATAL_OUT
Definition: Logger.h:347
void add(int64_t amount)
Add to memory used.
Definition: MemoryTracker.h:53
void ensure(size_t len)
Ensure space for additional data Will grow the space to 1.5 of the needed space with existing data un...
Definition: DynamicBuffer.h:82
Filesystem * m_filesys
Definition: CellStoreV2.h:135
virtual void serialize(uint8_t *buf)
Serializes this trailer to the given buffer;.
uint8_t * add_unchecked(const void *data, size_t len)
Adds additional data without boundary checks.
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV2.cc:88
virtual void create(const String &name, uint32_t flags, int32_t bufsz, int32_t replication, int64_t blksz, DispatchHandler *handler)=0
Creates a file asynchronously.
#define HT_DEBUG_OUT
Definition: Logger.h:261
int64_t get_ts64()
Returns the current time in nanoseconds as a 64bit number.
Definition: Time.cc:40
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
void add_entry(const SerializedKey key, int64_t offset)
Definition: CellStoreV2.cc:565
int code() const
Returns the error code.
Definition: Error.h:391
void reserve(size_t len, bool nocopy=false)
Reserve space for additional data Will grow the space to exactly what's needed.
Definition: DynamicBuffer.h:95
#define HT_THROW2(_code_, _ex_, _msg_)
Definition: Error.h:484
CellStoreTrailerV2 m_trailer
Definition: CellStoreV2.h:141
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV2.cc:308