0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
LoadDataSource.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 #include <Common/Compat.h>
22 #include "LoadDataSource.h"
23 
24 #include <Common/DynamicBuffer.h>
25 #include <Common/Error.h>
26 #include <Common/FileUtils.h>
27 #include <Common/Logger.h>
28 #include <Common/System.h>
29 #include <Common/Time.h>
30 
31 #include <Hypertable/Lib/Key.h>
33 
34 #include <boost/algorithm/string.hpp>
35 #include <boost/algorithm/string/predicate.hpp>
36 #include <boost/iostreams/filter/gzip.hpp>
37 #include <boost/shared_array.hpp>
38 #include <boost/spirit/include/classic_core.hpp>
39 #include <boost/spirit/include/classic_assign_actor.hpp>
40 
41 #include <cctype>
42 #include <cerrno>
43 #include <cstdlib>
44 #include <cstring>
45 #include <ctime>
46 #include <fstream>
47 #include <iostream>
48 
49 extern "C" {
50 #include <strings.h>
51 #include <sys/types.h>
52 #include <unistd.h>
53 }
54 
55 using namespace boost::iostreams;
56 using namespace boost::spirit::classic;
57 using namespace Hypertable;
58 using namespace std;
59 
60 
61 LoadDataSource::LoadDataSource(const string &header_fname,
62  int row_uniquify_chars,
63  int load_flags)
64  : m_type_mask(0), m_cur_line(0), m_line_buffer(0),
65  m_row_key_buffer(0), m_hyperformat(false), m_leading_timestamps(false),
66  m_timestamp_index(-1), m_timestamp(AUTO_ASSIGN), m_offset(0),
67  m_zipped(false), m_rsgen(0), m_header_fname(header_fname),
68  m_row_uniquify_chars(row_uniquify_chars),
69  m_load_flags(load_flags), m_source_size(0), m_first_line_cached(false),
70  m_field_separator('\t')
71 {
72  if (row_uniquify_chars)
73  m_rsgen = new FixedRandomStringGenerator(row_uniquify_chars);
74 
75  // Verify existence of header file
76  if (m_header_fname != "") {
78  HT_THROWF(Error::FILE_NOT_FOUND, "Unable to open header file '%s'",
79  m_header_fname.c_str());
80  }
81 
82  return;
83 }
84 
86  string three_column_header = format("#row%ccolumn%cvalue",
88  string four_column_header = format("#timestamp%crow%ccolumn%cvalue",
90  string header = "";
91  if (m_header_fname != "") {
92  std::ifstream in(m_header_fname.c_str());
93  getline(in, header);
94  }
96  // force autodetect
97  size_t tabs = 0;
98  getline(m_fin, m_first_line);
99  for (const char *ptr = m_first_line.c_str(); *ptr; ptr++) {
100  if (*ptr == m_field_separator)
101  tabs++;
102  }
103  if (tabs == 2) {
104  if (m_first_line.compare(three_column_header))
105  m_first_line_cached = true;
106  header = three_column_header;
107  }
108  else if (tabs == 3) {
109  if (m_first_line.compare(four_column_header))
110  m_first_line_cached = true;
111  header = four_column_header;
112  }
113  else
115  "Untable to autodetect format, expected 2 or 3 field separators, got %d", (int)tabs);
116  }
117  else {
118  // autodetect
119  getline(m_fin, m_first_line);
120  if (m_first_line[0] == '#')
121  header = m_first_line;
122  else {
123  size_t tabs = 0;
124  for (const char *ptr = m_first_line.c_str(); *ptr; ptr++) {
125  if (*ptr == m_field_separator)
126  tabs++;
127  }
128  if (tabs == 2)
129  header = "#row\tcolumn\tvalue";
130  else if (tabs == 3)
131  header = "#timestamp\trow\tcolumn\tvalue";
132  else
134  "Untable to autodetect format, expected 2 or 3 tabs, "
135  "got %d", (int)tabs);
136  m_first_line_cached = true;
137  }
138  }
139 
140  return header;
141 }
142 
143 void
144 LoadDataSource::init(const std::vector<String> &key_columns,
145  const string &timestamp_column,
146  char field_separator) {
147  string header;
148  m_field_separator = field_separator;
149  init_src();
150  header = get_header();
151  parse_header(header, key_columns, timestamp_column);
152 }
153 
154 void
155 LoadDataSource::parse_header(const string &header,
156  const std::vector<String> &key_columns,
157  const string &timestamp_column)
158 {
159  string line, column_name;
160  char *base, *ptr, *colon_ptr;
161  int index = 0;
162  KeyComponentInfo key_comps;
163  ColumnInfo cinfo;
164 
165  // Assuming max column size of 256
166  m_type_mask = new uint32_t [257];
167  memset(m_type_mask, 0, 257*sizeof(uint32_t));
168 
169  base = (char *)header.c_str();
170  if (*base == '#') {
171  base++;
172  while (isspace(*base))
173  base++;
174  }
175 
176  ptr = strchr(base, m_field_separator);
177 
178  while (base) {
179 
180  if (ptr)
181  *ptr++ = 0;
182 
183  colon_ptr = strchr(base, ':');
184  if (colon_ptr) {
185  cinfo.family = String(base, colon_ptr - base);
186  cinfo.qualifier = colon_ptr+1;
187  }
188  else {
189  cinfo.family = base;
190  cinfo.qualifier = "";
191  }
192 
193  m_column_info.push_back(cinfo);
194 
195  if (timestamp_column != "" && timestamp_column == cinfo.family) {
196  m_timestamp_index = index;
197  m_type_mask[index] |= TIMESTAMP;
198  }
199 
200  index++;
201  HT_EXPECT(index < 256, Error::TOO_MANY_COLUMNS);
202 
203  if (ptr) {
204  base = ptr;
205  ptr = strchr(base, m_field_separator);
206  }
207  else
208  base = 0;
209  }
210 
211 
215  for (size_t i=0; i<key_columns.size(); i++) {
216  size_t j;
217  const char *ptr;
218 
219  key_comps.clear();
220 
221  if (boost::algorithm::starts_with(key_columns[i], "\\%"))
222  column_name = key_columns[i].substr(1);
223  else if (boost::algorithm::starts_with(key_columns[i], "%0")) {
224  key_comps.pad_character = '0';
225  column_name = key_columns[i].substr(2);
226  ptr = column_name.c_str();
227  key_comps.width = atoi(ptr);
228  while (isdigit(*ptr))
229  ptr++;
230  column_name = String(ptr);
231  }
232  else if (boost::algorithm::starts_with(key_columns[i], "%-")) {
233  key_comps.left_justify = true;
234  column_name = key_columns[i].substr(2);
235  ptr = column_name.c_str();
236  key_comps.width = atoi(ptr);
237  while (isdigit(*ptr))
238  ptr++;
239  column_name = String(ptr);
240  }
241  else if (boost::algorithm::starts_with(key_columns[i], "%")) {
242  column_name = key_columns[i].substr(1);
243  ptr = column_name.c_str();
244  key_comps.width = atoi(ptr);
245  while (isdigit(*ptr))
246  ptr++;
247  column_name = String(ptr);
248  }
249  else
250  column_name = key_columns[i];
251 
252  for (j=0; j<m_column_info.size(); j++) {
253  if (m_column_info[j].family == column_name) {
254  key_comps.index = j;
255  m_key_comps.push_back(key_comps);
256  m_type_mask[j] |= ROW_KEY;
257  break;
258  }
259  }
260  if (j == m_column_info.size()) {
261  cout << "ERROR: key column '" << column_name
262  << "' not found in input file" << endl;
263  exit(EXIT_FAILURE);
264  }
265  }
266 
267  if (m_key_comps.empty()) {
268  key_comps.clear();
269  m_key_comps.push_back(key_comps);
270  m_type_mask[0] |= ROW_KEY;
271  }
272 
273  if (m_column_info.size() == 3 || m_column_info.size() == 4) {
274  size_t i=m_column_info.size()-3;
275  if ((m_column_info[i].family == "rowkey" ||
276  m_column_info[i].family == "row") &&
277  (m_column_info[i+1].family == "columnkey" ||
278  m_column_info[i+1].family == "column") &&
279  m_column_info[i+2].family == "value") {
280  if (i == 0 || m_column_info[0].family == "timestamp") {
281  m_hyperformat = true;
282  m_leading_timestamps = (i==1);
283  }
284  }
285  }
286 
287  m_next_value = m_column_info.size();
288  m_limit = 0;
289 
290  if (!m_hyperformat && m_column_info.size() < 2)
292  "No columns specified in load file");
293 
294  m_cur_line = 1;
295 }
296 
300 bool
301 LoadDataSource::next(KeySpec *keyp, uint8_t **valuep, uint32_t *value_lenp,
302  bool *is_deletep, uint32_t *consumedp)
303 {
304  string line;
305  int index;
306  char *base, *ptr, *colon;
307 
308  if (consumedp)
309  *consumedp = 0;
310 
311  *is_deletep = false;
312  keyp->flag = FLAG_INSERT;
313 
314  if (m_hyperformat) {
315 
316  while (get_next_line(line)) {
317  m_cur_line++;
318 
319  if (consumedp && !m_zipped)
320  *consumedp += line.length() + 1;
321 
323 
324  m_line_buffer.add(line.c_str(), strlen(line.c_str())+1);
325 
326  base = (char *)m_line_buffer.base;
327 
331  if (m_leading_timestamps) {
332  if ((ptr = strchr(base, m_field_separator)) == 0) {
333  cerr << "warning: too few fields on line " << m_cur_line << endl;
334  continue;
335  }
336  *ptr++ = 0;
337 
338  int64_t timestamp;
339  if (!parse_date_format(base, timestamp)) {
340  cerr << "warn: invalid timestamp format on line " << m_cur_line
341  << ", skipping..." << endl;
342  continue;
343  }
344  keyp->timestamp = (int64_t)timestamp;
345 
346  base = ptr;
347  }
348  else
349  keyp->timestamp = AUTO_ASSIGN;
350 
354  if ((ptr = strchr(base, m_field_separator)) == 0) {
355  cerr << "warning: too few fields on line " << m_cur_line << endl;
356  continue;
357  }
358  if (m_rsgen) {
359  keyp->row_len = (ptr-base) + m_row_uniquify_chars + 1;
361  m_row_key_buffer.ensure(keyp->row_len + 1);
362  m_row_key_buffer.add_unchecked(base, ptr-base);
364  m_rsgen->write((char *)m_row_key_buffer.ptr);
365  keyp->row = m_row_key_buffer.base;
366  }
367  else {
368  keyp->row = base;
369  keyp->row_len = ptr - base;
370  }
371  if (keyp->row_len == 0) {
372  cerr << "warning: zero-lengthed row key on line "
373  << m_cur_line << ", skipping..."
374  << endl;
375  continue;
376  }
377  *ptr++ = 0;
378  base = ptr;
379 
383  if ((ptr = strchr(base, m_field_separator)) == 0) {
384  cerr << "warning: too few fields on line " << m_cur_line << endl;
385  continue;
386  }
387  *ptr = 0;
388 
389  if ((colon = strchr(base, ':')) != 0) {
390  *colon++ = 0;
391  if (colon < ptr) {
392  keyp->column_qualifier = colon;
393  keyp->column_qualifier_len = ptr - colon;
394  }
395  else {
396  keyp->column_qualifier = 0;
397  keyp->column_qualifier_len = 0;
398  }
399  }
400  else {
401  keyp->column_qualifier = 0;
402  keyp->column_qualifier_len = 0;
403  }
404  keyp->column_family = *base ? base : 0;
405  ptr++;
406 
410  base = ptr;
411  *valuep = (uint8_t *)base;
412  if ((ptr = strchr(base, m_field_separator)) == 0) {
413  *value_lenp = strlen((char *)*valuep);
414  }
415  else {
416  *value_lenp = ptr-base;
417  *ptr++ = 0;
418  if (!strcmp(ptr, "DELETE")) {
419  if (keyp->column_family == 0)
420  keyp->flag = FLAG_DELETE_ROW;
421  else {
424  else
425  keyp->flag = FLAG_DELETE_CELL;
426  }
427  *is_deletep = true;
428  }
429  else if (!strcmp(ptr, "DELETE_ROW")) {
430  if (keyp->column_family != 0)
431  cerr << "warning: column family specified with DELETE_ROW on line "
432  << m_cur_line << endl;
433  keyp->flag = FLAG_DELETE_ROW;
434  *is_deletep = true;
435  }
436  else if (!strcmp(ptr, "DELETE_COLUMN_FAMILY")) {
438  *is_deletep = true;
439  }
440  else if (!strcmp(ptr, "DELETE_CELL")) {
441  keyp->flag = FLAG_DELETE_CELL;
442  *is_deletep = true;
443  }
444  else if (!strcmp(ptr, "DELETE_CELL_VERSION")) {
446  *is_deletep = true;
447  }
448  else {
449  cerr << "warning: too many fields on line " << m_cur_line << endl;
450  }
451  }
452 
453  if (m_zipped && consumedp) {
454  *consumedp = incr_consumed();
455  }
456 
457  return true;
458  }
459  }
460  else {
461 
462  *is_deletep = false;
463 
464  // skip timestamp and rowkey (if needed)
465  // issue 720: do not insert NULL values
466  while (m_next_value < m_limit) {
468  m_next_value++;
469  else
470  break;
471  }
472 
473  // get from parsed cells if available
474  if (m_next_value > 0 && m_next_value < m_limit) {
475  keyp->row = m_row_key_buffer.base;
476  keyp->row_len = m_row_key_buffer.fill();
477  keyp->column_family = m_column_info[m_next_value].family.c_str();
478  keyp->timestamp = m_timestamp;
479 
480  // clear these, just in case they were set by the client
481  if (m_column_info[m_next_value].qualifier.empty()) {
482  keyp->column_qualifier = 0;
483  keyp->column_qualifier_len = 0;
484  }
485  else {
486  keyp->column_qualifier = m_column_info[m_next_value].qualifier.c_str();
487  keyp->column_qualifier_len =
488  m_column_info[m_next_value].qualifier.length();
489  }
490  if (m_values[m_next_value] == 0) {
491  *valuep = 0;
492  *value_lenp = 0;
493  }
494  else {
495  *valuep = (uint8_t *)m_values[m_next_value];
496  *value_lenp = strlen(m_values[m_next_value]);
497  }
498  m_next_value++;
499 
500  if (m_zipped && consumedp) {
501  *consumedp = incr_consumed();
502  }
503 
504  return true;
505  }
506 
507  while (get_next_line(line)) {
508  m_cur_line++;
509  index = 0;
510 
511  if (consumedp && !m_zipped)
512  *consumedp += line.length() + 1;
513 
514  boost::trim_right_if(line, boost::is_any_of("\n"));
515  if (line.length() == 0)
516  continue;
517 
519  m_line_buffer.add(line.c_str(), strlen(line.c_str())+1);
520  m_values.clear();
521 
522  base = (char *)m_line_buffer.base;
523 
524  while ((ptr = strchr(base, m_field_separator)) != 0) {
525  *ptr++ = 0;
526 
527  if (strlen(base) == 0 || !strcmp(base, "NULL") ||
528  !strcmp(base, "\\N")) {
529  if (m_type_mask[index]) {
530  cout << "WARNING: Required key or timestamp field not found "
531  "on line " << m_cur_line << ", skipping ..." << endl << flush;
532  continue;
533  }
534  m_values.push_back(0);
535  }
536  else
537  m_values.push_back(base);
538 
539  base = ptr;
540  index++;
541  }
542  if (strlen(base) == 0 || !strcmp(base, "NULL") || !strcmp(base, "\\N"))
543  m_values.push_back(0);
544  else
545  m_values.push_back(base);
546 
547  if (!m_hyperformat && m_values.size() != m_column_info.size()) {
548  cerr << "warn: field count on line " << m_cur_line << " does not match header, skipping..." << endl;
549  continue;
550  }
551 
552  m_limit = std::min(m_values.size(), m_column_info.size());
553 
558  if (!add_row_component(0))
559  continue;
560 
561  size_t i;
562  for (i=1; i<m_key_comps.size(); i++) {
563  m_row_key_buffer.add(" ", 1);
564  if (!add_row_component(i))
565  break;
566  }
567  if (i<m_key_comps.size())
568  continue; // rowkey not found. warning in add_row_component.
569 
571  *m_row_key_buffer.ptr = 0;
572 
573  if (m_timestamp_index >= 0) {
574 
575  if (m_values.size() <= (size_t)m_timestamp_index) {
576  cerr << "warn: timestamp field not found on line " << m_cur_line
577  << ", skipping..." << endl;
578  continue;
579  }
580 
582  cerr << "warn: invalid timestamp format on line " << m_cur_line
583  << ", skipping..." << endl;
584  continue;
585  }
586  }
587  else
589 
590  m_next_value = 0;
592  m_next_value++;
593 
594  if (m_rsgen) {
596  keyp->row = m_row_key_buffer.base;
599  m_rsgen->write((char *)m_row_key_buffer.ptr);
601  }
602  else {
603  keyp->row = m_row_key_buffer.base;
604  keyp->row_len = m_row_key_buffer.fill();
605  }
606 
607  keyp->column_family = m_column_info[m_next_value].family.c_str();
608  keyp->timestamp = m_timestamp;
609 
610  if (m_column_info[m_next_value].qualifier.empty()) {
611  keyp->column_qualifier = 0;
612  keyp->column_qualifier_len = 0;
613  }
614  else {
615  keyp->column_qualifier = m_column_info[m_next_value].qualifier.c_str();
616  keyp->column_qualifier_len =
617  m_column_info[m_next_value].qualifier.length();
618  }
619 
620  if (m_values[m_next_value] == 0) {
621  *valuep = 0;
622  *value_lenp = 0;
623  }
624  else {
625  *valuep = (uint8_t *)m_values[m_next_value];
626  *value_lenp = strlen(m_values[m_next_value]);
627  }
628  m_next_value++;
629 
630  if (m_zipped && consumedp) {
631  *consumedp = incr_consumed();
632  }
633 
634  return true;
635  }
636 
637  }
638 
639  return false;
640 }
641 
643 {
644  const char *value = m_values[m_key_comps[index].index];
645  size_t value_len = 0;
646 
647  if (value == 0) {
648  cout << "WARNING: Required key field not found on line " << m_cur_line
649  << ", skipping ..." << endl << flush;
650  return false;
651  }
652 
653  value_len = strlen(value);
654 
655  if ((size_t)m_key_comps[index].index >= m_values.size() || value == 0) {
656  cout << "WARNING: Required key field not found on line " << m_cur_line
657  << ", skipping ..." << endl << flush;
658  return false;
659  }
660 
661  if ((size_t)m_key_comps[index].width > value_len) {
662  size_t padding = m_key_comps[index].width - value_len;
663  m_row_key_buffer.ensure(m_key_comps[index].width);
664  if (m_key_comps[index].left_justify) {
665  m_row_key_buffer.add(value, value_len);
666  memset(m_row_key_buffer.ptr, m_key_comps[index].pad_character, padding);
667  m_row_key_buffer.ptr += padding;
668  }
669  else {
670  memset(m_row_key_buffer.ptr, m_key_comps[index].pad_character, padding);
671  m_row_key_buffer.ptr += padding;
672  m_row_key_buffer.add(value, value_len);
673  }
674  }
675  else
676  m_row_key_buffer.add(value, value_len);
677 
678  return true;
679 
680 }
681 
682 bool LoadDataSource::parse_date_format(const char *str, int64_t &timestamp)
683 {
684  int ival;
685  const char *ptr = str;
686  char *end_ptr;
687  struct tm tm;
688  time_t tt;
689  int64_t ns;
690  int64_t sec;
691 
692  ns = (int64_t)strtoll(ptr, &end_ptr, 10);
693  if (*end_ptr == 0) {
694  timestamp = ns;
695  return true;
696  }
697  else if ((end_ptr - ptr) != 4)
698  return false;
699 
700  tm.tm_year = ns - 1900;
701  ptr = end_ptr + 1;
702  ns = 0;
703 
707  if ((ival = strtol(ptr, &end_ptr, 10)) == 0 || (end_ptr - ptr) != 2 ||
708  *end_ptr != '-')
709  return false;
710 
711  tm.tm_mon = ival - 1;
712  ptr = end_ptr + 1;
713 
717  if ((ival = strtol(ptr, &end_ptr, 10)) == 0 || (end_ptr - ptr) != 2 ||
718  (*end_ptr != ' ' && *end_ptr != 'T'))
719  return false;
720 
721  tm.tm_mday = ival;
722  ptr = end_ptr + 1;
723 
727  ival = strtol(ptr, &end_ptr, 10);
728  if ((end_ptr - ptr) != 2 || *end_ptr != ':')
729  return false;
730  tm.tm_hour = ival;
731 
732  ptr = end_ptr + 1;
733 
737  ival = strtol(ptr, &end_ptr, 10);
738  if ((end_ptr - ptr) != 2 || *end_ptr != ':')
739  return false;
740  tm.tm_min = ival;
741 
742  ptr = end_ptr + 1;
743 
748  if(!parse_sec(ptr, &end_ptr, sec))
749  return false;
750  tm.tm_sec = 0;
751 
752  tm.tm_isdst = -1;
753 #if !defined(__sun__)
754  tm.tm_gmtoff = System::tm_gmtoff;
755  tm.tm_zone = (char *)System::tm_zone.c_str();
756 #endif
757 
758  if ((tt = mktime(&tm)) == (time_t)-1)
759  return false;
760 
761  ptr = end_ptr + 1;
762  // add integer nanoseconds
763  if (*end_ptr == ':') {
764  ns = strtoul(ptr, &end_ptr, 10);
765  }
766 
767  timestamp = ((int64_t)tt * 1000000000LL) + sec + ns;
768 
769  return true;
770 }
771 
772 bool LoadDataSource::parse_sec(const char *str, char **end_ptr, int64_t &ns)
773 {
774  uint_parser<unsigned int, 10, 2, 2> uint2_p;
775  int64_t int_seconds=0;
776  double decimal_seconds=0;
777  *end_ptr = (char*) str;
778  parse_info<> info = parse(str,
779  (uint2_p[assign_a(int_seconds)] >> !real_p[assign_a(decimal_seconds)]));
780  ns = (int64_t)int_seconds * 1000000000LL + (int64_t)(decimal_seconds *
781  ((double) 1000000000LL));
782  if (info.hit)
783  *end_ptr += info.length - 1 ;
784  return info.hit;
785 }
int64_t timestamp
Definition: KeySpec.h:130
Retrieves system information (hardware, installation directory, etc)
bool should_skip(int idx, const uint32_t *masks)
virtual void parse_header(const String &header, const std::vector< String > &key_columns, const std::string &timestamp_column)
static const uint32_t FLAG_DELETE_ROW
Definition: KeySpec.h:40
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
static String tm_zone
Timezone abbreviation.
Definition: System.h:126
String format(const char *fmt,...)
Returns a String using printf like format facilities Vanilla snprintf is about 1.5x faster than this...
Definition: String.cc:37
static const uint32_t FLAG_INSERT
Definition: KeySpec.h:47
virtual void init_src()=0
Po::typed_value< String > * str(String *v=0)
Definition: Properties.h:166
const char * column_qualifier
Definition: KeySpec.h:128
static bool exists(const String &fname)
Checks if a file or directory exists.
Definition: FileUtils.cc:420
static const uint32_t FLAG_DELETE_CELL
Definition: KeySpec.h:42
STL namespace.
size_t column_qualifier_len
Definition: KeySpec.h:129
virtual void init(const std::vector< String > &key_columns, const std::string &timestamp_column, char field_separator)
const void * row
Definition: KeySpec.h:125
uint8_t * ptr
Pointer to the end of the used part of the buffer.
std::vector< KeyComponentInfo > m_key_comps
static const uint32_t FLAG_DELETE_COLUMN_FAMILY
Definition: KeySpec.h:41
static long tm_gmtoff
Seconds east of UTC.
Definition: System.h:123
#define HT_EXPECT(_e_, _code_)
Definition: Logger.h:388
File system utility functions.
A dynamic, resizable memory buffer.
std::vector< ColumnInfo > m_column_info
boost::iostreams::filtering_istream m_fin
uint8_t * add(const void *data, size_t len)
Adds more data WITH boundary checks; if required the buffer is resized and existing data is preserved...
Logging routines and macros.
bool get_next_line(String &line)
std::vector< const char * > m_values
Compatibility Macros for C/C++.
Time related declarations.
bool parse_date_format(const char *str, int64_t &timestamp)
Hypertable definitions
bool add_row_component(int index)
void clear()
Clears the buffer.
virtual uint64_t incr_consumed()=0
FixedRandomStringGenerator * m_rsgen
#define HT_THROWF(_code_, _fmt_,...)
Definition: Error.h:490
uint8_t * base
Pointer to the allocated memory buffer.
size_t fill() const
Returns the size of the used portion.
Definition: DynamicBuffer.h:70
bool single_cell_format(int flags)
Definition: LoadDataFlags.h:47
static const int64_t AUTO_ASSIGN
Definition: KeySpec.h:38
virtual bool next(KeySpec *keyp, uint8_t **valuep, uint32_t *value_lenp, bool *is_deletep, uint32_t *consumedp)
const char * column_family
Definition: KeySpec.h:127
Error codes, Exception handling, error logging.
#define HT_THROW(_code_, _msg_)
Definition: Error.h:478
static const uint32_t FLAG_DELETE_CELL_VERSION
Definition: KeySpec.h:43
void ensure(size_t len)
Ensure space for additional data Will grow the space to 1.5 of the needed space with existing data un...
Definition: DynamicBuffer.h:82
uint8_t * add_unchecked(const void *data, size_t len)
Adds additional data without boundary checks.
bool parse_sec(const char *str, char **end_ptr, int64_t &ns)