0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
Utility.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
26 
27 #include <Common/Compat.h>
28 #include "Utility.h"
29 
31 
32 #include <Hypertable/Lib/Key.h>
33 #include <Hypertable/Lib/KeySpec.h>
37 #include <Hypertable/Lib/Schema.h>
41 
42 #include <Hyperspace/Session.h>
43 
44 #include <AsyncComm/CommAddress.h>
45 
46 #include <Common/FailureInducer.h>
47 #include <Common/ScopeGuard.h>
48 #include <Common/StatusPersister.h>
49 #include <Common/StringExt.h>
50 #include <Common/md5.h>
51 #include <Common/Time.h>
52 
53 using namespace Hyperspace;
54 
55 namespace Hypertable { namespace Utility {
56 
57 void get_table_server_set(ContextPtr &context, const String &id,
58  const String &row, StringSet &servers) {
59  String start_row, end_row;
60  ScanSpec scan_spec;
61  RowInterval ri;
62  TableScannerPtr scanner;
63  Cell cell;
64  String location;
65 
66  if (context->test_mode) {
67  context->get_available_servers(servers);
68  return;
69  }
70 
71  if (row.empty()) {
72  start_row = format("%s:", id.c_str());
73  scan_spec.row_limit = 0;
74  }
75  else {
76  start_row = format("%s:%s", id.c_str(), row.c_str());
77  scan_spec.row_limit = 1;
78  }
79 
80  end_row = format("%s:%s", id.c_str(), Key::END_ROW_MARKER);
81 
82  scan_spec.max_versions = 1;
83  scan_spec.columns.clear();
84  scan_spec.columns.push_back("Location");
85 
86  ri.start = start_row.c_str();
87  ri.end = end_row.c_str();
88  scan_spec.row_intervals.push_back(ri);
89 
90  scanner.reset( context->metadata_table->create_scanner(scan_spec) );
91 
92  while (scanner->next(cell)) {
93  location = String((const char *)cell.value, cell.value_len);
94  boost::trim(location);
95  if (location != "!")
96  servers.insert(location);
97  }
98 }
99 
100 bool table_exists(ContextPtr &context, const String &name, String &id) {
101  bool is_namespace;
102 
103  id = "";
104 
105  if (!context->namemap->name_to_id(name, id, &is_namespace) ||
106  is_namespace)
107  return false;
108 
109  String tablefile = context->toplevel_dir + "/tables/" + id;
110 
111  try {
112  if (context->hyperspace->attr_exists(tablefile, "x"))
113  return true;
114  }
115  catch (Exception &e) {
118  return false;
119  HT_THROW2(e.code(), e, name);
120  }
121  return false;
122 }
123 
124 TableParts get_index_parts(const std::string &schema) {
125  uint8_t parts {};
126  SchemaPtr s(Schema::new_instance(schema));
127 
128  for (auto cf_spec : s->get_column_families()) {
129  if (cf_spec && !cf_spec->get_deleted()) {
130  if (cf_spec->get_value_index())
131  parts |= TableParts::VALUE_INDEX;
132  if (cf_spec->get_qualifier_index())
134  }
135  }
136  return TableParts(parts);
137 }
138 
139 
140 bool table_exists(ContextPtr &context, const String &id) {
141 
142  String tablefile = context->toplevel_dir + "/tables/" + id;
143 
144  try {
145  if (context->hyperspace->attr_exists(tablefile, "x"))
146  return true;
147  }
148  catch (Exception &e) {
151  return false;
152  HT_THROW2(e.code(), e, id);
153  }
154  return false;
155 }
156 
157 
158 void verify_table_name_availability(ContextPtr &context, const String &name, String &id) {
159  bool is_namespace;
160 
161  id = "";
162 
163  if (!context->namemap->name_to_id(name, id, &is_namespace))
164  return;
165 
166  if (is_namespace)
168 
169  String tablefile = context->toplevel_dir + "/tables/" + id;
170 
171  try {
172  if (context->hyperspace->attr_exists(tablefile, "x"))
174  }
175  catch (Exception &e) {
177  HT_THROW2(e.code(), e, id);
178  }
179 }
180 
181 
182 void create_table_in_hyperspace(ContextPtr &context, const String &name,
183  const String &schema_str, TableIdentifierManaged *table) {
184  String table_name = name;
185 
186  // Strip leading '/'
187  if (table_name[0] == '/')
188  table_name = table_name.substr(1);
189 
190  String table_id;
191  Utility::verify_table_name_availability(context, table_name, table_id);
192 
193  if (table_id == "")
194  context->namemap->add_mapping(table_name, table_id, 0, true);
195 
196  HT_MAYBE_FAIL("Utility-create-table-in-hyperspace-1");
197 
198 
199 
201  table_id == TableIdentifier::METADATA_ID);
202 
203  // Create table file
204  String tablefile = context->toplevel_dir + "/tables/" + table_id;
206 
207  // Write schema attribute
208  context->hyperspace->attr_set(tablefile, oflags, "schema", schema_str.c_str(),
209  schema_str.length());
210 
211  HT_MAYBE_FAIL("Utility-create-table-in-hyperspace-2");
212 
213  // Create /hypertable/tables/<table>/<accessGroup> directories
214  // for this table in DFS
215  String table_basedir = context->toplevel_dir + "/tables/" + table_id + "/";
216 
217  SchemaPtr schema(Schema::new_instance(schema_str));
218 
219  for (auto ag_spec : schema->get_access_groups()) {
220  String agdir = table_basedir + ag_spec->get_name();
221  context->dfs->mkdirs(agdir);
222  }
223 
224  table->set_id(table_id);
225  table->generation = schema->get_generation();
226 
227 }
228 
229 void prepare_index(ContextPtr &context, const String &name,
230  const String &schema_str, bool qualifier,
231  String &index_name, String &index_schema_str)
232 {
233  // load the schema of the primary table
234  SchemaPtr primary_schema( Schema::new_instance(schema_str) );
235 
236  // create a new schema and fill it
237  AccessGroupSpec *ag_spec = new AccessGroupSpec("default");
238  ColumnFamilySpec *new_cf_spec = new ColumnFamilySpec("v1");
239  new_cf_spec->set_access_group("default");
240  ag_spec->add_column(new_cf_spec);
241 
242  Schema *index_schema = new Schema();
243 
244  // Merge defaults
245  index_schema->access_group_defaults().merge( primary_schema->access_group_defaults() );
246  index_schema->column_family_defaults().merge( primary_schema->column_family_defaults() );
247 
248  index_schema->add_access_group(ag_spec);
249  index_schema->set_group_commit_interval(
250  primary_schema->get_group_commit_interval());
251  index_schema->set_version(1);
252  int64_t generation = get_ts64();
253  index_schema->update_generation(generation);
254 
255  // the index table name is prepended with ^, and the qualified
256  // index with ^^
257  index_name = Filesystem::dirname(name);
258  if (qualifier) {
259  if (index_name == "/")
260  index_name += String("^^") + Filesystem::basename(name);
261  else
262  index_name += String("/^^") + Filesystem::basename(name);
263  }
264  else {
265  if (index_name == "/")
266  index_name += String("^") + Filesystem::basename(name);
267  else
268  index_name += String("/^") + Filesystem::basename(name);
269  }
270 
271  index_schema_str = index_schema->render_xml();
272  delete index_schema;
273 }
274 
276 
277  if (context->test_mode) {
278  HT_WARN("Skipping create_table_write_metadata due to TEST MODE");
279  return;
280  }
281 
282  TableMutatorPtr mutator_ptr(context->metadata_table->create_mutator());
283 
284  String metadata_key_str = String(table->id) + ":" + Key::END_ROW_MARKER;
285  String start_row;
286  KeySpec key;
287  key.row = metadata_key_str.c_str();
288  key.row_len = metadata_key_str.length();
289  key.column_qualifier = 0;
290  key.column_qualifier_len = 0;
291  key.column_family = "StartRow";
292 
293  if (table->is_metadata())
294  mutator_ptr->set(key, Key::END_ROOT_ROW, strlen(Key::END_ROOT_ROW));
295  else
296  mutator_ptr->set(key, 0, 0);
297 
298  mutator_ptr->flush();
299 }
300 
304 bool next_available_server(ContextPtr &context, String &location, bool urgent) {
306  if (!context->rsc_manager->next_available_server(rsc, urgent))
307  return false;
308  location = rsc->location();
309  return true;
310 }
311 
312 
313 void create_table_load_range(ContextPtr &context, const String &location,
314  TableIdentifier &table, RangeSpec &range, bool needs_compaction) {
315  Lib::RangeServer::Client rsc(context->comm);
316  CommAddress addr;
317 
318  if (context->test_mode) {
320  HT_WARNF("Skipping %s::load_range() because in TEST MODE", location.c_str());
321  HT_ASSERT(context->rsc_manager->find_server_by_location(location, rsc));
322  if (!rsc->connected())
324  return;
325  }
326 
327  addr.set_proxy(location);
328 
329  try {
330  RangeState range_state;
331  int64_t split_size = context->props->get_i64("Hypertable.RangeServer.Range.SplitSize");
332 
333  if (table.is_metadata())
334  range_state.soft_limit = context->props->get_i64("Hypertable.RangeServer.Range.MetadataSplitSize", split_size);
335  else {
336  range_state.soft_limit = split_size;
337  if (context->props->get_bool("Hypertable.Master.Split.SoftLimitEnabled"))
338  range_state.soft_limit /= std::min(64, (int)context->rsc_manager->server_count()*2);
339  }
340 
341  rsc.load_range(addr, table, range, range_state, needs_compaction);
342  }
343  catch (Exception &e) {
345  HT_THROW2F(e.code(), e, "Problem loading range %s[%s..%s] in %s",
346  table.id, range.start_row, range.end_row, location.c_str());
347  }
348 }
349 
350 void create_table_acknowledge_range(ContextPtr &context, const String &location,
351  TableIdentifier &table, RangeSpec &range) {
352  Lib::RangeServer::Client rsc(context->comm);
353  CommAddress addr;
354 
355  if (context->test_mode) {
357  HT_WARNF("Skipping %s::acknowledge_load() because in TEST MODE", location.c_str());
358  HT_ASSERT(context->rsc_manager->find_server_by_location(location, rsc));
359  if (!rsc->connected())
361  return;
362  }
363 
364  addr.set_proxy(location);
365 
366  QualifiedRangeSpec qrs(table, range);
367  vector<QualifiedRangeSpec *> range_vec;
368  map<QualifiedRangeSpec, int> response_map;
369  range_vec.push_back(&qrs);
370  rsc.acknowledge_load(addr, range_vec, response_map);
371  map<QualifiedRangeSpec, int>::iterator it = response_map.begin();
372  if (it->second != Error::OK &&
373  it->second != Error::TABLE_NOT_FOUND &&
375  HT_THROW(it->second, "Problem acknowledging load range");
376 }
377 
378 
379 int64_t range_hash_code(const TableIdentifier &table, const RangeSpec &range, const String &qualifier) {
380  if (!qualifier.empty())
381  return md5_hash(qualifier.c_str()) ^ md5_hash(table.id) ^ md5_hash(range.start_row) ^ md5_hash(range.end_row);
382  return md5_hash(table.id) ^ md5_hash(range.start_row) ^ md5_hash(range.end_row);
383 }
384 
385 String range_hash_string(const TableIdentifier &table, const RangeSpec &range, const String &qualifier) {
386  return String("") + range_hash_code(table, range, qualifier);
387 }
388 
390  DynamicBuffer value(0);
391  String location;
392  uint64_t root_handle=0;
393  String toplevel_dir = context->props->get_str("Hypertable.Directory");
394 
395  try {
396  HT_ON_SCOPE_EXIT(&Hyperspace::close_handle_ptr, context->hyperspace, &root_handle);
397  root_handle = context->hyperspace->open(toplevel_dir + "/root", OPEN_FLAG_READ);
398  context->hyperspace->attr_get(root_handle, "Location", value);
399  location = (const char *)value.base;
400  }
401  catch (Exception &e) {
402  HT_ERROR_OUT << "Unable to read root location -" << e.what() << HT_END;
403  HT_THROW(e.code(), e.what());
404  }
405  return location;
406 }
407 
408 bool status(ContextPtr &context, Timer &timer, Status &status) {
409  if (context->startup_in_progress())
411  else if (context->shutdown_in_progress())
413  else if (!context->master_file->lock_acquired())
415  else if (context->quorum_reached) {
416  size_t connected_servers = context->available_server_count();
417  size_t total_servers = context->rsc_manager->server_count();
418  if (connected_servers < total_servers) {
419  size_t failover_pct
420  = context->props->get_i32("Hypertable.Failover.Quorum.Percentage");
421  size_t quorum = ((total_servers * failover_pct) + 99) / 100;
422  if (connected_servers == 0)
424  "RangeServer recovery blocked because 0 servers available.");
425  else if (connected_servers < quorum)
427  format("RangeServer recovery blocked (%d servers "
428  "available, quorum of %d is required)",
429  (int)connected_servers, (int)quorum));
430  else
431  status.set(Status::Code::WARNING, "RangeServer recovery in progress");
432  }
433  else {
434  context->dfs->status(status, &timer);
435  Status::Code code;
436  string text;
437  status.get(&code, text);
438  if (code != Status::Code::OK)
439  status.set(code, format("FsBroker %s", text.c_str()));
440  else
441  StatusPersister::get(status);
442  }
443  }
444  return status.get() == Status::Code::OK;
445 }
446 
448  try {
449  Lib::RangeServer::Client rsc(context->comm);
450  rsc.shutdown(addr);
451  }
452  catch (Exception &e) {
453  }
454 }
455 
456 }}
std::set< String > StringSet
STL Set managing Strings.
Definition: StringExt.h:42
#define HT_THROW2F(_code_, _ex_, _fmt_,...)
Definition: Error.h:494
void set_version(int32_t version)
Sets version number.
Definition: Schema.h:222
bool next_available_server(ContextPtr &context, String &location, bool urgent)
Gets name of next available server.
Definition: Utility.cc:304
Schema specification.
Definition: Schema.h:52
#define HT_WARNF(msg,...)
Definition: Logger.h:290
void prepare_index(ContextPtr &context, const String &name, const String &schema_str, bool qualifier, String &index_name, String &index_schema_str)
Prepares index schema and table name.
Definition: Utility.cc:229
The FailureInducer simulates errors.
int64_t md5_hash(const char *input)
Returns a 64-bit hash checksum of a null terminated input buffer.
Definition: md5.cc:388
static const char * METADATA_ID
Range specification.
Definition: RangeSpec.h:40
Holds Nagios-style program status information.
Definition: Status.h:42
std::shared_ptr< RangeServerConnection > RangeServerConnectionPtr
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
String format(const char *fmt,...)
Returns a String using printf like format facilities Vanilla snprintf is about 1.5x faster than this...
Definition: String.cc:37
static constexpr const char * SERVER_IS_SHUTTING_DOWN
Definition: Status.h:62
static const char * METADATA_NAME
Declarations for TableIdentifier and TableIdentifierManaged.
const char * column_qualifier
Definition: KeySpec.h:128
void acknowledge_load(const CommAddress &addr, const vector< QualifiedRangeSpec * > &ranges, std::map< QualifiedRangeSpec, int > &response_map)
Issues a synchronous "acknowledge load" request for multiple ranges.
Definition: Client.cc:163
Column family specification.
uint64_t soft_limit
Soft split size limit.
Definition: RangeState.h:108
Declarations for CommAddress.
void create_table_load_range(ContextPtr &context, const String &location, TableIdentifier &table, RangeSpec &range, bool needs_compaction)
Loads a table's initial range.
Definition: Utility.cc:313
void create_table_acknowledge_range(ContextPtr &context, const String &location, TableIdentifier &table, RangeSpec &range)
Calls RangeServer::acknowledge_load for a range.
Definition: Utility.cc:350
int64_t range_hash_code(const TableIdentifier &table, const RangeSpec &range, const String &qualifier)
Returns a hash code for a range with an optional qualifer string.
Definition: Utility.cc:379
#define HT_ON_SCOPE_EXIT(...)
Definition: ScopeGuard.h:301
size_t column_qualifier_len
Definition: KeySpec.h:129
const void * row
Definition: KeySpec.h:125
Declarations for RangeServerClient.
std::shared_ptr< TableScanner > TableScannerPtr
Smart pointer to TableScanner.
Definition: TableScanner.h:124
void set(Code code, const std::string &text)
Sets status code and text.
Definition: Status.h:101
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
Represents a set of table parts (sub-tables).
Definition: TableParts.h:47
Code
Enumeration for status codes.
Definition: Status.h:47
Declarations for Schema.
Represents a row interval.
Definition: RowInterval.h:38
std::shared_ptr< Context > ContextPtr
Smart pointer to Context.
Definition: Context.h:265
Hyperspace definitions
#define HT_ASSERT(_e_)
Definition: Logger.h:396
String range_hash_string(const TableIdentifier &table, const RangeSpec &range, const String &qualifier)
Returns string representation of hash code for a range with an optional qualifer string.
Definition: Utility.cc:385
Wrapper for TableIdentifier providing member storage.
void verify_table_name_availability(ContextPtr &context, const String &name, String &id)
Checks if table name is available.
Definition: Utility.cc:158
Scan predicate and control specification.
Definition: ScanSpec.h:56
std::shared_ptr< TableMutator > TableMutatorPtr
Smart pointer to TableMutator.
Definition: TableMutator.h:257
const char * end_row
Definition: RangeSpec.h:60
Declarations for QualifiedRangeSpec and QualifiedRangeSpecManaged.
Open file for writing.
Definition: Session.h:73
static constexpr const char * STANDBY
Definition: Status.h:63
bool status(ContextPtr &context, Timer &timer, Status &status)
Runs a status check on the master.
Definition: Utility.cc:408
void set_proxy(const String &str)
Sets address type to CommAddress::PROXY and proxy name to p.
Definition: CommAddress.h:76
void shutdown(const CommAddress &addr)
Issues a "shutdown" request.
Definition: Client.cc:545
void close_handle_ptr(SessionPtr hyperspace, uint64_t *handlep)
Definition: Session.cc:1400
Compatibility Macros for C/C++.
AccessGroupOptions & access_group_defaults()
Returns reference to default access group options.
Definition: Schema.h:381
#define HT_END
Definition: Logger.h:220
#define HT_ERROR_OUT
Definition: Logger.h:301
static String basename(String name, char separator= '/')
A posix-compliant basename() which strips directory names from a filename.
Definition: Filesystem.cc:154
Time related declarations.
void merge(const ColumnFamilyOptions &other)
Merges options from another ColumnFamilyOptions object.
Declarations for RangeSpec and RangeSpecManaged.
Access group specification.
bool table_exists(ContextPtr &context, const String &id)
Checks if table ID exists.
Definition: Utility.cc:140
Hypertable definitions
static Schema * new_instance(const std::string &buf)
Creates schema object from XML schema string.
Definition: Schema.cc:202
void set_id(const std::string &new_name)
static void get(Status &status)
Gets persistent status.
Declarations for general-purpose utility functions.
void add_access_group(AccessGroupSpec *ag)
Adds access group specification.
Definition: Schema.cc:354
const char * start_row
Definition: RangeSpec.h:59
void create_table_in_hyperspace(ContextPtr &context, const String &name, const String &schema_str, TableIdentifierManaged *table)
Creates a table in Hyperspace.
Definition: Utility.cc:182
void set_access_group(const std::string &ag)
Sets access group.
ColumnFamilyOptions & column_family_defaults()
Returns reference to default column family options.
Definition: Schema.h:392
void create_table_write_metadata(ContextPtr &context, TableIdentifier *table)
Creates initial METADATA table entry for table.
Definition: Utility.cc:275
Client interface to RangeServer.
Definition: Client.h:63
uint8_t * base
Pointer to the allocated memory buffer.
void get(Code *code, std::string &text) const
Gets status code and text.
Definition: Status.h:111
void get_table_server_set(ContextPtr &context, const String &id, const String &row, StringSet &servers)
Gets set of servers holding ranges for a given table.
Definition: Utility.cc:57
String root_range_location(ContextPtr &context)
Returns location of root METADATA range.
Definition: Utility.cc:389
RowIntervals row_intervals
Definition: ScanSpec.h:275
A timer class to keep timeout states across AsyncComm related calls.
Definition: Timer.h:44
This is a generic exception class for Hypertable.
Definition: Error.h:314
void set_group_commit_interval(int32_t interval)
Sets group commit interval.
Definition: Schema.h:364
static const char * END_ROOT_ROW
Definition: Key.h:50
Declarations for StatusPersister.
Qualified (with table identifier) range specification.
uint32_t value_len
Definition: Cell.h:72
Create file if it does not exist.
Definition: Session.h:77
void shutdown_rangeserver(ContextPtr &context, CommAddress &addr)
Sends a shutdown command to a rangeserver.
Definition: Utility.cc:447
static String dirname(String name, char separator= '/')
A posix-compliant dirname() which strips the last component from a file name.
Definition: Filesystem.cc:127
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
TableParts get_index_parts(const std::string &schema)
Gets index parts specified in schema.
Definition: Utility.cc:124
Range state.
Definition: RangeState.h:48
void load_range(const CommAddress &addr, const TableIdentifier &table, const RangeSpec &range_spec, const RangeState &range_state, bool needs_compaction)
Issues a synchronous "load range" request.
Definition: Client.cc:125
#define HT_MAYBE_FAIL(_label_)
Open file for reading.
Definition: Session.h:71
#define HT_WARN(msg)
Definition: Logger.h:289
Encapsulates decomposed key and value.
Definition: Cell.h:32
String extensions and helpers: sets, maps, append operators etc.
const char * column_family
Definition: KeySpec.h:127
#define HT_THROW(_code_, _msg_)
Definition: Error.h:478
void add_column(ColumnFamilySpec *cf)
Adds column family specification.
static const char * END_ROW_MARKER
Definition: Key.h:49
md5 digest routines.
Declarations for Context.
static constexpr const char * SERVER_IS_COMING_UP
Definition: Status.h:61
const std::string render_xml(bool with_ids=false)
Renders schema in XML format.
Definition: Schema.cc:265
Address abstraction to hold either proxy name or IPv4:port address.
Definition: CommAddress.h:52
const uint8_t * value
Definition: Cell.h:71
int64_t get_ts64()
Returns the current time in nanoseconds as a 64bit number.
Definition: Time.cc:40
int code() const
Returns the error code.
Definition: Error.h:391
void update_generation(int64_t generation)
Updates generation and assigns column family IDs.
Definition: Schema.cc:244
void merge(const AccessGroupOptions &other)
Merges options from another AccessGroupOptions object.
#define HT_THROW2(_code_, _ex_, _msg_)
Definition: Error.h:484
Executes user-defined functions when leaving the current scope.