0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
RangeServer.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
25 
26 #ifndef Hypertable_RangeServer_RangeServer_h
27 #define Hypertable_RangeServer_RangeServer_h
28 
50 
51 #include <Hypertable/Lib/Cells.h>
59 
60 #include <Hyperspace/Session.h>
61 
63 #include <AsyncComm/Clock.h>
64 #include <AsyncComm/Comm.h>
66 #include <AsyncComm/Event.h>
68 
69 #include <Common/Filesystem.h>
70 #include <Common/Logger.h>
72 #include <Common/MetricsProcess.h>
73 #include <Common/Properties.h>
74 
75 #include <map>
76 #include <memory>
77 #include <mutex>
78 
79 using namespace Hyperspace::Lib;
80 using namespace Hyperspace;
81 using namespace Hypertable::RangeServer;
82 
83 namespace Hypertable {
84 namespace RangeServer { class ConnectionHandler; }
85 class UpdateRecTable;
86 namespace Apps {
87 
93 
94  class RangeServer {
95  public:
98  virtual ~RangeServer();
99 
100  // range server protocol implementations
101  void compact(ResponseCallback *, const TableIdentifier &,
102  const char *row, int32_t flags);
103  void create_scanner(Response::Callback::CreateScanner *,
104  const TableIdentifier &,
105  const RangeSpec &, const ScanSpec &,
106  QueryCache::Key *);
107  void destroy_scanner(ResponseCallback *cb, int32_t scanner_id);
108  void fetch_scanblock(Response::Callback::CreateScanner *, int32_t scanner_id);
109  void load_range(ResponseCallback *, const TableIdentifier &,
110  const RangeSpec &, const RangeState &,
111  bool needs_compaction);
112  void acknowledge_load(Response::Callback::AcknowledgeLoad *cb,
113  const vector<QualifiedRangeSpec> &specs);
114  void update_schema(ResponseCallback *, const TableIdentifier &,
115  const char *);
116 
119  void update(Response::Callback::Update *cb, uint64_t cluster_id,
120  const TableIdentifier &table, uint32_t count,
121  StaticBuffer &buffer, uint32_t flags);
122  void batch_update(std::vector<UpdateRecTable *> &updates,
123  ClockT::time_point expire_time);
124 
125  void commit_log_sync(ResponseCallback *cb, uint64_t cluster_id,
126  const TableIdentifier &table);
127 
130  void drop_table(ResponseCallback *cb, const TableIdentifier &table);
131 
132  void dump(ResponseCallback *, const char *, bool);
133 
135  void dump_pseudo_table(ResponseCallback *cb, const TableIdentifier &table,
136  const char *pseudo_table, const char *outfile);
137  void get_statistics(Response::Callback::GetStatistics *cb,
138  const std::vector<SystemVariable::Spec> &specs,
139  uint64_t generation);
140 
141  void drop_range(ResponseCallback *, const TableIdentifier &,
142  const RangeSpec &);
143 
144  void relinquish_range(ResponseCallback *, const TableIdentifier &,
145  const RangeSpec &);
146  void heapcheck(ResponseCallback *, const char *);
147 
148  void metadata_sync(ResponseCallback *, const char *, uint32_t flags, std::vector<const char *> columns);
149 
150  void replay_fragments(ResponseCallback *, int64_t op_id,
151  const String &location, int32_t plan_generation, int32_t type,
152  const vector<int32_t> &fragments,
153  const RangeServerRecovery::ReceiverPlan &receiver_plan, int32_t replay_timeout);
154 
155  void phantom_load(ResponseCallback *, const String &location,
156  int32_t plan_generation,
157  const vector<int32_t> &fragments,
158  const vector<QualifiedRangeSpec> &specs,
159  const vector<RangeState> &states);
160 
161  void phantom_update(Response::Callback::PhantomUpdate *, const String &location,
162  int32_t plan_generation, const QualifiedRangeSpec &range,
163  int32_t fragment, EventPtr &event);
164 
165  void phantom_prepare_ranges(ResponseCallback *, int64_t op_id,
166  const String &location, int32_t plan_generation,
167  const vector<QualifiedRangeSpec> &ranges);
168 
169  void phantom_commit_ranges(ResponseCallback *, int64_t op_id,
170  const String &location, int32_t plan_generation,
171  const vector<QualifiedRangeSpec> &ranges);
172 
173  void set_state(ResponseCallback *cb,
174  const std::vector<SystemVariable::Spec> &specs,
175  int64_t generation);
176 
180  void table_maintenance_enable(ResponseCallback *cb,
181  const TableIdentifier &table);
182 
190  void table_maintenance_disable(ResponseCallback *cb,
191  const TableIdentifier &table);
192 
198  void wait_for_maintenance(ResponseCallback *cb);
199 
200  // Other methods
201  void group_commit();
202  void do_maintenance();
203 
204  MaintenanceSchedulerPtr &get_scheduler() { return m_maintenance_scheduler; }
205 
206  ApplicationQueuePtr get_application_queue() { return m_app_queue; }
207 
208  void master_change();
209 
211  return m_log_replay_barrier->user_complete();
212  }
213 
215 
216  void shutdown();
217 
218  void write_profile_data(const String &line) {
219  if (m_profile_query) {
220  std::lock_guard<std::mutex> lock(m_profile_mutex);
221  m_profile_query_out << line << "\n";
222  }
223  }
224 
225  private:
226 
227  void initialize(PropertiesPtr &);
228  void local_recover();
229  void decode_table_id(const uint8_t **bufp, size_t *remainp, TableIdentifier *tid);
230  typedef std::map<String, SchemaPtr> TableSchemaMap;
231  void get_table_schemas(TableSchemaMap &table_schemas);
232  static void map_table_schemas(const String &parent, const std::vector<DirEntryAttr> &listing,
233  TableSchemaMap &table_schemas);
234  void replay_load_range(TableInfoMap &replay_map,
235  MetaLogEntityRangePtr &range_entity);
236  void replay_log(TableInfoMap &replay_map, CommitLogReaderPtr &log_reader);
237 
238  void verify_schema(TableInfoPtr &, uint32_t generation, const TableSchemaMap *table_schemas=0);
239 
240  bool live(const vector<QualifiedRangeSpec> &ranges);
241  bool live(const QualifiedRangeSpec &spec);
242 
243  void group_commit_add(EventPtr &event, uint64_t cluster_id,
244  SchemaPtr &schema, const TableIdentifier &table,
245  uint32_t count, StaticBuffer &buffer, uint32_t flags);
246 
252  std::lock_guard<std::mutex> lock(m_mutex);
253  bool old_value = m_get_statistics_outstanding;
254  m_get_statistics_outstanding = value;
255  return old_value;
256  }
257 
261 
264 
267 
270 
272  Filesystem::Flags m_log_flush_method_meta {};
273 
275  Filesystem::Flags m_log_flush_method_user {};
276 
278 
281 
283  bool m_verbose {};
284 
286  bool m_startup {true};
287 
289  bool m_shutdown {};
290 
291  typedef map<String, PhantomRangeMapPtr> FailoverPhantomRangeMap;
292  FailoverPhantomRangeMap m_failover_map;
294 
299  std::shared_ptr<ConnectionHandler> m_master_connection_handler;
302 
305  uint32_t m_scanner_ttl;
306  uint64_t m_bytes_loaded;
308 
311 
313  int64_t m_stats_last_timestamp {};
314 
316  bool m_get_statistics_outstanding {};
317 
320 
323 
326 
330  int64_t m_scanner_buffer_size {};
331  time_t m_last_metrics_update {};
332  time_t m_next_metrics_update {};
333  double m_loadavg_accum {};
334  uint64_t m_page_in_accum {};
335  uint64_t m_page_out_accum {};
337  size_t m_metric_samples {};
338  size_t m_cores {};
340  CellsBuilder *m_pending_metrics_updates {};
341  std::chrono::steady_clock::time_point m_last_control_file_check;
342  int32_t m_control_file_check_interval {};
343  std::ofstream m_profile_query_out;
344  bool m_profile_query {};
346 
349 
352  };
353 
355  typedef std::shared_ptr<RangeServer> RangeServerPtr;
356 
358 
359 }}
360 
361 #endif // Hypertable_RangeServer_RangeServer_h
A memory buffer of static size.
Definition: StaticBuffer.h:45
Declarations for Status response callback.
static std::mutex mutex
Definition: Logger.cc:43
void initialize(const String &name)
Public initialization function - creates a singleton instance of LogWriter.
Definition: Logger.cc:45
Declarations for LogReplayBarrier.
Range specification.
Definition: RangeSpec.h:40
Abstract base class for a filesystem.
std::shared_ptr< StatsRangeServer > StatsRangeServerPtr
Smart pointer to StatsRangeServer.
UpdatePipelinePtr m_update_pipeline_metadata
Update pipeline for METADTA table.
Definition: RangeServer.h:263
std::chrono::steady_clock::time_point m_last_control_file_check
Definition: RangeServer.h:341
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
bool test_and_set_get_statistics_outstanding(bool value)
Performs a "test and set" operation on m_get_statistics_outstanding.
Definition: RangeServer.h:251
Declarations for MetaLogEntityRange.
std::map< String, SchemaPtr > TableSchemaMap
Definition: RangeServer.h:230
chrono::time_point< fast_clock > time_point
Definition: fast_clock.h:42
std::shared_ptr< UpdatePipeline > UpdatePipelinePtr
Smart pointer to UpdatePipeline.
Declarations for MetricsCollectorGanglia.
Declarations for TableIdentifier and TableIdentifierManaged.
Holds updates destined for a specific table.
Flags
Enumeration type for append flags.
Definition: Filesystem.h:76
Program options handling.
std::shared_ptr< Event > EventPtr
Smart pointer to Event.
Definition: Event.h:228
std::ofstream m_profile_query_out
Definition: RangeServer.h:343
Hyperspace::SessionPtr m_hyperspace
Definition: RangeServer.h:301
Declarations for ConnectionInitializer.
MaintenanceSchedulerPtr & get_scheduler()
Definition: RangeServer.h:204
Declarations for GroupCommitInterface.
Declarations for Context.
Declarations for Event.
std::shared_ptr< Context > ContextPtr
Smart pointer to Context.
Definition: Context.h:265
Hyperspace definitions
std::shared_ptr< CommitLogReader > CommitLogReaderPtr
Smart pointer to CommitLogReader.
std::shared_ptr< Client > ClientPtr
Definition: Client.h:156
Declaration of ClockT.
std::shared_ptr< LoadStatistics > LoadStatisticsPtr
Shared smart pointer to LoadStatistics.
Scan predicate and control specification.
Definition: ScanSpec.h:56
Declarations for TableInfoMap.
Response callback for status function.
Definition: Status.h:46
ScannerMap m_scanner_map
Outstanding scanner map.
Definition: RangeServer.h:304
Hash key to query cache.
Definition: QueryCache.h:54
Declarations for QualifiedRangeSpec and QualifiedRangeSpecManaged.
PropertiesPtr m_props
Configuration properties.
Definition: RangeServer.h:280
Declarations for RangeState.
StatsRangeServerPtr m_stats
Definition: RangeServer.h:309
std::shared_ptr< Session > SessionPtr
Definition: Session.h:734
void write_profile_data(const String &line)
Definition: RangeServer.h:218
Declarations for MetricsProcess.
bool status(ContextPtr &context, Timer &timer, Status &status)
Runs a status check on the master.
Definition: Utility.cc:408
Declarations for MaintenanceScheduler.
TimerHandlerPtr m_timer_handler
Smart pointer to timer handler.
Definition: RangeServer.h:325
LockSequencer m_existence_file_sequencer
Definition: RangeServer.h:298
UpdatePipelinePtr m_update_pipeline_system
Update pipeline for other (non-METADATA) system tables.
Definition: RangeServer.h:266
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
Logging routines and macros.
GroupCommitInterfacePtr m_group_commit
Definition: RangeServer.h:327
ConnectionManagerPtr m_conn_manager
Definition: RangeServer.h:295
std::shared_ptr< ConnectionHandler > m_master_connection_handler
Definition: RangeServer.h:299
std::shared_ptr< TimerHandler > TimerHandlerPtr
Smart pointer to TimerHandler.
Definition: TimerHandler.h:226
ApplicationQueuePtr get_application_queue()
Definition: RangeServer.h:206
std::shared_ptr< LogReplayBarrier > LogReplayBarrierPtr
Smart pointer to LogReplayBarrier.
Declarations for TimerHandler.
Map to hold outstanding scanners.
Definition: ScannerMap.h:45
std::shared_ptr< QueryCache > QueryCachePtr
Smart pointer to QueryCache.
Definition: QueryCache.h:252
NameIdMapperPtr m_namemap
Table name-to-ID mapper
Definition: RangeServer.h:319
Lib::Master::ClientPtr m_master_client
Definition: RangeServer.h:300
std::shared_ptr< MetricsCollectorGanglia > MetricsCollectorGangliaPtr
Smart pointer to MetricsCollectorGanglia.
Declarations for RangeSpec and RangeSpecManaged.
This class is used to generate and deliver standard responses back to a client.
UpdatePipelinePtr m_update_pipeline_user
Update pipeline for USER tables.
Definition: RangeServer.h:269
Hypertable definitions
Declarations for TableInfo.
std::shared_ptr< MetaLogEntityRange > MetaLogEntityRangePtr
Smart pointer to MetaLogEntityRange.
std::shared_ptr< TableInfo > TableInfoPtr
Smart pointer to TableInfo.
Definition: TableInfo.h:312
Declarations for UpdatePipeline.
ApplicationQueuePtr m_app_queue
Definition: RangeServer.h:296
Declarations for Comm.
MetricsCollectorGangliaPtr m_ganglia_collector
Ganglia metrics collector.
Definition: RangeServer.h:348
Computes and publishes general process metrics.
std::shared_ptr< GroupCommitTimerHandler > GroupCommitTimerHandlerPtr
Shared smart pointer to GroupCommitTimerHandler.
FailoverPhantomRangeMap m_failover_map
Definition: RangeServer.h:292
LoadStatisticsPtr m_load_statistics
Definition: RangeServer.h:310
Declarations for MasterClient This file contains declarations for MasterClient, a client interface cl...
Handles incoming Master requests.
MaintenanceSchedulerPtr m_maintenance_scheduler
Smart pointer to maintenance scheduler.
Definition: RangeServer.h:322
Qualified (with table identifier) range specification.
std::shared_ptr< RangeServer > RangeServerPtr
Shared smart pointer to RangeServer.
Definition: RangeServer.h:355
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
map< String, PhantomRangeMapPtr > FailoverPhantomRangeMap
Definition: RangeServer.h:291
Range state.
Definition: RangeState.h:48
Declarations for ApplicationQueue.
std::shared_ptr< MaintenanceScheduler > MaintenanceSchedulerPtr
Smart pointer to MaintenanceScheduler.
std::shared_ptr< ConnectionManager > ConnectionManagerPtr
Smart pointer to ConnectionManager.
Declarations for ResponseCallback.
std::shared_ptr< ApplicationQueue > ApplicationQueuePtr
Shared smart pointer to ApplicationQueue object.
Declarations for LoadStatistics.
std::shared_ptr< GroupCommitInterface > GroupCommitInterfacePtr
Smart pointer to GroupCommitInterface.
std::shared_ptr< NameIdMapper > NameIdMapperPtr
Smart pointer to NameIdMapper.
Definition: NameIdMapper.h:121
Manages live range map and set of log names that can be safely removed.
Definition: TableInfoMap.h:67
GroupCommitTimerHandlerPtr m_group_commit_timer_handler
Definition: RangeServer.h:328
MetricsProcess m_metrics_process
Process metrics.
Definition: RangeServer.h:351
Declarations for ScannerMap.
LogReplayBarrierPtr m_log_replay_barrier
Definition: RangeServer.h:260
RangeServer recovery receiver plan.
Definition: ReceiverPlan.h:48