0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
MaintenancePrioritizerLogCleanup.cc
Go to the documentation of this file.
1 
21 #include "Common/Compat.h"
22 #include "Common/StringExt.h"
23 
24 #include <iostream>
25 
26 #include "Global.h"
27 #include "MaintenanceFlag.h"
29 
30 using namespace Hypertable;
31 using namespace std;
32 
33 void
34 MaintenancePrioritizerLogCleanup::prioritize(std::vector<RangeData> &range_data,
35  MemoryState &memory_state,
36  int32_t priority, String *trace) {
37  LoadStatistics::Bundle load_stats;
38  std::vector<RangeData> range_data_root;
39  std::vector<RangeData> range_data_metadata;
40  std::vector<RangeData> range_data_system;
41  std::vector<RangeData> range_data_user;
42 
43  for (size_t i=0; i<range_data.size(); i++) {
44  if (range_data[i].range->is_root())
45  range_data_root.push_back(range_data[i]);
46  else if (range_data[i].data->is_metadata)
47  range_data_metadata.push_back(range_data[i]);
48  else if (range_data[i].data->is_system)
49  range_data_system.push_back(range_data[i]);
50  else
51  range_data_user.push_back(range_data[i]);
52  }
53 
54  m_uninitialized_ranges_seen = false;
55 
59  if (!range_data_root.empty())
60  assign_priorities(range_data_root, Global::root_log,
62  memory_state, priority, trace);
63 
64 
68  if (!range_data_metadata.empty())
69  assign_priorities(range_data_metadata, Global::metadata_log,
71  memory_state, priority, trace);
72 
76  Global::load_statistics->get(&load_stats);
77  int64_t prune_threshold = (int64_t)(load_stats.update_mbps * (double)Global::log_prune_threshold_max);
78  if (prune_threshold < Global::log_prune_threshold_min)
79  prune_threshold = Global::log_prune_threshold_min;
80  else if (prune_threshold > Global::log_prune_threshold_max)
81  prune_threshold = Global::log_prune_threshold_max;
82  if (trace)
83  *trace += format("%d prune threshold\t%lld\n", __LINE__, (Lld)prune_threshold);
84 
88  if (!range_data_system.empty())
89  assign_priorities(range_data_system, Global::system_log, prune_threshold,
90  memory_state, priority, trace);
91 
96  if (!range_data_user.empty())
97  assign_priorities(range_data_user, Global::user_log, prune_threshold,
98  memory_state, priority, trace);
99 
100  if (m_uninitialized_ranges_seen == false)
101  m_initialization_complete = true;
102 
107  if (load_stats.update_bytes == 0 ||
108  (load_stats.update_bytes < 1000000 && load_stats.scan_count > 20)) {
109  if (memory_state.balance < memory_state.limit) {
110  int64_t available = memory_state.limit - memory_state.balance;
111  if (Global::block_cache) {
112  int64_t block_cache_available = Global::block_cache->available();
113  if (block_cache_available < available) {
114  HT_INFOF("Increasing block cache limit by %lld",
115  (Lld)available - block_cache_available);
116  Global::block_cache->increase_limit(available - block_cache_available);
117  }
118  }
119  }
120  }
121 
122 }
123 
124 
130 void
131 MaintenancePrioritizerLogCleanup::assign_priorities(std::vector<RangeData> &range_data,
132  CommitLogPtr &log, int64_t prune_threshold, MemoryState &memory_state,
133  int32_t &priority, String *trace) {
134 
135  // 1. Schedule deferred initialization tasks
136  schedule_initialization_operations(range_data, priority);
137 
138  // 2. Schedule in-progress relinquish and/or split operations
139  schedule_inprogress_operations(range_data, memory_state, priority, trace);
140 
141  // 3. Schedule splits and relinquishes
142  schedule_splits_and_relinquishes(range_data, memory_state, priority, trace);
143 
144  // 4. Schedule compactions for log cleaning purposes
145  schedule_necessary_compactions(range_data, log, prune_threshold,
146  memory_state, priority, trace);
147 
148 }
POD-style structure to hold statistics.
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
String format(const char *fmt,...)
Returns a String using printf like format facilities Vanilla snprintf is about 1.5x faster than this...
Definition: String.cc:37
STL namespace.
static int64_t log_prune_threshold_max
Definition: Global.h:96
uint64_t update_bytes
Bytes updated.
static CommitLogPtr root_log
Definition: Global.h:80
Compatibility Macros for C/C++.
static int64_t log_prune_threshold_min
Definition: Global.h:95
void increase_limit(int64_t amount)
void assign_priorities(std::vector< RangeData > &range_data, CommitLogPtr &log, int64_t prune_threshold, MemoryState &memory_state, int32_t &priority, String *trace)
Hypertable definitions
long long int Lld
Shortcut for printf formats.
Definition: String.h:53
static CommitLogPtr system_log
Definition: Global.h:78
double update_mbps
Megabytes/s updated.
#define HT_INFOF(msg,...)
Definition: Logger.h:272
std::shared_ptr< CommitLog > CommitLogPtr
Smart pointer to CommitLog.
Definition: CommitLog.h:223
static CommitLogPtr user_log
Definition: Global.h:77
static LoadStatisticsPtr load_statistics
Definition: Global.h:72
virtual void prioritize(std::vector< RangeData > &range_data, MemoryState &memory_state, int32_t prioritize, String *trace)
Declarations for MaintenanceFlag This file contains declarations that are part of the MaintenanceFlag...
String extensions and helpers: sets, maps, append operators etc.
static Hypertable::FileBlockCache * block_cache
Definition: Global.h:90
static CommitLogPtr metadata_log
Definition: Global.h:79