0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
MaintenancePrioritizerLowMemory.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 #include "Common/Compat.h"
22 #include "Common/Config.h"
23 #include "Common/ScopeGuard.h"
24 #include "Common/StringExt.h"
25 
26 #include <iostream>
27 
28 #include "Global.h"
29 #include "MaintenanceFlag.h"
31 
32 using namespace Hypertable;
33 using namespace Hypertable::Config;
34 using namespace std;
35 
36 void
37 MaintenancePrioritizerLowMemory::prioritize(std::vector<RangeData> &range_data,
38  MemoryState &memory_state,
39  int32_t priority, String *trace) {
40  LoadStatistics::Bundle load_stats;
41  std::vector<RangeData> range_data_root;
42  std::vector<RangeData> range_data_metadata;
43  std::vector<RangeData> range_data_system;
44  std::vector<RangeData> range_data_user;
45 
46  for (size_t i=0; i<range_data.size(); i++) {
47  if (range_data[i].range->is_root())
48  range_data_root.push_back(range_data[i]);
49  else if (range_data[i].data->is_metadata)
50  range_data_metadata.push_back(range_data[i]);
51  else if (range_data[i].data->is_system)
52  range_data_system.push_back(range_data[i]);
53  else
54  range_data_user.push_back(range_data[i]);
55  }
56 
57  m_uninitialized_ranges_seen = false;
58 
62  if (!range_data_root.empty()) {
63  assign_priorities_all(range_data_root, Global::root_log,
65  memory_state, priority, trace);
66  schedule_initialization_operations(range_data_root, priority);
67  }
68 
69 
73  if (!range_data_metadata.empty()) {
74  assign_priorities_all(range_data_metadata, Global::metadata_log,
75  Global::log_prune_threshold_min, memory_state,
76  priority, trace);
77  schedule_initialization_operations(range_data_metadata, priority);
78  }
79 
83  Global::load_statistics->get(&load_stats);
84  int64_t prune_threshold = (int64_t)(load_stats.update_mbps * (double)Global::log_prune_threshold_max);
85  if (prune_threshold < Global::log_prune_threshold_min)
86  prune_threshold = Global::log_prune_threshold_min;
87  else if (prune_threshold > Global::log_prune_threshold_max)
88  prune_threshold = Global::log_prune_threshold_max;
89  if (trace)
90  *trace += format("%d prune threshold\t%lld\n", __LINE__, (Lld)prune_threshold);
91 
95  if (!range_data_system.empty()) {
96  assign_priorities_all(range_data_system, Global::system_log, prune_threshold,
97  memory_state, priority, trace);
98  schedule_initialization_operations(range_data_system, priority);
99  }
100 
104  if (!range_data_user.empty()) {
105 
106  if (schedule_inprogress_operations(range_data_user, memory_state, priority, trace))
107  schedule_splits_and_relinquishes(range_data, memory_state, priority, trace);
108 
109  assign_priorities_user(range_data_user, load_stats, memory_state,
110  priority, trace);
111 
112  schedule_necessary_compactions(range_data_user, Global::user_log,
113  prune_threshold, memory_state, priority,
114  trace);
115 
116  schedule_initialization_operations(range_data_user, priority);
117  }
118 
119  if (m_uninitialized_ranges_seen == false)
120  m_initialization_complete = true;
121 
122 }
123 
124 
132 void
134  CommitLogPtr &log, int64_t prune_threshold, MemoryState &memory_state,
135  int32_t &priority, String *trace) {
136 
137  if (!schedule_inprogress_operations(range_data, memory_state, priority, trace))
138  return;
139 
140  if (!schedule_splits_and_relinquishes(range_data, memory_state, priority, trace))
141  return;
142 
143  if (!schedule_necessary_compactions(range_data, log, prune_threshold,
144  memory_state, priority, trace))
145  return;
146 
147 }
148 
149 
171  std::vector<RangeData> &range_data, LoadStatistics::Bundle &load_stats,
172  MemoryState &memory_state, int32_t &priority, String *trace) {
173 
174  if (!purge_shadow_caches(range_data, memory_state, priority, trace))
175  return;
176 
177  if (load_stats.update_bytes < 500000 && load_stats.scan_count > 10) {
178 
179  HT_INFOF("READ workload prioritization (update_bytes=%llu, scan_count=%u)",
180  (Llu)load_stats.update_bytes, (unsigned)load_stats.scan_count);
181 
182  if (!compact_cellcaches(range_data, memory_state, priority, trace))
183  return;
184 
185  if (Global::block_cache) {
187  memory_state.decrement_needed( Global::block_cache->decrease_limit(memory_state.needed) );
188  if (!memory_state.need_more())
189  return;
190  }
191 
192  if (!purge_cellstore_indexes(range_data, memory_state, priority, trace))
193  return;
194 
195  }
196  else {
197 
198  HT_INFOF("WRITE workload prioritization (update_bytes=%llu, scan_count=%u)",
199  (Llu)load_stats.update_bytes, (unsigned)load_stats.scan_count);
200 
201  if (Global::block_cache) {
203  memory_state.decrement_needed( Global::block_cache->decrease_limit(memory_state.needed) );
204  if (!memory_state.need_more())
205  return;
206  }
207 
208  if (!purge_cellstore_indexes(range_data, memory_state, priority, trace))
209  return;
210 
211  if (!compact_cellcaches(range_data, memory_state, priority, trace))
212  return;
213 
214  }
215 
216 }
POD-style structure to hold statistics.
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
String format(const char *fmt,...)
Returns a String using printf like format facilities Vanilla snprintf is about 1.5x faster than this...
Definition: String.cc:37
long long unsigned int Llu
Shortcut for printf formats.
Definition: String.h:50
virtual void prioritize(std::vector< RangeData > &range_data, MemoryState &memory_state, int32_t priority, String *trace)
STL namespace.
static int64_t log_prune_threshold_max
Definition: Global.h:96
void assign_priorities_user(std::vector< RangeData > &range_data, LoadStatistics::Bundle &load_stats, MemoryState &memory_state, int32_t &priority, String *trace)
Memory freeing algorithm:
uint64_t update_bytes
Bytes updated.
static CommitLogPtr root_log
Definition: Global.h:80
Compatibility Macros for C/C++.
static int64_t log_prune_threshold_min
Definition: Global.h:95
void cap_memory_use()
Sets limit to memory currently used, it will not reduce the limit below min_memory.
Hypertable definitions
void assign_priorities_all(std::vector< RangeData > &range_data, CommitLogPtr &log, int64_t prune_threshold, MemoryState &memory_state, int32_t &priority, String *trace)
Memory freeing algorithm:
long long int Lld
Shortcut for printf formats.
Definition: String.h:53
static CommitLogPtr system_log
Definition: Global.h:78
double update_mbps
Megabytes/s updated.
#define HT_INFOF(msg,...)
Definition: Logger.h:272
std::shared_ptr< CommitLog > CommitLogPtr
Smart pointer to CommitLog.
Definition: CommitLog.h:223
static CommitLogPtr user_log
Definition: Global.h:77
static LoadStatisticsPtr load_statistics
Definition: Global.h:72
Configuration settings.
Declarations for MaintenanceFlag This file contains declarations that are part of the MaintenanceFlag...
String extensions and helpers: sets, maps, append operators etc.
static Hypertable::FileBlockCache * block_cache
Definition: Global.h:90
static CommitLogPtr metadata_log
Definition: Global.h:79
Executes user-defined functions when leaving the current scope.