0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
AccessGroup.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
26 
27 #ifndef Hypertable_RangeServer_AccessGroup_h
28 #define Hypertable_RangeServer_AccessGroup_h
29 
37 
39 #include <Hypertable/Lib/Schema.h>
41 
42 #include <Common/PageArena.h>
43 #include <Common/String.h>
44 #include <Common/StringExt.h>
45 
46 #include <boost/thread/condition.hpp>
47 
48 #include <condition_variable>
49 #include <cstdio>
50 #include <ctime>
51 #include <fstream>
52 #include <iostream>
53 #include <memory>
54 #include <mutex>
55 #include <queue>
56 #include <set>
57 #include <vector>
58 
59 namespace Hypertable {
60 
63 
65  class AccessGroup {
66 
67  public:
68 
70  public:
71  void *user_data;
79  };
80 
82  public:
83  void *user_data;
89  int64_t mem_used;
90  int64_t mem_allocated;
91  uint64_t cell_count;
92  int64_t disk_used;
93  int64_t disk_estimate;
95  int64_t key_bytes;
96  int64_t value_bytes;
97  uint32_t file_count;
98  int32_t deletes;
108  bool in_memory;
109  bool gc_needed;
111  bool end_merge;
112  };
113 
114  class Hints {
115  public:
117  void clear() {
118  ag_name.clear();
120  disk_usage = 0;
121  files.clear();
122  }
123  bool operator==(const Hints &other) const {
124  return ag_name == other.ag_name &&
126  disk_usage == other.disk_usage &&
127  files == other.files;
128  }
131  uint64_t disk_usage;
133  };
134 
135  AccessGroup(const TableIdentifier *identifier, SchemaPtr &schema,
136  AccessGroupSpec *ag_spec, const RangeSpec *range,
137  const Hints *hints=nullptr);
138 
142  void add(const Key &key, const ByteString value);
143 
145 
147 
156 
157  void update_schema(SchemaPtr &schema, AccessGroupSpec *ag_spec);
158 
159  void lock() {
160  m_mutex.lock();
161  m_cell_cache_manager->lock();
162  }
163 
164  void unlock() {
165  m_cell_cache_manager->unlock();
166  m_mutex.unlock();
167  }
168 
170 
171  bool include_in_scan(ScanContext *scan_ctx);
172  uint64_t disk_usage();
173  uint64_t memory_usage();
174  void space_usage(int64_t *memp, int64_t *diskp);
175 
176  void load_cellstore(CellStorePtr &cellstore);
177 
179  std::lock_guard<std::mutex> lock(m_mutex);
181  }
182 
186  if (!m_in_memory &&
190  }
191 
192  void measure_garbage(double *total, double *garbage);
193 
194  void run_compaction(int maintenance_flags, Hints *hints);
195 
196  uint64_t purge_memory(MaintenanceFlag::Map &subtask_map);
197 
198  MaintenanceData *get_maintenance_data(ByteArena &arena, time_t now,
199  int flags);
200 
201  void stage_compaction();
202 
203  void unstage_compaction();
204 
205  const char *get_name() { return m_name.c_str(); }
206 
207  const char *get_full_name() { return m_full_name.c_str(); }
208 
209  void shrink(String &split_row, bool drop_high, Hints *hints);
210 
211  void get_file_data(String &file_list, int64_t *block_countp, bool include_blocked) {
212  m_file_tracker.get_file_data(file_list, block_countp, include_blocked);
213  }
214 
215  void release_files(const std::vector<String> &files);
216 
218  void recovery_finalize() { m_recovering = false; }
219 
220  void dump_keys(std::ofstream &out);
221 
228  void dump_garbage_tracker_statistics(std::ofstream &out);
229 
230  void set_next_csid(uint32_t nid) {
231  if (nid > m_next_cs_id) {
232  m_next_cs_id = nid;
234  }
235  }
236 
237  void load_hints(Hints *hints);
238 
239  String describe();
240 
241  private:
242 
244  void merge_caches();
245  void range_dir_initialize();
246  void recompute_compression_ratio(int64_t *total_index_entriesp=0);
247 
248  bool find_merge_run(size_t *indexp=0, size_t *lenp=0);
249 
258  void get_merge_info(bool &needs_merging, bool &end_merge) {
259  size_t index, length;
260  needs_merging = find_merge_run(&index, &length);
261  if (needs_merging)
262  end_merge = (index + length) == m_stores.size();
263  }
264 
266 
270  std::condition_variable m_outstanding_scanner_cond;
274  std::set<uint8_t> m_column_families;
282  std::vector<CellStoreInfo> m_stores;
285  uint32_t m_next_cs_id {};
286  uint64_t m_disk_usage {};
287  float m_compression_ratio {1.0};
294  bool m_is_root {};
295  bool m_in_memory {};
296  bool m_recovering {};
298  bool m_end_merge {};
299  bool m_dirty {};
301  };
302 
303  typedef std::shared_ptr<AccessGroup> AccessGroupPtr;
304 
305  std::ostream &operator<<(std::ostream &os, const AccessGroup::MaintenanceData &mdata);
306 
308 
309 } // namespace Hypertable
310 
311 #endif // Hypertable_RangeServer_AccessGroup_h
MaintenanceData * get_maintenance_data(ByteArena &arena, time_t now, int flags)
Definition: AccessGroup.cc:344
void update_cellstore_info(std::vector< CellStoreInfo > &stores, time_t t=0, bool collection_performed=true)
Updates stored data statistics from current set of CellStores.
std::set< uint8_t > m_column_families
Definition: AccessGroup.h:274
static std::mutex mutex
Definition: Logger.cc:43
Cell list scanner over a buffer of cells.
void set_next_csid(uint32_t nid)
Definition: AccessGroup.h:230
void get_file_data(String &file_list, int64_t *block_countp, bool include_blocked)
Definition: AccessGroup.h:211
Range specification.
Definition: RangeSpec.h:40
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
Declarations for MergeScannerAccessGroup.
PageArena memory allocator.
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
void measure_garbage(double *total, double *garbage)
Definition: AccessGroup.cc:458
Tracks access group garbage and signals when collection is needed.
std::mutex m_outstanding_scanner_mutex
Definition: AccessGroup.h:269
int32_t m_outstanding_scanner_count
Definition: AccessGroup.h:271
const char * get_name()
Definition: AccessGroup.h:205
Declarations for TableIdentifier and TableIdentifierManaged.
void update_schema(SchemaPtr &schema, AccessGroupSpec *ag_spec)
Currently supports only adding and deleting column families from AccessGroup.
Definition: AccessGroup.cc:98
CellStoreMaintenanceData * csdata
Definition: AccessGroup.h:86
Maps object pointers to bit fields.
void load_hints(Hints *hints)
Definition: AccessGroup.cc:846
Scan context information.
Definition: ScanContext.h:52
bool operator==(const Hints &other) const
Definition: AccessGroup.h:123
bool include_in_scan(ScanContext *scan_ctx)
Definition: AccessGroup.cc:268
int64_t m_latest_stored_revision_hint
Definition: AccessGroup.h:291
void split_row_estimate_data_stored(CellList::SplitRowDataMapT &split_row_data)
Definition: AccessGroup.cc:285
Declarations for Schema.
static const int64_t TIMESTAMP_MIN
Definition: KeySpec.h:34
void run_compaction(int maintenance_flags, Hints *hints)
Definition: AccessGroup.cc:487
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
Wrapper for TableIdentifier providing member storage.
Declarations for CellStore.
void get_merge_info(bool &needs_merging, bool &end_merge)
Gets merging compaction information.
Definition: AccessGroup.h:258
int64_t m_earliest_cached_revision
Definition: AccessGroup.h:288
int64_t m_latest_stored_revision
Definition: AccessGroup.h:290
Tracks files that are live or referenced for purposes of maintaining the 'Files' METADATA column...
void merge_caches()
Assumes mutex is locked.
void get_file_data(String &file_list, int64_t *block_countp, bool include_blocked)
Returns ' ' separated list of files, suitable for writing into the 'Files' column of METADATA...
void set_next_csid(uint32_t nid)
std::map< const char *, int64_t, LtCstr, SplitRowDataAlloc > SplitRowDataMapT
Definition: CellList.h:66
std::condition_variable m_outstanding_scanner_cond
Definition: AccessGroup.h:270
void split_row_estimate_data_cached(CellList::SplitRowDataMapT &split_row_data)
Definition: AccessGroup.cc:279
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
void shrink(String &split_row, bool drop_high, Hints *hints)
Definition: AccessGroup.cc:899
void populate_cellstore_index_pseudo_table_scanner(CellListScannerBuffer *scanner)
Populates scanner with data for .cellstore.index pseudo table.
Definition: AccessGroup.cc:293
std::ostream & operator<<(std::ostream &os, const crontab_entry &entry)
Helper function to write crontab_entry to an ostream.
Definition: Crontab.cc:301
std::shared_ptr< AccessGroup > AccessGroupPtr
Definition: AccessGroup.h:303
void dump_garbage_tracker_statistics(std::ofstream &out)
Prints human-readable representation of garbage tracker state to an output stream.
void space_usage(int64_t *memp, int64_t *diskp)
Definition: AccessGroup.cc:312
void add(const Key &key, const ByteString value)
Adds a key/value pair.
Definition: AccessGroup.cc:153
void purge_stored_cells_from_cache()
Definition: AccessGroup.cc:867
Declarations for RangeSpec and RangeSpecManaged.
Access group specification.
std::shared_ptr< CellStore > CellStorePtr
Smart pointer to CellStore.
Definition: CellStore.h:340
Hypertable definitions
CellCacheManagerPtr m_cell_cache_manager
Definition: AccessGroup.h:284
static const int64_t TIMESTAMP_MAX
Definition: KeySpec.h:35
LiveFileTracker m_file_tracker
Definition: AccessGroup.h:292
AccessGroupGarbageTracker m_garbage_tracker
Definition: AccessGroup.h:293
const char * get_full_name()
Definition: AccessGroup.h:207
Provides access to internal components of opaque key.
Definition: Key.h:40
std::vector< CellStoreInfo > m_stores
Definition: AccessGroup.h:282
PropertiesPtr m_cellstore_props
Definition: AccessGroup.h:283
void recompute_compression_ratio(int64_t *total_index_entriesp=0)
A String class based on std::string.
Declarations for CellCacheManager.
TableIdentifierManaged m_identifier
Definition: AccessGroup.h:272
std::shared_ptr< Schema > SchemaPtr
Smart pointer to Schema.
Definition: Schema.h:465
AccessGroup(const TableIdentifier *identifier, SchemaPtr &schema, AccessGroupSpec *ag_spec, const RangeSpec *range, const Hints *hints=nullptr)
Definition: AccessGroup.cc:57
bool find_merge_run(size_t *indexp=0, size_t *lenp=0)
MergeScannerAccessGroup * create_scanner(ScanContext *scan_ctx)
Definition: AccessGroup.cc:187
Declarations for MaintenanceFlag This file contains declarations that are part of the MaintenanceFlag...
void dump_keys(std::ofstream &out)
void release_files(const std::vector< String > &files)
Access group.
Definition: AccessGroup.h:65
std::shared_ptr< CellCacheManager > CellCacheManagerPtr
Smart pointer to CellCacheManager.
String extensions and helpers: sets, maps, append operators etc.
int64_t m_earliest_cached_revision_saved
Definition: AccessGroup.h:289
uint64_t purge_memory(MaintenanceFlag::Map &subtask_map)
Definition: AccessGroup.cc:320
void load_cellstore(CellStorePtr &cellstore)
Definition: AccessGroup.cc:436
Declarations for AccessGroupGarbageTracker.
Merge scanner for access groups.