0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
BlockCompressionCodecLzo.cc
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
26 
27 #include <Common/Compat.h>
28 
30 
31 #include <Common/DynamicBuffer.h>
32 #include <Common/Error.h>
33 #include <Common/Logger.h>
34 #include <Common/Checksum.h>
35 
36 #include <ThirdParty/lzo/minilzo.h>
37 
38 using namespace Hypertable;
39 
40 namespace {
41  const uint8_t fence_marker[4] = { 0x11, 0x11, 0x11, 0x11 };
42 }
43 
44 
46  : m_workmem(new uint8_t [LZO1X_1_MEM_COMPRESS + 4]) {
47  if (lzo_init() != LZO_E_OK)
49  "Problem initializing lzo library");
50  memcpy(&m_workmem[LZO1X_1_MEM_COMPRESS], fence_marker, 4);
51  set_args(args);
52 }
53 
54 
56 }
57 
58 
59 
63 void
65  DynamicBuffer &output, BlockHeader &header, size_t reserve) {
66  uint32_t avail_out = (input.fill() + input.fill() / 16 + 64 + 3 + 4);
67  int ret;
68  lzo_uint out_len = avail_out;
69  uint8_t *fence_ptr = 0;
70 
71  output.clear();
72  output.reserve(header.encoded_length() + avail_out + reserve);
73 
74  fence_ptr = output.base + header.encoded_length() + (avail_out-4);
75  memcpy(fence_ptr, fence_marker, 4);
76 
77  ret = lzo1x_1_compress(input.base, input.fill(), output.base+header.encoded_length(),
78  &out_len, m_workmem.get());
79  assert(ret == LZO_E_OK);
80  (void)ret;
81 
82  /* check for an incompressible block */
83  if (out_len >= input.fill()) {
84  header.set_compression_type(NONE);
85  memcpy(output.base+header.encoded_length(), input.base, input.fill());
86  header.set_data_length(input.fill());
87  header.set_data_zlength(input.fill());
88  }
89  else {
90  header.set_compression_type(LZO);
91  header.set_data_length(input.fill());
92  header.set_data_zlength(out_len);
93  }
94  header.set_data_checksum(fletcher32(output.base + header.encoded_length(),
95  header.get_data_zlength()));
96 
97  output.ptr = output.base;
98  header.encode(&output.ptr);
99  output.ptr += header.get_data_zlength();
100 
101  HT_ASSERT(!memcmp(fence_ptr, fence_marker, 4));
102  HT_ASSERT(!memcmp(&m_workmem[LZO1X_1_MEM_COMPRESS], fence_marker, 4));
103 }
104 
105 
109 void
111  DynamicBuffer &output, BlockHeader &header) {
112  int ret;
113  const uint8_t *msg_ptr = input.base;
114  size_t remaining = input.fill();
115  lzo_uint new_len;
116 
117  header.decode(&msg_ptr, &remaining);
118 
119  if (header.get_data_zlength() > remaining) {
120  HT_ERRORF("Block decompression error, header zlength = %d, actual = %d",
121  (int)header.get_data_zlength(), (int)remaining);
123  }
124 
125  uint32_t checksum = fletcher32(msg_ptr, header.get_data_zlength());
126  if (checksum != header.get_data_checksum()) {
127  HT_ERRORF("Compressed block checksum mismatch header=%u, computed=%u",
128  header.get_data_checksum(), checksum);
130  }
131 
132  output.reserve(header.get_data_length());
133 
134  // check compress bit
135  if (header.get_compression_type() == NONE)
136  memcpy(output.base, msg_ptr, header.get_data_length());
137  else {
138  new_len = header.get_data_length();
139  ret = lzo1x_decompress(msg_ptr, header.get_data_zlength(), output.base,
140  &new_len, 0);
141  if (ret != LZO_E_OK || new_len != header.get_data_length()) {
142  HT_ERRORF("Lzo decompression error, rval = %d", ret);
144  }
145  }
146  output.ptr = output.base + header.get_data_length();
147 }
BlockCompressionCodecLzo(const Args &args)
Constructor.
void set_data_length(uint32_t length)
Sets the uncompressed data length field.
Definition: BlockHeader.h:91
void set_data_checksum(uint32_t checksum)
Sets the checksum field.
Definition: BlockHeader.h:113
std::unique_ptr< uint8_t[]> m_workmem
Working memory buffer used by deflate() and inflate()
virtual void set_args(const Args &args)
Sets arguments to control compression behavior.
uint16_t get_compression_type()
Gets the compression type field.
Definition: BlockHeader.h:128
void set_data_zlength(uint32_t zlength)
Sets the compressed data length field.
Definition: BlockHeader.h:101
uint8_t * ptr
Pointer to the end of the used part of the buffer.
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
uint32_t get_data_checksum()
Gets the checksum field.
Definition: BlockHeader.h:118
std::vector< String > Args
Compression codec argument vector.
#define HT_ASSERT(_e_)
Definition: Logger.h:396
uint32_t fletcher32(const void *data8, size_t len8)
Compute fletcher32 checksum for arbitary data.
Definition: Checksum.cc:42
uint32_t get_data_length()
Gets the uncompressed data length field.
Definition: BlockHeader.h:96
A dynamic, resizable memory buffer.
Logging routines and macros.
Compatibility Macros for C/C++.
virtual void decode(const uint8_t **bufp, size_t *remainp)
Decodes serialized block header.
Definition: BlockHeader.cc:104
Hypertable definitions
Implementation of checksum routines.
void clear()
Clears the buffer.
void set_compression_type(uint16_t type)
Sets the compression type field.
Definition: BlockHeader.h:123
virtual void deflate(const DynamicBuffer &input, DynamicBuffer &output, BlockHeader &header, size_t reserve=0)
Compresses a buffer using the LZO algorithm.
uint8_t * base
Pointer to the allocated memory buffer.
virtual void inflate(const DynamicBuffer &input, DynamicBuffer &output, BlockHeader &header)
Decompresses a buffer compressed with the LZO algorithm.
size_t fill() const
Returns the size of the used portion.
Definition: DynamicBuffer.h:70
virtual size_t encoded_length()
Returns length of serizlized block header.
Definition: BlockHeader.cc:76
#define HT_ERRORF(msg,...)
Definition: Logger.h:300
uint32_t get_data_zlength()
Gets the compressed data length field.
Definition: BlockHeader.h:106
Declarations for BlockCompressionCodecLzo.
Base class for block headers.
Definition: BlockHeader.h:48
Error codes, Exception handling, error logging.
#define HT_THROW(_code_, _msg_)
Definition: Error.h:478
void reserve(size_t len, bool nocopy=false)
Reserve space for additional data Will grow the space to exactly what's needed.
Definition: DynamicBuffer.h:95
virtual void encode(uint8_t **bufp)
Encodes serialized representation of block header.
Definition: BlockHeader.cc:82