libdap++  Updated for version 3.12.0
DAP4StreamMarshaller.cc
Go to the documentation of this file.
1 // DAP4StreamMarshaller.cc
2 
3 // -*- mode: c++; c-basic-offset:4 -*-
4 
5 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
6 // Access Protocol.
7 
8 // Copyright (c) 2012 OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 //
27 // Portions of this code are from Google's great protocol buffers library and
28 // are copyrighted as follows:
29 
30 // Protocol Buffers - Google's data interchange format
31 // Copyright 2008 Google Inc. All rights reserved.
32 // http://code.google.com/p/protobuf/
33 
34 
35 #include "DAP4StreamMarshaller.h"
36 
37 #include <stdint.h> // for the Google protobuf code
38 #include <byteswap.h>
39 
40 #include <iostream>
41 #include <sstream>
42 #include <iomanip>
43 
44 using namespace std;
45 
46 //#define DODS_DEBUG 1
47 
48 #include "dods-datatypes.h"
49 #include "util.h"
50 #include "debug.h"
51 
52 namespace libdap {
53 
54 static inline bool is_host_big_endian()
55 {
56 #ifdef COMPUTE_ENDIAN_AT_RUNTIME
57 
58  dods_int16 i = 0x0100;
59  char *c = reinterpret_cast<char*>(&i);
60  return *c;
61 
62 #else
63 
64 #ifdef WORDS_BIGENDIAN
65  return true;
66 #else
67  return false;
68 #endif
69 
70 #endif
71 }
72 
73 
74 // From the Google protobuf library
75 inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) {
76  // Splitting into 32-bit pieces gives better performance on 32-bit
77  // processors.
78  uint32_t part0 = static_cast<uint32_t>(value );
79  uint32_t part1 = static_cast<uint32_t>(value >> 28);
80  uint32_t part2 = static_cast<uint32_t>(value >> 56);
81 
82  int size;
83 
84  // Here we can't really optimize for small numbers, since the value is
85  // split into three parts. Checking for numbers < 128, for instance,
86  // would require three comparisons, since you'd have to make sure part1
87  // and part2 are zero. However, if the caller is using 64-bit integers,
88  // it is likely that they expect the numbers to often be very large, so
89  // we probably don't want to optimize for small numbers anyway. Thus,
90  // we end up with a hard coded binary search tree...
91  if (part2 == 0) {
92  if (part1 == 0) {
93  if (part0 < (1 << 14)) {
94  if (part0 < (1 << 7)) {
95  size = 1; goto size1;
96  } else {
97  size = 2; goto size2;
98  }
99  } else {
100  if (part0 < (1 << 21)) {
101  size = 3; goto size3;
102  } else {
103  size = 4; goto size4;
104  }
105  }
106  } else {
107  if (part1 < (1 << 14)) {
108  if (part1 < (1 << 7)) {
109  size = 5; goto size5;
110  } else {
111  size = 6; goto size6;
112  }
113  } else {
114  if (part1 < (1 << 21)) {
115  size = 7; goto size7;
116  } else {
117  size = 8; goto size8;
118  }
119  }
120  }
121  } else {
122  if (part2 < (1 << 7)) {
123  size = 9; goto size9;
124  } else {
125  size = 10; goto size10;
126  }
127  }
128 
129  // GOOGLE_LOG(FATAL) << "Can't get here.";
130 
131  size10: target[9] = static_cast<uint8_t>((part2 >> 7) | 0x80);
132  size9 : target[8] = static_cast<uint8_t>((part2 ) | 0x80);
133  size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80);
134  size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80);
135  size6 : target[5] = static_cast<uint8_t>((part1 >> 7) | 0x80);
136  size5 : target[4] = static_cast<uint8_t>((part1 ) | 0x80);
137  size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80);
138  size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80);
139  size2 : target[1] = static_cast<uint8_t>((part0 >> 7) | 0x80);
140  size1 : target[0] = static_cast<uint8_t>((part0 ) | 0x80);
141 
142  target[size-1] &= 0x7F;
143  return target + size;
144 }
145 
153 DAP4StreamMarshaller::DAP4StreamMarshaller(ostream &out, bool write_data) :
154  d_out(out), d_ctx(0), d_write_data(write_data), d_checksum_ctx_valid(false)
155 {
156  // XDR is used if the call std::numeric_limits<double>::is_iec559()
157  // returns false indicating that the compiler is not using IEEE 754.
158  // If it is, we just write out the bytes. Why malloc()? Because
159  // xdr_destroy is going to call free() for us.
160  d_ieee754_buf = (char*)malloc(sizeof(dods_float64));
161  if (!d_ieee754_buf)
162  throw InternalErr(__FILE__, __LINE__, "Could not create DAP4StreamMarshaller");
163  xdrmem_create(&d_scalar_sink, d_ieee754_buf, sizeof(dods_float64), XDR_ENCODE);
164 
165  // This will cause exceptions to be thrown on i/o errors. The exception
166  // will be ostream::failure
167  out.exceptions(ostream::failbit | ostream::badbit);
168 
169  d_ctx = EVP_MD_CTX_create();
170 }
171 
173 {
174  // Free the buffer this contains. The xdr_destroy() macro does not
175  // free the XDR struct (which is fine since we did not dynamically
176  // allocate it).
177  xdr_destroy (&d_scalar_sink);
178 
179  EVP_MD_CTX_destroy(d_ctx);
180 }
181 
188 string
190 {
191  return (is_host_big_endian()) ? "big": "little";
192 }
193 
199 {
200  if (EVP_DigestInit_ex(d_ctx, EVP_md5(), 0) == 0)
201  throw Error("Failed to initialize checksum object.");
202 
203  d_checksum_ctx_valid = true;
204 }
205 
209 void DAP4StreamMarshaller::m_compute_checksum()
210 {
211  if (d_checksum_ctx_valid) {
212  // '...Final()' 'erases' the context so the next call without a reset
213  // returns a bogus value.
214  d_checksum_ctx_valid = false;
215 
216  // For MD5, the array md holds the 16 digits of the hash as values.
217  // The loop below turns that into a nice 32-digit hex number; see
218  // put_checksum() for a version that writes the 128-bit value without
219  // that conversion.
220  unsigned int md_len;
221  int status = EVP_DigestFinal_ex(d_ctx, &d_md[0], &md_len);
222  if (status == 0 || md_len != c_md5_length)
223  throw Error("Error computing the checksum (checksum computation).");
224  }
225 }
226 
233 {
234  if (d_checksum_ctx_valid) {
235  m_compute_checksum();
236  }
237 
238  ostringstream oss;
239  oss.setf(ios::hex, ios::basefield);
240  for (unsigned int i = 0; i < c_md5_length; ++i) {
241  oss << setfill('0') << setw(2) << (unsigned int) d_md[i];
242  }
243 
244  return oss.str();
245 }
246 
248 {
249  if (d_checksum_ctx_valid) {
250  m_compute_checksum();
251  }
252 
253  d_out.write(reinterpret_cast<char*>(&d_md[0]), c_md5_length);
254 }
255 
256 void DAP4StreamMarshaller::checksum_update(const void *data, unsigned long len)
257 {
258  if (!d_checksum_ctx_valid)
259  throw InternalErr(__FILE__, __LINE__, "Invalid checksum context (checksum update).");
260 
261  if (EVP_DigestUpdate(d_ctx, data, len) == 0) {
262  d_checksum_ctx_valid = false;
263  throw Error("Error computing the checksum (checksum update).");
264  }
265 }
266 
268 {
269  checksum_update(&val, sizeof(dods_byte));
270 
271  if (d_write_data) {
272  DBG( std::cerr << "put_byte: " << val << std::endl );
273 
274  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_byte));
275  }
276 }
277 
279 {
280  checksum_update(&val, sizeof(dods_int8));
281 
282  if (d_write_data) {
283  DBG( std::cerr << "put_int8: " << val << std::endl );
284 
285  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int8));
286  }
287 }
288 
290 {
291  checksum_update(&val, sizeof(dods_int16));
292 
293  if (d_write_data)
294  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int16));
295 }
296 
298 {
299  checksum_update(&val, sizeof(dods_int32));
300 
301  if (d_write_data)
302  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int32));
303 }
304 
306 {
307  checksum_update(&val, sizeof(dods_int64));
308 
309  if (d_write_data)
310  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int64));
311 }
312 
314 {
315  checksum_update(&val, sizeof(dods_float32));
316 
317  if (d_write_data) {
318  if (std::numeric_limits<float>::is_iec559 ) {
319  DBG2(cerr << "Native rep is ieee754." << endl);
320  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float32));
321  }
322  else {
323  if (!xdr_setpos(&d_scalar_sink, 0))
324  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
325 
326  if (!xdr_float(&d_scalar_sink, &val))
327  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
328 
329  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float32))
330  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
331 
332  // If this is a little-endian host, twiddle the bytes
333  static bool twiddle_bytes = !is_host_big_endian();
334  if (twiddle_bytes) {
335  dods_int32 *i = reinterpret_cast<dods_int32*>(&d_ieee754_buf);
336  *i = bswap_32(*i);
337  }
338 
339  d_out.write(d_ieee754_buf, sizeof(dods_float32));
340  }
341  }
342 }
343 
345 {
346  checksum_update(&val, sizeof(dods_float64));
347 
348  if (d_write_data) {
349  if (std::numeric_limits<double>::is_iec559)
350  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float64));
351  else {
352  if (!xdr_setpos(&d_scalar_sink, 0))
353  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
354 
355  if (!xdr_double(&d_scalar_sink, &val))
356  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
357 
358  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float64))
359  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
360 
361  // If this is a little-endian host, twiddle the bytes
362  static bool twiddle_bytes = !is_host_big_endian();
363  if (twiddle_bytes) {
364  dods_int64 *i = reinterpret_cast<dods_int64*>(&d_ieee754_buf);
365  *i = bswap_64(*i);
366  }
367 
368  d_out.write(d_ieee754_buf, sizeof(dods_float64));
369  }
370  }
371 }
372 
374 {
375  checksum_update(&val, sizeof(dods_uint16));
376 
377  if (d_write_data)
378  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint16));
379 }
380 
382 {
383  checksum_update(&val, sizeof(dods_uint32));
384 
385  if (d_write_data)
386  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint32));
387 }
388 
390 {
391  checksum_update(&val, sizeof(dods_uint64));
392 
393  if (d_write_data)
394  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint64));
395 }
396 
397 
398 void DAP4StreamMarshaller::put_str(const string &val)
399 {
400  checksum_update(val.c_str(), val.length());
401 
402  if (d_write_data) {
403  put_length_prefix(val.length());
404  d_out.write(val.data(), val.length());
405  }
406 }
407 
408 void DAP4StreamMarshaller::put_url(const string &val)
409 {
410  put_str(val);
411 }
412 
413 void DAP4StreamMarshaller::put_opaque(char *val, unsigned int len)
414 {
415  checksum_update(val, len);
416 
417  if (d_write_data) {
418  put_length_prefix(len);
419  d_out.write(val, len);
420  }
421 }
422 
424 {
425  if (d_write_data) {
426  DBG2(cerr << "val: " << val << endl);
427 
428  vector<uint8_t> target(sizeof(dods_uint64) + 1, 0);
429  uint8_t* to_send = WriteVarint64ToArrayInline(val, &target[0]);
430  d_out.write(reinterpret_cast<char*>(&target[0]), to_send - &target[0]);
431 
432  DBG2(cerr << "varint: " << hex << *(uint64_t*)&target[0] << dec << endl);
433  }
434 }
435 
436 void DAP4StreamMarshaller::put_vector(char *val, unsigned int num)
437 {
438  checksum_update(val, num);
439 
440  d_out.write(val, num);
441 }
442 
453 void DAP4StreamMarshaller::put_varying_vector(char *val, unsigned int num)
454 {
455  put_opaque(val, num);
456 }
457 
463 void DAP4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type)
464 {
465  dods_uint64 size = num * width;
466  // This is a leak!!! xdr_destroy does not free 'buf'.
467  //char *buf = (char*)malloc(size);
468  vector<char> buf(size);
469  XDR xdr;
470  xdrmem_create(&xdr, &buf[0], size, XDR_ENCODE);
471  try {
472  if(!xdr_array(&xdr, &val, (unsigned int *)&num, size, width, XDRUtils::xdr_coder(type)))
473  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
474 
475  if (xdr_getpos(&xdr) != size)
476  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
477 
478  // If this is a little-endian host, twiddle the bytes
479  static bool twiddle_bytes = !is_host_big_endian();
480  if (twiddle_bytes) {
481  if (width == 4) {
482  dods_float32 *lbuf = reinterpret_cast<dods_float32*>(&buf[0]);
483  while (num--) {
484  dods_int32 *i = reinterpret_cast<dods_int32*>(lbuf++);
485  *i = bswap_32(*i);
486  }
487  }
488  else { // width == 8
489  dods_float64 *lbuf = reinterpret_cast<dods_float64*>(&buf[0]);
490  while (num--) {
491  dods_int64 *i = reinterpret_cast<dods_int64*>(lbuf++);
492  *i = bswap_64(*i);
493  }
494  }
495  }
496 
497  d_out.write(&buf[0], size);
498  }
499  catch (...) {
500  xdr_destroy(&xdr);
501  throw;
502  }
503  xdr_destroy(&xdr);
504 }
505 
506 void DAP4StreamMarshaller::put_vector(char *val, unsigned int num, int width, Type type)
507 {
508  checksum_update(val, num * width);
509 
510  if (d_write_data) {
511  if (type == dods_float32_c && !std::numeric_limits<float>::is_iec559) {
512  // If not using IEEE 754, use XDR to get it that way.
513  m_serialize_reals(val, num, 4, type);
514  }
515  else if (type == dods_float64_c && !std::numeric_limits<double>::is_iec559) {
516  m_serialize_reals(val, num, 8, type);
517  }
518  else {
519  d_out.write(val, num * width);
520  }
521  }
522 }
523 
535 void DAP4StreamMarshaller::put_varying_vector(char *val, unsigned int num, int width, Type type)
536 {
537  put_length_prefix(num);
538  put_vector(val, num, width, type);
539 }
540 
541 void DAP4StreamMarshaller::dump(ostream &strm) const
542 {
543  strm << DapIndent::LMarg << "DAP4StreamMarshaller::dump - (" << (void *) this << ")" << endl;
544 }
545 
546 } // namespace libdap
547 
virtual void put_uint16(dods_uint16 val)
uint8_t dods_byte
static const unsigned int c_md5_length
int64_t dods_int64
virtual void put_varying_vector(char *val, unsigned int num)
DINT64 dods_int64
virtual void put_float64(dods_float64 val)
virtual void put_float32(dods_float32 val)
virtual void put_vector(char *val, unsigned int num)
virtual void dump(ostream &strm) const
dump the contents of this object to the specified ostream
virtual void put_opaque(char *val, unsigned int len)
Type
Identifies the data type.
Definition: BaseType.h:137
uint16_t dods_uint16
#define DBG2(x)
Definition: debug.h:73
uint64_t dods_uint64
virtual string get_endian() const
A class for software fault reporting.
Definition: InternalErr.h:64
uint8_t * WriteVarint64ToArrayInline(uint64_t value, uint8_t *target)
#define DBG(x)
Definition: debug.h:58
virtual void put_int32(dods_int32 val)
double dods_float64
virtual void checksum_update(const void *data, unsigned long len)
virtual void put_int16(dods_int16 val)
uint32_t dods_uint32
virtual void put_int8(dods_int8 val)
virtual void put_byte(dods_byte val)
virtual void put_url(const string &val)
virtual void put_uint64(dods_uint64 val)
static ostream & LMarg(ostream &strm)
Definition: DapIndent.cc:78
int16_t dods_int16
virtual void put_uint32(dods_uint32 val)
virtual void put_str(const string &val)
virtual void put_int64(dods_int64 val)
DFLOAT64 dods_float64
static xdrproc_t xdr_coder(const Type &t)
Returns a function used to encode elements of an array.
Definition: XDRUtils.cc:143
A class for error processing.
Definition: Error.h:90
DINT16 dods_int16
virtual void put_length_prefix(dods_uint64 val)
int32_t dods_int32