Back to index

tor  0.2.3.18-rc
Classes | Defines | Functions | Variables
torgzip.c File Reference

A simple in-memory gzip implementation. More...

#include "orconfig.h"
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include "torint.h"
#include "util.h"
#include "torlog.h"
#include "torgzip.h"
#include <zlib.h>

Go to the source code of this file.

Classes

struct  tor_zlib_state_t
 Internal state for an incremental zlib compression/decompression. More...

Defines

#define _LARGEFILE64_SOURCE   0
#define _LFS64_LARGEFILE   0
#define _FILE_OFFSET_BITS   0
#define off64_t   int64_t
#define MAX_UNCOMPRESSION_FACTOR   25
#define CHECK_FOR_COMPRESSION_BOMB_AFTER   (1024*64)

Functions

int is_gzip_supported (void)
 Return true iff we support gzip-based compression.
static INLINE int method_bits (compress_method_t method)
 Return the 'bits' value to tell zlib to use method.
static int is_compression_bomb (size_t size_in, size_t size_out)
 Return true if uncompressing an input of size in_size to an input of size at least size_out looks like a compression bomb.
int tor_gzip_compress (char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method)
 Given in_len bytes at in, compress them into a newly allocated buffer, using the method described in method.
int tor_gzip_uncompress (char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method, int complete_only, int protocol_warn_level)
 Given zero or more zlib-compressed or gzip-compressed strings of total length in_len bytes at in, uncompress them into a newly allocated buffer, using the method described in method.
compress_method_t detect_compression_method (const char *in, size_t in_len)
 Try to tell whether the in_len-byte string in in is likely to be compressed or not.
tor_zlib_state_ttor_zlib_new (int compress, compress_method_t method)
 Construct and return a tor_zlib_state_t object using method.
tor_zlib_output_t tor_zlib_process (tor_zlib_state_t *state, char **out, size_t *out_len, const char **in, size_t *in_len, int finish)
 Compress/decompress some bytes using state.
void tor_zlib_free (tor_zlib_state_t *state)
 Deallocate state.

Variables

static int gzip_is_supported = -1
 Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; set to -1 if we haven't checked yet.

Detailed Description

A simple in-memory gzip implementation.

Definition in file torgzip.c.


Class Documentation

struct tor_zlib_state_t

Internal state for an incremental zlib compression/decompression.

The body of this struct is not exposed.

Definition at line 390 of file torgzip.c.

Class Members
int compress True if we are compressing; false if we are inflating.
size_t input_so_far Number of bytes read so far. Used to detect zlib bombs.
size_t output_so_far Number of bytes written so far. Used to detect zlib bombs.

Define Documentation

#define _FILE_OFFSET_BITS   0

Definition at line 41 of file torgzip.c.

#define _LARGEFILE64_SOURCE   0

Definition at line 35 of file torgzip.c.

#define _LFS64_LARGEFILE   0

Definition at line 38 of file torgzip.c.

#define CHECK_FOR_COMPRESSION_BOMB_AFTER   (1024*64)

Definition at line 94 of file torgzip.c.

#define MAX_UNCOMPRESSION_FACTOR   25

Definition at line 93 of file torgzip.c.

#define off64_t   int64_t

Definition at line 44 of file torgzip.c.


Function Documentation

compress_method_t detect_compression_method ( const char *  in,
size_t  in_len 
)

Try to tell whether the in_len-byte string in in is likely to be compressed or not.

If it is, return the likeliest compression method. Otherwise, return UNKNOWN_METHOD.

Definition at line 376 of file torgzip.c.

{
  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
    return GZIP_METHOD;
  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
             (ntohs(get_uint16(in)) % 31) == 0) {
    return ZLIB_METHOD;
  } else {
    return UNKNOWN_METHOD;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

static int is_compression_bomb ( size_t  size_in,
size_t  size_out 
) [static]

Return true if uncompressing an input of size in_size to an input of size at least size_out looks like a compression bomb.

Definition at line 100 of file torgzip.c.

{
  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
    return 0;

  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
}

Here is the caller graph for this function:

int is_gzip_supported ( void  )

Return true iff we support gzip-based compression.

Otherwise, we need to use zlib.

Definition at line 56 of file torgzip.c.

{
  if (gzip_is_supported >= 0)
    return gzip_is_supported;

  if (!strcmpstart(ZLIB_VERSION, "0.") ||
      !strcmpstart(ZLIB_VERSION, "1.0") ||
      !strcmpstart(ZLIB_VERSION, "1.1"))
    gzip_is_supported = 0;
  else
    gzip_is_supported = 1;

  return gzip_is_supported;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static INLINE int method_bits ( compress_method_t  method) [static]

Return the 'bits' value to tell zlib to use method.

Definition at line 73 of file torgzip.c.

{
  /* Bits+16 means "use gzip" in zlib >= 1.2 */
  return method == GZIP_METHOD ? 15+16 : 15;
}

Here is the caller graph for this function:

int tor_gzip_compress ( char **  out,
size_t *  out_len,
const char *  in,
size_t  in_len,
compress_method_t  method 
)

Given in_len bytes at in, compress them into a newly allocated buffer, using the method described in method.

Store the compressed string in *out, and its length in *out_len. Return 0 on success, -1 on failure.

Definition at line 114 of file torgzip.c.

{
  struct z_stream_s *stream = NULL;
  size_t out_size, old_size;
  off_t offset;

  tor_assert(out);
  tor_assert(out_len);
  tor_assert(in);
  tor_assert(in_len < UINT_MAX);

  *out = NULL;

  if (method == GZIP_METHOD && !is_gzip_supported()) {
    /* Old zlib version don't support gzip in deflateInit2 */
    log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
    goto err;
  }

  stream = tor_malloc_zero(sizeof(struct z_stream_s));
  stream->zalloc = Z_NULL;
  stream->zfree = Z_NULL;
  stream->opaque = NULL;
  stream->next_in = (unsigned char*) in;
  stream->avail_in = (unsigned int)in_len;

  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
                   method_bits(method),
                   8, Z_DEFAULT_STRATEGY) != Z_OK) {
    log_warn(LD_GENERAL, "Error from deflateInit2: %s",
             stream->msg?stream->msg:"<no message>");
    goto err;
  }

  /* Guess 50% compression. */
  out_size = in_len / 2;
  if (out_size < 1024) out_size = 1024;
  *out = tor_malloc(out_size);
  stream->next_out = (unsigned char*)*out;
  stream->avail_out = (unsigned int)out_size;

  while (1) {
    switch (deflate(stream, Z_FINISH))
      {
      case Z_STREAM_END:
        goto done;
      case Z_OK:
        /* In case zlib doesn't work as I think .... */
        if (stream->avail_out >= stream->avail_in+16)
          break;
      case Z_BUF_ERROR:
        offset = stream->next_out - ((unsigned char*)*out);
        old_size = out_size;
        out_size *= 2;
        if (out_size < old_size) {
          log_warn(LD_GENERAL, "Size overflow in compression.");
          goto err;
        }
        *out = tor_realloc(*out, out_size);
        stream->next_out = (unsigned char*)(*out + offset);
        if (out_size - offset > UINT_MAX) {
          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
                   "uncompressing.");
          goto err;
        }
        stream->avail_out = (unsigned int)(out_size - offset);
        break;
      default:
        log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
                 stream->msg ? stream->msg : "<no message>");
        goto err;
      }
  }
 done:
  *out_len = stream->total_out;
#ifdef OPENBSD
  /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
   *    third-party API!"
   * "Oh, that trick will just make people do unsafe casts to the unsigned
   *    type in their cross-platform code!"
   * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
   *    the newly unsigned field isn't negative." */
  tor_assert(stream->total_out >= 0);
#endif
  if (((size_t)stream->total_out) > out_size + 4097) {
    /* If we're wasting more than 4k, don't. */
    *out = tor_realloc(*out, stream->total_out + 1);
  }
  if (deflateEnd(stream)!=Z_OK) {
    log_warn(LD_BUG, "Error freeing gzip structures");
    goto err;
  }
  tor_free(stream);

  if (is_compression_bomb(*out_len, in_len)) {
    log_warn(LD_BUG, "We compressed something and got an insanely high "
          "compression factor; other Tors would think this was a zlib bomb.");
    goto err;
  }

  return 0;
 err:
  if (stream) {
    deflateEnd(stream);
    tor_free(stream);
  }
  tor_free(*out);
  return -1;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int tor_gzip_uncompress ( char **  out,
size_t *  out_len,
const char *  in,
size_t  in_len,
compress_method_t  method,
int  complete_only,
int  protocol_warn_level 
)

Given zero or more zlib-compressed or gzip-compressed strings of total length in_len bytes at in, uncompress them into a newly allocated buffer, using the method described in method.

Store the uncompressed string in *out, and its length in *out_len. Return 0 on success, -1 on failure.

If complete_only is true, we consider a truncated input as a failure; otherwise we decompress as much as we can. Warn about truncated or corrupt inputs at protocol_warn_level.

Definition at line 238 of file torgzip.c.

{
  struct z_stream_s *stream = NULL;
  size_t out_size, old_size;
  off_t offset;
  int r;

  tor_assert(out);
  tor_assert(out_len);
  tor_assert(in);
  tor_assert(in_len < UINT_MAX);

  if (method == GZIP_METHOD && !is_gzip_supported()) {
    /* Old zlib version don't support gzip in inflateInit2 */
    log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
    return -1;
  }

  *out = NULL;

  stream = tor_malloc_zero(sizeof(struct z_stream_s));
  stream->zalloc = Z_NULL;
  stream->zfree = Z_NULL;
  stream->opaque = NULL;
  stream->next_in = (unsigned char*) in;
  stream->avail_in = (unsigned int)in_len;

  if (inflateInit2(stream,
                   method_bits(method)) != Z_OK) {
    log_warn(LD_GENERAL, "Error from inflateInit2: %s",
             stream->msg?stream->msg:"<no message>");
    goto err;
  }

  out_size = in_len * 2;  /* guess 50% compression. */
  if (out_size < 1024) out_size = 1024;
  if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
    goto err;

  *out = tor_malloc(out_size);
  stream->next_out = (unsigned char*)*out;
  stream->avail_out = (unsigned int)out_size;

  while (1) {
    switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
      {
      case Z_STREAM_END:
        if (stream->avail_in == 0)
          goto done;
        /* There may be more compressed data here. */
        if ((r = inflateEnd(stream)) != Z_OK) {
          log_warn(LD_BUG, "Error freeing gzip structures");
          goto err;
        }
        if (inflateInit2(stream, method_bits(method)) != Z_OK) {
          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
                   stream->msg?stream->msg:"<no message>");
          goto err;
        }
        break;
      case Z_OK:
        if (!complete_only && stream->avail_in == 0)
          goto done;
        /* In case zlib doesn't work as I think.... */
        if (stream->avail_out >= stream->avail_in+16)
          break;
      case Z_BUF_ERROR:
        if (stream->avail_out > 0) {
          log_fn(protocol_warn_level, LD_PROTOCOL,
                 "possible truncated or corrupt zlib data");
          goto err;
        }
        offset = stream->next_out - (unsigned char*)*out;
        old_size = out_size;
        out_size *= 2;
        if (out_size < old_size) {
          log_warn(LD_GENERAL, "Size overflow in uncompression.");
          goto err;
        }
        if (is_compression_bomb(in_len, out_size)) {
          log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
                   "not proceeding.");
          goto err;
        }
        if (out_size >= SIZE_T_CEILING) {
          log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
          goto err;
        }
        *out = tor_realloc(*out, out_size);
        stream->next_out = (unsigned char*)(*out + offset);
        if (out_size - offset > UINT_MAX) {
          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
                   "uncompressing.");
          goto err;
        }
        stream->avail_out = (unsigned int)(out_size - offset);
        break;
      default:
        log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
                 stream->msg ? stream->msg : "<no message>");
        goto err;
      }
  }
 done:
  *out_len = stream->next_out - (unsigned char*)*out;
  r = inflateEnd(stream);
  tor_free(stream);
  if (r != Z_OK) {
    log_warn(LD_BUG, "Error freeing gzip structures");
    goto err;
  }

  /* NUL-terminate output. */
  if (out_size == *out_len)
    *out = tor_realloc(*out, out_size + 1);
  (*out)[*out_len] = '\0';

  return 0;
 err:
  if (stream) {
    inflateEnd(stream);
    tor_free(stream);
  }
  if (*out) {
    tor_free(*out);
  }
  return -1;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void tor_zlib_free ( tor_zlib_state_t state)

Deallocate state.

Definition at line 499 of file torgzip.c.

{
  if (!state)
    return;

  if (state->compress)
    deflateEnd(&state->stream);
  else
    inflateEnd(&state->stream);

  tor_free(state);
}

Here is the caller graph for this function:

tor_zlib_state_t* tor_zlib_new ( int  compress,
compress_method_t  method 
)

Construct and return a tor_zlib_state_t object using method.

If compress, it's for compression; otherwise it's for decompression.

Definition at line 404 of file torgzip.c.

{
  tor_zlib_state_t *out;

  if (method == GZIP_METHOD && !is_gzip_supported()) {
    /* Old zlib version don't support gzip in inflateInit2 */
    log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
    return NULL;
 }

 out = tor_malloc_zero(sizeof(tor_zlib_state_t));
 out->stream.zalloc = Z_NULL;
 out->stream.zfree = Z_NULL;
 out->stream.opaque = NULL;
 out->compress = compress;
 if (compress) {
   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
                    method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
     goto err;
 } else {
   if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
     goto err;
 }
 return out;

 err:
 tor_free(out);
 return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

tor_zlib_output_t tor_zlib_process ( tor_zlib_state_t state,
char **  out,
size_t *  out_len,
const char **  in,
size_t *  in_len,
int  finish 
)

Compress/decompress some bytes using state.

Read up to *in_len bytes from *in, and write up to *out_len bytes to *out, adjusting the values as we go. If finish is true, we've reached the end of the input.

Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. Return TOR_ZLIB_OK if we're processed everything from the input. Return TOR_ZLIB_BUF_FULL if we're out of space on out. Return TOR_ZLIB_ERR if the stream is corrupt.

Definition at line 445 of file torgzip.c.

{
  int err;
  tor_assert(*in_len <= UINT_MAX);
  tor_assert(*out_len <= UINT_MAX);
  state->stream.next_in = (unsigned char*) *in;
  state->stream.avail_in = (unsigned int)*in_len;
  state->stream.next_out = (unsigned char*) *out;
  state->stream.avail_out = (unsigned int)*out_len;

  if (state->compress) {
    err = deflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
  } else {
    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
  }

  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);

  *out = (char*) state->stream.next_out;
  *out_len = state->stream.avail_out;
  *in = (const char *) state->stream.next_in;
  *in_len = state->stream.avail_in;

  if (! state->compress &&
      is_compression_bomb(state->input_so_far, state->output_so_far)) {
    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
    return TOR_ZLIB_ERR;
  }

  switch (err)
    {
    case Z_STREAM_END:
      return TOR_ZLIB_DONE;
    case Z_BUF_ERROR:
      if (state->stream.avail_in == 0)
        return TOR_ZLIB_OK;
      return TOR_ZLIB_BUF_FULL;
    case Z_OK:
      if (state->stream.avail_out == 0 || finish)
        return TOR_ZLIB_BUF_FULL;
      return TOR_ZLIB_OK;
    default:
      log_warn(LD_GENERAL, "Gzip returned an error: %s",
               state->stream.msg ? state->stream.msg : "<no message>");
      return TOR_ZLIB_ERR;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

int gzip_is_supported = -1 [static]

Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; set to -1 if we haven't checked yet.

Definition at line 51 of file torgzip.c.