build: vendor libmpack source from libmpack/libmpack 22b1fd90285117c995728511f9525d29520a8c82

9 files changed, 1809 insertions, 0 deletions

diff --git a/src/mpack/LICENSE-MIT b/src/mpack/LICENSE-MIT
new file mode 100644
index 0000000000..030ba872c5
--- /dev/null
+++ b/src/mpack/LICENSE-MIT
@@ -0,0 +1,22 @@
+Copyright (c) 2016 Thiago de Arruda
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
diff --git a/src/mpack/conv.c b/src/mpack/conv.c
new file mode 100644
index 0000000000..203b13fadb
--- /dev/null
+++ b/src/mpack/conv.c
@@ -0,0 +1,375 @@
+#include "conv.h"
+
+static int mpack_fits_single(double v);
+static mpack_value_t mpack_pack_ieee754(double v, unsigned m, unsigned e);
+static int mpack_is_be(void) FPURE;
+static double mpack_fmod_pow2_32(double a);
+
+
+#define POW2(n) \
+  ((double)(1 << (n / 2)) * (double)(1 << (n / 2)) * (double)(1 << (n % 2)))
+
+#define MPACK_SWAP_VALUE(val)                                  \
+  do {                                                         \
+    mpack_uint32_t lo = val.lo;                                \
+    val.lo = val.hi;                                           \
+    val.hi = lo;                                               \
+  } while (0)
+
+MPACK_API mpack_token_t mpack_pack_nil(void)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_NIL;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_boolean(unsigned v)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_BOOLEAN;
+  rv.data.value.lo = v ? 1 : 0;
+  rv.data.value.hi = 0;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_uint(mpack_uintmax_t v)
+{
+  mpack_token_t rv;
+  rv.data.value.lo = v & 0xffffffff;
+  rv.data.value.hi = (mpack_uint32_t)((v >> 31) >> 1);
+  rv.type = MPACK_TOKEN_UINT;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_sint(mpack_sintmax_t v)
+{
+  if (v < 0) {
+    mpack_token_t rv;
+    mpack_uintmax_t tc = -((mpack_uintmax_t)(v + 1)) + 1;
+    tc = ~tc + 1;
+    rv = mpack_pack_uint(tc);
+    rv.type = MPACK_TOKEN_SINT;
+    return rv;
+  }
+
+  return mpack_pack_uint((mpack_uintmax_t)v);
+}
+
+MPACK_API mpack_token_t mpack_pack_float_compat(double v)
+{
+  /* ieee754 single-precision limits to determine if "v" can be fully
+   * represented in 4 bytes */
+  mpack_token_t rv;
+
+  if (mpack_fits_single(v)) {
+    rv.length = 4;
+    rv.data.value = mpack_pack_ieee754(v, 23, 8);
+  } else {
+    rv.length = 8;
+    rv.data.value = mpack_pack_ieee754(v, 52, 11);
+  }
+
+  rv.type = MPACK_TOKEN_FLOAT;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_float_fast(double v)
+{
+  /* ieee754 single-precision limits to determine if "v" can be fully
+   * represented in 4 bytes */
+  mpack_token_t rv;
+
+  if (mpack_fits_single(v)) {
+    union {
+      float f;
+      mpack_uint32_t m;
+    } conv;
+    conv.f = (float)v;
+    rv.length = 4;
+    rv.data.value.lo = conv.m;
+    rv.data.value.hi = 0;
+  } else {
+    union {
+      double d;
+      mpack_value_t m;
+    } conv;
+    conv.d = v;
+    rv.length = 8;
+    rv.data.value = conv.m;
+    if (mpack_is_be()) {
+      MPACK_SWAP_VALUE(rv.data.value);
+    }
+  }
+
+  rv.type = MPACK_TOKEN_FLOAT;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_number(double v)
+{
+  mpack_token_t tok;
+  double vabs;
+  vabs = v < 0 ? -v : v;
+  assert(v <= 9007199254740991. && v >= -9007199254740991.);
+  tok.data.value.hi = (mpack_uint32_t)(vabs / POW2(32));
+  tok.data.value.lo = (mpack_uint32_t)mpack_fmod_pow2_32(vabs);
+
+  if (v < 0) {
+    /* Compute the two's complement */
+    tok.type = MPACK_TOKEN_SINT;
+    tok.data.value.hi = ~tok.data.value.hi;
+    tok.data.value.lo = ~tok.data.value.lo + 1;
+    if (!tok.data.value.lo) tok.data.value.hi++;
+    if (tok.data.value.lo == 0 && tok.data.value.hi == 0) tok.length = 1;
+    else if (tok.data.value.lo < 0x80000000) tok.length = 8;
+    else if (tok.data.value.lo < 0xffff7fff) tok.length = 4;
+    else if (tok.data.value.lo < 0xffffff7f) tok.length = 2;
+    else tok.length = 1;
+  } else {
+    tok.type = MPACK_TOKEN_UINT;
+    if (tok.data.value.hi) tok.length = 8;
+    else if (tok.data.value.lo > 0xffff) tok.length = 4;
+    else if (tok.data.value.lo > 0xff) tok.length = 2;
+    else tok.length = 1;
+  }
+
+  if (mpack_unpack_number(tok) != v) {
+    return mpack_pack_float(v);
+  }
+
+  return tok;
+}
+
+MPACK_API mpack_token_t mpack_pack_chunk(const char *p, mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_CHUNK;
+  rv.data.chunk_ptr = p;
+  rv.length = l;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_str(mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_STR;
+  rv.length = l;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_bin(mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_BIN;
+  rv.length = l;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_ext(int t, mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_EXT;
+  rv.length = l;
+  rv.data.ext_type = t;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_array(mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_ARRAY;
+  rv.length = l;
+  return rv;
+}
+
+MPACK_API mpack_token_t mpack_pack_map(mpack_uint32_t l)
+{
+  mpack_token_t rv;
+  rv.type = MPACK_TOKEN_MAP;
+  rv.length = l;
+  return rv;
+}
+
+MPACK_API bool mpack_unpack_boolean(mpack_token_t t)
+{
+  return t.data.value.lo || t.data.value.hi;
+}
+
+MPACK_API mpack_uintmax_t mpack_unpack_uint(mpack_token_t t)
+{
+  return (((mpack_uintmax_t)t.data.value.hi << 31) << 1) | t.data.value.lo;
+}
+
+/* unpack signed integer without relying on two's complement as internal
+ * representation */
+MPACK_API mpack_sintmax_t mpack_unpack_sint(mpack_token_t t)
+{
+  mpack_uint32_t hi = t.data.value.hi;
+  mpack_uint32_t lo = t.data.value.lo;
+  mpack_uintmax_t rv = lo;
+  assert(t.length <= sizeof(mpack_sintmax_t));
+
+  if (t.length == 8) {
+    rv |= (((mpack_uintmax_t)hi) << 31) << 1;
+  }
+  /* reverse the two's complement so that lo/hi contain the absolute value.
+   * note that we have to mask ~rv so that it reflects the two's complement
+   * of the appropriate byte length */
+  rv = (~rv & (((mpack_uintmax_t)1 << ((t.length * 8) - 1)) - 1)) + 1;
+  /* negate and return the absolute value, making sure mpack_sintmax_t can
+   * represent the positive cast. */
+  return -((mpack_sintmax_t)(rv - 1)) - 1;
+}
+
+MPACK_API double mpack_unpack_float_compat(mpack_token_t t)
+{
+  mpack_uint32_t sign;
+  mpack_sint32_t exponent, bias;
+  unsigned mantbits;
+  unsigned expbits;
+  double mant;
+
+  if (t.data.value.lo == 0 && t.data.value.hi == 0)
+    /* nothing to do */
+    return 0;
+
+  if (t.length == 4) mantbits = 23, expbits = 8;
+  else mantbits = 52, expbits = 11;
+  bias = (1 << (expbits - 1)) - 1;
+
+  /* restore sign/exponent/mantissa */
+  if (mantbits == 52) {
+    sign = t.data.value.hi >> 31;
+    exponent = (t.data.value.hi >> 20) & ((1 << 11) - 1);
+    mant = (t.data.value.hi & ((1 << 20) - 1)) * POW2(32);
+    mant += t.data.value.lo;
+  } else {
+    sign = t.data.value.lo >> 31;
+    exponent = (t.data.value.lo >> 23) & ((1 << 8) - 1);
+    mant = t.data.value.lo & ((1 << 23) - 1);
+  }
+
+  mant /= POW2(mantbits);
+  if (exponent) mant += 1.0; /* restore leading 1 */
+  else exponent = 1; /* subnormal */
+  exponent -= bias;
+
+  /* restore original value */
+  while (exponent > 0) mant *= 2.0, exponent--;
+  while (exponent < 0) mant /= 2.0, exponent++;
+  return mant * (sign ? -1 : 1);
+}
+
+MPACK_API double mpack_unpack_float_fast(mpack_token_t t)
+{
+  if (t.length == 4) {
+    union {
+      float f;
+      mpack_uint32_t m;
+    } conv;
+    conv.m = t.data.value.lo;
+    return conv.f;
+  } else {
+    union {
+      double d;
+      mpack_value_t m;
+    } conv;
+    conv.m = t.data.value;
+    
+    if (mpack_is_be()) {
+      MPACK_SWAP_VALUE(conv.m);
+    }
+
+    return conv.d;
+  }
+}
+
+MPACK_API double mpack_unpack_number(mpack_token_t t)
+{
+  double rv;
+  mpack_uint32_t hi, lo;
+  if (t.type == MPACK_TOKEN_FLOAT) return mpack_unpack_float(t);
+  assert(t.type == MPACK_TOKEN_UINT || t.type == MPACK_TOKEN_SINT);
+  hi = t.data.value.hi;
+  lo = t.data.value.lo;
+  if (t.type == MPACK_TOKEN_SINT) {
+    /* same idea as mpack_unpack_sint, except here we shouldn't rely on
+     * mpack_uintmax_t having 64-bits, operating on the 32-bit words separately.
+     */
+    if (!hi) {
+      assert(t.length <= 4);
+      hi = 0;
+      lo = (~lo & (((mpack_uint32_t)1 << ((t.length * 8) - 1)) - 1));
+    } else {
+      hi = ~hi;
+      lo = ~lo;
+    }
+    lo++;
+    if (!lo) hi++;
+  }
+  rv = (double)lo + POW2(32) * hi;
+  return t.type == MPACK_TOKEN_SINT ? -rv : rv;
+}
+
+static int mpack_fits_single(double v)
+{
+  return (float)v == v;
+}
+
+static mpack_value_t mpack_pack_ieee754(double v, unsigned mantbits,
+    unsigned expbits)
+{
+  mpack_value_t rv = {0, 0};
+  mpack_sint32_t exponent, bias = (1 << (expbits - 1)) - 1;
+  mpack_uint32_t sign;
+  double mant;
+
+  if (v == 0) {
+    rv.lo = 0;
+    rv.hi = 0;
+    goto end;
+  }
+
+  if (v < 0) sign = 1, mant = -v;
+  else sign = 0, mant = v;
+
+  exponent = 0;
+  while (mant >= 2.0) mant /= 2.0, exponent++;
+  while (mant < 1.0 && exponent > -(bias - 1)) mant *= 2.0, exponent--;
+
+  if (mant < 1.0) exponent = -bias; /* subnormal value */
+  else mant = mant - 1.0; /* remove leading 1 */
+  exponent += bias;
+  mant *= POW2(mantbits);
+
+  if (mantbits == 52) {
+    rv.hi = (mpack_uint32_t)(mant / POW2(32));
+    rv.lo = (mpack_uint32_t)(mant - rv.hi * POW2(32));
+    rv.hi |= ((mpack_uint32_t)exponent << 20) | (sign << 31);
+  } else if (mantbits == 23) {
+    rv.hi = 0;
+    rv.lo = (mpack_uint32_t)mant;
+    rv.lo |= ((mpack_uint32_t)exponent << 23) | (sign << 31);
+  }
+
+end:
+  return rv;
+}
+
+static int mpack_is_be(void)
+{
+  union {
+    mpack_uint32_t i;
+    char c[sizeof(mpack_uint32_t)];
+  } test;
+
+  test.i = 1;
+  return test.c[0] == 0;
+}
+
+/* this simplified version of `fmod` that returns the remainder of double
+ * division by 0xffffffff, which is enough for our purposes */
+static double mpack_fmod_pow2_32(double a)
+{
+  return a - ((double)(mpack_uint32_t)(a / POW2(32)) * POW2(32));
+}
diff --git a/src/mpack/conv.h b/src/mpack/conv.h
new file mode 100644
index 0000000000..f022c74efa
--- /dev/null
+++ b/src/mpack/conv.h
@@ -0,0 +1,55 @@
+#ifndef MPACK_CONV_H
+#define MPACK_CONV_H
+
+#include "core.h"
+
+#if ULLONG_MAX == 0xffffffffffffffff
+typedef long long mpack_sintmax_t;
+typedef unsigned long long mpack_uintmax_t;
+#elif UINT64_MAX == 0xffffffffffffffff
+typedef int64_t mpack_sintmax_t;
+typedef uint64_t mpack_uintmax_t;
+#else
+typedef mpack_sint32_t mpack_sintmax_t;
+typedef mpack_uint32_t mpack_uintmax_t;
+#endif
+
+#ifndef bool
+# define bool unsigned
+#endif
+
+MPACK_API mpack_token_t mpack_pack_nil(void) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_boolean(unsigned v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_uint(mpack_uintmax_t v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_sint(mpack_sintmax_t v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_float_compat(double v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_float_fast(double v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_number(double v) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_chunk(const char *p, mpack_uint32_t l)
+  FUNUSED FPURE FNONULL;
+MPACK_API mpack_token_t mpack_pack_str(mpack_uint32_t l) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_bin(mpack_uint32_t l) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_ext(int type, mpack_uint32_t l)
+  FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_array(mpack_uint32_t l) FUNUSED FPURE;
+MPACK_API mpack_token_t mpack_pack_map(mpack_uint32_t l) FUNUSED FPURE;
+MPACK_API bool mpack_unpack_boolean(mpack_token_t t) FUNUSED FPURE;
+MPACK_API mpack_uintmax_t mpack_unpack_uint(mpack_token_t t) FUNUSED FPURE;
+MPACK_API mpack_sintmax_t mpack_unpack_sint(mpack_token_t t) FUNUSED FPURE;
+MPACK_API double mpack_unpack_float_fast(mpack_token_t t) FUNUSED FPURE;
+MPACK_API double mpack_unpack_float_compat(mpack_token_t t) FUNUSED FPURE;
+MPACK_API double mpack_unpack_number(mpack_token_t t) FUNUSED FPURE;
+
+/* The mpack_{pack,unpack}_float_fast functions should work in 99% of the
+ * platforms. When compiling for a platform where floats don't use ieee754 as
+ * the internal format, pass
+ * -Dmpack_{pack,unpack}_float=mpack_{pack,unpack}_float_compat to the
+ *  compiler.*/
+#ifndef mpack_pack_float
+# define mpack_pack_float mpack_pack_float_fast
+#endif
+#ifndef mpack_unpack_float
+# define mpack_unpack_float mpack_unpack_float_fast
+#endif
+
+#endif  /* MPACK_CONV_H */
diff --git a/src/mpack/core.c b/src/mpack/core.c
new file mode 100644
index 0000000000..ce529f0dca
--- /dev/null
+++ b/src/mpack/core.c
@@ -0,0 +1,575 @@
+#include <string.h>
+
+#include "core.h"
+
+#define UNUSED(p) (void)p;
+#define ADVANCE(buf, buflen) ((*buflen)--, (unsigned char)*((*buf)++))
+#define TLEN(val, range_start) ((mpack_uint32_t)(1 << (val - range_start)))
+#ifndef MIN
+# define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
+
+static int mpack_rtoken(const char **buf, size_t *buflen,
+    mpack_token_t *tok);
+static int mpack_rpending(const char **b, size_t *nl, mpack_tokbuf_t *tb);
+static int mpack_rvalue(mpack_token_type_t t, mpack_uint32_t l,
+    const char **b, size_t *bl, mpack_token_t *tok);
+static int mpack_rblob(mpack_token_type_t t, mpack_uint32_t l,
+    const char **b, size_t *bl, mpack_token_t *tok);
+static int mpack_wtoken(const mpack_token_t *tok, char **b, size_t *bl);
+static int mpack_wpending(char **b, size_t *bl, mpack_tokbuf_t *tb);
+static int mpack_wpint(char **b, size_t *bl, mpack_value_t v);
+static int mpack_wnint(char **b, size_t *bl, mpack_value_t v);
+static int mpack_wfloat(char **b, size_t *bl, const mpack_token_t *v);
+static int mpack_wstr(char **buf, size_t *buflen, mpack_uint32_t len);
+static int mpack_wbin(char **buf, size_t *buflen, mpack_uint32_t len);
+static int mpack_wext(char **buf, size_t *buflen, int type,
+    mpack_uint32_t len);
+static int mpack_warray(char **buf, size_t *buflen, mpack_uint32_t len);
+static int mpack_wmap(char **buf, size_t *buflen, mpack_uint32_t len);
+static int mpack_w1(char **b, size_t *bl, mpack_uint32_t v);
+static int mpack_w2(char **b, size_t *bl, mpack_uint32_t v);
+static int mpack_w4(char **b, size_t *bl, mpack_uint32_t v);
+static mpack_value_t mpack_byte(unsigned char b);
+static int mpack_value(mpack_token_type_t t, mpack_uint32_t l,
+    mpack_value_t v, mpack_token_t *tok);
+static int mpack_blob(mpack_token_type_t t, mpack_uint32_t l, int et,
+    mpack_token_t *tok);
+
+MPACK_API void mpack_tokbuf_init(mpack_tokbuf_t *tokbuf)
+{
+  tokbuf->ppos = 0;
+  tokbuf->plen = 0;
+  tokbuf->passthrough = 0;
+}
+
+MPACK_API int mpack_read(mpack_tokbuf_t *tokbuf, const char **buf,
+    size_t *buflen, mpack_token_t *tok)
+{
+  int status;
+  size_t initial_ppos, ptrlen, advanced;
+  const char *ptr, *ptr_save;
+  assert(*buf && *buflen);
+
+  if (tokbuf->passthrough) {
+    /* pass data from str/bin/ext directly as a MPACK_TOKEN_CHUNK, adjusting
+     * *buf and *buflen */
+    tok->type = MPACK_TOKEN_CHUNK;
+    tok->data.chunk_ptr = *buf;
+    tok->length = MIN((mpack_uint32_t)*buflen, tokbuf->passthrough);
+    tokbuf->passthrough -= tok->length;
+    *buf += tok->length;
+    *buflen -= tok->length;
+    goto done;
+  }
+
+  initial_ppos = tokbuf->ppos;
+
+  if (tokbuf->plen) {
+    if (!mpack_rpending(buf, buflen, tokbuf)) {
+      return MPACK_EOF;
+    }
+    ptr = tokbuf->pending;
+    ptrlen = tokbuf->ppos;
+  } else {
+    ptr = *buf;
+    ptrlen = *buflen;
+  }
+
+  ptr_save = ptr;
+
+  if ((status = mpack_rtoken(&ptr, &ptrlen, tok))) {
+    if (status != MPACK_EOF) return MPACK_ERROR;
+    /* need more data */
+    assert(!tokbuf->plen);
+    /* read the remainder of *buf to tokbuf->pending so it can be parsed
+     * later with more data. only required when tokbuf->plen == 0 or else
+     * it would have been done already. */
+    tokbuf->plen = tok->length + 1;
+    assert(tokbuf->plen <= sizeof(tokbuf->pending));
+    tokbuf->ppos = 0;
+    status = mpack_rpending(buf, buflen, tokbuf);
+    assert(!status);
+    return MPACK_EOF;
+  }
+
+  advanced = (size_t)(ptr - ptr_save) - initial_ppos;
+  tokbuf->plen = tokbuf->ppos = 0;
+  *buflen -= advanced; 
+  *buf += advanced;
+
+  if (tok->type > MPACK_TOKEN_MAP) {
+    tokbuf->passthrough = tok->length;
+  }
+
+done:
+  return MPACK_OK;
+}
+
+MPACK_API int mpack_write(mpack_tokbuf_t *tokbuf, char **buf, size_t *buflen,
+    const mpack_token_t *t)
+{
+  int status;
+  char *ptr;
+  size_t ptrlen;
+  mpack_token_t tok = tokbuf->plen ? tokbuf->pending_tok : *t;
+  assert(*buf && *buflen);
+
+  if (tok.type == MPACK_TOKEN_CHUNK) {
+    size_t written, pending, count;
+    if (!tokbuf->plen) tokbuf->ppos = 0;
+    written = tokbuf->ppos;
+    pending = tok.length - written;
+    count = MIN(pending, *buflen);
+    memcpy(*buf, tok.data.chunk_ptr + written, count);
+    *buf += count;
+    *buflen -= count;
+    tokbuf->ppos += count;
+    tokbuf->plen = count == pending ? 0 : tok.length;
+    if (count == pending) {
+      return MPACK_OK;
+    } else {
+      tokbuf->pending_tok = tok;
+      return MPACK_EOF;
+    }
+  }
+
+  if (tokbuf->plen) return mpack_wpending(buf, buflen, tokbuf);
+
+  if (*buflen < MPACK_MAX_TOKEN_LEN) {
+    ptr = tokbuf->pending;
+    ptrlen = sizeof(tokbuf->pending);
+  } else {
+    ptr = *buf;
+    ptrlen = *buflen;
+  }
+
+  if ((status = mpack_wtoken(&tok, &ptr, &ptrlen))) return status;
+
+  if (*buflen < MPACK_MAX_TOKEN_LEN) {
+    size_t toklen = sizeof(tokbuf->pending) - ptrlen;
+    size_t write_cnt = MIN(toklen, *buflen);
+    memcpy(*buf, tokbuf->pending, write_cnt);
+    *buf += write_cnt;
+    *buflen -= write_cnt;
+    if (write_cnt < toklen) {
+      assert(!*buflen);
+      tokbuf->plen = toklen;
+      tokbuf->ppos = write_cnt;
+      tokbuf->pending_tok = tok;
+      return MPACK_EOF;
+    }
+  } else {
+    *buflen -= (size_t)(ptr - *buf);
+    *buf = ptr;
+  }
+
+  return MPACK_OK;
+}
+
+static int mpack_rtoken(const char **buf, size_t *buflen,
+    mpack_token_t *tok)
+{
+  unsigned char t = ADVANCE(buf, buflen);
+  if (t < 0x80) {
+    /* positive fixint */
+    return mpack_value(MPACK_TOKEN_UINT, 1, mpack_byte(t), tok);
+  } else if (t < 0x90) {
+    /* fixmap */
+    return mpack_blob(MPACK_TOKEN_MAP, t & 0xf, 0, tok);
+  } else if (t < 0xa0) {
+    /* fixarray */
+    return mpack_blob(MPACK_TOKEN_ARRAY, t & 0xf, 0, tok);
+  } else if (t < 0xc0) {
+    /* fixstr */
+    return mpack_blob(MPACK_TOKEN_STR, t & 0x1f, 0, tok);
+  } else if (t < 0xe0) {
+    switch (t) {
+      case 0xc0:  /* nil */
+        return mpack_value(MPACK_TOKEN_NIL, 0, mpack_byte(0), tok);
+      case 0xc2:  /* false */
+        return mpack_value(MPACK_TOKEN_BOOLEAN, 1, mpack_byte(0), tok);
+      case 0xc3:  /* true */
+        return mpack_value(MPACK_TOKEN_BOOLEAN, 1, mpack_byte(1), tok);
+      case 0xc4:  /* bin 8 */
+      case 0xc5:  /* bin 16 */
+      case 0xc6:  /* bin 32 */
+        return mpack_rblob(MPACK_TOKEN_BIN, TLEN(t, 0xc4), buf, buflen, tok);
+      case 0xc7:  /* ext 8 */
+      case 0xc8:  /* ext 16 */
+      case 0xc9:  /* ext 32 */
+        return mpack_rblob(MPACK_TOKEN_EXT, TLEN(t, 0xc7), buf, buflen, tok);
+      case 0xca:  /* float 32 */
+      case 0xcb:  /* float 64 */
+        return mpack_rvalue(MPACK_TOKEN_FLOAT, TLEN(t, 0xc8), buf, buflen, tok);
+      case 0xcc:  /* uint 8 */
+      case 0xcd:  /* uint 16 */
+      case 0xce:  /* uint 32 */
+      case 0xcf:  /* uint 64 */
+        return mpack_rvalue(MPACK_TOKEN_UINT, TLEN(t, 0xcc), buf, buflen, tok);
+      case 0xd0:  /* int 8 */
+      case 0xd1:  /* int 16 */
+      case 0xd2:  /* int 32 */
+      case 0xd3:  /* int 64 */
+        return mpack_rvalue(MPACK_TOKEN_SINT, TLEN(t, 0xd0), buf, buflen, tok);
+      case 0xd4:  /* fixext 1 */
+      case 0xd5:  /* fixext 2 */
+      case 0xd6:  /* fixext 4 */
+      case 0xd7:  /* fixext 8 */
+      case 0xd8:  /* fixext 16 */
+        if (*buflen == 0) {
+          /* require only one extra byte for the type code */
+          tok->length = 1;
+          return MPACK_EOF;
+        }
+        tok->length = TLEN(t, 0xd4);
+        tok->type = MPACK_TOKEN_EXT;
+        tok->data.ext_type = ADVANCE(buf, buflen);
+        return MPACK_OK;
+      case 0xd9:  /* str 8 */
+      case 0xda:  /* str 16 */
+      case 0xdb:  /* str 32 */
+        return mpack_rblob(MPACK_TOKEN_STR, TLEN(t, 0xd9), buf, buflen, tok);
+      case 0xdc:  /* array 16 */
+      case 0xdd:  /* array 32 */
+        return mpack_rblob(MPACK_TOKEN_ARRAY, TLEN(t, 0xdb), buf, buflen, tok);
+      case 0xde:  /* map 16 */
+      case 0xdf:  /* map 32 */
+        return mpack_rblob(MPACK_TOKEN_MAP, TLEN(t, 0xdd), buf, buflen, tok);
+      default:
+        return MPACK_ERROR;
+    }
+  } else {
+    /* negative fixint */
+    return mpack_value(MPACK_TOKEN_SINT, 1, mpack_byte(t), tok);
+  }
+}
+
+static int mpack_rpending(const char **buf, size_t *buflen,
+    mpack_tokbuf_t *state)
+{
+  size_t count;
+  assert(state->ppos < state->plen);
+  count = MIN(state->plen - state->ppos, *buflen);
+  memcpy(state->pending + state->ppos, *buf, count);
+  state->ppos += count;
+  if (state->ppos < state->plen) {
+    /* consume buffer since no token will be parsed yet. */
+    *buf += *buflen;
+    *buflen = 0;
+    return 0;
+  }
+  return 1;
+}
+
+static int mpack_rvalue(mpack_token_type_t type, mpack_uint32_t remaining,
+    const char **buf, size_t *buflen, mpack_token_t *tok)
+{
+  if (*buflen < remaining) {
+    tok->length = remaining;
+    return MPACK_EOF;
+  }
+
+  mpack_value(type, remaining, mpack_byte(0), tok);
+
+  while (remaining) {
+    mpack_uint32_t byte = ADVANCE(buf, buflen), byte_idx, byte_shift;
+    byte_idx = (mpack_uint32_t)--remaining;
+    byte_shift = (byte_idx % 4) * 8;
+    tok->data.value.lo |= byte << byte_shift;
+    if (remaining == 4) {
+      /* unpacked the first half of a 8-byte value, shift what was parsed to the
+       * "hi" field and reset "lo" for the trailing 4 bytes. */
+      tok->data.value.hi = tok->data.value.lo;
+      tok->data.value.lo = 0;
+    }
+  }
+
+  if (type == MPACK_TOKEN_SINT) {
+    mpack_uint32_t hi = tok->data.value.hi;
+    mpack_uint32_t lo = tok->data.value.lo;
+    mpack_uint32_t msb = (tok->length == 8 && hi >> 31) ||
+                         (tok->length == 4 && lo >> 31) ||
+                         (tok->length == 2 && lo >> 15) ||
+                         (tok->length == 1 && lo >> 7);
+    if (!msb) {
+      tok->type = MPACK_TOKEN_UINT;
+    }
+  }
+
+  return MPACK_OK;
+}
+
+static int mpack_rblob(mpack_token_type_t type, mpack_uint32_t tlen,
+    const char **buf, size_t *buflen, mpack_token_t *tok)
+{
+  mpack_token_t l;
+  mpack_uint32_t required = tlen + (type == MPACK_TOKEN_EXT ? 1 : 0);
+
+  if (*buflen < required) {
+    tok->length = required;
+    return MPACK_EOF;
+  }
+
+  l.data.value.lo = 0;
+  mpack_rvalue(MPACK_TOKEN_UINT, tlen, buf, buflen, &l);
+  tok->type = type;
+  tok->length = l.data.value.lo;
+
+  if (type == MPACK_TOKEN_EXT) {
+    tok->data.ext_type = ADVANCE(buf, buflen);
+  }
+
+  return MPACK_OK;
+}
+
+static int mpack_wtoken(const mpack_token_t *tok, char **buf,
+    size_t *buflen)
+{
+  switch (tok->type) {
+    case MPACK_TOKEN_NIL:
+      return mpack_w1(buf, buflen, 0xc0);
+    case MPACK_TOKEN_BOOLEAN:
+      return mpack_w1(buf, buflen, tok->data.value.lo ? 0xc3 : 0xc2);
+    case MPACK_TOKEN_UINT:
+      return mpack_wpint(buf, buflen, tok->data.value);
+    case MPACK_TOKEN_SINT:
+      return mpack_wnint(buf, buflen, tok->data.value);
+    case MPACK_TOKEN_FLOAT:
+      return mpack_wfloat(buf, buflen, tok);
+    case MPACK_TOKEN_BIN:
+      return mpack_wbin(buf, buflen, tok->length);
+    case MPACK_TOKEN_STR:
+      return mpack_wstr(buf, buflen, tok->length);
+    case MPACK_TOKEN_EXT:
+      return mpack_wext(buf, buflen, tok->data.ext_type, tok->length);
+    case MPACK_TOKEN_ARRAY:
+      return mpack_warray(buf, buflen, tok->length);
+    case MPACK_TOKEN_MAP:
+      return mpack_wmap(buf, buflen, tok->length);
+    default:
+      return MPACK_ERROR;
+  }
+}
+
+static int mpack_wpending(char **buf, size_t *buflen, mpack_tokbuf_t *state)
+{
+  size_t count;
+  assert(state->ppos < state->plen);
+  count = MIN(state->plen - state->ppos, *buflen);
+  memcpy(*buf, state->pending + state->ppos, count);
+  state->ppos += count;
+  *buf += count;
+  *buflen -= count;
+  if (state->ppos == state->plen) {
+    state->plen = 0;
+    return MPACK_OK;
+  }
+  return MPACK_EOF;
+}
+
+static int mpack_wpint(char **buf, size_t *buflen, mpack_value_t val)
+{
+  mpack_uint32_t hi = val.hi;
+  mpack_uint32_t lo = val.lo;
+
+  if (hi) {
+    /* uint 64 */
+    return mpack_w1(buf, buflen, 0xcf) ||
+           mpack_w4(buf, buflen, hi)   ||
+           mpack_w4(buf, buflen, lo);
+  } else if (lo > 0xffff) {
+    /* uint 32 */
+    return mpack_w1(buf, buflen, 0xce) ||
+           mpack_w4(buf, buflen, lo);
+  } else if (lo > 0xff) {
+    /* uint 16 */
+    return mpack_w1(buf, buflen, 0xcd) ||
+           mpack_w2(buf, buflen, lo);
+  } else if (lo > 0x7f) {
+    /* uint 8 */
+    return mpack_w1(buf, buflen, 0xcc) ||
+           mpack_w1(buf, buflen, lo);
+  } else {
+    return mpack_w1(buf, buflen, lo);
+  }
+}
+
+static int mpack_wnint(char **buf, size_t *buflen, mpack_value_t val)
+{
+  mpack_uint32_t hi = val.hi;
+  mpack_uint32_t lo = val.lo;
+
+  if (lo < 0x80000000) {
+    /* int 64 */
+    return mpack_w1(buf, buflen, 0xd3) ||
+           mpack_w4(buf, buflen, hi)   ||
+           mpack_w4(buf, buflen, lo);
+  } else if (lo < 0xffff7fff) {
+    /* int 32 */
+    return mpack_w1(buf, buflen, 0xd2) ||
+           mpack_w4(buf, buflen, lo);
+  } else if (lo < 0xffffff7f) {
+    /* int 16 */
+    return mpack_w1(buf, buflen, 0xd1) ||
+           mpack_w2(buf, buflen, lo);
+  } else if (lo < 0xffffffe0) {
+    /* int 8 */
+    return mpack_w1(buf, buflen, 0xd0) ||
+           mpack_w1(buf, buflen, lo);
+  } else {
+    /* negative fixint */
+    return mpack_w1(buf, buflen, (mpack_uint32_t)(0x100 + lo));
+  }
+}
+
+static int mpack_wfloat(char **buf, size_t *buflen,
+    const mpack_token_t *tok)
+{
+  if (tok->length == 4) {
+    return mpack_w1(buf, buflen, 0xca) ||
+           mpack_w4(buf, buflen, tok->data.value.lo);
+  } else if (tok->length == 8) {
+    return mpack_w1(buf, buflen, 0xcb) ||
+           mpack_w4(buf, buflen, tok->data.value.hi) ||
+           mpack_w4(buf, buflen, tok->data.value.lo);
+  } else {
+    return MPACK_ERROR;
+  }
+}
+
+static int mpack_wstr(char **buf, size_t *buflen, mpack_uint32_t len)
+{
+  if (len < 0x20) {
+    return mpack_w1(buf, buflen, 0xa0 | len);
+  } else if (len < 0x100) {
+    return mpack_w1(buf, buflen, 0xd9) ||
+           mpack_w1(buf, buflen, len);
+  } else if (len < 0x10000) {
+    return mpack_w1(buf, buflen, 0xda) ||
+           mpack_w2(buf, buflen, len);
+  } else {
+    return mpack_w1(buf, buflen, 0xdb) ||
+           mpack_w4(buf, buflen, len);
+  }
+}
+
+static int mpack_wbin(char **buf, size_t *buflen, mpack_uint32_t len)
+{
+  if (len < 0x100) {
+    return mpack_w1(buf, buflen, 0xc4) ||
+           mpack_w1(buf, buflen, len);
+  } else if (len < 0x10000) {
+    return mpack_w1(buf, buflen, 0xc5) ||
+           mpack_w2(buf, buflen, len);
+  } else {
+    return mpack_w1(buf, buflen, 0xc6) ||
+           mpack_w4(buf, buflen, len);
+  }
+}
+
+static int mpack_wext(char **buf, size_t *buflen, int type,
+    mpack_uint32_t len)
+{
+  mpack_uint32_t t;
+  assert(type >= 0 && type < 0x80);
+  t = (mpack_uint32_t)type;
+  switch (len) {
+    case 1: mpack_w1(buf, buflen, 0xd4); return mpack_w1(buf, buflen, t);
+    case 2: mpack_w1(buf, buflen, 0xd5); return mpack_w1(buf, buflen, t);
+    case 4: mpack_w1(buf, buflen, 0xd6); return mpack_w1(buf, buflen, t);
+    case 8: mpack_w1(buf, buflen, 0xd7); return mpack_w1(buf, buflen, t);
+    case 16: mpack_w1(buf, buflen, 0xd8); return mpack_w1(buf, buflen, t);
+    default:
+      if (len < 0x100) {
+        return mpack_w1(buf, buflen, 0xc7) ||
+               mpack_w1(buf, buflen, len)  ||
+               mpack_w1(buf, buflen, t);
+      } else if (len < 0x10000) {
+        return mpack_w1(buf, buflen, 0xc8) ||
+               mpack_w2(buf, buflen, len)  ||
+               mpack_w1(buf, buflen, t);
+      } else {
+        return mpack_w1(buf, buflen, 0xc9) ||
+               mpack_w4(buf, buflen, len)  ||
+               mpack_w1(buf, buflen, t);
+      }
+  }
+}
+
+static int mpack_warray(char **buf, size_t *buflen, mpack_uint32_t len)
+{
+  if (len < 0x10) {
+    return mpack_w1(buf, buflen, 0x90 | len);
+  } else if (len < 0x10000) {
+    return mpack_w1(buf, buflen, 0xdc) ||
+           mpack_w2(buf, buflen, len);
+  } else {
+    return mpack_w1(buf, buflen, 0xdd) ||
+           mpack_w4(buf, buflen, len);
+  }
+}
+
+static int mpack_wmap(char **buf, size_t *buflen, mpack_uint32_t len)
+{
+  if (len < 0x10) {
+    return mpack_w1(buf, buflen, 0x80 | len);
+  } else if (len < 0x10000) {
+    return mpack_w1(buf, buflen, 0xde) ||
+           mpack_w2(buf, buflen, len);
+  } else {
+    return mpack_w1(buf, buflen, 0xdf) ||
+           mpack_w4(buf, buflen, len);
+  }
+}
+
+static int mpack_w1(char **b, size_t *bl, mpack_uint32_t v)
+{
+  (*bl)--;
+  *(*b)++ = (char)(v & 0xff);
+  return MPACK_OK;
+}
+
+static int mpack_w2(char **b, size_t *bl, mpack_uint32_t v)
+{
+  *bl -= 2;
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+  return MPACK_OK;
+}
+
+static int mpack_w4(char **b, size_t *bl, mpack_uint32_t v)
+{
+  *bl -= 4;
+  *(*b)++ = (char)((v >> 24) & 0xff);
+  *(*b)++ = (char)((v >> 16) & 0xff);
+  *(*b)++ = (char)((v >> 8) & 0xff);
+  *(*b)++ = (char)(v & 0xff);
+  return MPACK_OK;
+}
+
+static int mpack_value(mpack_token_type_t type, mpack_uint32_t length,
+    mpack_value_t value, mpack_token_t *tok)
+{
+  tok->type = type;
+  tok->length = length;
+  tok->data.value = value;
+  return MPACK_OK;
+}
+
+static int mpack_blob(mpack_token_type_t type, mpack_uint32_t length,
+    int ext_type, mpack_token_t *tok)
+{
+  tok->type = type;
+  tok->length = length;