Squashed 'src/univalue/' changes from 5839ac3..2740c4f

2740c4f Merge branch '2015_11_escape_plan' into bitcoin
7482163 Add new testcase to Makefile.am
46098ee Version 1.0.1.
ccf3575 parser: Ensure multiple values cannot follow each other
eb6cd64 Omit Obj/Arr open token from jsonTokenIsValue() test
bfef9e2 Makefile.am: list recently added test data, fail{35,36}.json
3e319f3 parser: Tighten array, object syntax checks.
c74185c parser: transform C++ variables into bitmask
f2568bc Prefer C++ STL vector .at() for accessing object values.
8eafa26 travis: run parallel 'make distcheck'
fd448da test: Improve tester diagnostics.  Add failing test case from #15
2158205 Use internal, locale-independent isspace(), isdigit() implementations.
2ab9ad4 travis: Make 'make distcheck' for more comprehensive checks.
3339191 Escape all control characters

git-subtree-dir: src/univalue
git-subtree-split: 2740c4f71242086a7eb3dc32f812546ba9fad913

.gitignore

@@ -18,6 +18,7 @@ univalue-config.h*
 test-driver
 libtool
 ltmain.sh
+test-suite.log
 
 *.a
 *.la

.travis.yml

@@ -36,7 +36,7 @@ script:
   - ./configure --cache-file=config.cache $UNIVALUE_CONFIG_ALL $UNIVALUE_CONFIG || ( cat config.log && false)
   - make -s $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL ; false )
   - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
-  - if [ "$RUN_TESTS" = "true" ]; then make check; fi
+  - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi
 
 matrix:
   fast_finish: true

Makefile.am

@@ -70,6 +70,9 @@ TEST_FILES = \
 	$(TEST_DATA_DIR)/fail32.json \
 	$(TEST_DATA_DIR)/fail33.json \
 	$(TEST_DATA_DIR)/fail34.json \
+	$(TEST_DATA_DIR)/fail35.json \
+	$(TEST_DATA_DIR)/fail36.json \
+	$(TEST_DATA_DIR)/fail37.json \
 	$(TEST_DATA_DIR)/fail3.json \
 	$(TEST_DATA_DIR)/fail4.json \
 	$(TEST_DATA_DIR)/fail5.json \
@@ -79,6 +82,7 @@ TEST_FILES = \
 	$(TEST_DATA_DIR)/fail9.json \
 	$(TEST_DATA_DIR)/pass1.json \
 	$(TEST_DATA_DIR)/pass2.json \
-	$(TEST_DATA_DIR)/pass3.json
+	$(TEST_DATA_DIR)/pass3.json \
+	$(TEST_DATA_DIR)/round1.json
 
 EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS)

configure.ac

@@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter
 m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()])
 
 
-AC_INIT([univalue], [1.0.0],
+AC_INIT([univalue], [1.0.1],
         [http://github.com/jgarzik/univalue/])
 
 dnl make the compilation flags quiet unless V=1 is used

gen/gen.cpp

@@ -8,7 +8,6 @@
 // $ ./gen > univalue_escapes.h
 //
 
-#include <ctype.h>
 #include <stdio.h>
 #include <string.h>
 #include "univalue.h"
@@ -16,10 +15,17 @@
 using namespace std;
 
 static bool initEscapes;
-static const char *escapes[256];
+static std::string escapes[256];
 
 static void initJsonEscape()
 {
+    // Escape all lower control characters (some get overridden with smaller sequences below)
+    for (int ch=0x00; ch<0x20; ++ch) {
+        char tmpbuf[20];
+        snprintf(tmpbuf, sizeof(tmpbuf), "\\u%04x", ch);
+        escapes[ch] = std::string(tmpbuf);
+    }
+
     escapes[(int)'"'] = "\\\"";
     escapes[(int)'\\'] = "\\\\";
     escapes[(int)'\b'] = "\\b";
@@ -27,6 +33,7 @@ static void initJsonEscape()
     escapes[(int)'\n'] = "\\n";
     escapes[(int)'\r'] = "\\r";
     escapes[(int)'\t'] = "\\t";
+    escapes[(int)'\x7f'] = "\\u007f"; // U+007F DELETE
 
     initEscapes = true;
 }
@@ -39,13 +46,13 @@ static void outputEscape()
 		"static const char *escapes[256] = {\n");
 
 	for (unsigned int i = 0; i < 256; i++) {
-		if (!escapes[i]) {
+		if (escapes[i].empty()) {
 			printf("\tNULL,\n");
 		} else {
 			printf("\t\"");
 
 			unsigned int si;
-			for (si = 0; si < strlen(escapes[i]); si++) {
+			for (si = 0; si < escapes[i].size(); si++) {
 				char ch = escapes[i][si];
 				switch (ch) {
 				case '"':

include/univalue.h

@@ -243,6 +243,39 @@ extern enum jtokentype getJsonToken(std::string& tokenVal,
                                     unsigned int& consumed, const char *raw);
 extern const char *uvTypeName(UniValue::VType t);
 
+static inline bool jsonTokenIsValue(enum jtokentype jtt)
+{
+    switch (jtt) {
+    case JTOK_KW_NULL:
+    case JTOK_KW_TRUE:
+    case JTOK_KW_FALSE:
+    case JTOK_NUMBER:
+    case JTOK_STRING:
+        return true;
+
+    default:
+        return false;
+    }
+
+    // not reached
+}
+
+static inline bool json_isspace(int ch)
+{
+    switch (ch) {
+    case 0x20:
+    case 0x09:
+    case 0x0a:
+    case 0x0d:
+        return true;
+
+    default:
+        return false;
+    }
+
+    // not reached
+}
+
 extern const UniValue NullUniValue;
 
 const UniValue& find_value( const UniValue& obj, const std::string& name);

lib/univalue.cpp

@@ -4,7 +4,6 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <stdint.h>
-#include <ctype.h>
 #include <errno.h>
 #include <iomanip>
 #include <limits>
@@ -21,7 +20,7 @@ static bool ParsePrechecks(const std::string& str)
 {
     if (str.empty()) // No empty string allowed
         return false;
-    if (str.size() >= 1 && (isspace(str[0]) || isspace(str[str.size()-1]))) // No padding allowed
+    if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed
         return false;
     if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed
         return false;
@@ -210,7 +209,7 @@ bool UniValue::pushKVs(const UniValue& obj)
 
     for (unsigned int i = 0; i < obj.keys.size(); i++) {
         keys.push_back(obj.keys[i]);
-        values.push_back(obj.values[i]);
+        values.push_back(obj.values.at(i));
     }
 
     return true;
@@ -234,7 +233,7 @@ bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t)
         if (idx < 0)
             return false;
 
-        if (values[idx].getType() != it->second)
+        if (values.at(idx).getType() != it->second)
             return false;
     }
 
@@ -250,7 +249,7 @@ const UniValue& UniValue::operator[](const std::string& key) const
     if (index < 0)
         return NullUniValue;
 
-    return values[index];
+    return values.at(index);
 }
 
 const UniValue& UniValue::operator[](unsigned int index) const
@@ -260,7 +259,7 @@ const UniValue& UniValue::operator[](unsigned int index) const
     if (index >= values.size())
         return NullUniValue;
 
-    return values[index];
+    return values.at(index);
 }
 
 const char *uvTypeName(UniValue::VType t)
@@ -278,15 +277,11 @@ const char *uvTypeName(UniValue::VType t)
     return NULL;
 }
 
-const UniValue& find_value( const UniValue& obj, const std::string& name)
+const UniValue& find_value(const UniValue& obj, const std::string& name)
 {
     for (unsigned int i = 0; i < obj.keys.size(); i++)
-    {
-        if( obj.keys[i] == name )
-        {
-            return obj.values[i];
-        }
-    }
+        if (obj.keys[i] == name)
+            return obj.values.at(i);
 
     return NullUniValue;
 }

lib/univalue_escapes.h

@@ -2,38 +2,38 @@
 #ifndef BITCOIN_UNIVALUE_UNIVALUE_ESCAPES_H
 #define BITCOIN_UNIVALUE_UNIVALUE_ESCAPES_H
 static const char *escapes[256] = {
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	"\\u0000",
+	"\\u0001",
+	"\\u0002",
+	"\\u0003",
+	"\\u0004",
+	"\\u0005",
+	"\\u0006",
+	"\\u0007",
 	"\\b",
 	"\\t",
 	"\\n",
-	NULL,
+	"\\u000b",
 	"\\f",
 	"\\r",
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	"\\u000e",
+	"\\u000f",
+	"\\u0010",
+	"\\u0011",
+	"\\u0012",
+	"\\u0013",
+	"\\u0014",
+	"\\u0015",
+	"\\u0016",
+	"\\u0017",
+	"\\u0018",
+	"\\u0019",
+	"\\u001a",
+	"\\u001b",
+	"\\u001c",
+	"\\u001d",
+	"\\u001e",
+	"\\u001f",
 	NULL,
 	NULL,
 	"\\\"",
@@ -129,7 +129,7 @@ static const char *escapes[256] = {
 	NULL,
 	NULL,
 	NULL,
-	NULL,
+	"\\u007f",
 	NULL,
 	NULL,
 	NULL,

lib/univalue_read.cpp

@@ -9,6 +9,11 @@
 
 using namespace std;
 
+static bool json_isdigit(int ch)
+{
+    return ((ch >= '0') && (ch <= '9'));
+}
+
 // convert hexadecimal string to unsigned integer
 static const char *hatoui(const char *first, const char *last,
                           unsigned int& out)
@@ -17,7 +22,7 @@ static const char *hatoui(const char *first, const char *last,
     for (; first != last; ++first)
     {
         int digit;
-        if (isdigit(*first))
+        if (json_isdigit(*first))
             digit = *first - '0';
 
         else if (*first >= 'a' && *first <= 'f')
@@ -44,7 +49,7 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
 
     const char *rawStart = raw;
 
-    while ((*raw) && (isspace(*raw)))             // skip whitespace
+    while ((*raw) && (json_isspace(*raw)))             // skip whitespace
         raw++;
 
     switch (*raw) {
@@ -113,18 +118,18 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
         const char *first = raw;
 
         const char *firstDigit = first;
-        if (!isdigit(*firstDigit))
+        if (!json_isdigit(*firstDigit))
             firstDigit++;
-        if ((*firstDigit == '0') && isdigit(firstDigit[1]))
+        if ((*firstDigit == '0') && json_isdigit(firstDigit[1]))
             return JTOK_ERR;
 
         numStr += *raw;                       // copy first char
         raw++;
 
-        if ((*first == '-') && (!isdigit(*raw)))
+        if ((*first == '-') && (!json_isdigit(*raw)))
             return JTOK_ERR;
 
-        while ((*raw) && isdigit(*raw)) {     // copy digits
+        while ((*raw) && json_isdigit(*raw)) {     // copy digits
             numStr += *raw;
             raw++;
         }
@@ -134,9 +139,9 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
             numStr += *raw;                   // copy .
             raw++;
 
-            if (!isdigit(*raw))
+            if (!json_isdigit(*raw))
                 return JTOK_ERR;
-            while ((*raw) && isdigit(*raw)) { // copy digits
+            while ((*raw) && json_isdigit(*raw)) { // copy digits
                 numStr += *raw;
                 raw++;
             }
@@ -152,9 +157,9 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
                 raw++;
             }
 
-            if (!isdigit(*raw))
+            if (!json_isdigit(*raw))
                 return JTOK_ERR;
-            while ((*raw) && isdigit(*raw)) { // copy digits
+            while ((*raw) && json_isdigit(*raw)) { // copy digits
                 numStr += *raw;
                 raw++;
             }
@@ -236,12 +241,23 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed,
     }
 }
 
+enum expect_bits {
+    EXP_OBJ_NAME = (1U << 0),
+    EXP_COLON = (1U << 1),
+    EXP_ARR_VALUE = (1U << 2),
+    EXP_VALUE = (1U << 3),
+    EXP_NOT_VALUE = (1U << 4),
+};
+
+#define expect(bit) (expectMask & (EXP_##bit))
+#define setExpect(bit) (expectMask |= EXP_##bit)
+#define clearExpect(bit) (expectMask &= ~EXP_##bit)
+
 bool UniValue::read(const char *raw)
 {
     clear();
 
-    bool expectName = false;
-    bool expectColon = false;
+    uint32_t expectMask = 0;
     vector<UniValue*> stack;
 
     string tokenVal;
@@ -256,6 +272,41 @@ bool UniValue::read(const char *raw)
             return false;
         raw += consumed;
 
+        bool isValueOpen = jsonTokenIsValue(tok) ||
+            tok == JTOK_OBJ_OPEN || tok == JTOK_ARR_OPEN;
+
+        if (expect(VALUE)) {
+            if (!isValueOpen)
+                return false;
+            clearExpect(VALUE);
+
+        } else if (expect(ARR_VALUE)) {
+            bool isArrValue = isValueOpen || (tok == JTOK_ARR_CLOSE);
+            if (!isArrValue)
+                return false;
+
+            clearExpect(ARR_VALUE);
+
+        } else if (expect(OBJ_NAME)) {
+            bool isObjName = (tok == JTOK_OBJ_CLOSE || tok == JTOK_STRING);
+            if (!isObjName)
+                return false;
+
+        } else if (expect(COLON)) {
+            if (tok != JTOK_COLON)
+                return false;
+            clearExpect(COLON);
+
+        } else if (!expect(COLON) && (tok == JTOK_COLON)) {
+            return false;
+        }
+
+        if (expect(NOT_VALUE)) {
+            if (isValueOpen)
+                return false;
+            clearExpect(NOT_VALUE);
+        }
+
         switch (tok) {
 
         case JTOK_OBJ_OPEN:
@@ -277,13 +328,15 @@ bool UniValue::read(const char *raw)
             }
 
             if (utyp == VOBJ)
-                expectName = true;
+                setExpect(OBJ_NAME);
+            else
+                setExpect(ARR_VALUE);
             break;
             }
 
         case JTOK_OBJ_CLOSE:
         case JTOK_ARR_CLOSE: {
-            if (!stack.size() || expectColon || (last_tok == JTOK_COMMA))
+            if (!stack.size() || (last_tok == JTOK_COMMA))
                 return false;
 
             VType utyp = (tok == JTOK_OBJ_CLOSE ? VOBJ : VARR);
@@ -292,37 +345,40 @@ bool UniValue::read(const char *raw)
                 return false;
 
             stack.pop_back();
-            expectName = false;
+            clearExpect(OBJ_NAME);
+            setExpect(NOT_VALUE);
             break;
             }
 
         case JTOK_COLON: {
-            if (!stack.size() || expectName || !expectColon)
+            if (!stack.size())
                 return false;
 
             UniValue *top = stack.back();
             if (top->getType() != VOBJ)
                 return false;
 
-            expectColon = false;
+            setExpect(VALUE);
             break;
             }
 
         case JTOK_COMMA: {
-            if (!stack.size() || expectName || expectColon ||
+            if (!stack.size() ||
                 (last_tok == JTOK_COMMA) || (last_tok == JTOK_ARR_OPEN))
                 return false;
 
             UniValue *top = stack.back();
             if (top->getType() == VOBJ)
-                expectName = true;
+                setExpect(OBJ_NAME);
+            else
+                setExpect(ARR_VALUE);
             break;
             }
 
         case JTOK_KW_NULL:
         case JTOK_KW_TRUE:
         case JTOK_KW_FALSE: {
-            if (!stack.size() || expectName || expectColon)
+            if (!stack.size())
                 return false;
 
             UniValue tmpVal;
@@ -342,17 +398,19 @@ bool UniValue::read(const char *raw)
             UniValue *top = stack.back();
             top->values.push_back(tmpVal);
 
+            setExpect(NOT_VALUE);
             break;
             }
 
         case JTOK_NUMBER: {
-            if (!stack.size() || expectName || expectColon)
+            if (!stack.size())
                 return false;
 
             UniValue tmpVal(VNUM, tokenVal);
             UniValue *top = stack.back();
             top->values.push_back(tmpVal);
 
+            setExpect(NOT_VALUE);
             break;
             }
 
@@ -362,15 +420,16 @@ bool UniValue::read(const char *raw)
 
             UniValue *top = stack.back();
 
-            if (expectName) {
+            if (expect(OBJ_NAME)) {
                 top->keys.push_back(tokenVal);
-                expectName = false;
-                expectColon = true;
+                clearExpect(OBJ_NAME);
+                setExpect(COLON);
             } else {
                 UniValue tmpVal(VSTR, tokenVal);
                 top->values.push_back(tmpVal);
             }
 
+            setExpect(NOT_VALUE);
             break;
             }
 

lib/univalue_write.cpp

@@ -2,7 +2,6 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-#include <ctype.h>
 #include <iomanip>
 #include <sstream>
 #include <stdio.h>
@@ -25,10 +24,10 @@ static string json_escape(const string& inS)
         if (escStr)
             outS += escStr;
 
-        else if (isprint(ch))
+        else if (ch < 0x80)
             outS += ch;
 
-        else {
+        else { // TODO handle UTF-8 properly
             char tmpesc[16];
             sprintf(tmpesc, "\\u%04x", ch);
             outS += tmpesc;
@@ -113,7 +112,7 @@ void UniValue::writeObject(unsigned int prettyIndent, unsigned int indentLevel,
         s += "\"" + json_escape(keys[i]) + "\":";
         if (prettyIndent)
             s += " ";
-        s += values[i].write(prettyIndent, indentLevel + 1);
+        s += values.at(i).write(prettyIndent, indentLevel + 1);
         if (i != (values.size() - 1))
             s += ",";
         if (prettyIndent)

test/.gitignore

@@ -1 +1,4 @@
 unitester
+
+*.trs
+*.log

test/fail35.json

@@ -0,0 +1 @@
+[ true true true [] [] [] ]

test/fail36.json

@@ -0,0 +1 @@
+{"a":}

test/fail37.json

@@ -0,0 +1 @@
+{"a":1 "b":2}

test/round1.json

@@ -0,0 +1 @@
+["\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f"]

test/unitester.cpp

@@ -19,24 +19,37 @@
 
 using namespace std;
 string srcdir(JSON_TEST_SRC);
+static bool test_failed = false;
 
-static void runtest(string filename, const string& jdata)
+#define d_assert(expr) { if (!(expr)) { test_failed = true; fprintf(stderr, "%s failed\n", filename.c_str()); } }
+
+static std::string rtrim(std::string s)
 {
-        fprintf(stderr, "test %s\n", filename.c_str());
+    s.erase(s.find_last_not_of(" \n\r\t")+1);
+    return s;
+}
 
+static void runtest(string filename, const string& jdata)
+{
         string prefix = filename.substr(0, 4);
 
-        bool wantPass = (prefix == "pass");
+        bool wantPass = (prefix == "pass") || (prefix == "roun");
         bool wantFail = (prefix == "fail");
+        bool wantRoundTrip = (prefix == "roun");
         assert(wantPass || wantFail);
 
         UniValue val;
         bool testResult = val.read(jdata);
 
         if (wantPass) {
-            assert(testResult == true);
+            d_assert(testResult == true);
         } else {
-            assert(testResult == false);
+            d_assert(testResult == false);
+        }
+
+        if (wantRoundTrip) {
+            std::string odata = val.write(0, 0);
+            assert(odata == rtrim(jdata));
         }
 }
 
@@ -92,6 +105,9 @@ static const char *filenames[] = {
         "fail32.json",
         "fail33.json",
         "fail34.json",
+        "fail35.json",
+        "fail36.json",
+        "fail37.json",
         "fail3.json",
         "fail4.json",                // extra comma
         "fail5.json",
@@ -102,6 +118,7 @@ static const char *filenames[] = {
         "pass1.json",
         "pass2.json",
         "pass3.json",
+        "round1.json",              // round-trip test
 };
 
 int main (int argc, char *argv[])
@@ -110,6 +127,6 @@ int main (int argc, char *argv[])
         runtest_file(filenames[fidx]);
     }
 
-    return 0;
+    return test_failed ? 1 : 0;
 }