Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

715 lines
17 KiB

5 years ago
//========= Copyright Valve Corporation, All rights reserved. =================//
//
// Read JSON-formatted data into KeyValues
//
//=============================================================================//
#include "tier1/keyvaluesjson.h"
#include "tier1/utlbuffer.h"
#include "tier1/strtools.h"
#include <stdint.h> // INT32_MIN defn
KeyValuesJSONParser::KeyValuesJSONParser( const CUtlBuffer &buf )
{
Init( (const char *)buf.Base(), buf.TellPut() );
}
KeyValuesJSONParser::KeyValuesJSONParser( const char *pszText, int cbSize )
{
Init( pszText, cbSize >= 0 ? cbSize : V_strlen(pszText) );
}
KeyValuesJSONParser::~KeyValuesJSONParser() {}
void KeyValuesJSONParser::Init( const char *pszText, int cbSize )
{
m_szErrMsg[0] = '\0';
m_nLine = 1;
m_cur = pszText;
m_end = pszText+cbSize;
m_eToken = kToken_Null;
NextToken();
}
KeyValues *KeyValuesJSONParser::ParseFile()
{
// A valid JSON object should contain a single object, surrounded by curly braces.
if ( m_eToken == kToken_EOF )
{
V_sprintf_safe( m_szErrMsg, "Input contains no data" );
return NULL;
}
if ( m_eToken == kToken_Err )
return NULL;
if ( m_eToken == '{' )
{
// Parse the the entire file as one big object
KeyValues *pResult = new KeyValues("");
if ( !ParseObject( pResult ) )
{
pResult->deleteThis();
return NULL;
}
if ( m_eToken == kToken_EOF )
return pResult;
pResult->deleteThis();
}
V_sprintf_safe( m_szErrMsg, "%s not expected here. A valid JSON document should be a single object, which begins with '{' and ends with '}'", GetTokenDebugText() );
return NULL;
}
bool KeyValuesJSONParser::ParseObject( KeyValues *pObject )
{
Assert( m_eToken == '{' );
int nOpenDelimLine = m_nLine;
NextToken();
KeyValues *pLastChild = NULL;
while ( m_eToken != '}' )
{
// Parse error?
if ( m_eToken == kToken_Err )
return false;
if ( m_eToken == kToken_EOF )
{
// Actually report the error at the line of the unmatched delimiter.
// There's no need to report the line number of the end of file, that is always
// useless.
m_nLine = nOpenDelimLine;
V_strcpy_safe( m_szErrMsg, "End of input was reached and '{' was not matched by '}'" );
return false;
}
// It must be a string, for the key name
if ( m_eToken != kToken_String )
{
V_sprintf_safe( m_szErrMsg, "%s not expected here; expected string for key name or '}'", GetTokenDebugText() );
return false;
}
KeyValues *pChildValue = new KeyValues( m_vecTokenChars.Base() );
NextToken();
// Expect and eat colon
if ( m_eToken != ':' )
{
V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ':'?", GetTokenDebugText() );
pChildValue->deleteThis();
return false;
}
NextToken();
// Recursively parse the value
if ( !ParseValue( pChildValue ) )
{
pChildValue->deleteThis();
return false;
}
// Add to parent.
pObject->AddSubkeyUsingKnownLastChild( pChildValue, pLastChild );
pLastChild = pChildValue;
// Eat the comma, if there is one. If no comma,
// then the other thing that could come next
// is the closing brace to close the object
// NOTE: We are allowing the extra comma after the last item
if ( m_eToken == ',' )
{
NextToken();
}
else if ( m_eToken != '}' )
{
V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ',' or '}'?", GetTokenDebugText() );
return false;
}
}
// Eat closing '}'
NextToken();
// Success
return true;
}
bool KeyValuesJSONParser::ParseArray( KeyValues *pArray )
{
Assert( m_eToken == '[' );
int nOpenDelimLine = m_nLine;
NextToken();
KeyValues *pLastChild = NULL;
int idx = 0;
while ( m_eToken != ']' )
{
// Parse error?
if ( m_eToken == kToken_Err )
return false;
if ( m_eToken == kToken_EOF )
{
// Actually report the error at the line of the unmatched delimiter.
// There's no need to report the line number of the end of file, that is always
// useless.
m_nLine = nOpenDelimLine;
V_strcpy_safe( m_szErrMsg, "End of input was reached and '[' was not matched by ']'" );
return false;
}
// Set a dummy key name based on the index
char szKeyName[ 32 ];
V_sprintf_safe( szKeyName, "%d", idx );
++idx;
KeyValues *pChildValue = new KeyValues( szKeyName );
// Recursively parse the value
if ( !ParseValue( pChildValue ) )
{
pChildValue->deleteThis();
return false;
}
// Add to parent.
pArray->AddSubkeyUsingKnownLastChild( pChildValue, pLastChild );
pLastChild = pChildValue;
// Handle a colon here specially. If one appears, the odds are they
// are trying to put object-like data inside of an array
if ( m_eToken == ':' )
{
V_sprintf_safe( m_szErrMsg, "':' not expected inside an array. ('[]' used when '{}' was intended?)" );
return false;
}
// Eat the comma, if there is one. If no comma,
// then the other thing that could come next
// is the closing brace to close the object
// NOTE: We are allowing the extra comma after the last item
if ( m_eToken == ',' )
{
NextToken();
}
else if ( m_eToken != ']' )
{
V_sprintf_safe( m_szErrMsg, "%s not expected here. Missing ',' or ']'?", GetTokenDebugText() );
return false;
}
}
// Eat closing ']'
NextToken();
// Success
return true;
}
bool KeyValuesJSONParser::ParseValue( KeyValues *pValue )
{
switch ( m_eToken )
{
case '{': return ParseObject( pValue );
case '[': return ParseArray( pValue );
case kToken_String:
pValue->SetString( NULL, m_vecTokenChars.Base() );
NextToken();
return true;
case kToken_NumberInt:
{
const char *pszNum = m_vecTokenChars.Base();
// Negative?
if ( *pszNum == '-' )
{
int64 val64 = V_atoi64( pszNum );
if ( val64 < INT32_MIN )
{
// !KLUDGE! KeyValues cannot support this!
V_sprintf_safe( m_szErrMsg, "%s is out of range for KeyValues, which doesn't support signed 64-bit numbers", pszNum );
return false;
}
pValue->SetInt( NULL, (int)val64 );
}
else
{
uint64 val64 = V_atoui64( pszNum );
if ( val64 > 0x7fffffffU )
{
pValue->SetUint64( NULL, val64 );
}
else
{
pValue->SetInt( NULL, (int)val64 );
}
}
NextToken();
return true;
}
case kToken_NumberFloat:
{
float f = V_atof( m_vecTokenChars.Base() );
pValue->SetFloat( NULL, f );
NextToken();
return true;
}
case kToken_True:
pValue->SetBool( NULL, true );
NextToken();
return true;
case kToken_False:
pValue->SetBool( NULL, false );
NextToken();
return true;
case kToken_Null:
pValue->SetPtr( NULL, NULL );
NextToken();
return true;
case kToken_Err:
return false;
}
V_sprintf_safe( m_szErrMsg, "%s not expected here; missing value?", GetTokenDebugText() );
return false;
}
void KeyValuesJSONParser::NextToken()
{
// Already in terminal state?
if ( m_eToken < 0 )
return;
// Clear token
m_vecTokenChars.SetCount(0);
// Scan until we hit the end of input
while ( m_cur < m_end )
{
// Next character?
char c = *m_cur;
switch (c)
{
// Whitespace? Eat it and keep parsing
case ' ':
case '\t':
++m_cur;
break;
// Newline? Eat it and advance line number
case '\n':
case '\r':
++m_nLine;
++m_cur;
// Eat \r\n or \n\r pair as a single character
if ( m_cur < m_end && *m_cur == ( '\n' + '\r' - c ) )
++m_cur;
break;
// Single-character JSON token?
case ':':
case '{':
case '}':
case '[':
case ']':
case ',':
m_eToken = c;
++m_cur;
return;
// String?
case '\"':
case '\'': // NOTE: We allow strings to be delimited by single quotes, which is not JSON compliant
ParseStringToken();
return;
case '-':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
ParseNumberToken();
return;
// Literal "true"
case 't':
if ( m_cur + 4 <= m_end && m_cur[1] == 'r' && m_cur[2] == 'u' && m_cur[3] == 'e' )
{
m_cur += 4;
m_eToken = kToken_True;
return;
}
goto unexpected_char;
// Literal "false"
case 'f':
if ( m_cur + 5 <= m_end && m_cur[1] == 'a' && m_cur[2] == 'l' && m_cur[3] == 's' && m_cur[4] == 'e' )
{
m_cur += 5;
m_eToken = kToken_False;
return;
}
goto unexpected_char;
// Literal "null"
case 'n':
if ( m_cur + 4 <= m_end && m_cur[1] == 'u' && m_cur[2] == 'l' && m_cur[3] == 'l' )
{
m_cur += 4;
m_eToken = kToken_Null;
return;
}
goto unexpected_char;
case '/':
// C++-style comment?
if ( m_cur < m_end && m_cur[1] == '/' )
{
m_cur += 2;
while ( m_cur < m_end && *m_cur != '\n' && *m_cur != '\r' )
++m_cur;
// Leave newline as the next character, we'll handle it above
break;
}
// | fall
// | through
// V
default:
unexpected_char:
if ( V_isprint(c) )
V_sprintf_safe( m_szErrMsg, "Unexpected character 0x%02x ('%c')", (uint8)c, c );
else
V_sprintf_safe( m_szErrMsg, "Unexpected character 0x%02x", (uint8)c );
m_eToken = kToken_Err;
return;
}
}
m_eToken = kToken_EOF;
}
void KeyValuesJSONParser::ParseNumberToken()
{
// Clear token
m_vecTokenChars.SetCount(0);
// Eat leading minus sign
if ( *m_cur == '-' )
{
m_vecTokenChars.AddToTail( '-' );
++m_cur;
}
if ( m_cur >= m_end )
{
V_strcpy_safe( m_szErrMsg, "Unexpected EOF while parsing number" );
m_eToken = kToken_Err;
return;
}
char c = *m_cur;
m_vecTokenChars.AddToTail( c );
bool bHasWholePart = false;
switch ( c )
{
case '0':
// Leading 0 cannot be followed by any more digits, as per JSON spec (and to make sure nobody tries to parse octal).
++m_cur;
bHasWholePart = true;
break;
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
bHasWholePart = true;
++m_cur;
// Accumulate digits until we hit a non-digit
while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' )
m_vecTokenChars.AddToTail( *(m_cur++) );
break;
case '.':
// strict JSON doesn't allow a number that starts with a decimal point, but we do
break;
}
// Assume this is integral, unless we hit a decimal point and/or exponent
m_eToken = kToken_NumberInt;
// Fractional portion?
if ( m_cur < m_end && *m_cur == '.' )
{
m_eToken = kToken_NumberFloat;
// Eat decimal point
m_vecTokenChars.AddToTail( *(m_cur++) );
// Accumulate digits until we hit a non-digit
bool bHasFractionPart = false;
while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' )
{
m_vecTokenChars.AddToTail( *(m_cur++) );
bHasFractionPart = true;
}
// Make sure we aren't just a single '.'
if ( !bHasWholePart && !bHasFractionPart )
{
m_vecTokenChars.AddToTail(0);
V_sprintf_safe( m_szErrMsg, "Invalid number starting with '%s'", m_vecTokenChars.Base() );
m_eToken = kToken_Err;
return;
}
}
// Exponent?
if ( m_cur < m_end && ( *m_cur == 'e' || *m_cur == 'E' ) )
{
m_eToken = kToken_NumberFloat;
// Eat 'e'
m_vecTokenChars.AddToTail( *(m_cur++) );
// Optional sign
if ( m_cur < m_end && ( *m_cur == '-' || *m_cur == '+' ) )
m_vecTokenChars.AddToTail( *(m_cur++) );
// Accumulate digits until we hit a non-digit
bool bHasExponentDigit = false;
while ( m_cur < m_end && *m_cur >= '0' && *m_cur <= '9' )
{
m_vecTokenChars.AddToTail( *(m_cur++) );
bHasExponentDigit = true;
}
if ( !bHasExponentDigit )
{
V_strcpy_safe( m_szErrMsg, "Bad exponent in floating point number" );
m_eToken = kToken_Err;
return;
}
}
// OK, We have parsed a valid number.
// Terminate token
m_vecTokenChars.AddToTail( '\0' );
// EOF? That's OK for now, at this lexical parsing level. We'll handle the error
// at the higher parse level, when expecting a comma or closing delimiter
if ( m_cur >= m_end )
return;
// Is the next thing a valid character? This is the most common case.
c = *m_cur;
if ( V_isspace( c ) || c == ',' || c == '}' || c == ']' || c == '/' )
return;
// Handle these guys as "tokens", to provide a slightly more meaningful error message
if ( c == '[' || c == '{' )
return;
// Anything else, treat the whole thing as an invalid numerical constant
if ( V_isprint(c) )
V_sprintf_safe( m_szErrMsg, "Number contains invalid character 0x%02x ('%c')", (uint8)c, c );
else
V_sprintf_safe( m_szErrMsg, "Number contains invalid character 0x%02x", (uint8)c );
m_eToken = kToken_Err;
}
void KeyValuesJSONParser::ParseStringToken()
{
char cDelim = *(m_cur++);
while ( m_cur < m_end )
{
char c = *(m_cur++);
if ( c == '\r' || c == '\n' )
{
V_sprintf_safe( m_szErrMsg, "Hit end of line before closing quote (%c)", c );
m_eToken = kToken_Err;
return;
}
if ( c == cDelim )
{
m_eToken = kToken_String;
m_vecTokenChars.AddToTail( '\0' );
return;
}
// Ordinary character? Just append it
if ( c != '\\' )
{
m_vecTokenChars.AddToTail( c );
continue;
}
// Escaped character.
// End of string? We'll handle it above
if ( m_cur >= m_end )
continue;
// Check table of allowed escape characters
switch (c)
{
case '\\':
case '/':
case '\'':
case '\"': m_vecTokenChars.AddToTail( c ); break;
case 'b': m_vecTokenChars.AddToTail( '\b' ); break;
case 'f': m_vecTokenChars.AddToTail( '\f' ); break;
case 'n': m_vecTokenChars.AddToTail( '\n' ); break;
case 'r': m_vecTokenChars.AddToTail( '\r' ); break;
case 't': m_vecTokenChars.AddToTail( '\t' ); break;
case 'u':
{
// Make sure are followed by exactly 4 hex digits
if ( m_cur + 4 > m_end || !V_isxdigit( m_cur[0] ) || !V_isxdigit( m_cur[1] ) || !V_isxdigit( m_cur[2] ) || !V_isxdigit( m_cur[3] ) )
{
V_sprintf_safe( m_szErrMsg, "\\u must be followed by exactly 4 hex digits" );
m_eToken = kToken_Err;
return;
}
// Parse the codepoint
uchar32 nCodePoint = 0;
for ( int n = 0 ; n < 4 ; ++n )
{
nCodePoint <<= 4;
char chHex = *(m_cur++);
if ( chHex >= '0' && chHex <= '9' )
nCodePoint += chHex - '0';
else if ( chHex >= 'a' && chHex <= 'a' )
nCodePoint += chHex + 0x0a - 'a';
else if ( chHex >= 'A' && chHex <= 'A' )
nCodePoint += chHex + 0x0a - 'A';
else
Assert( false ); // inconceivable, due to above
}
// Encode it in UTF-8
char utf8Encode[8];
int r = Q_UChar32ToUTF8( nCodePoint, utf8Encode );
if ( r < 0 || r > 4 )
{
V_sprintf_safe( m_szErrMsg, "Invalid code point \\u%04x", nCodePoint );
m_eToken = kToken_Err;
return;
}
for ( int i = 0 ; i < r ; ++i )
m_vecTokenChars.AddToTail( utf8Encode[i] );
} break;
default:
if ( V_isprint(c) )
V_sprintf_safe( m_szErrMsg, "Invalid escape character 0x%02x ('\\%c')", (uint8)c, c );
else
V_sprintf_safe( m_szErrMsg, "Invalid escape character 0x%02x", (uint8)c );
m_eToken = kToken_Err;
return;
}
}
V_sprintf_safe( m_szErrMsg, "Hit end of input before closing quote (%c)", cDelim );
m_eToken = kToken_Err;
}
const char *KeyValuesJSONParser::GetTokenDebugText()
{
switch ( m_eToken )
{
case kToken_EOF: return "<EOF>";
case kToken_String: return "<string>";
case kToken_NumberInt:
case kToken_NumberFloat: return "<number>";
case kToken_True: return "'true'";
case kToken_False: return "'false'";
case kToken_Null: return "'null'";
case '{': return "'{'";
case '}': return "'}'";
case '[': return "'['";
case ']': return "']'";
case ':': return "':'";
case ',': return "','";
}
// We shouldn't ever need to ask for a debug string for the error token,
// and anything else is an error
Assert( false );
return "<parse error>";
}
#ifdef _DEBUG
static void JSONTest_ParseValid( const char *pszData )
{
KeyValuesJSONParser parser( pszData );
KeyValues *pFile = parser.ParseFile();
Assert( pFile );
pFile->deleteThis();
}
static void JSONTest_ParseInvalid( const char *pszData, const char *pszExpectedErrMsgSnippet, int nExpectedFailureLine )
{
KeyValuesJSONParser parser( pszData );
KeyValues *pFile = parser.ParseFile();
Assert( pFile == NULL );
Assert( V_stristr( parser.m_szErrMsg, pszExpectedErrMsgSnippet ) != NULL );
Assert( parser.m_nLine == nExpectedFailureLine );
}
void TestKeyValuesJSONParser()
{
JSONTest_ParseValid( "{}" );
JSONTest_ParseValid( R"JSON({
"key": "string_value",
"pos_int32": 123,
"pos_int64": 123456789012,
"neg_int32": -456,
"float": -45.23,
"pos_exponent": 1e30,
"neg_exponent": 1e-16,
"decimal_and_exponent": 1.e+30,
"no_leading_zero": .7, // we support this, even though strict JSON says it's no good
"zero": 0,
"true_value": true,
"false_value": false,
"null_value": null,
"with_escaped": "\r \t \n",
"unicode": "\u1234 \\u12f3",
"array_of_ints": [ 1, 2, 3, -45 ],
"empty_array": [],
"array_with_stuff_inside": [
{}, // this is a comment.
[ 0.45, {}, "hello!" ],
{ "id": 0 },
// Trailing comma above. Comment here
],
})JSON" );
JSONTest_ParseInvalid( "{ \"key\": 123", "missing", 1 );
JSONTest_ParseInvalid( "{ \"key\": 123.4f }", "number", 1 );
}
#endif