Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

588 lines
20 KiB

5 years ago
//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
//=============================================================================
#include "dmserializerbinary.h"
#include "datamodel/idatamodel.h"
#include "datamodel.h"
#include "datamodel/dmelement.h"
#include "datamodel/dmattributevar.h"
#include "dmattributeinternal.h"
#include "dmelementdictionary.h"
#include "tier1/utlbuffer.h"
#include "DmElementFramework.h"
//-----------------------------------------------------------------------------
// Forward declarations
//-----------------------------------------------------------------------------
class CUtlBuffer;
class CBaseSceneObject;
//-----------------------------------------------------------------------------
// special element indices
//-----------------------------------------------------------------------------
enum
{
ELEMENT_INDEX_NULL = -1,
ELEMENT_INDEX_EXTERNAL = -2,
};
//-----------------------------------------------------------------------------
// Serialization class for Binary output
//-----------------------------------------------------------------------------
class CDmSerializerBinary : public IDmSerializer
{
public:
CDmSerializerBinary() {}
// Inherited from IDMSerializer
virtual const char *GetName() const { return "binary"; }
virtual const char *GetDescription() const { return "Binary"; }
virtual bool StoresVersionInFile() const { return true; }
virtual bool IsBinaryFormat() const { return true; }
virtual int GetCurrentVersion() const { return 2; }
virtual bool Serialize( CUtlBuffer &buf, CDmElement *pRoot );
virtual bool Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
const char *pSourceFormatName, int nFormatVersion,
DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot );
private:
// Methods related to serialization
void SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid );
void SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute );
void SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute );
bool SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement );
bool SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement );
bool SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& dict, unsigned short *symbolToIndexMap, CDmElement *pElement);
// Methods related to unserialization
DmElementHandle_t UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList );
void UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList );
void UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList );
bool UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable );
bool UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable );
};
//-----------------------------------------------------------------------------
// Singleton instance
//-----------------------------------------------------------------------------
static CDmSerializerBinary s_DMSerializerBinary;
void InstallBinarySerializer( IDataModel *pFactory )
{
pFactory->AddSerializer( &s_DMSerializerBinary );
}
//-----------------------------------------------------------------------------
// Write out the index of the element to avoid looks at read time
//-----------------------------------------------------------------------------
void CDmSerializerBinary::SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid )
{
if ( hElement == DMELEMENT_HANDLE_INVALID )
{
buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle
}
else
{
CDmElement *pElement = g_pDataModel->GetElement( hElement );
if ( pElement )
{
if ( pElement->GetFileId() == fileid )
{
buf.PutInt( list.Find( pElement ) );
}
else
{
buf.PutInt( ELEMENT_INDEX_EXTERNAL );
char idstr[ 40 ];
UniqueIdToString( pElement->GetId(), idstr, sizeof( idstr ) );
buf.PutString( idstr );
}
}
else
{
DmObjectId_t *pId = NULL;
DmElementReference_t *pRef = g_pDataModelImp->FindElementReference( hElement, &pId );
if ( pRef && pId )
{
buf.PutInt( ELEMENT_INDEX_EXTERNAL );
char idstr[ 40 ];
UniqueIdToString( *pId, idstr, sizeof( idstr ) );
buf.PutString( idstr );
}
else
{
buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle
}
}
}
}
//-----------------------------------------------------------------------------
// Writes out element attributes
//-----------------------------------------------------------------------------
void CDmSerializerBinary::SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute )
{
SerializeElementIndex( buf, list, pAttribute->GetValue< DmElementHandle_t >(), pAttribute->GetOwner()->GetFileId() );
}
//-----------------------------------------------------------------------------
// Writes out element array attributes
//-----------------------------------------------------------------------------
void CDmSerializerBinary::SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute )
{
DmFileId_t fileid = pAttribute->GetOwner()->GetFileId();
CDmrElementArray<> vec( pAttribute );
int nCount = vec.Count();
buf.PutInt( nCount );
for ( int i = 0; i < nCount; ++i )
{
SerializeElementIndex( buf, list, vec.GetHandle(i), fileid );
}
}
//-----------------------------------------------------------------------------
// Writes out all attributes
//-----------------------------------------------------------------------------
bool CDmSerializerBinary::SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement )
{
// Collect the attributes to be written
CDmAttribute **ppAttributes = ( CDmAttribute** )_alloca( pElement->AttributeCount() * sizeof( CDmAttribute* ) );
int nAttributes = 0;
for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
{
if ( pAttribute->IsFlagSet( FATTRIB_DONTSAVE | FATTRIB_STANDARD ) )
continue;
ppAttributes[ nAttributes++ ] = pAttribute;
}
// Now write them all out in reverse order, since FirstAttribute is actually the *last* attribute for perf reasons
buf.PutInt( nAttributes );
for ( int i = nAttributes - 1; i >= 0; --i )
{
CDmAttribute *pAttribute = ppAttributes[ i ];
Assert( pAttribute );
buf.PutShort( symbolToIndexMap[ pAttribute->GetNameSymbol() ] );
buf.PutChar( pAttribute->GetType() );
switch( pAttribute->GetType() )
{
default:
pAttribute->Serialize( buf );
break;
case AT_ELEMENT:
SerializeElementAttribute( buf, list, pAttribute );
break;
case AT_ELEMENT_ARRAY:
SerializeElementArrayAttribute( buf, list, pAttribute );
break;
}
}
return buf.IsValid();
}
bool CDmSerializerBinary::SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement )
{
SerializeAttributes( buf, list, symbolToIndexMap, pElement );
return buf.IsValid();
}
bool CDmSerializerBinary::SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement )
{
buf.PutShort( symbolToIndexMap[ pElement->GetType() ] );
buf.PutString( pElement->GetName() );
buf.Put( &pElement->GetId(), sizeof(DmObjectId_t) );
return buf.IsValid();
}
void MarkSymbol( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, UtlSymId_t sym )
{
if ( symbolToIndexMap[ sym ] != 0xffff )
return;
symbolToIndexMap[ sym ] = nSymbols;
indexToSymbolMap[ nSymbols ] = sym;
nSymbols++;
}
void MarkSymbols( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, CDmElement *pElement )
{
MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pElement->GetType() );
for ( CDmAttribute *pAttr = pElement->FirstAttribute(); pAttr; pAttr = pAttr->NextAttribute() )
{
MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pAttr->GetNameSymbol() );
}
}
bool CDmSerializerBinary::Serialize( CUtlBuffer &outBuf, CDmElement *pRoot )
{
// Save elements, attribute links
CDmElementSerializationDictionary dict;
dict.BuildElementList( pRoot, true );
// TODO - consider allowing dmxconvert to skip the collection step, since the only datamodel symbols will be the ones from the file
unsigned short nTotalSymbols = g_pDataModelImp->GetSymbolCount();
UtlSymId_t *indexToSymbolMap = ( UtlSymId_t * )stackalloc( nTotalSymbols * sizeof( UtlSymId_t ) );
unsigned short *symbolToIndexMap = ( unsigned short* )stackalloc( nTotalSymbols * sizeof( unsigned short ) );
V_memset( indexToSymbolMap, 0xff, nTotalSymbols * sizeof( UtlSymId_t ) );
V_memset( symbolToIndexMap, 0xff, nTotalSymbols * sizeof( unsigned short ) );
// collect list of attribute names and element types into string table
unsigned short nUsedSymbols = 0;
DmElementDictHandle_t i;
for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
{
MarkSymbols( indexToSymbolMap, symbolToIndexMap, nUsedSymbols, dict.GetRootElement( i ) );
}
Assert( nUsedSymbols <= nTotalSymbols );
// write out the symbol table for this file (may be significantly smaller than datamodel's full symbol table)
outBuf.PutShort( nUsedSymbols );
for ( int si = 0; si < nUsedSymbols; ++si )
{
UtlSymId_t sym = indexToSymbolMap[ si ];
const char *pStr = g_pDataModel->GetString( sym );
outBuf.PutString( pStr );
}
// First write out the dictionary of all elements (to avoid later stitching up in unserialize)
outBuf.PutInt( dict.RootElementCount() );
for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
{
SaveElementDict( outBuf, symbolToIndexMap, dict.GetRootElement( i ) );
}
// Now write out the attributes of each of those elements
for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
{
SaveElement( outBuf, dict, symbolToIndexMap, dict.GetRootElement( i ) );
}
return true;
}
//-----------------------------------------------------------------------------
// Reads an element index and converts it to a handle (local or external)
//-----------------------------------------------------------------------------
DmElementHandle_t CDmSerializerBinary::UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList )
{
int nElementIndex = buf.GetInt();
Assert( nElementIndex < elementList.Count() );
if ( nElementIndex == ELEMENT_INDEX_EXTERNAL )
{
char idstr[ 40 ];
buf.GetString( idstr );
DmObjectId_t id;
UniqueIdFromString( &id, idstr, sizeof( idstr ) );
return g_pDataModelImp->FindOrCreateElementHandle( id );
}
Assert( nElementIndex >= 0 || nElementIndex == ELEMENT_INDEX_NULL );
if ( nElementIndex < 0 || !elementList[ nElementIndex ] )
return DMELEMENT_HANDLE_INVALID;
return elementList[ nElementIndex ]->GetHandle();
}
//-----------------------------------------------------------------------------
// Reads an element attribute
//-----------------------------------------------------------------------------
void CDmSerializerBinary::UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList )
{
DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList );
if ( !pAttribute )
return;
pAttribute->SetValue( hElement );
}
//-----------------------------------------------------------------------------
// Reads an element array attribute
//-----------------------------------------------------------------------------
void CDmSerializerBinary::UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList )
{
int nElementCount = buf.GetInt();
if ( !pAttribute || pAttribute->GetType() != AT_ELEMENT_ARRAY )
{
// Parse past the data
for ( int i = 0; i < nElementCount; ++i )
{
UnserializeElementIndex( buf, elementList );
}
return;
}
CDmrElementArray<> array( pAttribute );
array.RemoveAll();
array.EnsureCapacity( nElementCount );
for ( int i = 0; i < nElementCount; ++i )
{
DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList );
array.AddToTail( hElement );
}
}
//-----------------------------------------------------------------------------
// Reads a single element
//-----------------------------------------------------------------------------
bool CDmSerializerBinary::UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable )
{
char nameBuf[ 1024 ];
int nAttributeCount = buf.GetInt();
for ( int i = 0; i < nAttributeCount; ++i )
{
const char *pName = NULL;
if ( symbolTable )
{
unsigned short nName = buf.GetShort();
pName = g_pDataModel->GetString( symbolTable[ nName ] );
}
else
{
buf.GetString( nameBuf );
pName = nameBuf;
}
DmAttributeType_t nAttributeType = (DmAttributeType_t)buf.GetChar();
Assert( pName != NULL && pName[ 0 ] != '\0' );
Assert( nAttributeType != AT_UNKNOWN );
CDmAttribute *pAttribute = pElement ? pElement->AddAttribute( pName, nAttributeType ) : NULL;
if ( pElement && !pAttribute )
{
Warning("Dm: Attempted to read an attribute (\"%s\") of an inappropriate type!\n", pName );
return false;
}
switch( nAttributeType )
{
default:
if ( !pAttribute )
{
SkipUnserialize( buf, nAttributeType );
}
else
{
pAttribute->Unserialize( buf );
}
break;
case AT_ELEMENT:
UnserializeElementAttribute( buf, pAttribute, elementList );
break;
case AT_ELEMENT_ARRAY:
UnserializeElementArrayAttribute( buf, pAttribute, elementList );
break;
}
}
return buf.IsValid();
}
struct DmIdPair_t
{
DmObjectId_t m_oldId;
DmObjectId_t m_newId;
DmIdPair_t &operator=( const DmIdPair_t &that )
{
CopyUniqueId( that.m_oldId, &m_oldId );
CopyUniqueId( that.m_newId, &m_newId );
return *this;
}
static unsigned int HashKey( const DmIdPair_t& that )
{
return *( unsigned int* )&that.m_oldId.m_Value;
}
static bool Compare( const DmIdPair_t& a, const DmIdPair_t& b )
{
return IsUniqueIdEqual( a.m_oldId, b.m_oldId );
}
};
DmElementHandle_t CreateElementWithFallback( const char *pType, const char *pName, DmFileId_t fileid, const DmObjectId_t &id )
{
DmElementHandle_t hElement = g_pDataModel->CreateElement( pType, pName, fileid, &id );
if ( hElement == DMELEMENT_HANDLE_INVALID )
{
Warning("Binary: Element uses unknown element type %s\n", pType );
hElement = g_pDataModel->CreateElement( "DmElement", pName, fileid, &id );
Assert( hElement != DMELEMENT_HANDLE_INVALID );
}
return hElement;
}
//-----------------------------------------------------------------------------
// Main entry point for the unserialization
//-----------------------------------------------------------------------------
bool CDmSerializerBinary::Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
const char *pSourceFormatName, int nSourceFormatVersion,
DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot )
{
Assert( !V_stricmp( pEncodingName, GetName() ) );
if ( V_stricmp( pEncodingName, GetName() ) != 0 )
return false;
Assert( nEncodingVersion >= 0 && nEncodingVersion <= 2 );
if ( nEncodingVersion < 0 || nEncodingVersion > 2 )
return false;
bool bReadSymbolTable = nEncodingVersion >= 2;
// Read string table
unsigned short nStrings = 0;
UtlSymId_t *symbolTable = NULL;
if ( bReadSymbolTable )
{
char stringBuf[ 256 ];
nStrings = buf.GetShort();
symbolTable = ( UtlSymId_t* )stackalloc( nStrings * sizeof( UtlSymId_t ) );
for ( int i = 0; i < nStrings; ++i )
{
buf.GetString( stringBuf );
symbolTable[ i ] = g_pDataModel->GetSymbol( stringBuf );
}
}
bool bSuccess = UnserializeElements( buf, fileid, idConflictResolution, ppRoot, symbolTable );
if ( !bSuccess )
return false;
return g_pDataModel->UpdateUnserializedElements( pSourceFormatName, nSourceFormatVersion, fileid, idConflictResolution, ppRoot );
}
bool CDmSerializerBinary::UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable )
{
*ppRoot = NULL;
// Read in the element count.
int nElementCount = buf.GetInt();
if ( !nElementCount )
return true;
int nMaxIdConflicts = min( nElementCount, g_pDataModel->GetAllocatedElementCount() );
int nExpectedIdCopyConflicts = ( idConflictResolution == CR_FORCE_COPY || idConflictResolution == CR_COPY_NEW ) ? nMaxIdConflicts : 0;
int nBuckets = min( 0x10000, max( 16, nExpectedIdCopyConflicts / 16 ) ); // CUtlHash can only address up to 65k buckets
CUtlHash< DmIdPair_t > idmap( nBuckets, 0, 0, DmIdPair_t::Compare, DmIdPair_t::HashKey );
// Read + create all elements
CUtlVector<CDmElement*> elementList( 0, nElementCount );
for ( int i = 0; i < nElementCount; ++i )
{
char pName[2048];
DmObjectId_t id;
char typeBuf[ 256 ];
const char *pType = NULL;
if ( symbolTable )
{
unsigned short nType = buf.GetShort();
pType = g_pDataModel->GetString( symbolTable[ nType ] );
}
else
{
buf.GetString( typeBuf );
pType = typeBuf;
}
buf.GetString( pName );
buf.Get( &id, sizeof(DmObjectId_t) );
if ( idConflictResolution == CR_FORCE_COPY )
{
DmIdPair_t idpair;
CopyUniqueId( id, &idpair.m_oldId );
CreateUniqueId( &idpair.m_newId );
idmap.Insert( idpair );
CopyUniqueId( idpair.m_newId, &id );
}
DmElementHandle_t hElement = DMELEMENT_HANDLE_INVALID;
DmElementHandle_t hExistingElement = g_pDataModel->FindElement( id );
if ( hExistingElement != DMELEMENT_HANDLE_INVALID )
{
// id is already in use - need to resolve conflict
if ( idConflictResolution == CR_DELETE_NEW )
{
elementList.AddToTail( g_pDataModel->GetElement( hExistingElement ) );
continue; // just don't create this element
}
else if ( idConflictResolution == CR_DELETE_OLD )
{
g_pDataModelImp->DeleteElement( hExistingElement, HR_NEVER ); // keep the handle around until CreateElementWithFallback
hElement = CreateElementWithFallback( pType, pName, fileid, id );
Assert( hElement == hExistingElement );
}
else if ( idConflictResolution == CR_COPY_NEW )
{
DmIdPair_t idpair;
CopyUniqueId( id, &idpair.m_oldId );
CreateUniqueId( &idpair.m_newId );
idmap.Insert( idpair );
hElement = CreateElementWithFallback( pType, pName, fileid, idpair.m_newId );
}
else
Assert( 0 );
}
// if not found, then create it
if ( hElement == DMELEMENT_HANDLE_INVALID )
{
hElement = CreateElementWithFallback( pType, pName, fileid, id );
}
CDmElement *pElement = g_pDataModel->GetElement( hElement );
CDmeElementAccessor::MarkBeingUnserialized( pElement, true );
elementList.AddToTail( pElement );
}
// The root is the 0th element
*ppRoot = elementList[ 0 ];
// Now read all attributes
for ( int i = 0; i < nElementCount; ++i )
{
CDmElement *pInternal = elementList[ i ];
UnserializeAttributes( buf, pInternal->GetFileId() == fileid ? pInternal : NULL, elementList, symbolTable );
}
for ( int i = 0; i < nElementCount; ++i )
{
CDmElement *pElement = elementList[ i ];
if ( pElement->GetFileId() == fileid )
{
// mark all unserialized elements as done unserializing, and call Resolve()
CDmeElementAccessor::MarkBeingUnserialized( pElement, false );
}
}
g_pDmElementFrameworkImp->RemoveCleanElementsFromDirtyList( );
return buf.IsValid();
}