Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

594 lines
18 KiB

5 years ago
//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
//=============================================================================
#include "MakeGameData.h"
static CUtlSymbolTable g_CriticalPreloadTable( 0, 32, true );
//-----------------------------------------------------------------------------
// Purpose: Compute preload data by file type. Calls into appropriate libraries
// to get the preload info. Libraries would use filename to generate
// the preload if there is a compilation step, otherwise the file buffer
// is a buffer loaded filename.
//-----------------------------------------------------------------------------
static bool GetPreloadBuffer( const char *pFilename, CUtlBuffer &fileBuffer, CUtlBuffer &preloadBuffer )
{
char fileExtension[MAX_PATH];
Q_ExtractFileExtension( pFilename, fileExtension, sizeof( fileExtension ) );
// adding an entire file IS ONLY for files that are expected to be read by the game as a single read
// NOT for files that have any seek pattern
bool bAddEntireFile = false;
// trivial small files, always add
if ( !Q_stricmp( fileExtension, "txt" ) ||
!Q_stricmp( fileExtension, "dat" ) ||
!Q_stricmp( fileExtension, "lst" ) ||
!Q_stricmp( fileExtension, "res" ) ||
!Q_stricmp( fileExtension, "vmt" ) ||
!Q_stricmp( fileExtension, "cfg" ) ||
!Q_stricmp( fileExtension, "bnf" ) ||
!Q_stricmp( fileExtension, "rc" ) ||
!Q_stricmp( fileExtension, "vbf" ) ||
!Q_stricmp( fileExtension, "vfe" ) ||
!Q_stricmp( fileExtension, "pcf" ) ||
!Q_stricmp( fileExtension, "inf" ) )
{
bAddEntireFile = true;
}
// critical resources get blindly added to the preload
if ( !bAddEntireFile && ( g_CriticalPreloadTable.Find( pFilename ) != UTL_INVAL_SYMBOL ) )
{
bAddEntireFile = true;
}
if ( bAddEntireFile && LZMA_IsCompressed( (unsigned char *)fileBuffer.Base() ) )
{
// sorry, not allowed to add entirely to preload if already compressed
// breaks the run-time filesystem due to inability to deliver file as-is
bAddEntireFile = false;
}
if ( bAddEntireFile )
{
if ( fileBuffer.TellMaxPut() >= 1*1024 )
{
// only compress preload files of reasonable size
unsigned int compressedSize = 0;
unsigned char *pCompressedOutput = LZMA_OpportunisticCompress( (unsigned char *)fileBuffer.Base(),
fileBuffer.TellMaxPut(), &compressedSize );
if ( pCompressedOutput )
{
// add as compressed
preloadBuffer.EnsureCapacity( compressedSize );
preloadBuffer.Put( pCompressedOutput, compressedSize );
free( pCompressedOutput );
return true;
}
}
// add entire file to preload section
preloadBuffer.EnsureCapacity( fileBuffer.TellMaxPut() );
preloadBuffer.Put( fileBuffer.Base(), fileBuffer.TellMaxPut() );
return true;
}
// Each library will fetch the optional preload data into caller's buffer
if ( !Q_stricmp( fileExtension, "wav" ) )
{
return GetPreloadData_WAV( pFilename, fileBuffer, preloadBuffer );
}
else if ( !Q_stricmp( fileExtension, "vtf" ) )
{
return GetPreloadData_VTF( pFilename, fileBuffer, preloadBuffer );
}
else if ( !Q_stricmp( fileExtension, "vcs" ) )
{
return GetPreloadData_VCS( pFilename, fileBuffer, preloadBuffer );
}
else if ( !Q_stricmp( fileExtension, "vhv" ) )
{
return GetPreloadData_VHV( pFilename, fileBuffer, preloadBuffer );
}
else if ( !Q_stricmp( fileExtension, "vtx" ) )
{
return GetPreloadData_VTX( pFilename, fileBuffer, preloadBuffer );
}
else if ( !Q_stricmp( fileExtension, "vvd" ) )
{
return GetPreloadData_VVD( pFilename, fileBuffer, preloadBuffer );
}
// others...
return false;
}
void SetupCriticalPreloadScript( const char *pModPath )
{
characterset_t breakSet;
CharacterSetBuild( &breakSet, "" );
// purge any prior entries
g_CriticalPreloadTable.RemoveAll();
// populate table
char szCriticaList[MAX_PATH];
char szToken[MAX_PATH];
V_ComposeFileName( pModPath, "scripts\\preload_xbox.xsc", szCriticaList, sizeof( szCriticaList ) );
CUtlBuffer criticalListBuffer;
if ( ReadFileToBuffer( szCriticaList, criticalListBuffer, true, true ) )
{
for ( ;; )
{
int nTokenSize = criticalListBuffer.ParseToken( &breakSet, szToken, sizeof( szToken ) );
if ( nTokenSize <= 0 )
{
break;
}
V_strlower( szToken );
V_FixSlashes( szToken, CORRECT_PATH_SEPARATOR );
if ( UTL_INVAL_SYMBOL == g_CriticalPreloadTable.Find( szToken ) )
{
g_CriticalPreloadTable.AddString( szToken );
}
}
}
}
CXZipTool::CXZipTool()
{
m_pZip = NULL;
m_hPreloadFile = INVALID_HANDLE_VALUE;
m_hOutputZipFile = INVALID_HANDLE_VALUE;
m_PreloadFilename[0] = '\0';
}
CXZipTool::~CXZipTool()
{
Reset();
}
void CXZipTool::Reset()
{
if ( m_hOutputZipFile != INVALID_HANDLE_VALUE )
{
CloseHandle( m_hOutputZipFile );
m_hOutputZipFile = INVALID_HANDLE_VALUE;
}
if ( m_hPreloadFile != INVALID_HANDLE_VALUE )
{
CloseHandle( m_hPreloadFile );
m_hPreloadFile = INVALID_HANDLE_VALUE;
}
if ( m_PreloadFilename[0] )
{
DeleteFile( m_PreloadFilename );
m_PreloadFilename[0] = '\0';
}
if ( m_pZip )
{
IZip::ReleaseZip( m_pZip );
m_pZip = NULL;
}
m_ZipPreloadDirectoryEntries.Purge();
m_ZipCRCList.Purge();
m_ZipPreloadRemapEntries.Purge();
}
//-----------------------------------------------------------------------------
// Purpose: Add a file buffer to the zip
//-----------------------------------------------------------------------------
bool CXZipTool::AddBuffer( const char *pFilename, CUtlBuffer &fileBuffer, bool bDoPreload )
{
if ( !m_pZip )
{
return false;
}
// safely strip unecessary prefix, otherise pollutes CRC
if ( !strnicmp( pFilename, ".\\", 2 ) )
{
pFilename += 2;
}
// scan for CRC collision now, not at runtime
CRCEntry_t crcEntry;
crcEntry.fileNameCRC = HashStringCaselessConventional( pFilename );
crcEntry.filename = pFilename;
int idx = m_ZipCRCList.Find( crcEntry );
if ( -1 != idx )
{
if ( !V_stricmp( pFilename, m_ZipCRCList[idx].filename.String() ) )
{
// file has already been added, ignore as succesful
return true;
}
Msg( "ERROR: CRC Collision: '%s' with '%s'\n", pFilename, m_ZipCRCList[idx].filename.String() );
return false;
}
else
{
// add unique entry to lists
// must track filenames in a non-sort manner
m_ZipCRCList.Insert( crcEntry );
}
// default, no preload entry
unsigned short preloadDir = INVALID_PRELOAD_ENTRY;
if ( bDoPreload )
{
CUtlBuffer preloadBuffer;
bool bHasPreload = GetPreloadBuffer( pFilename, fileBuffer, preloadBuffer );
int preloadSize = preloadBuffer.TellMaxPut();
if ( bHasPreload && preloadSize > 0 )
{
if ( m_ZipPreloadDirectoryEntries.Count() >= 65534 )
{
Msg( "ERROR: Preload section FULL!, skipping %s\n", pFilename );
return FALSE;
}
// Initialize the entry header
ZIP_PreloadDirectoryEntry entry;
memset( &entry, 0, sizeof( entry ) );
entry.Length = preloadSize;
entry.DataOffset = SetFilePointer( m_hPreloadFile, 0, NULL, FILE_CURRENT );
// Add the directory entry to the preload table
preloadDir = m_ZipPreloadDirectoryEntries.AddToTail( entry );
// Append the preload data to the preload file
DWORD numBytesWritten;
BOOL bOK = WriteFile( m_hPreloadFile, preloadBuffer.Base(), preloadSize, &numBytesWritten, NULL );
if ( !bOK || preloadSize != numBytesWritten )
{
Msg( "ERROR: writing %d preload bytes of '%s'\n", preloadSize, pFilename );
return false;
}
if ( !g_bQuiet )
{
// Spew it
if ( LZMA_IsCompressed( (unsigned char *)preloadBuffer.Base() ) )
{
unsigned int actualSize = LZMA_GetActualSize( (unsigned char *)preloadBuffer.Base() );
Msg( "Preload: '%s': Compressed:%u Actual:%u\n", pFilename, preloadSize, actualSize );
}
else
{
Msg( "Preload: '%s': Length:%u\n", pFilename, preloadSize );
}
}
}
}
unsigned int fileSize = fileBuffer.TellMaxPut();
if ( fileSize > 0 )
{
// order in zip is sorted, not sequential
m_pZip->AddBufferToZip( pFilename, fileBuffer.Base(), fileSize, false );
// track the file in the zip directory to it's preload entry
// order in preload is sequential as buffers are added
preloadRemap_t remap;
remap.filename = pFilename;
remap.preloadDirIndex = preloadDir;
m_ZipPreloadRemapEntries.AddToTail( remap );
if ( !g_bQuiet )
{
Msg( "File: '%s': Length:%u\n", pFilename, fileSize );
}
}
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Load a file and add it to the zip
//-----------------------------------------------------------------------------
bool CXZipTool::AddFile( const char *pFilename, bool bDoPreload )
{
if ( !m_pZip )
{
return false;
}
FILE* pFile = fopen( pFilename, "rb" );
if( !pFile )
{
Msg( "ERROR: failed to open file: '%s'\n", pFilename );
return false;
}
// Get the length of the file
fseek( pFile, 0, SEEK_END );
unsigned fileSize = ftell( pFile );
fseek( pFile, 0, SEEK_SET);
// read file to buffer
CUtlBuffer fileBuffer;
fileBuffer.EnsureCapacity( fileSize );
fread( fileBuffer.Base(), fileSize, 1, pFile );
fclose( pFile );
fileBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, fileSize );
return AddBuffer( pFilename, fileBuffer, bDoPreload );
}
//-----------------------------------------------------------------------------
// Purpose: Add the preload section and save out the zip file
//-----------------------------------------------------------------------------
bool CXZipTool::End()
{
if ( !m_pZip )
{
return false;
}
// Add the preload section to the zip
if ( m_ZipPreloadDirectoryEntries.Count() )
{
CUtlBuffer sectionBuffer;
CByteswap byteSwap;
// pc tools write 360 native data
byteSwap.ActivateByteSwapping( IsPC() );
// determine the preload data footprint
unsigned int preloadDataSize = SetFilePointer( m_hPreloadFile, 0, NULL, FILE_CURRENT );
// finalize header
m_ZipPreloadHeader.DirectoryEntries = m_ZipPreloadRemapEntries.Count();
m_ZipPreloadHeader.PreloadDirectoryEntries = m_ZipPreloadDirectoryEntries.Count();
// determine the total section size ( treated as a single file inside zip )
unsigned int sectionSize = sizeof( ZIP_PreloadHeader ) +
m_ZipPreloadHeader.PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) +
m_ZipPreloadHeader.DirectoryEntries * sizeof( unsigned short ) +
preloadDataSize;
sectionSize = AlignValue( sectionSize, m_ZipPreloadHeader.Alignment );
sectionBuffer.EnsureCapacity( sectionSize );
// save data in order
// save the header
byteSwap.SwapFieldsToTargetEndian( &m_ZipPreloadHeader );
sectionBuffer.Put( &m_ZipPreloadHeader, sizeof( ZIP_PreloadHeader ) );
// fixup and save the preload directory
for ( int i=0; i<m_ZipPreloadDirectoryEntries.Count(); i++ )
{
ZIP_PreloadDirectoryEntry entry = m_ZipPreloadDirectoryEntries[i];
byteSwap.SwapFieldsToTargetEndian( &entry );
sectionBuffer.Put( &entry, sizeof( ZIP_PreloadDirectoryEntry ) );
}
// generate remap table
char fileName[MAX_PATH];
int fileSize;
int zipIndex = -1;
unsigned short *pRemapTable = (unsigned short *)malloc( m_ZipPreloadRemapEntries.Count() * sizeof( unsigned short ) );
for ( int i=0; i<m_ZipPreloadRemapEntries.Count(); i++ )
{
// zip files get iterated in the same order they are serialized to disk
fileName[0] = '\0';
fileSize = 0;
zipIndex = m_pZip->GetNextFilename( zipIndex, fileName, sizeof( fileName ), fileSize );
// find the file in the preload remap table
bool bFound = false;
int j;
for ( j=0; j<m_ZipPreloadRemapEntries.Count(); j++ )
{
if ( !Q_stricmp( fileName, m_ZipPreloadRemapEntries[j].filename.String() ) )
{
bFound = true;
break;
}
}
if ( !bFound )
{
// shouldn't happen, every file in the zip has a matching preload remap entry, that is valid or marked invalid
Msg( "ERROR: file '%s' was expected to have an entry in preload table\n", fileName );
}
// the remap table is used to go find a file's preload data (if available)
pRemapTable[i] = m_ZipPreloadRemapEntries[j].preloadDirIndex;
}
for ( int i=0; i<m_ZipPreloadRemapEntries.Count(); i++ )
{
unsigned short s = pRemapTable[i];
sectionBuffer.PutShort( BigShort( s ) );
}
free( pRemapTable );
// get and save preload data
void *pPreloadData = malloc( preloadDataSize );
SetFilePointer( m_hPreloadFile, 0, NULL, FILE_BEGIN );
DWORD numBytesRead;
BOOL bOK = ReadFile( m_hPreloadFile, pPreloadData, preloadDataSize, &numBytesRead, NULL );
if ( !bOK || numBytesRead != preloadDataSize )
{
Msg( "ERROR: failed to read %d bytes from temporary preload file\n", preloadDataSize );
}
CloseHandle( m_hPreloadFile );
m_hPreloadFile = INVALID_HANDLE_VALUE;
sectionBuffer.Put( pPreloadData, preloadDataSize );
free( pPreloadData );
// cannot have written more than was pre-calced, unless code was broken
Assert( (unsigned int)sectionBuffer.TellMaxPut() <= sectionSize );
while( (unsigned int)sectionBuffer.TellMaxPut() < sectionSize )
{
// pad to final alignment
sectionBuffer.PutChar( 0 );
}
m_pZip->AddBufferToZip( PRELOAD_SECTION_NAME, sectionBuffer.Base(), sectionBuffer.TellMaxPut(), false );
}
else
{
// Clear the preload section placeholder
m_pZip->RemoveFileFromZip( PRELOAD_SECTION_NAME );
}
m_pZip->SaveToDisk( m_hOutputZipFile );
Reset();
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Create the zip file
//-----------------------------------------------------------------------------
bool CXZipTool::Begin( const char *pZipFileName, unsigned int alignment )
{
// get the volume of the target zip
char drivePath[MAX_PATH];
_splitpath( pZipFileName, drivePath, NULL, NULL, NULL );
m_pZip = IZip::CreateZip( drivePath, true );
if ( alignment )
{
// making an aligned zip that uses an optimized (but incompatible) format
m_pZip->ForceAlignment( true, false, alignment );
}
// Open the output file
m_hOutputZipFile = CreateFile( pZipFileName, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
if ( m_hOutputZipFile == INVALID_HANDLE_VALUE )
{
Msg( "ERROR: failed to create zip file '%s'\n", pZipFileName );
return false;
}
// Create a temporary file for storing the preloaded data
scriptlib->MakeTemporaryFilename( g_szModPath, m_PreloadFilename, sizeof( m_PreloadFilename ) );
m_hPreloadFile = CreateFile( m_PreloadFilename, GENERIC_READ|GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
if ( m_hPreloadFile == INVALID_HANDLE_VALUE )
{
Msg( "ERROR: failed to create temporary file '%s' for preload data\n", m_PreloadFilename );
CloseHandle( m_hOutputZipFile );
m_hOutputZipFile = INVALID_HANDLE_VALUE;
return false;
}
memset( &m_ZipPreloadHeader, 0, sizeof( ZIP_PreloadHeader ) );
m_ZipPreloadHeader.Version = PRELOAD_HDR_VERSION;
m_ZipPreloadHeader.Alignment = alignment;
// Add a placeholder for the preload section
m_pZip->AddBufferToZip( PRELOAD_SECTION_NAME, NULL, 0, false );
preloadRemap_t remap;
remap.filename = PRELOAD_SECTION_NAME;
remap.preloadDirIndex = INVALID_PRELOAD_ENTRY;
m_ZipPreloadRemapEntries.AddToTail( remap );
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Dump the preload contents
//-----------------------------------------------------------------------------
void CXZipTool::SpewPreloadInfo( const char *pZipName )
{
IZip *pZip = IZip::CreateZip( NULL, true );
HANDLE hZipFile = pZip->ParseFromDisk( pZipName );
if ( !hZipFile )
{
Msg( "Bad or missing zip file, failed to open '%s'\n", pZipName );
return;
}
CUtlBuffer preloadBuffer;
if ( !pZip->ReadFileFromZip( hZipFile, PRELOAD_SECTION_NAME, false, preloadBuffer ) )
{
Msg( "No preload info for '%s'\n", pZipName );
return;
}
preloadBuffer.ActivateByteSwapping( IsPC() );
ZIP_PreloadHeader header;
preloadBuffer.GetObjects( &header );
// get the dir table
ZIP_PreloadDirectoryEntry *pDir = (ZIP_PreloadDirectoryEntry *)malloc( header.PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) );
preloadBuffer.GetObjects( pDir, header.PreloadDirectoryEntries );
// get the remap table
unsigned short *pRemap = (unsigned short *)malloc( header.DirectoryEntries * sizeof( unsigned short ) );
for ( unsigned int i = 0; i < header.DirectoryEntries; i++ )
{
pRemap[i] = preloadBuffer.GetShort();
}
int zipIndex = -1;
int fileSize;
char fileName[MAX_PATH];
// iterate preload entries sequentially
CUtlDict< unsigned int, int > sizes( true );
for ( unsigned int i = 0; i < header.DirectoryEntries; i++ )
{
fileName[0] = '\0';
fileSize = 0;
zipIndex = pZip->GetNextFilename( zipIndex, fileName, sizeof( fileName ), fileSize );
unsigned short zipPreloadDirIndex = pRemap[i];
if ( zipPreloadDirIndex == INVALID_PRELOAD_ENTRY )
{
continue;
}
Msg( "Offset: 0x%8.8x Length: %5d %s (%d)\n", pDir[zipPreloadDirIndex].DataOffset, pDir[zipPreloadDirIndex].Length, fileName, fileSize );
// total preload sizes by extension
const char *pExt = V_GetFileExtension( fileName );
if ( !pExt )
{
pExt = "???";
}
int iIndex = sizes.Find( pExt );
if ( iIndex == sizes.InvalidIndex() )
{
iIndex = sizes.Insert( pExt );
sizes[iIndex] = 0;
}
sizes[iIndex] += pDir[zipPreloadDirIndex].Length;
}
Msg( "\n" );
Msg( "Preload Size: %.2f MB\n", (float)preloadBuffer.TellMaxPut()/(1024.0f * 1024.0f) );
Msg( "Zip Entries: %d\n", header.DirectoryEntries );
Msg( "Preload Entries: %d\n", header.PreloadDirectoryEntries );
// dump each extension's total size, necessary for debugging who is the largest contributor
for ( int i = 0; i < sizes.Count(); i++ )
{
Msg( "Extension: '%3s' %d bytes (%.2f%s)\n", sizes.GetElementName( i ), sizes[i], (float)sizes[i]/(float)preloadBuffer.TellMaxPut() * 100.0f, "%%" );
}
Msg( "\n" );
free( pRemap );
free( pDir );
IZip::ReleaseZip( pZip );
}