d4708
/
source-engine
mirror of https://github.com/nillerusr/source-engine.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
280 Commits
25 Branches
6 Tags
221 MiB
 
 
 
 
 
 
Tree: 41414854db
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '41414854db'
${ noResults }
source-engine/utils/shadercompile/shadercompile.cpp

2699 lines
74 KiB
Raw Blame History

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
// $NoKeywords: $
//
//=============================================================================//
// vmpi_bareshell.cpp : Defines the entry point for the console application.
//
#include <windows.h>
#include <conio.h>
#include <process.h>
#include "vmpi.h"
#include "filesystem.h"
#include "vmpi_filesystem.h"
#include "vmpi_distribute_work.h"
#include "vmpi_tools_shared.h"
#include "cmdlib.h"
#include "utlvector.h"
#include "Utlhash.h"
#include "UtlBuffer.h"
#include "utlstring.h"
#include "utlbinaryblock.h"
#include "tier2/utlstreambuffer.h"
#include "UtlLinkedList.h"
#include "UtlStringMap.h"
#include "tier0/icommandline.h"
#include "tier1/strtools.h"
#include "vstdlib/jobthread.h"
#include "threads.h"
#include "tier0/dbg.h"
#include "tier1/smartptr.h"
#include "interface.h"
#include "ishadercompiledll.h"
#include <direct.h>
#include "io.h"
#include <sys/types.h>
#include <sys/stat.h>
#include "materialsystem/shader_vcs_version.h"
#include "ilaunchabledll.h"
#include <tier1/diff.h>
#include "utlnodehash.h"
#include "lzma/lzma.h"
#include "mathlib/mathlib.h"
#include "tier1/checksum_crc.h"
#include "tier0/tslist.h"
#include "tools_minidump.h"
#include "cmdsink.h"
#include "d3dxfxc.h"
#include "subprocess.h"
#include "cfgprocessor.h"
// Type conversions should be controlled by programmer explicitly - shadercompile makes use of 64-bit integer arithmetics
#pragma warning( error : 4244 )
static inline uint32 uint64_as_uint32( uint64 x )
{
Assert( x < uint64( uint32( ~0 ) ) );
return uint32( x );
}
static inline UtlSymId_t int_as_symid( int x )
{
Assert( ( sizeof( UtlSymId_t ) >= sizeof( int ) ) || ( x >= 0 && x < ( int )( unsigned int )( UtlSymId_t(~0) ) ) );
return UtlSymId_t( x );
}
// VMPI packets
#define STARTWORK_PACKETID 5
#define WORKUNIT_PACKETID 6
#define ERRMSG_PACKETID 7
#define SHADERHADERROR_PACKETID 8
#define MACHINE_NAME 9
#ifdef _DEBUG
//#define DEBUGFP
#endif
// Dealing with job list
namespace
{
CArrayAutoPtr< CfgProcessor::CfgEntryInfo > g_arrCompileEntries;
uint64 g_numShaders = 0, g_numCompileCommands = 0, g_numStaticCombos = 0;
uint64 g_nStaticCombosPerWorkUnit = 0, g_numCompletedStaticCombos = 0, g_numCommandsCompleted = 0;
uint64 g_numSkippedStaticCombos = 0;
CfgProcessor::CfgEntryInfo const * GetEntryByStaticComboNum( uint64 nStaticCombo, uint64 *pnStaticCombo )
{
CfgProcessor::CfgEntryInfo const *pInfo;
uint64 nRemainStaticCombos = nStaticCombo;
for ( pInfo = g_arrCompileEntries.Get(); pInfo && pInfo->m_szName; ++ pInfo )
{
if ( nRemainStaticCombos >= pInfo->m_numStaticCombos )
nRemainStaticCombos -= pInfo->m_numStaticCombos;
else
break;
}
if ( pnStaticCombo )
*pnStaticCombo = nRemainStaticCombos;
return pInfo;
}
}; // `anonymous` namespace
char * PrettyPrintNumber( uint64 k )
{
static char chCompileString[50] = {0};
char *pchPrint = chCompileString + sizeof( chCompileString ) - 3;
for ( uint64 j = 0; k > 0; k /= 10, ++ j )
{
( j && !( j % 3 ) ) ? ( * pchPrint -- = ',' ) : 0;
* pchPrint -- = '0' + char( k % 10 );
}
( * ++ pchPrint ) ? 0 : ( * pchPrint = 0 );
return pchPrint;
}
const char *g_pShaderPath = NULL;
char g_WorkerTempPath[MAX_PATH];
char g_ExeDir[MAX_PATH];
#ifdef DEBUGFP
FILE *g_WorkerDebugFp = NULL;
#endif
bool g_bGotStartWorkPacket = false;
double g_flStartTime;
bool g_bVerbose = false;
bool g_bIsX360 = false;
bool g_bSuppressWarnings = false;
FORCEINLINE long AsTargetLong( long x ) { return ( ( g_bIsX360 ) ? ( BigLong( x ) ) : ( x ) ); }
struct ShaderInfo_t
{
ShaderInfo_t() { memset( this, 0, sizeof( *this ) ); }
uint64 m_nShaderCombo;
uint64 m_nTotalShaderCombos;
const char *m_pShaderName;
const char *m_pShaderSrc;
unsigned m_CentroidMask;
uint64 m_nDynamicCombos;
uint64 m_nStaticCombo;
unsigned m_Flags; // from IShader.h
char m_szShaderModel[ 12 ];
};
void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo );
struct CByteCodeBlock
{
CByteCodeBlock *m_pNext, *m_pPrev;
int m_nCRC32;
uint64 m_nComboID;
size_t m_nCodeSize;
uint8 *m_ByteCode;
CByteCodeBlock( void )
{
m_ByteCode = NULL;
}
CByteCodeBlock( void const *pByteCode, size_t nCodeSize, uint64 nComboID )
{
m_ByteCode = new uint8[nCodeSize];
m_nComboID = nComboID;
m_nCodeSize = nCodeSize;
memcpy( m_ByteCode, pByteCode, nCodeSize );
m_nCRC32 = CRC32_ProcessSingleBuffer( pByteCode, nCodeSize );
}
~CByteCodeBlock( void )
{
if ( m_ByteCode )
delete[] m_ByteCode;
}
};
static int __cdecl CompareDynamicComboIDs( CByteCodeBlock * const *pA, CByteCodeBlock * const *pB )
{
if ( (*pA)->m_nComboID < (*pB)->m_nComboID )
return -1;
if ( (*pA)->m_nComboID > (*pB)->m_nComboID )
return 1;
return 0;
}
struct CStaticCombo // all the data for one static combo
{
CStaticCombo *m_pNext, *m_pPrev;
uint64 m_nStaticComboID;
CUtlVector< CByteCodeBlock* > m_DynamicCombos;
struct PackedCode : protected CArrayAutoPtr<uint8> {
size_t GetLength() const { if( uint8 *pb = Get() ) return *reinterpret_cast<size_t *>( pb ); else return 0; }
uint8 *GetData() const { if( uint8 *pb = Get() ) return pb + sizeof( size_t ); else return NULL; }
uint8 *AllocData( size_t len ) { Delete(); if ( len ) { Attach( new uint8[ len + sizeof( size_t ) ] ); *reinterpret_cast<size_t *>( Get() ) = len; } return GetData(); }
} m_abPackedCode; // Packed code for entire static combo
uint64 Key( void ) const
{
return m_nStaticComboID;
}
CStaticCombo( uint64 nComboID )
{
m_nStaticComboID = nComboID;
}
~CStaticCombo( void )
{
m_DynamicCombos.PurgeAndDeleteElements();
}
void AddDynamicCombo( uint64 nComboID, void const *pComboData, size_t nCodeSize )
{
CByteCodeBlock *pNewBlock = new CByteCodeBlock( pComboData, nCodeSize, nComboID );
m_DynamicCombos.AddToTail( pNewBlock );
}
void SortDynamicCombos( void )
{
m_DynamicCombos.Sort( CompareDynamicComboIDs );
}
uint8 *AllocPackedCodeBlock( size_t nPackedCodeSize )
{
return m_abPackedCode.AllocData( nPackedCodeSize );
}
};
typedef CUtlNodeHash<CStaticCombo, 7097, uint64> StaticComboNodeHash_t;
template <>
inline StaticComboNodeHash_t **Construct( StaticComboNodeHash_t ** pMemory )
{
return ::new( pMemory ) StaticComboNodeHash_t *( NULL ); // Explicitly new with NULL
}
struct CShaderMap : public CUtlStringMap<StaticComboNodeHash_t *> {
;
} g_ShaderByteCode;
CStaticCombo * StaticComboFromDictAdd( char const *pszShaderName, uint64 nStaticComboId )
{
StaticComboNodeHash_t *& rpNodeHash = g_ShaderByteCode[ pszShaderName ];
if ( !rpNodeHash )
{
rpNodeHash = new StaticComboNodeHash_t;
}
// search for this static combo. make it if not found
CStaticCombo *pStaticCombo = rpNodeHash->FindByKey( nStaticComboId );
if ( !pStaticCombo )
{
pStaticCombo = new CStaticCombo( nStaticComboId );
rpNodeHash->Add( pStaticCombo );
}
return pStaticCombo;
}
CStaticCombo * StaticComboFromDict( char const *pszShaderName, uint64 nStaticComboId )
{
if ( StaticComboNodeHash_t *pNodeHash = g_ShaderByteCode[ pszShaderName ] )
return pNodeHash->FindByKey( nStaticComboId );
else
return NULL;
}
CUtlStringMap<ShaderInfo_t> g_ShaderToShaderInfo;
class CompilerMsgInfo
{
public:
CompilerMsgInfo() : m_numTimesReported( 0 ) {}
public:
void SetMsgReportedCommand( char const *szCommand, int numTimesReported = 1 ) { if ( !m_numTimesReported ) m_sFirstCommand = szCommand; m_numTimesReported += numTimesReported; }
public:
char const * GetFirstCommand() const { return m_sFirstCommand.String(); }
int GetNumTimesReported() const { return m_numTimesReported; }
protected:
CUtlString m_sFirstCommand;
int m_numTimesReported;
};
CUtlStringMap<bool> g_Master_ShaderHadError;
CUtlStringMap<bool> g_Master_ShaderWrittenToDisk;
CUtlStringMap<CompilerMsgInfo> g_Master_CompilerMsgInfo;
namespace Threading
{
enum Mode { eSingleThreaded = 0, eMultiThreaded = 1 };
// A special object that makes single-threaded code incur no penalties
// and multithreaded code to be synchronized properly.
template < class MT_MUTEX_TYPE = CThreadFastMutex >
class CSwitchableMutex
{
public:
public:
FORCEINLINE explicit CSwitchableMutex( Mode eMode, MT_MUTEX_TYPE *pMtMutex = NULL ) : m_pMtx( pMtMutex ), m_pUseMtx( eMode ? pMtMutex : NULL ) {}
public:
FORCEINLINE void SetMtMutex( MT_MUTEX_TYPE *pMtMutex ) { m_pMtx = pMtMutex; m_pUseMtx = ( m_pUseMtx ? pMtMutex : NULL ); }
FORCEINLINE void SetThreadedMode( Mode eMode ) { m_pUseMtx = ( eMode ? m_pMtx : NULL ); }
public:
FORCEINLINE void Lock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Lock(); }
FORCEINLINE void Unlock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Unlock(); }
FORCEINLINE bool TryLock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->TryLock(); else return true; }
FORCEINLINE bool AssertOwnedByCurrentThread() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->AssertOwnedByCurrentThread(); else return true; }
FORCEINLINE void SetTrace( bool b ) { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->SetTrace( b ); }
FORCEINLINE uint32 GetOwnerId() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetOwnerId(); else return 0; }
FORCEINLINE int GetDepth() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetDepth(); else return 0; }
private:
MT_MUTEX_TYPE *m_pMtx;
CInterlockedPtr< MT_MUTEX_TYPE > m_pUseMtx;
};
namespace Private
{
typedef CThreadMutex MtMutexType_t;
MtMutexType_t g_mtxSyncObjMT;
}; // namespace Private
CSwitchableMutex< Private::MtMutexType_t > g_mtxGlobal( eSingleThreaded, &Private::g_mtxSyncObjMT );
class CGlobalMutexAutoLock
{
public:
CGlobalMutexAutoLock() { g_mtxGlobal.Lock(); }
~CGlobalMutexAutoLock() { g_mtxGlobal.Unlock(); }
};
}; // namespace Threading
// Access to global data should be synchronized by these global locks
#define GLOBAL_DATA_MTX_LOCK() Threading::g_mtxGlobal.Lock()
#define GLOBAL_DATA_MTX_UNLOCK() Threading::g_mtxGlobal.Unlock()
#define GLOBAL_DATA_MTX_LOCK_AUTO Threading::CGlobalMutexAutoLock UNIQUE_ID;
CDispatchReg g_DistributeWorkReg( WORKUNIT_PACKETID, DistributeWorkDispatch );
unsigned long VMPI_Stats_GetJobWorkerID( void )
{
return 0;
}
bool StartWorkDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
g_bGotStartWorkPacket = true;
return true;
}
CDispatchReg g_StartWorkReg( STARTWORK_PACKETID, StartWorkDispatch );
// Consume all characters for which (isspace) is true
template < typename T >
char * ConsumeCharacters( char *szString, T pred )
{
if ( szString )
{
while ( *szString && pred( *szString ) )
{
++ szString;
}
}
return szString;
}
char * FindNext( char *szString, char *szSearchSet )
{
bool bFound = (szString == NULL);
char *szNext = NULL;
if ( szString && szSearchSet )
{
for ( ; *szSearchSet; ++ szSearchSet )
{
if ( char *szTmp = strchr( szString, *szSearchSet ) )
{
szNext = bFound ? ( min( szNext, szTmp ) ) : szTmp;
bFound = true;
}
}
}
return bFound ? szNext : ( szString + strlen( szString ) );
}
char * FindLast( char *szString, char *szSearchSet )
{
bool bFound = (szString != NULL);
char *szNext = NULL;
if ( szString && szSearchSet )
{
for ( ; *szSearchSet; ++ szSearchSet )
{
if ( char *szTmp = strrchr( szString, *szSearchSet ) )
{
szNext = bFound ? ( max( szNext, szTmp ) ) : szTmp;
bFound = true;
}
}
}
return bFound ? szNext : ( szString + strlen( szString ) );
}
void ErrMsgDispatchMsgLine( char const *szCommand, char *szMsgLine, char const *szShaderName = NULL )
{
// When the filename is specified in front of the message, make sure it is truncated to the bare name only
if ( V_isalpha( *szMsgLine ) && szMsgLine[1] == ':' )
{
// Preceded by drive letter
szMsgLine += 2;
}
// Trim the path from the msg
// e.g. make string
// c:\temp\shadercompiletemp\1234\myfile.fxc(435): warning X3083: Truncating ...
// look like
// myfile.fxc(435): warning X3083: Truncating ...
// which will be both readable and same coming from different worker machines
char *szEndFileLinePlant = FindNext( szMsgLine, ":" );
if ( ':' == *szEndFileLinePlant )
{
*szEndFileLinePlant = 0;
if ( char *szLastSlash = FindLast( szMsgLine, "\\/" ) )
{
if ( *szLastSlash )
{
*szLastSlash = 0;
szMsgLine = szLastSlash + 1;
}
}
*szEndFileLinePlant = ':';
}
// If the shader file name is not given in the message add it
if ( szShaderName )
{
static char chFitLongMsgLine[4096];
if ( *szMsgLine == '(' )
{
sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine );
szMsgLine = chFitLongMsgLine;
}
else if ( !strncmp( szMsgLine, "memory(", 7 ) )
{
sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine+6 );
szMsgLine = chFitLongMsgLine;
}
}
// Now store the message with the command it was generated from
g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand );
}
void ErrMsgDispatchInt( char *szMessage, char const *szShaderName = NULL )
{
// First line is the command number "szCommand"
char *szCommand = ConsumeCharacters( szMessage, isspace );
char *szMessageListing = FindNext(szCommand, "\r\n");
char chTerminator = *szMessageListing;
*( szMessageListing ++ ) = 0;
// Now come the command lines actually
while ( chTerminator )
{
char *szMsgText = ConsumeCharacters( szMessageListing, isspace );
szMessageListing = FindNext( szMsgText, "\r\n" );
chTerminator = *szMessageListing;
*( szMessageListing ++ ) = 0;
if( *szMsgText )
{
// Trim command at redirection character if present
* FindNext( szCommand, ">" ) = 0;
ErrMsgDispatchMsgLine( szCommand, szMsgText, szShaderName );
}
}
}
//
// BUFFER:
// 1 byte = * = buffer type
//
// string = message
// 1 byte = \n = newline delimiting the message
//
// string = command that first encountered the message
// 1 byte = \n = newline delimiting the command
//
// string = printed number of times the message was encountered
// 1 byte = \n = newline delimiting the number
//
// 1 byte = 0 = null-terminator for the buffer
//
bool ErrMsgDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
bool bInvalidPkgRetCode = true;
// Parse the err msg packet
char *szMsgLine = pBuf->data + 1;
char *szCommand = FindNext( szMsgLine, "\n" );
if ( !*szCommand )
return bInvalidPkgRetCode;
*( szCommand ++ ) = 0;
char *szNumTimesReported = FindNext( szCommand, "\n" );
if ( !*szNumTimesReported )
return bInvalidPkgRetCode;
*( szNumTimesReported ++ ) = 0;
char *szTerminator = FindNext( szNumTimesReported, "\n" );
if ( !*szTerminator )
return bInvalidPkgRetCode;
*( szTerminator ++ ) = 0;
// Set the msg info
g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand, atoi( szNumTimesReported ) );
return true;
}
CDispatchReg g_ErrMsgReg( ERRMSG_PACKETID, ErrMsgDispatch );
void ShaderHadErrorDispatchInt( char const *szShader )
{
g_Master_ShaderHadError[ szShader ] = true;
}
//
// BUFFER:
// 1 byte = * = buffer type
//
// string = shader name
// 1 byte = 0 = null-terminator for the name
//
bool ShaderHadErrorDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
ShaderHadErrorDispatchInt( pBuf->data + 1 );
return true;
}
CDispatchReg g_ShaderHadErrorReg( SHADERHADERROR_PACKETID, ShaderHadErrorDispatch );
void DebugOut( const char *pMsg, ... )
{
if (g_bVerbose)
{
char msg[2048];
va_list marker;
va_start( marker, pMsg );
_vsnprintf( msg, sizeof( msg ), pMsg, marker );
va_end( marker );
Msg( "%s", msg );
#ifdef DEBUGFP
fprintf( g_WorkerDebugFp, "%s", msg );
fflush( g_WorkerDebugFp );
#endif
}
}
void Vmpi_Worker_DefaultDisconnectHandler( int procID, const char *pReason )
{
Msg( "Master disconnected.\n ");
DebugOut( "Master disconnected.\n" );
TerminateProcess( GetCurrentProcess(), 1 );
}
typedef void ( * DisconnectHandlerFn_t )( int procID, const char *pReason );
DisconnectHandlerFn_t g_fnDisconnectHandler = Vmpi_Worker_DefaultDisconnectHandler;
// Worker should implement this so it will quit nicely when the master disconnects.
void MyDisconnectHandler( int procID, const char *pReason )
{
// If we're a worker, then it's a fatal error if we lose the connection to the master.
if ( !g_bMPIMaster && g_fnDisconnectHandler )
{
(* g_fnDisconnectHandler)( procID, pReason );
}
}
// new format:
// ver#
// total shader combos
// total dynamic combos
// flags
// centroid mask
// total non-skipped static combos
// [ (sorted by static combo id)
// static combo id
// file offset of packed dynamic combo
// ]
// 0xffffffff (sentinel key)
// end of file offset (so can tell compressed size of last combo)
//
// # of duplicate static combos (if version >= 6 )
// [ (sorted by static combo id)
// static combo id
// id of static bombo which is identical
// ]
//
// each packed dynamic combo for a given static combo is stored as a series of compressed blocks.
// block 1:
// ulong blocksize (high bit set means uncompressed)
// block data
// block2..
// 0xffffffff indicates no more blocks for this combo
//
// each block, when uncompressed, holds one or more dynamic combos:
// dynamic combo id (full id if v<6, dynamic combo id only id >=6)
// size of shader
// ..
// there is no terminator - the size of the uncompressed shader tells you when to stop
// this record is then bzip2'd.
// qsort driver function
// returns negative number if idA is less than idB, positive when idA is greater than idB
// and zero if the ids are equal
static int __cdecl CompareDupComboIndices( const StaticComboAliasRecord_t *pA, const StaticComboAliasRecord_t *pB )
{
if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
return -1;
if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
return 1;
return 0;
}
static void FlushCombos( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, MessageBuffer *pBuf )
{
if ( !pDynamicComboBuffer->TellPut() )
// Nothing to do here
return;
size_t nCompressedSize;
uint8 *pCompressedShader = LZMA_OpportunisticCompress( reinterpret_cast<uint8 *> ( pDynamicComboBuffer->Base() ),
pDynamicComboBuffer->TellPut(),
&nCompressedSize );
// high 2 bits of length =
// 00 = bzip2 compressed
// 10 = uncompressed
// 01 = lzma compressed
// 11 = unused
if ( ! pCompressedShader )
{
// it grew
long lFlagSize = AsTargetLong( 0x80000000 | pDynamicComboBuffer->TellPut() );
pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
pBuf->write( pDynamicComboBuffer->Base(), pDynamicComboBuffer->TellPut() );
*pnTotalFlushedSize += sizeof( lFlagSize ) + pDynamicComboBuffer->TellPut();
}
else
{
long lFlagSize = AsTargetLong( 0x40000000 | nCompressedSize );
pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
pBuf->write( pCompressedShader, nCompressedSize );
delete[] pCompressedShader;
*pnTotalFlushedSize += sizeof( lFlagSize ) + nCompressedSize;
}
pDynamicComboBuffer->Clear(); // start over
}
static void OutputDynamicCombo( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
MessageBuffer *pBuf, uint64 nComboID, int nComboSize,
uint8 *pComboCode )
{
if ( pDynamicComboBuffer->TellPut() + nComboSize+16 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
{
FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
}
pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) );
pDynamicComboBuffer->PutInt( nComboSize );
// pDynamicComboBuffer->PutInt( CRC32_ProcessSingleBuffer( pComboCode, nComboSize ) );
pDynamicComboBuffer->Put( pComboCode, nComboSize );
}
static void OutputDynamicComboDup( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
MessageBuffer *pBuf, uint64 nComboID, uint64 nBaseCombo )
{
if ( pDynamicComboBuffer->TellPut() + 8 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
{
FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
}
pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) | 0x80000000 );
pDynamicComboBuffer->PutInt( uint64_as_uint32( nBaseCombo ) );
}
void GetVCSFilenames( char *pszMainOutFileName, ShaderInfo_t const &si )
{
sprintf( pszMainOutFileName, "%s\\shaders\\fxc", g_pShaderPath );
struct _stat buf;
if( _stat( pszMainOutFileName, &buf ) == -1 )
{
printf( "mkdir %s\n", pszMainOutFileName );
// doh. . need to make the directory that the vcs file is going to go into.
_mkdir( pszMainOutFileName );
}
strcat( pszMainOutFileName, "\\" );
strcat( pszMainOutFileName, si.m_pShaderName );
if ( g_bIsX360 )
{
strcat( pszMainOutFileName, ".360" );
}
strcat( pszMainOutFileName, ".vcs" ); // Different extensions for main output file
// Check status of vcs file...
if( _stat( pszMainOutFileName, &buf ) != -1 )
{
// The file exists, let's see if it's writable.
if( !( buf.st_mode & _S_IWRITE ) )
{
// It isn't writable. . we'd better change its permissions (or check it out possibly)
printf( "Warning: making %s writable!\n", pszMainOutFileName );
_chmod( pszMainOutFileName, _S_IREAD | _S_IWRITE );
}
}
}
// WriteShaderFiles
//
// should be called either on the main thread or
// on the async writing thread.
//
// So the function WriteShaderFiles should not be reentrant, however the
// data that it uses might be updated by the main thread when built pieces
// are received from the workers.
//
#define STATIC_COMBO_HASH_SIZE 73
struct StaticComboAuxInfo_t : StaticComboRecord_t
{
uint32 m_nCRC32; // CRC32 of packed data
struct CStaticCombo *m_pByteCode;
};
static int __cdecl CompareComboIds( const StaticComboAuxInfo_t *pA, const StaticComboAuxInfo_t *pB )
{
if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
return -1;
if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
return 1;
return 0;
}
static void WriteShaderFiles( const char *pShaderName )
{
if ( !g_Master_ShaderWrittenToDisk.Defined( pShaderName ) )
g_Master_ShaderWrittenToDisk[ pShaderName ] = true;
else
return;
bool bShaderFailed = g_Master_ShaderHadError.Defined( pShaderName );
char const *szShaderFileOperation = bShaderFailed ? "Removing failed" : "Writing";
//
// Progress indication
//
if ( g_numCommandsCompleted < g_numCompileCommands )
{
static char chProgress[] = { '/', '-', '\\', '|' };
static int iProgressSymbol = 0;
Msg( "\b%c", chProgress[ ( ++ iProgressSymbol ) % 4 ] );
}
else
{
char chShaderName[33];
Q_snprintf( chShaderName, 29, "%s...", pShaderName );
sprintf( chShaderName + sizeof( chShaderName ) - 5, "..." );
Msg( "\r%s %s \r", szShaderFileOperation, chShaderName );
}
//
// Retrieve the data we are going to operate on
// from global variables under lock.
//
GLOBAL_DATA_MTX_LOCK();
StaticComboNodeHash_t *pByteCodeArray;
{
StaticComboNodeHash_t *&rp = g_ShaderByteCode[pShaderName]; // Get a static combo pointer, reset it as well
pByteCodeArray = rp;
rp = NULL;
/*
Assert( pByteCodeArray );
if ( !pByteCodeArray )
ShaderHadErrorDispatchInt( pShaderName );
*/
}
ShaderInfo_t shaderInfo = g_ShaderToShaderInfo[pShaderName];
if ( !shaderInfo.m_pShaderName )
{
for ( CfgProcessor::CfgEntryInfo const *pAnalyze = g_arrCompileEntries.Get() ;
pAnalyze->m_szName ;
++ pAnalyze )
{
if ( !strcmp( pAnalyze->m_szName, pShaderName ) )
{
Shader_ParseShaderInfoFromCompileCommands( pAnalyze, shaderInfo );
g_ShaderToShaderInfo[ pShaderName ] = shaderInfo;
break;
}
}
}
GLOBAL_DATA_MTX_UNLOCK();
if ( !shaderInfo.m_pShaderName )
return;
//
// Shader vcs file name
//
char szVCSfilename[MAX_PATH];
GetVCSFilenames( szVCSfilename, shaderInfo );
if ( bShaderFailed )
{
DebugOut( "Removing failed shader file \"%s\".\n", szVCSfilename );
unlink( szVCSfilename );
return;
}
if ( !pByteCodeArray )
return;
DebugOut( "%s : %I64u combos centroid mask: 0x%x numDynamicCombos: %I64u flags: 0x%x\n",
pShaderName, shaderInfo.m_nTotalShaderCombos,
shaderInfo.m_CentroidMask, shaderInfo.m_nDynamicCombos, shaderInfo.m_Flags );
//
// Static combo headers
//
CUtlVector< StaticComboAuxInfo_t > StaticComboHeaders;
StaticComboHeaders.EnsureCapacity( 1 + pByteCodeArray->Count() ); // we know how much ram we need
CUtlVector< int > comboIndicesHashedByCRC32[STATIC_COMBO_HASH_SIZE];
CUtlVector< StaticComboAliasRecord_t > duplicateCombos;
// now, lets fill in our combo headers, sort, and write
for( int nChain = 0 ; nChain < NELEMS( pByteCodeArray->m_HashChains) ; nChain++ )
{
for( CStaticCombo *pStatic = pByteCodeArray->m_HashChains[ nChain ].m_pHead;
pStatic;
pStatic = pStatic->m_pNext )
{
if ( pStatic->m_abPackedCode.GetLength() )
{
StaticComboAuxInfo_t Hdr;
Hdr.m_nStaticComboID = uint64_as_uint32( pStatic->m_nStaticComboID );
Hdr.m_nFileOffset = 0; // fill in later
Hdr.m_nCRC32 = CRC32_ProcessSingleBuffer( pStatic->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetLength() );
int nHashIdx = Hdr.m_nCRC32 % STATIC_COMBO_HASH_SIZE;
Hdr.m_pByteCode = pStatic;
// now, see if we have an identical static combo
bool bIsDuplicate = false;
for( int i = 0; i < comboIndicesHashedByCRC32[nHashIdx].Count() ; i++ )
{
StaticComboAuxInfo_t const &check = StaticComboHeaders[comboIndicesHashedByCRC32[nHashIdx][i]];
if (
( check.m_nCRC32 == Hdr.m_nCRC32 ) &&
( check.m_pByteCode->m_abPackedCode.GetLength() == pStatic->m_abPackedCode.GetLength() ) &&
( memcmp( check.m_pByteCode->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetData(), check.m_pByteCode->m_abPackedCode.GetLength() ) == 0 )
)
{
// this static combo is the same as another one!!
StaticComboAliasRecord_t aliasHdr;
aliasHdr.m_nStaticComboID = Hdr.m_nStaticComboID;
aliasHdr.m_nSourceStaticCombo = check.m_nStaticComboID;
duplicateCombos.AddToTail( aliasHdr );
bIsDuplicate = true;
break;
}
}
if ( ! bIsDuplicate )
{
StaticComboHeaders.AddToTail( Hdr );
comboIndicesHashedByCRC32[nHashIdx].AddToTail( StaticComboHeaders.Count() - 1 );
}
}
}
}
// add sentinel key
StaticComboAuxInfo_t Hdr;
Hdr.m_nStaticComboID = 0xffffffff;
Hdr.m_nFileOffset = 0;
StaticComboHeaders.AddToTail( Hdr );
// now, sort. sentinel key will end up at end
StaticComboHeaders.Sort( CompareComboIds );
// Set the CRC to zero for now. . will patch in copyshaders.pl with the correct CRC.
unsigned int crc32 = 0;
//
// Shader file stream buffer
//
CUtlStreamBuffer ShaderFile( szVCSfilename, NULL ); // Streaming buffer for vcs file (since this can blow memory)
ShaderFile.SetBigEndian( g_bIsX360 ); // Swap the header bytes to X360 format
// ------ Header --------------
ShaderFile.PutInt( SHADER_VCS_VERSION_NUMBER ); // Version
ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nTotalShaderCombos ) );
ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nDynamicCombos ) );
ShaderFile.PutUnsignedInt( shaderInfo.m_Flags );
ShaderFile.PutUnsignedInt( shaderInfo.m_CentroidMask );
ShaderFile.PutUnsignedInt( StaticComboHeaders.Count() );
ShaderFile.PutUnsignedInt( crc32 );
// static combo dictionary
int nDictionaryOffset= ShaderFile.TellPut();
// we will re write this one we know the offsets
ShaderFile.Put( StaticComboHeaders.Base(), sizeof( StaticComboRecord_t ) * StaticComboHeaders.Count() ); // dummy write, 8 bytes per static combo
ShaderFile.PutUnsignedInt( duplicateCombos.Count() );
// now, write out all duplicate header records
// sort duplicate combo records for binary search
duplicateCombos.Sort( CompareDupComboIndices );
for( int i = 0; i < duplicateCombos.Count(); i++ )
{
ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nStaticComboID );
ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nSourceStaticCombo );
}
// now, write out all static combos
for( int i=0 ; i<StaticComboHeaders.Count(); i++ )
{
StaticComboRecord_t &SRec = StaticComboHeaders[i];
SRec.m_nFileOffset = ShaderFile.TellPut();
if ( SRec.m_nStaticComboID != 0xffffffff ) // sentinel key?
{
CStaticCombo *pStatic=pByteCodeArray->FindByKey( SRec.m_nStaticComboID );
Assert( pStatic );
// Put the packed chunk of code for this static combo
if ( size_t nPackedLen = pStatic->m_abPackedCode.GetLength() )
ShaderFile.Put( pStatic->m_abPackedCode.GetData(), nPackedLen );
ShaderFile.PutInt( 0xffffffff ); // end of dynamic combos
}
if ( g_bIsX360 )
{
SRec.m_nFileOffset = BigLong( SRec.m_nFileOffset );
SRec.m_nStaticComboID = BigLong( SRec.m_nStaticComboID );
}
}
ShaderFile.Close();
//
// Re-writing the combo header
//
{
FILE *Handle=fopen( szVCSfilename, "rb+" );
if (! Handle )
printf(" failed to re-open %s\n",szVCSfilename );
fseek( Handle, nDictionaryOffset, SEEK_SET );
// now, rewrite header. data is already byte-swapped appropriately
for( int i = 0; i < StaticComboHeaders.Count(); i++ )
{
fwrite( &( StaticComboHeaders[i].m_nStaticComboID ), 4, 1, Handle );
fwrite( &( StaticComboHeaders[i].m_nFileOffset ), 4, 1, Handle );
}
fclose( Handle );
}
// Finalize, free memory
delete pByteCodeArray;
if ( g_numCommandsCompleted >= g_numCompileCommands )
{
Msg( "\r \r" );
}
}
// pBuf is ready to read the results written to the buffer in ProcessWorkUnitFn.
// work is done. .master gets it back this way.
// compiled code in pBuf
void Master_ReceiveWorkUnitFn( uint64 iWorkUnit, MessageBuffer *pBuf, int iWorker )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
comboEnd = min( g_numStaticCombos, comboEnd );
char const *chLastShaderName = "";
ShaderInfo_t siLastShaderInfo;
memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );
siLastShaderInfo.m_pShaderName = chLastShaderName;
uint64 nComboOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;
for( uint64 iCombo = comboStart; iCombo ++ < comboEnd;
( ( ! nComboOfTheEntry -- ) ? ( ++ pEntry, nComboOfTheEntry = pEntry->m_numStaticCombos - 1 ) : 0 ) )
{
Assert( nComboOfTheEntry < pEntry->m_numStaticCombos );
// Read length
int len;
pBuf->read( &len, sizeof( len ) );
// Length can indicate the number of skips to make
if ( len <= 0 )
{
// remember how many static combos get skipped
g_numSkippedStaticCombos += -len;
// then we skip as instructed
for ( int64 numSkips = - len - 1;
numSkips > 0; )
{
if ( numSkips <= nComboOfTheEntry )
{
nComboOfTheEntry -= numSkips;
iCombo += numSkips;
numSkips = 0;
}
else
{
numSkips -= nComboOfTheEntry + 1;
iCombo += nComboOfTheEntry + 1;
++ pEntry;
nComboOfTheEntry = pEntry->m_numStaticCombos - 1;
}
}
if ( iCombo < comboEnd )
continue;
else
break;
}
// Shader code arrived
char const *chShaderName = pEntry->m_szName;
// If starting new shader remember shader info
if ( chLastShaderName != chShaderName )
{
Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );
chLastShaderName = chShaderName;
g_ShaderToShaderInfo[ chLastShaderName ] = siLastShaderInfo;
}
// Read buffer
uint8 *pCodeBuffer = StaticComboFromDictAdd( chShaderName, nComboOfTheEntry )->AllocPackedCodeBlock( len );
if ( pCodeBuffer )
pBuf->read( pCodeBuffer, len );
}
}
//
// A function that will wait for right Ctrl+Alt+Shift to be held down simultaneously.
// This is useful for debugging short-lived processes and gives time for debugger to
// get attached.
//
void DebugSafeWaitPoint( bool bForceWait = false )
{
static bool s_bDebuggerAttached = ( CommandLine()->FindParm( "-debugwait" ) == 0 );
if ( bForceWait )
{
s_bDebuggerAttached = false;
}
if ( !s_bDebuggerAttached )
{
Msg( "Waiting for right Ctrl+Alt+Shift to continue..." );
while ( !s_bDebuggerAttached )
{
Msg( "." );
Sleep(1000);
if ( short( GetAsyncKeyState( VK_RCONTROL ) ) < 0 &&
short( GetAsyncKeyState( VK_RSHIFT ) ) < 0 &&
short( GetAsyncKeyState( VK_RMENU ) ) < 0 )
{
s_bDebuggerAttached = true;
}
}
Msg( " ok.\n" );
}
}
// same as "system", but doesn't pop up a window
void MySystem( char const * const pCommand, CmdSink::IResponse **ppResponse )
{
// Trap the command in InterceptFxc
if ( InterceptFxc::TryExecuteCommand( pCommand, ppResponse ) )
{
Sleep( 0 );
return;
}
unlink( "shader.o" );
FILE *batFp = fopen( "temp.bat", "w" );
fprintf( batFp, "%s\n", pCommand );
fclose( batFp );
STARTUPINFO si;
PROCESS_INFORMATION pi;
ZeroMemory( &si, sizeof(si) );
si.cb = sizeof(si);
ZeroMemory( &pi, sizeof(pi) );
// Start the child process.
if( !CreateProcess( NULL, // No module name (use command line).
"temp.bat", // Command line.
NULL, // Process handle not inheritable.
NULL, // Thread handle not inheritable.
FALSE, // Set handle inheritance to FALSE.
IDLE_PRIORITY_CLASS | CREATE_NO_WINDOW, // No creation flags.
NULL, // Use parent's environment block.
g_WorkerTempPath, // Use parent's starting directory.
&si, // Pointer to STARTUPINFO structure.
&pi ) // Pointer to PROCESS_INFORMATION structure.
)
{
Error( "CreateProcess failed." );
Assert( 0 );
}
// Wait until child process exits.
WaitForSingleObject( pi.hProcess, INFINITE );
// Close process and thread handles.
CloseHandle( pi.hProcess );
CloseHandle( pi.hThread );
}
// Assemble a reply package to the master from the compiled bytecode
// return the length of the package.
size_t AssembleWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
MessageBuffer *pBuf )
{
GLOBAL_DATA_MTX_LOCK();
CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ];
GLOBAL_DATA_MTX_UNLOCK();
size_t nBytesWritten = 0;
if ( pStComboRec && pStComboRec->m_DynamicCombos.Count() )
{
CUtlBuffer ubDynamicComboBuffer;
ubDynamicComboBuffer.SetBigEndian( g_bIsX360 );
pStComboRec->SortDynamicCombos();
// iterate over all dynamic combos.
for(int i = 0 ; i < pStComboRec->m_DynamicCombos.Count(); i++ )
{
CByteCodeBlock *pCode = pStComboRec->m_DynamicCombos[i];
// check if we have already output an identical combo
bool bDup = false;
#if 0
// check for duplicate bytecode. actually doesn't save much because bzip does a good
// job compressing dupes.
for( int j = 0; j < i; j++ )
{
if (
( pCode->m_nCRC32 == pStComboRec->m_DynamicCombos[j]->m_nCRC32 ) &&
( pCode->m_nCodeSize == pStComboRec->m_DynamicCombos[j]->m_nCodeSize ) &&
( memcmp( pCode->m_ByteCode, pStComboRec->m_DynamicCombos[i]->m_ByteCode, pCode->m_nCodeSize ) == 0 )
) // identical bytecode?
{
bDup = true;
OutputDynamicComboDup( &nBytesWritten, &ubDynamicComboBuffer,
pBuf, pCode->m_nComboID,
pStComboRec->m_DynamicCombos[j]->m_nComboID );
}
}
#endif
if ( ! bDup )
OutputDynamicCombo( &nBytesWritten, &ubDynamicComboBuffer,
pBuf, pCode->m_nComboID,
pCode->m_nCodeSize, pCode->m_ByteCode );
}
FlushCombos( &nBytesWritten, &ubDynamicComboBuffer, pBuf );
}
// Time to limit amount of prints
static float s_fLastInfoTime = 0;
float fCurTime = ( float ) Plat_FloatTime();
GLOBAL_DATA_MTX_LOCK();
if ( pStComboRec )
pByteCodeArray->DeleteByKey( nComboOfEntry );
if( fabs( fCurTime - s_fLastInfoTime ) > 1.f )
{
Msg( "\rCompiling %s [ %2llu remaining ] ... \r",
pEntry->m_szName, nComboOfEntry );
s_fLastInfoTime = fCurTime;
}
GLOBAL_DATA_MTX_UNLOCK();
return nBytesWritten;
}
// Copy a reply package to the master from the compiled bytecode
// return the length of the data copied.
size_t CopyWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
MessageBuffer *pBuf, int nSkipsSoFar )
{
GLOBAL_DATA_MTX_LOCK();
CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ]; // Get a static combo pointer
GLOBAL_DATA_MTX_UNLOCK();
int len = pStComboRec ? pStComboRec->m_abPackedCode.GetLength() : NULL;
if ( len )
{
if ( nSkipsSoFar )
{
pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
}
pBuf->write( &len, sizeof( len ) );
if ( len )
pBuf->write( pStComboRec->m_abPackedCode.GetData(), len );
}
if ( pStComboRec )
{
GLOBAL_DATA_MTX_LOCK();
pByteCodeArray->DeleteByKey( nComboOfEntry );
GLOBAL_DATA_MTX_UNLOCK();
}
return size_t( len );
}
template < typename TMutexType >
class CWorkerAccumState : public CParallelProcessorBase < CWorkerAccumState < TMutexType > >
{
friend ThisParallelProcessorBase_t;
private:
static bool & DisconnectState() { static bool sb = false; return sb; }
static void Special_DisconnectHandler( int procID, const char *pReason ) { DisconnectState() = true; }
public:
explicit CWorkerAccumState( TMutexType *pMutex ) :
m_pMutex( pMutex ), m_iFirstCommand( 0 ), m_iNextCommand( 0 ),
m_iEndCommand( 0 ), m_iLastFinished( 0 ),
m_hCombo( NULL ),
m_fnOldDisconnectHandler( g_fnDisconnectHandler ),
m_autoRestoreDisconnectHandler( g_fnDisconnectHandler )
{
DisconnectState() = false;
}
~CWorkerAccumState() { QuitSubs(); }
void RangeBegin( uint64 iFirstCommand, uint64 iEndCommand );
void RangeFinished( void );
void ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo );
void ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo );
void HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse );
public:
using ThisParallelProcessorBase_t::Run;
public:
bool OnProcess();
bool OnProcessST();
protected:
TMutexType *m_pMutex;
protected:
struct SubProcess
{
DWORD dwIndex;
DWORD dwSvcThreadId;
uint64 iRunningCommand;
PROCESS_INFORMATION pi;
SubProcessKernelObjects *pCommObjs;
};
CThreadLocal < SubProcess * > m_lpSubProcessInfo;
CUtlVector < SubProcess * > m_arrSubProcessInfos;
uint64 m_iFirstCommand;
uint64 m_iNextCommand;
uint64 m_iEndCommand;
uint64 m_iLastFinished;
CfgProcessor::ComboHandle m_hCombo;
DisconnectHandlerFn_t m_fnOldDisconnectHandler;
CAutoPushPop< DisconnectHandlerFn_t > m_autoRestoreDisconnectHandler;
void QuitSubs( void );
void TryToPackageData( uint64 iCommandNumber );
void PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs );
};
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeBegin( uint64 iFirstCommand, uint64 iEndCommand )
{
m_iFirstCommand = iFirstCommand;
m_iNextCommand = iFirstCommand;
m_iEndCommand = iEndCommand;
m_iLastFinished = iFirstCommand;
m_hCombo = NULL;
CfgProcessor::Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
g_fnDisconnectHandler = Special_DisconnectHandler;
// Notify all connected sub-processes that the master is still alive
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
if ( void *pvMemory = shrmem.Lock() )
{
strcpy( ( char * ) pvMemory, "keepalive" );
shrmem.Unlock();
}
}
}
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeFinished( void )
{
if( !DisconnectState() )
{
// Finish packaging data
TryToPackageData( m_iEndCommand - 1 );
}
else
{
// Master disconnected
QuitSubs();
}
g_fnDisconnectHandler = m_fnOldDisconnectHandler;
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::QuitSubs( void )
{
CUtlVector < HANDLE > m_arrWait;
m_arrWait.EnsureCapacity( m_arrSubProcessInfos.Count() );
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
if ( void *pvMemory = shrmem.Lock() )
{
strcpy( ( char * ) pvMemory, "quit" );
shrmem.Unlock();
}
m_arrWait.AddToTail( pSp->pi.hProcess );
}
}
if ( m_arrWait.Count() )
{
DWORD dwWait = WaitForMultipleObjects( m_arrWait.Count(), m_arrWait.Base(), TRUE, 2 * 1000 );
if ( WAIT_TIMEOUT == dwWait )
{
Warning( "Timed out while waiting for sub-processes to shut down!\n" );
}
}
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
CloseHandle( pSp->pi.hThread );
CloseHandle( pSp->pi.hProcess );
delete pSp->pCommObjs;
delete pSp;
}
}
if ( DisconnectState() )
Vmpi_Worker_DefaultDisconnectHandler( 0, "Master disconnected during compilation." );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs )
{
SubProcess *pSp = m_lpSubProcessInfo.Get();
SubProcessKernelObjects *pCommObjs = NULL;
if ( pSp )
{
pCommObjs = pSp->pCommObjs;
}
else
{
pSp = new SubProcess;
m_lpSubProcessInfo.Set( pSp );
pSp->dwSvcThreadId = ThreadGetCurrentId();
char chBaseNameBuffer[0x30];
sprintf( chBaseNameBuffer, "SHCMPL_SUB_%08X_%08llX_%08X", pSp->dwSvcThreadId, (long long)time( NULL ), GetCurrentProcessId() );
pCommObjs = pSp->pCommObjs = new SubProcessKernelObjects_Create( chBaseNameBuffer );
ZeroMemory( &pSp->pi, sizeof( pSp->pi ) );
STARTUPINFO si;
ZeroMemory( &si, sizeof( si ) );
si.cb = sizeof( si );
char chCommandLine[0x100];
sprintf( chCommandLine, "\"%s\\shadercompile.exe\" -subprocess %s", g_WorkerTempPath, chBaseNameBuffer );
#ifdef _DEBUG
V_strncat( chCommandLine, " -allowdebug", sizeof( chCommandLine ) );
#endif
BOOL bCreateResult = CreateProcess( NULL, chCommandLine, NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, g_WorkerTempPath, &si, &pSp->pi );
( void ) bCreateResult;
Assert( bCreateResult && "CreateProcess failed?" );
m_pMutex->Lock();
pSp->dwIndex = m_arrSubProcessInfos.AddToTail( pSp );
m_pMutex->Unlock();
}
if ( ppSp ) *ppSp = pSp;
if ( ppCommObjs ) *ppCommObjs = pCommObjs;
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo )
{
// DebugOut( "threaded: running: \"%s\"\n", szCommand );
SubProcessKernelObjects *pCommObjs = NULL;
PrepareSubProcess( NULL, &pCommObjs );
// Execute the command
SubProcessKernelObjects_Memory shrmem( pCommObjs );
{
void *pvMemory = shrmem.Lock();
Assert( pvMemory );
Combo_FormatCommand( hCombo, ( char * ) pvMemory );
shrmem.Unlock();
}
// Obtain the command response
{
void const *pvMemory = shrmem.Lock();
Assert( pvMemory );
// TODO: Vitaliy :: TEMP fix:
// Usually what happens if we fail to lock here is
// when our subprocess dies and to recover we will
// attempt to restart on another worker.
if ( !pvMemory )
// ::RaiseException( GetLastError(), EXCEPTION_NONCONTINUABLE, 0, NULL );
TerminateProcess( GetCurrentProcess(), 1 );
CmdSink::IResponse *pResponse;
if ( pvMemory )
pResponse = new CSubProcessResponse( pvMemory );
else
pResponse = new CmdSink::CResponseError;
HandleCommandResponse( hCombo, pResponse );
shrmem.Unlock();
}
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo )
{
CmdSink::IResponse *pResponse = NULL;
{
char chBuffer[ 4096 ];
Combo_FormatCommand( hCombo, chBuffer );
DebugOut( "running: \"%s\"\n", chBuffer );
MySystem( chBuffer, &pResponse );
}
HandleCommandResponse( hCombo, pResponse );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse )
{
VMPI_HandleSocketErrors();
if ( !pResponse )
pResponse = new CmdSink::CResponseFiles( "shader.o", "output.txt" );
// Command info
CfgProcessor::CfgEntryInfo const *pEntryInfo = Combo_GetEntryInfo( hCombo );
uint64 iComboIndex = Combo_GetComboNum( hCombo );
uint64 iCommandNumber = Combo_GetCommandNum( hCombo );
if ( pResponse->Succeeded() )
{
GLOBAL_DATA_MTX_LOCK();
uint64 nStComboIdx = iComboIndex / pEntryInfo->m_numDynamicCombos;
uint64 nDyComboIdx = iComboIndex - ( nStComboIdx * pEntryInfo->m_numDynamicCombos );
StaticComboFromDictAdd( pEntryInfo->m_szName, nStComboIdx )->AddDynamicCombo( nDyComboIdx , pResponse->GetResultBuffer(), pResponse->GetResultBufferLen() );
GLOBAL_DATA_MTX_UNLOCK();
}
// Tell the master that this shader failed
if ( !pResponse->Succeeded() )
{
GLOBAL_DATA_MTX_LOCK();
ShaderHadErrorDispatchInt( pEntryInfo->m_szName );
GLOBAL_DATA_MTX_UNLOCK();
}
// Process listing even if the shader succeeds for warnings
char const *szListing = pResponse->GetListing();
if ( ( !g_bSuppressWarnings && szListing ) || !pResponse->Succeeded() )
{
char chCommandNumber[50];
sprintf( chCommandNumber, "%I64u", iCommandNumber );
char chUnreportedListing[0xFF];
if ( !szListing )
{
sprintf( chUnreportedListing, "(0): error 0000: Compiler failed without error description, latest version of fxc.exe might give a description." );
szListing = chUnreportedListing;
}
// Send the listing for dispatch
CUtlBinaryBlock errMsg;
errMsg.SetLength(
strlen( chCommandNumber ) + 1 + // command + newline
strlen( szListing ) + 1 + // listing + newline
1 // null-terminator
);
sprintf( ( char * ) errMsg.Get(), "%s\n%s\n", chCommandNumber, szListing );
GLOBAL_DATA_MTX_LOCK();
ErrMsgDispatchInt( ( char * ) errMsg.Get(), pEntryInfo->m_szShaderFileName );
GLOBAL_DATA_MTX_UNLOCK();
}
// Maybe zip things up
TryToPackageData( iCommandNumber );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::TryToPackageData( uint64 iCommandNumber )
{
m_pMutex->Lock();
uint64 iFinishedByNow = iCommandNumber + 1;
// Check if somebody is running an earlier command
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
if ( pSp->iRunningCommand < iCommandNumber )
{
iFinishedByNow = 0;
break;
}
}
}
uint64 iLastFinished = m_iLastFinished;
if ( iFinishedByNow > m_iLastFinished )
{
m_iLastFinished = iFinishedByNow;
m_pMutex->Unlock();
}
else
{
m_pMutex->Unlock();
return;
}
CfgProcessor::ComboHandle hChBegin = CfgProcessor::Combo_GetCombo( iLastFinished );
CfgProcessor::ComboHandle hChEnd = CfgProcessor::Combo_GetCombo( iFinishedByNow );
Assert( hChBegin && hChEnd );
CfgProcessor::CfgEntryInfo const *pInfoBegin = Combo_GetEntryInfo( hChBegin );
CfgProcessor::CfgEntryInfo const *pInfoEnd = Combo_GetEntryInfo( hChEnd );
uint64 nComboBegin = Combo_GetComboNum( hChBegin ) / pInfoBegin->m_numDynamicCombos;
uint64 nComboEnd = Combo_GetComboNum( hChEnd ) / pInfoEnd->m_numDynamicCombos;
for ( ; pInfoBegin && (
( pInfoBegin->m_iCommandStart < pInfoEnd->m_iCommandStart ) ||
( nComboBegin > nComboEnd ) ); )
{
// Zip this combo
MessageBuffer mbPacked;
size_t nPackedLength = AssembleWorkerReplyPackage( pInfoBegin, nComboBegin, &mbPacked );
if ( nPackedLength )
{
// Packed buffer
GLOBAL_DATA_MTX_LOCK();
uint8 *pCodeBuffer = StaticComboFromDictAdd( pInfoBegin->m_szName,
nComboBegin )->AllocPackedCodeBlock( nPackedLength );
GLOBAL_DATA_MTX_UNLOCK();
if ( pCodeBuffer )
mbPacked.read( pCodeBuffer, nPackedLength );
}
// Next iteration
if ( ! nComboBegin -- )
{
Combo_Free( hChBegin );
if ( ( hChBegin = CfgProcessor::Combo_GetCombo( pInfoBegin->m_iCommandEnd ) ) != NULL )
{
pInfoBegin = Combo_GetEntryInfo( hChBegin );
nComboBegin = pInfoBegin->m_numStaticCombos - 1;
}
}
}
Combo_Free( hChBegin );
Combo_Free( hChEnd );
}
template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcess()
{
m_pMutex->Lock();
CfgProcessor::ComboHandle hThreadCombo = m_hCombo ? Combo_Alloc( m_hCombo ) : NULL;
m_pMutex->Unlock();
uint64 iThreadCommand = ~uint64(0);
SubProcess *pSp = NULL;
PrepareSubProcess( &pSp, NULL );
for ( ; ; )
{
m_pMutex->Lock();
if ( DisconnectState() )
Combo_Free( m_hCombo );
if ( m_hCombo )
{
Combo_Assign( hThreadCombo, m_hCombo );
pSp->iRunningCommand = Combo_GetCommandNum( hThreadCombo );
Combo_GetNext( iThreadCommand, m_hCombo, m_iEndCommand );
}
else
{
Combo_Free( hThreadCombo );
iThreadCommand = ~uint64(0);
pSp->iRunningCommand = ~uint64(0);
}
m_pMutex->Unlock();
if ( hThreadCombo )
{
ExecuteCompileCommandThreaded( hThreadCombo );
}
else
break;
}
Combo_Free( hThreadCombo );
return false;
}
template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcessST()
{
while ( m_hCombo )
{
ExecuteCompileCommand( m_hCombo );
Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
}
return false;
}
//
// Worker_ProcessCommandRange_Singleton
//
class Worker_ProcessCommandRange_Singleton
{
public:
static Worker_ProcessCommandRange_Singleton *& Instance() { static Worker_ProcessCommandRange_Singleton *s_ptr = NULL; return s_ptr; }
static Worker_ProcessCommandRange_Singleton * GetInstance() { Worker_ProcessCommandRange_Singleton *p = Instance(); Assert( p ); return p; }
public:
Worker_ProcessCommandRange_Singleton() { Assert( !Instance() ); Instance() = this; Startup(); }
~Worker_ProcessCommandRange_Singleton() { Assert( Instance() == this ); Instance() = NULL; Shutdown(); }
public:
void ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd );
protected:
void Startup( void );
void Shutdown( void );
//
// Multi-threaded section
protected:
struct MT {
MT() : pWorkerObj( NULL ), pThreadPool( NULL ) {}
typedef CThreadFastMutex MultiThreadMutex_t;
MultiThreadMutex_t mtx;
typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
WorkerClass_t *pWorkerObj;
IThreadPool *pThreadPool;
ThreadPoolStartParams_t tpsp;
} m_MT;
//
// Single-threaded section
protected:
struct ST {
ST() : pWorkerObj( NULL ) {}
typedef CThreadNullMutex MultiThreadMutex_t;
MultiThreadMutex_t mtx;
typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
WorkerClass_t *pWorkerObj;
} m_ST;
};
void Worker_ProcessCommandRange_Singleton::Startup( void )
{
bool bInitializedThreadPool = false;
CPUInformation const &cpu = *GetCPUInformation();
if ( cpu.m_nLogicalProcessors > 1 )
{
// Attempt to initialize thread pool
m_MT.pThreadPool = g_pThreadPool;
if ( m_MT.pThreadPool )
{
m_MT.tpsp.bIOThreads = false;
m_MT.tpsp.nThreads = cpu.m_nLogicalProcessors - 1;
if ( m_MT.pThreadPool->Start( m_MT.tpsp ) )
{
if ( m_MT.pThreadPool->NumThreads() >= 1 )
{
// Make sure that our mutex is in multi-threaded mode
Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );
m_MT.pWorkerObj = new MT::WorkerClass_t( &m_MT.mtx );
bInitializedThreadPool = true;
}
else
{
m_MT.pThreadPool->Stop();
}
}
if ( !bInitializedThreadPool )
m_MT.pThreadPool = NULL;
}
}
// Otherwise initialize single-threaded mode
if ( !bInitializedThreadPool )
{
m_ST.pWorkerObj = new ST::WorkerClass_t( &m_ST.mtx );
}
}
void Worker_ProcessCommandRange_Singleton::Shutdown( void )
{
if ( m_MT.pThreadPool )
{
if( m_MT.pWorkerObj )
delete m_MT.pWorkerObj;
m_MT.pThreadPool->Stop();
m_MT.pThreadPool = NULL;
}
else
{
if ( m_ST.pWorkerObj )
delete m_ST.pWorkerObj;
}
}
void Worker_ProcessCommandRange_Singleton::ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
if ( m_MT.pThreadPool )
{
MT::WorkerClass_t *pWorkerObj = m_MT.pWorkerObj;
pWorkerObj->RangeBegin( shaderStart, shaderEnd );
pWorkerObj->Run();
pWorkerObj->RangeFinished();
}
else
{
ST::WorkerClass_t *pWorkerObj = m_ST.pWorkerObj;
pWorkerObj->RangeBegin( shaderStart, shaderEnd );
pWorkerObj->OnProcessST();
pWorkerObj->RangeFinished();
}
}
// You must process the work unit range.
void Worker_ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
Worker_ProcessCommandRange_Singleton::GetInstance()->ProcessCommandRange( shaderStart, shaderEnd );
}
// You must append data to pBuf with the work unit results.
void Worker_ProcessWorkUnitFn( int iThread, uint64 iWorkUnit, MessageBuffer *pBuf )
{
uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
comboEnd = min( g_numStaticCombos, comboEnd );
// Determine the commands required to be executed:
uint64 nComboOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = NULL;
pEntry = GetEntryByStaticComboNum( comboEnd, &nComboOfTheEntry );
uint64 commandEnd = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;
Assert( commandEnd <= g_numCompileCommands );
pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
uint64 commandStart = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;
// Compile all the shader combos
Worker_ProcessCommandRange( commandStart, commandEnd );
nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;
// Copy off the reply packages
int nSkipsSoFar = 0;
for ( uint64 kCombo = comboStart; kCombo < comboEnd; ++ kCombo )
{
size_t nCpBytes = CopyWorkerReplyPackage( pEntry, nComboOfTheEntry, pBuf, nSkipsSoFar );
if ( nCpBytes )
nSkipsSoFar = 0;
else
-- nSkipsSoFar;
if ( nComboOfTheEntry == 0 )
{
++pEntry;
nComboOfTheEntry = pEntry->m_numStaticCombos;
}
nComboOfTheEntry--;
}
if ( nSkipsSoFar )
{
pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
}
//////////////////////////////////////////////////////////////////////////
//
// Now deliver all our accumulated spew to the master
//
//////////////////////////////////////////////////////////////////////////
// Failed shaders
for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
{
char const *szShaderName = g_Master_ShaderHadError.String( k );
if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
continue;
int const len = strlen( szShaderName );
CUtlBinaryBlock bb;
bb.SetLength( 1 + len + 1 );
sprintf( ( char * ) bb.Get(), "%c%s", SHADERHADERROR_PACKETID, szShaderName );
VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
VMPI_HandleSocketErrors();
}
// Compiler spew
for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
{
char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];
char const * const szFirstCmd = cmi.GetFirstCommand();
int const numReported = cmi.GetNumTimesReported();
char chNumReported[0x40];
sprintf( chNumReported, "%d", numReported );
CUtlBinaryBlock bb;
bb.SetLength( 1 + strlen(szMsg) + 1 + strlen( szFirstCmd ) + 1 + strlen( chNumReported ) + 1 + 1 );
sprintf( ( char * ) bb.Get(), "%c%s\n%s\n%s\n", ERRMSG_PACKETID, szMsg, szFirstCmd, chNumReported );
VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
VMPI_HandleSocketErrors();
}
// Clean all reported msgs
g_Master_CompilerMsgInfo.Purge();
}
void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo )
{
if ( CfgProcessor::ComboHandle hCombo = CfgProcessor::Combo_GetCombo( pEntry->m_iCommandStart ) )
{
char cmd[ 4096 ];
Combo_FormatCommand( hCombo, cmd );
{
memset( &shaderInfo, 0, sizeof( ShaderInfo_t ) );
const char *pCentroidMask = strstr( cmd, "/DCENTROIDMASK=" );
const char *pFlags = strstr( cmd, "/DFLAGS=0x" );
const char *pShaderModel = strstr( cmd, "/DSHADER_MODEL_" );
if( !pCentroidMask || !pFlags || !pShaderModel )
{
Assert( !"!pCentroidMask || !pFlags || !pShaderModel" );
return;
}
sscanf( pCentroidMask + strlen( "/DCENTROIDMASK=" ), "%u", &shaderInfo.m_CentroidMask );
sscanf( pFlags + strlen( "/DFLAGS=0x" ), "%x", &shaderInfo.m_Flags );
// Copy shader model
pShaderModel += strlen( "/DSHADER_MODEL_" );
for ( char *pszSm = shaderInfo.m_szShaderModel, * const pszEnd = pszSm + sizeof( shaderInfo.m_szShaderModel ) - 1;
pszSm < pszEnd ; ++ pszSm )
{
char &rchLastChar = (*pszSm = *pShaderModel ++);
if ( !rchLastChar ||
V_isspace( rchLastChar ) ||
'=' == rchLastChar )
{
rchLastChar = 0;
break;
}
}
shaderInfo.m_nShaderCombo = 0;
shaderInfo.m_nTotalShaderCombos = pEntry->m_numCombos;
shaderInfo.m_nDynamicCombos = pEntry->m_numDynamicCombos;
shaderInfo.m_nStaticCombo = 0;
shaderInfo.m_pShaderName = pEntry->m_szName;
shaderInfo.m_pShaderSrc = pEntry->m_szShaderFileName;
}
Combo_Free( hCombo );
}
}
void Worker_GetLocalCopyOfShaders( void )
{
// Create virtual files for all of the stuff that we need to compile the shader
// make sure and prefix the file name so that it doesn't find it locally.
char filename[1024];
sprintf( filename, "%s\\uniquefilestocopy.txt", g_pShaderPath );
CUtlInplaceBuffer bffr( 0, 0, CUtlBuffer::TEXT_BUFFER );
if( !g_pFileSystem->ReadFile( filename, NULL, bffr ) )
{
fprintf( stderr, "Can't open uniquefilestocopy.txt!\n" );
exit( -1 );
}
while( char *pszLineToCopy = bffr.InplaceGetLinePtr() )
{
V_MakeAbsolutePath( filename, sizeof( filename ), pszLineToCopy, g_pShaderPath );
if ( g_bVerbose )
printf( "getting local copy of shader: \"%s\" (\"%s\")\n", pszLineToCopy, filename );
CUtlBuffer fileBuf;
if ( !g_pFileSystem->ReadFile( filename, NULL, fileBuf ) )
{
Warning( "Can't find \"%s\"\n", filename );
continue;
}
// Grab just the filename.
char justFilename[MAX_PATH];
char *pLastSlash = max( strrchr( pszLineToCopy, '/' ), strrchr( pszLineToCopy, '\\' ) );
if ( pLastSlash )
Q_strncpy( justFilename, pLastSlash + 1, sizeof( justFilename ) );
else
Q_strncpy( justFilename, pszLineToCopy, sizeof( justFilename ) );
sprintf( filename, "%s%s", g_WorkerTempPath, justFilename );
if ( g_bVerbose )
printf( "creating \"%s\"\n", filename );
FILE *fp3 = fopen( filename, "wb" );
if ( !fp3 )
{
Error( "Can't open '%s' for writing.", pszLineToCopy );
continue;
}
fwrite( fileBuf.Base(), 1, fileBuf.GetBytesRemaining(), fp3 );
fclose( fp3 );
// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
// while we're running since they're in the temp path.
static CUtlVector< FILE * > s_arrHackedFiles;
static struct X_s_arrHackedFiles { ~X_s_arrHackedFiles() {
for ( int k = 0; k < s_arrHackedFiles.Count(); ++ k )
fclose( s_arrHackedFiles[k] );
} } s_autoCloseHackedFiles;
/* THIS IS THE EVIL LINE ----> */ FILE *fHack = fopen( filename, "r" );
s_arrHackedFiles.AddToTail( fHack );
// -- END of EVIL
}
}
void Worker_GetLocalCopyOfBinary( const char *pFilename )
{
CUtlBuffer fileBuf;
char tmpFilename[MAX_PATH];
sprintf( tmpFilename, "%s\\%s", g_ExeDir, pFilename );
if ( g_bVerbose )
printf( "trying to open: %s\n", tmpFilename );
FILE *fp = fopen( tmpFilename, "rb" );
if( !fp )
{
Assert( 0 );
fprintf( stderr, "Can't open %s!\n", pFilename );
exit( -1 );
}
fseek( fp, 0, SEEK_END );
int fileLen = ftell( fp );
fseek( fp, 0, SEEK_SET );
fileBuf.EnsureCapacity( fileLen );
int nBytesRead = fread( fileBuf.Base(), 1, fileLen, fp );
fclose( fp );
fileBuf.SeekPut( CUtlBuffer::SEEK_HEAD, nBytesRead );
char newFilename[MAX_PATH];
sprintf( newFilename, "%s%s", g_WorkerTempPath, pFilename );
FILE *fp2 = fopen( newFilename, "wb" );
if( !fp2 )
{
Assert( 0 );
fprintf( stderr, "Can't open %s!\n", newFilename );
exit( -1 );
}
fwrite( fileBuf.Base(), 1, fileLen, fp2 );
fclose( fp2 );
// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
// while we're running since they're in the temp path.
fopen( newFilename, "r" );
}
void Worker_GetLocalCopyOfBinaries( void )
{
Worker_GetLocalCopyOfBinary( "mysql_wrapper.dll" ); // This is necessary so VMPI doesn't run in SDK mode.
Worker_GetLocalCopyOfBinary( "vstdlib.dll" );
Worker_GetLocalCopyOfBinary( "tier0.dll" );
}
void Shared_ParseListOfCompileCommands( void )
{
// double tt_start = Plat_FloatTime();
char fileListFileName[1024];
sprintf( fileListFileName, "%s\\filelist.txt", g_pShaderPath );
CUtlInplaceBuffer bffr( 0, 0, CUtlInplaceBuffer::TEXT_BUFFER );
if( !g_pFileSystem->ReadFile( fileListFileName, NULL, bffr) )
{
DebugOut( "Can't open %s!\n", fileListFileName );
fprintf( stderr, "Can't open %s!\n", fileListFileName );
exit( -1 );
}
CfgProcessor::ReadConfiguration( &bffr );
CfgProcessor::DescribeConfiguration( g_arrCompileEntries );
for ( CfgProcessor::CfgEntryInfo const *pInfo = g_arrCompileEntries.Get();
pInfo && pInfo->m_szName; ++ pInfo )
{
++ g_numShaders;
g_numStaticCombos += pInfo->m_numStaticCombos;
g_numCompileCommands = pInfo->m_iCommandEnd;
}
// double tt_end = Plat_FloatTime();
Msg( "\rCompiling %s commands. \r", PrettyPrintNumber( g_numCompileCommands ) );
}
void SetupExeDir( int argc, char **argv )
{
strcpy( g_ExeDir, argv[0] );
Q_StripFilename( g_ExeDir );
if ( g_ExeDir[0] == 0 )
{
Q_strncpy( g_ExeDir, ".\\", sizeof( g_ExeDir ) );
}
Q_FixSlashes( g_ExeDir );
}
void SetupPaths( int argc, char **argv )
{
GetTempPath( sizeof( g_WorkerTempPath ), g_WorkerTempPath );
strcat( g_WorkerTempPath, "shadercompiletemp\\" );
char tmp[MAX_PATH];
sprintf( tmp, "rd /s /q \"%s\"", g_WorkerTempPath );
system( tmp );
_mkdir( g_WorkerTempPath );
// printf( "g_WorkerTempPath: \"%s\"\n", g_WorkerTempPath );
CommandLine()->CreateCmdLine( argc, argv );
g_pShaderPath = CommandLine()->ParmValue( "-shaderpath", "" );
g_bVerbose = CommandLine()->FindParm("-verbose") != 0;
}
void SetupDebugFile( void )
{
#ifdef DEBUGFP
const char *pComputerName = getenv( "COMPUTERNAME" );
char filename[MAX_PATH];
sprintf( filename, "\\\\fileserver\\user\\gary\\debug\\%s.txt", pComputerName );
g_WorkerDebugFp = fopen( filename, "w" );
Assert( g_WorkerDebugFp );
DebugOut( "opened debug file\n" );
#endif
}
void CompileShaders_NoVMPI()
{
Worker_ProcessCommandRange_Singleton pcr;
//
// We will iterate on the cfg entries and process them
//
for ( CfgProcessor::CfgEntryInfo const *pEntry = g_arrCompileEntries.Get();
pEntry && pEntry->m_szName; ++ pEntry )
{
//
// Stick the shader info
//
ShaderInfo_t siLastShaderInfo;
memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );
Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );
g_ShaderToShaderInfo[ pEntry->m_szName ] = siLastShaderInfo;
//
// Compile stuff
//
Worker_ProcessCommandRange( pEntry->m_iCommandStart, pEntry->m_iCommandEnd );
//
// Now when the whole shader is finished we can write it
//
char const *szShaderToWrite = pEntry->m_szName;
g_numCommandsCompleted = g_numCompileCommands;
WriteShaderFiles( szShaderToWrite );
g_numCommandsCompleted = pEntry->m_iCommandEnd;
}
Msg( "\r \r" );
}
class CDistributeShaderCompileMaster : public IWorkUnitDistributorCallbacks
{
public:
CDistributeShaderCompileMaster( void );
~CDistributeShaderCompileMaster( void );
public:
virtual void OnWorkUnitsCompleted( uint64 numWorkUnits );
private:
void ThreadProc( void );
friend DWORD WINAPI CDistributeShaderCompileMaster::ThreadProcAdapter( LPVOID pvArg );
static DWORD WINAPI ThreadProcAdapter( LPVOID pvArg ) { reinterpret_cast< CDistributeShaderCompileMaster * >( pvArg )->ThreadProc(); return 0; }
private:
HANDLE m_hThread;
HANDLE m_hEvent;
CThreadFastMutex m_mtx;
BOOL m_bRunning;
private:
CfgProcessor::CfgEntryInfo const *m_pAnalyzeShaders;
CUtlVector< char const * > m_arrShaderNamesToWrite;
};
CDistributeShaderCompileMaster::CDistributeShaderCompileMaster( void ) :
m_hThread( NULL ),
m_hEvent( NULL ),
m_bRunning( TRUE )
{
m_hEvent = CreateEvent( NULL, FALSE, FALSE, NULL );
m_hThread = CreateThread( NULL, 0, ThreadProcAdapter, reinterpret_cast< LPVOID >(this), 0, NULL );
m_pAnalyzeShaders = g_arrCompileEntries.Get();
}
CDistributeShaderCompileMaster::~CDistributeShaderCompileMaster( void )
{
m_bRunning = FALSE;
SetEvent( m_hEvent );
WaitForSingleObject( m_hThread, INFINITE );
CloseHandle( m_hThread );
CloseHandle( m_hEvent );
}
void CDistributeShaderCompileMaster::OnWorkUnitsCompleted( uint64 numWorkUnits )
{
// Make sure that our mutex is in multi-threaded mode
Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );
// Figure out how many commands have completed based on work units
g_numCompletedStaticCombos = numWorkUnits * g_nStaticCombosPerWorkUnit;
uint64 numStaticCombosOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( g_numCompletedStaticCombos, &numStaticCombosOfTheEntry );
g_numCommandsCompleted = pEntry->m_iCommandStart + numStaticCombosOfTheEntry * pEntry->m_numDynamicCombos;
// Iterate over the shaders yet to be written and see if we can queue them
for ( ; m_pAnalyzeShaders->m_szName &&
m_pAnalyzeShaders->m_iCommandEnd <= g_numCommandsCompleted;
++ m_pAnalyzeShaders
)
{
m_mtx.Lock();
m_arrShaderNamesToWrite.AddToTail( m_pAnalyzeShaders->m_szName );
SetEvent( m_hEvent );
m_mtx.Unlock();
}
}
void CDistributeShaderCompileMaster::ThreadProc( void )
{
for ( ; m_bRunning; )
{
WaitForSingleObject( m_hEvent, INFINITE );
// Do a pump of shaders to write
for ( int numShadersWritten = 0; /* forever */ ; ++ numShadersWritten )
{
m_mtx.Lock();
char const * szShaderToWrite = NULL;
if ( m_arrShaderNamesToWrite.Count() > numShadersWritten )
szShaderToWrite = m_arrShaderNamesToWrite[ numShadersWritten ];
else
m_arrShaderNamesToWrite.RemoveAll();
m_mtx.Unlock();
if ( !szShaderToWrite )
break;
// We have the shader to write asynchronously
WriteShaderFiles( szShaderToWrite );
}
}
}
int ShaderCompile_Main( int argc, char* argv[] )
{
InstallSpewFunction();
g_bSuppressPrintfOutput = false;
g_flStartTime = Plat_FloatTime();
SetupDebugFile();
numthreads = 1; // managed specifically in Worker_ProcessCommandRange_Singleton::Startup
/*
Special section of code implementing "-subprocess" flag
*/
if ( int iSubprocess = CommandLine()->FindParm( "-subprocess" ) )
{
char const *szSubProcessData = CommandLine()->GetParm( 1 + iSubprocess );
return ShaderCompile_Subprocess_Main( szSubProcessData );
}
// This needs to get called before VMPI is setup because in SDK mode, VMPI will change the args around.
SetupExeDir( argc, argv );
g_bIsX360 = CommandLine()->FindParm( "-x360" ) != 0;
// g_bSuppressWarnings = g_bIsX360;
bool bShouldUseVMPI = ( CommandLine()->FindParm( "-nompi" ) == 0 );
if ( bShouldUseVMPI )
{
// Master, start accepting connections.
// Worker, make a connection.
DebugOut( "Before VMPI_Init\n" );
g_bSuppressPrintfOutput = true;
VMPIRunMode mode = VMPI_RUN_NETWORKED;
if ( !VMPI_Init( argc, argv, "dependency_info_shadercompile.txt", MyDisconnectHandler, mode ) )
{
g_bSuppressPrintfOutput = false;
DebugOut( "MPI_Init failed.\n" );
Error( "MPI_Init failed." );
}
extern void VMPI_SetWorkUnitsPartitionSize( int numWusToDeal );
VMPI_SetWorkUnitsPartitionSize( 32 );
}
SetupPaths( argc, argv );
g_bSuppressPrintfOutput = false;
DebugOut( "After VMPI_Init\n" );
// Setting up the minidump handlers
if ( bShouldUseVMPI && !g_bMPIMaster )
SetupToolsMinidumpHandler( VMPI_ExceptionFilter );
else
SetupDefaultToolsMinidumpHandler();
if ( CommandLine()->FindParm( "-game" ) == 0 )
{
// Used with filesystem_stdio.dll
FileSystem_Init( NULL, 0, FS_INIT_COMPATIBILITY_MODE );
}
else
{
// SDK uses this since it only has filesystem_steam.dll.
FileSystem_Init( NULL, 0, FS_INIT_FULL );
}
DebugOut( "After VMPI_FileSystem_Init\n" );
Shared_ParseListOfCompileCommands();
DebugOut( "After Shared_ParseListOfCompileCommands\n" );
if ( bShouldUseVMPI )
{
// Partition combos
g_nStaticCombosPerWorkUnit = 0;
if ( g_numStaticCombos )
{
if ( g_numStaticCombos <= 1024 )
g_nStaticCombosPerWorkUnit = 1;
else if ( g_numStaticCombos > 1024 * 10 )
g_nStaticCombosPerWorkUnit = 10;
else
g_nStaticCombosPerWorkUnit = g_numStaticCombos / 1024;
}
uint64 nWorkUnits;
if( g_nStaticCombosPerWorkUnit == 0 )
{
nWorkUnits = 1;
g_nStaticCombosPerWorkUnit = g_numStaticCombos;
}
else
{
nWorkUnits = g_numStaticCombos / g_nStaticCombosPerWorkUnit + 1;
}
DebugOut( "Before conditional\n" );
if ( g_bMPIMaster )
{
// Send all of the workers the complete list of work to do.
DebugOut( "Before STARTWORK_PACKETID\n" );
char packetID = STARTWORK_PACKETID;
VMPI_SendData( &packetID, sizeof( packetID ), VMPI_PERSISTENT );
// Compile master distribution tracker
CDistributeShaderCompileMaster dscm;
g_pDistributeWorkCallbacks = &dscm;
{
char chCommands[50], chStaticCombos[50], chNumWorkUnits[50];
sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
sprintf( chNumWorkUnits, "%s", PrettyPrintNumber( nWorkUnits ) );
Msg( "\rCompiling %s commands in %s work units.\n", chCommands, chNumWorkUnits );
}
// nWorkUnits is how many work units. . .1000 is good.
// The work unit number impies which combo to do.
DebugOut( "Before DistributeWork\n" );
DistributeWork( nWorkUnits, WORKUNIT_PACKETID, NULL, Master_ReceiveWorkUnitFn );
g_pDistributeWorkCallbacks = NULL;
}
else
{
// wait until we get a packet from the master to start doing stuff.
MessageBuffer buf;
DebugOut( "Before VMPI_DispatchUntil\n" );
while ( !g_bGotStartWorkPacket )
{
VMPI_DispatchNextMessage();
}
DebugOut( "after VMPI_DispatchUntil\n" );
DebugOut( "Before Worker_GetLocalCopyOfShaders\n" );
Worker_GetLocalCopyOfShaders();
DebugOut( "Before Worker_GetLocalCopyOfBinaries\n" );
Worker_GetLocalCopyOfBinaries();
DebugOut( "Before _chdir\n" );
_chdir( g_WorkerTempPath );
// nWorkUnits is how many work units. . .1000 is good.
// The work unit number impies which combo to do.
DebugOut( "Before DistributeWork\n" );
// Allows calling into ProcessCommandRange inside the worker function
{
Worker_ProcessCommandRange_Singleton pcr;
DistributeWork( nWorkUnits, WORKUNIT_PACKETID, Worker_ProcessWorkUnitFn, NULL );
}
}
g_bSuppressPrintfOutput = true;
g_bSuppressPrintfOutput = false;
}
else // no VMPI
{
Worker_GetLocalCopyOfShaders();
Worker_GetLocalCopyOfBinaries();
_chdir( g_WorkerTempPath );
{
char chCommands[50], chStaticCombos[50];
sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
Msg( "\rCompiling %s commands in %s static combos.\n", chCommands, chStaticCombos );
}
CompileShaders_NoVMPI();
}
Msg( "\r \r" );
if ( g_bMPIMaster || !bShouldUseVMPI )
{
char str[ 4096 ];
// Write everything that succeeded
int nStrings = g_ShaderByteCode.GetNumStrings();
for( int i = 0; i < nStrings; i++ )
{
WriteShaderFiles( g_ShaderByteCode.String(i) );
}
// Write all the errors
//////////////////////////////////////////////////////////////////////////
//
// Now deliver all our accumulated spew to the output
//
//////////////////////////////////////////////////////////////////////////
bool bValveVerboseComboErrors = ( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) &&
atoi( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) ) ) ? true : false;
// Compiler spew
for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
{
char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];
char const * const szFirstCmd = cmi.GetFirstCommand();
int const numReported = cmi.GetNumTimesReported();
uint64 iFirstCommand = _strtoui64( szFirstCmd, NULL, 10 );
CfgProcessor::ComboHandle hCombo = NULL;
CfgProcessor::CfgEntryInfo const *pComboEntryInfo = NULL;
if ( CfgProcessor::Combo_GetNext( iFirstCommand, hCombo, g_numCompileCommands ) )
{
Combo_FormatCommand( hCombo, str );
pComboEntryInfo = Combo_GetEntryInfo( hCombo );
Combo_Free( hCombo );
}
else
{
sprintf( str, "cmd # %s", szFirstCmd );
}
Msg( "\n%s\n", szMsg );
Msg( " Reported %d time(s), example command:\n", numReported);
if ( bValveVerboseComboErrors )
{
Msg( " Verbose Description:\n" );
if ( pComboEntryInfo )
{
Msg( " Src File: %s\n", pComboEntryInfo->m_szShaderFileName );
Msg( " Tgt File: %s\n", pComboEntryInfo->m_szName );
}
// Between /DSHADERCOMBO= and /Dmain
char const *pBegin = strstr( str, "/DSHADERCOMBO=" );
char const *pEnd = strstr( str, "/Dmain" );
if ( pBegin )
{
pBegin += strlen( "/DSHADERCOMBO=" ) ;
char const *pSpace = strchr( pBegin, ' ' );
if ( pSpace )
Msg( " Combo # : %.*s\n", ( pSpace - pBegin ), pBegin );
}
if ( !pEnd )
pEnd = str + strlen( str );
while ( pBegin && *pBegin && !V_isspace( *pBegin ) )
++ pBegin;
while ( pBegin && *pBegin && V_isspace( *pBegin ) )
++ pBegin;
// Now parse all combo defines in [pBegin, pEnd]
while ( pBegin && *pBegin && ( pBegin < pEnd ) )
{
char const *pDefine = strstr( pBegin, "/D" );
if ( !pDefine || pDefine >= pEnd )
break;
char const *pEqSign = strchr( pDefine, '=' );
if ( !pEqSign || pEqSign >= pEnd )
break;
char const *pSpace = strchr( pEqSign, ' ' );
if ( !pSpace || pSpace >= pEnd )
pSpace = pEnd;
pBegin = pSpace;
Msg( " %.*s %.*s\n",
( pSpace - pEqSign - 1 ), pEqSign + 1,
( pEqSign - pDefine - 2 ), pDefine + 2 );
}
}
Msg( " %s\n", str );
}
// Failed shaders summary
for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
{
char const *szShaderName = g_Master_ShaderHadError.String( k );
if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
continue;
Msg( "FAILED: %s\n", szShaderName );
}
//
// End
//
double end = Plat_FloatTime();
GetHourMinuteSecondsString( (int)( end - g_flStartTime ), str, sizeof( str ) );
DebugOut( "%s elapsed\n", str );
DebugOut( "Precise timing = %.5f\n", ( end - g_flStartTime ) );
if ( bShouldUseVMPI )
{
VMPI_FileSystem_Term();
DebugOut( "Before VMPI_Finalize\n" );
VMPI_Finalize();
}
}
return g_Master_ShaderHadError.GetNumStrings();
}
class CShaderCompileDLL : public IShaderCompileDLL
{
int main( int argc, char **argv );
};
int CShaderCompileDLL::main( int argc, char **argv )
{
return ShaderCompile_Main( argc, argv );
}
EXPOSE_SINGLE_INTERFACE( CShaderCompileDLL, IShaderCompileDLL, SHADER_COMPILE_INTERFACE_VERSION );
class CLaunchableDLL : public ILaunchableDLL
{
int main( int argc, char **argv )
{
return ShaderCompile_Main( argc, argv );
}
};
EXPOSE_SINGLE_INTERFACE( CLaunchableDLL, ILaunchableDLL, LAUNCHABLE_DLL_INTERFACE_VERSION );