//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
// $NoKeywords: $
//
//=============================================================================//
// vmpi_bareshell.cpp : Defines the entry point for the console application.
//
#include <windows.h>
#include <conio.h>
#include <process.h>
#include "vmpi.h"
#include "filesystem.h"
#include "vmpi_filesystem.h"
#include "vmpi_distribute_work.h"
#include "vmpi_tools_shared.h"
#include "cmdlib.h"
#include "utlvector.h"
#include "Utlhash.h"
#include "UtlBuffer.h"
#include "utlstring.h"
#include "utlbinaryblock.h"
#include "tier2/utlstreambuffer.h"
#include "UtlLinkedList.h"
#include "UtlStringMap.h"
#include "tier0/icommandline.h"
#include "tier1/strtools.h"
#include "vstdlib/jobthread.h"
#include "threads.h"
#include "tier0/dbg.h"
#include "tier1/smartptr.h"
#include "interface.h"
#include "ishadercompiledll.h"
#include <direct.h>
#include "io.h"
#include <sys/types.h>
#include <sys/stat.h>
#include "materialsystem/shader_vcs_version.h"
#include "ilaunchabledll.h"
#include <tier1/diff.h>
#include "utlnodehash.h"
#include "lzma/lzma.h"
#include "mathlib/mathlib.h"
#include "tier1/checksum_crc.h"
#include "tier0/tslist.h"
#include "tools_minidump.h"
#include "cmdsink.h"
#include "d3dxfxc.h"
#include "subprocess.h"
#include "cfgprocessor.h"
// Type conversions should be controlled by programmer explicitly - shadercompile makes use of 64-bit integer arithmetics
#pragma warning( error : 4244 )
static inline uint32 uint64_as_uint32( uint64 x )
{
Assert( x < uint64( uint32( ~0 ) ) );
return uint32( x );
}
static inline UtlSymId_t int_as_symid( int x )
{
Assert( ( sizeof( UtlSymId_t ) >= sizeof( int ) ) || ( x >= 0 && x < ( int )( unsigned int )( UtlSymId_t(~0) ) ) );
return UtlSymId_t( x );
}
// VMPI packets
#define STARTWORK_PACKETID 5
#define WORKUNIT_PACKETID 6
#define ERRMSG_PACKETID 7
#define SHADERHADERROR_PACKETID 8
#define MACHINE_NAME 9
#ifdef _DEBUG
//#define DEBUGFP
#endif
// Dealing with job list
namespace
{
CArrayAutoPtr< CfgProcessor::CfgEntryInfo > g_arrCompileEntries;
uint64 g_numShaders = 0, g_numCompileCommands = 0, g_numStaticCombos = 0;
uint64 g_nStaticCombosPerWorkUnit = 0, g_numCompletedStaticCombos = 0, g_numCommandsCompleted = 0;
uint64 g_numSkippedStaticCombos = 0;
CfgProcessor::CfgEntryInfo const * GetEntryByStaticComboNum( uint64 nStaticCombo, uint64 *pnStaticCombo )
{
CfgProcessor::CfgEntryInfo const *pInfo;
uint64 nRemainStaticCombos = nStaticCombo;
for ( pInfo = g_arrCompileEntries.Get(); pInfo && pInfo->m_szName; ++ pInfo )
{
if ( nRemainStaticCombos >= pInfo->m_numStaticCombos )
nRemainStaticCombos -= pInfo->m_numStaticCombos;
else
break;
}
if ( pnStaticCombo )
*pnStaticCombo = nRemainStaticCombos;
return pInfo;
}
}; // `anonymous` namespace
char * PrettyPrintNumber( uint64 k )
{
static char chCompileString[50] = {0};
char *pchPrint = chCompileString + sizeof( chCompileString ) - 3;
for ( uint64 j = 0; k > 0; k /= 10, ++ j )
{
( j && !( j % 3 ) ) ? ( * pchPrint -- = ',' ) : 0;
* pchPrint -- = '0' + char( k % 10 );
}
( * ++ pchPrint ) ? 0 : ( * pchPrint = 0 );
return pchPrint;
}
const char *g_pShaderPath = NULL;
char g_WorkerTempPath[MAX_PATH];
char g_ExeDir[MAX_PATH];
#ifdef DEBUGFP
FILE *g_WorkerDebugFp = NULL;
#endif
bool g_bGotStartWorkPacket = false;
double g_flStartTime;
bool g_bVerbose = false;
bool g_bIsX360 = false;
bool g_bSuppressWarnings = false;
FORCEINLINE long AsTargetLong( long x ) { return ( ( g_bIsX360 ) ? ( BigLong( x ) ) : ( x ) ); }
struct ShaderInfo_t
{
ShaderInfo_t() { memset( this, 0, sizeof( *this ) ); }
uint64 m_nShaderCombo;
uint64 m_nTotalShaderCombos;
const char *m_pShaderName;
const char *m_pShaderSrc;
unsigned m_CentroidMask;
uint64 m_nDynamicCombos;
uint64 m_nStaticCombo;
unsigned m_Flags; // from IShader.h
char m_szShaderModel[ 12 ];
};
void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo );
struct CByteCodeBlock
{
CByteCodeBlock *m_pNext, *m_pPrev;
int m_nCRC32;
uint64 m_nComboID;
size_t m_nCodeSize;
uint8 *m_ByteCode;
CByteCodeBlock( void )
{
m_ByteCode = NULL;
}
CByteCodeBlock( void const *pByteCode, size_t nCodeSize, uint64 nComboID )
{
m_ByteCode = new uint8[nCodeSize];
m_nComboID = nComboID;
m_nCodeSize = nCodeSize;
memcpy( m_ByteCode, pByteCode, nCodeSize );
m_nCRC32 = CRC32_ProcessSingleBuffer( pByteCode, nCodeSize );
}
~CByteCodeBlock( void )
{
if ( m_ByteCode )
delete[] m_ByteCode;
}
};
static int __cdecl CompareDynamicComboIDs( CByteCodeBlock * const *pA, CByteCodeBlock * const *pB )
{
if ( (*pA)->m_nComboID < (*pB)->m_nComboID )
return -1;
if ( (*pA)->m_nComboID > (*pB)->m_nComboID )
return 1;
return 0;
}
struct CStaticCombo // all the data for one static combo
{
CStaticCombo *m_pNext, *m_pPrev;
uint64 m_nStaticComboID;
CUtlVector< CByteCodeBlock* > m_DynamicCombos;
struct PackedCode : protected CArrayAutoPtr<uint8> {
size_t GetLength() const { if( uint8 *pb = Get() ) return *reinterpret_cast<size_t *>( pb ); else return 0; }
uint8 *GetData() const { if( uint8 *pb = Get() ) return pb + sizeof( size_t ); else return NULL; }
uint8 *AllocData( size_t len ) { Delete(); if ( len ) { Attach( new uint8[ len + sizeof( size_t ) ] ); *reinterpret_cast<size_t *>( Get() ) = len; } return GetData(); }
} m_abPackedCode; // Packed code for entire static combo
uint64 Key( void ) const
{
return m_nStaticComboID;
}
CStaticCombo( uint64 nComboID )
{
m_nStaticComboID = nComboID;
}
~CStaticCombo( void )
{
m_DynamicCombos.PurgeAndDeleteElements();
}
void AddDynamicCombo( uint64 nComboID, void const *pComboData, size_t nCodeSize )
{
CByteCodeBlock *pNewBlock = new CByteCodeBlock( pComboData, nCodeSize, nComboID );
m_DynamicCombos.AddToTail( pNewBlock );
}
void SortDynamicCombos( void )
{
m_DynamicCombos.Sort( CompareDynamicComboIDs );
}
uint8 *AllocPackedCodeBlock( size_t nPackedCodeSize )
{
return m_abPackedCode.AllocData( nPackedCodeSize );
}
};
typedef CUtlNodeHash<CStaticCombo, 7097, uint64> StaticComboNodeHash_t;
template <>
inline StaticComboNodeHash_t **Construct( StaticComboNodeHash_t ** pMemory )
{
return ::new( pMemory ) StaticComboNodeHash_t *( NULL ); // Explicitly new with NULL
}
struct CShaderMap : public CUtlStringMap<StaticComboNodeHash_t *> {
;
} g_ShaderByteCode;
CStaticCombo * StaticComboFromDictAdd( char const *pszShaderName, uint64 nStaticComboId )
{
StaticComboNodeHash_t *& rpNodeHash = g_ShaderByteCode[ pszShaderName ];
if ( !rpNodeHash )
{
rpNodeHash = new StaticComboNodeHash_t;
}
// search for this static combo. make it if not found
CStaticCombo *pStaticCombo = rpNodeHash->FindByKey( nStaticComboId );
if ( !pStaticCombo )
{
pStaticCombo = new CStaticCombo( nStaticComboId );
rpNodeHash->Add( pStaticCombo );
}
return pStaticCombo;
}
CStaticCombo * StaticComboFromDict( char const *pszShaderName, uint64 nStaticComboId )
{
if ( StaticComboNodeHash_t *pNodeHash = g_ShaderByteCode[ pszShaderName ] )
return pNodeHash->FindByKey( nStaticComboId );
else
return NULL;
}
CUtlStringMap<ShaderInfo_t> g_ShaderToShaderInfo;
class CompilerMsgInfo
{
public:
CompilerMsgInfo() : m_numTimesReported( 0 ) {}
public:
void SetMsgReportedCommand( char const *szCommand, int numTimesReported = 1 ) { if ( !m_numTimesReported ) m_sFirstCommand = szCommand; m_numTimesReported += numTimesReported; }
public:
char const * GetFirstCommand() const { return m_sFirstCommand.String(); }
int GetNumTimesReported() const { return m_numTimesReported; }
protected:
CUtlString m_sFirstCommand;
int m_numTimesReported;
};
CUtlStringMap<bool> g_Master_ShaderHadError;
CUtlStringMap<bool> g_Master_ShaderWrittenToDisk;
CUtlStringMap<CompilerMsgInfo> g_Master_CompilerMsgInfo;
namespace Threading
{
enum Mode { eSingleThreaded = 0, eMultiThreaded = 1 };
// A special object that makes single-threaded code incur no penalties
// and multithreaded code to be synchronized properly.
template < class MT_MUTEX_TYPE = CThreadFastMutex >
class CSwitchableMutex
{
public:
public:
FORCEINLINE explicit CSwitchableMutex( Mode eMode, MT_MUTEX_TYPE *pMtMutex = NULL ) : m_pMtx( pMtMutex ), m_pUseMtx( eMode ? pMtMutex : NULL ) {}
public:
FORCEINLINE void SetMtMutex( MT_MUTEX_TYPE *pMtMutex ) { m_pMtx = pMtMutex; m_pUseMtx = ( m_pUseMtx ? pMtMutex : NULL ); }
FORCEINLINE void SetThreadedMode( Mode eMode ) { m_pUseMtx = ( eMode ? m_pMtx : NULL ); }
public:
FORCEINLINE void Lock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Lock(); }
FORCEINLINE void Unlock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Unlock(); }
FORCEINLINE bool TryLock() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->TryLock(); else return true; }
FORCEINLINE bool AssertOwnedByCurrentThread() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->AssertOwnedByCurrentThread(); else return true; }
FORCEINLINE void SetTrace( bool b ) { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->SetTrace( b ); }
FORCEINLINE uint32 GetOwnerId() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetOwnerId(); else return 0; }
FORCEINLINE int GetDepth() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetDepth(); else return 0; }
private:
MT_MUTEX_TYPE *m_pMtx;
CInterlockedPtr< MT_MUTEX_TYPE > m_pUseMtx;
};
namespace Private
{
typedef CThreadMutex MtMutexType_t;
MtMutexType_t g_mtxSyncObjMT;
}; // namespace Private
CSwitchableMutex< Private::MtMutexType_t > g_mtxGlobal( eSingleThreaded, &Private::g_mtxSyncObjMT );
class CGlobalMutexAutoLock
{
public:
CGlobalMutexAutoLock() { g_mtxGlobal.Lock(); }
~CGlobalMutexAutoLock() { g_mtxGlobal.Unlock(); }
};
}; // namespace Threading
// Access to global data should be synchronized by these global locks
#define GLOBAL_DATA_MTX_LOCK() Threading::g_mtxGlobal.Lock()
#define GLOBAL_DATA_MTX_UNLOCK() Threading::g_mtxGlobal.Unlock()
#define GLOBAL_DATA_MTX_LOCK_AUTO Threading::CGlobalMutexAutoLock UNIQUE_ID;
CDispatchReg g_DistributeWorkReg( WORKUNIT_PACKETID, DistributeWorkDispatch );
unsigned long VMPI_Stats_GetJobWorkerID( void )
{
return 0;
}
bool StartWorkDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
g_bGotStartWorkPacket = true;
return true;
}
CDispatchReg g_StartWorkReg( STARTWORK_PACKETID, StartWorkDispatch );
// Consume all characters for which (isspace) is true
template < typename T >
char * ConsumeCharacters( char *szString, T pred )
{
if ( szString )
{
while ( *szString && pred( *szString ) )
{
++ szString;
}
}
return szString;
}
char * FindNext( char *szString, char *szSearchSet )
{
bool bFound = (szString == NULL);
char *szNext = NULL;
if ( szString && szSearchSet )
{
for ( ; *szSearchSet; ++ szSearchSet )
{
if ( char *szTmp = strchr( szString, *szSearchSet ) )
{
szNext = bFound ? ( min( szNext, szTmp ) ) : szTmp;
bFound = true;
}
}
}
return bFound ? szNext : ( szString + strlen( szString ) );
}
char * FindLast( char *szString, char *szSearchSet )
{
bool bFound = (szString != NULL);
char *szNext = NULL;
if ( szString && szSearchSet )
{
for ( ; *szSearchSet; ++ szSearchSet )
{
if ( char *szTmp = strrchr( szString, *szSearchSet ) )
{
szNext = bFound ? ( max( szNext, szTmp ) ) : szTmp;
bFound = true;
}
}
}
return bFound ? szNext : ( szString + strlen( szString ) );
}
void ErrMsgDispatchMsgLine( char const *szCommand, char *szMsgLine, char const *szShaderName = NULL )
{
// When the filename is specified in front of the message, make sure it is truncated to the bare name only
if ( V_isalpha( *szMsgLine ) && szMsgLine[1] == ':' )
{
// Preceded by drive letter
szMsgLine += 2;
}
// Trim the path from the msg
// e.g. make string
// c:\temp\shadercompiletemp\1234\myfile.fxc(435): warning X3083: Truncating ...
// look like
// myfile.fxc(435): warning X3083: Truncating ...
// which will be both readable and same coming from different worker machines
char *szEndFileLinePlant = FindNext( szMsgLine, ":" );
if ( ':' == *szEndFileLinePlant )
{
*szEndFileLinePlant = 0;
if ( char *szLastSlash = FindLast( szMsgLine, "\\/" ) )
{
if ( *szLastSlash )
{
*szLastSlash = 0;
szMsgLine = szLastSlash + 1;
}
}
*szEndFileLinePlant = ':';
}
// If the shader file name is not given in the message add it
if ( szShaderName )
{
static char chFitLongMsgLine[4096];
if ( *szMsgLine == '(' )
{
sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine );
szMsgLine = chFitLongMsgLine;
}
else if ( !strncmp( szMsgLine, "memory(", 7 ) )
{
sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine+6 );
szMsgLine = chFitLongMsgLine;
}
}
// Now store the message with the command it was generated from
g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand );
}
void ErrMsgDispatchInt( char *szMessage, char const *szShaderName = NULL )
{
// First line is the command number "szCommand"
char *szCommand = ConsumeCharacters( szMessage, isspace );
char *szMessageListing = FindNext(szCommand, "\r\n");
char chTerminator = *szMessageListing;
*( szMessageListing ++ ) = 0;
// Now come the command lines actually
while ( chTerminator )
{
char *szMsgText = ConsumeCharacters( szMessageListing, isspace );
szMessageListing = FindNext( szMsgText, "\r\n" );
chTerminator = *szMessageListing;
*( szMessageListing ++ ) = 0;
if( *szMsgText )
{
// Trim command at redirection character if present
* FindNext( szCommand, ">" ) = 0;
ErrMsgDispatchMsgLine( szCommand, szMsgText, szShaderName );
}
}
}
//
// BUFFER:
// 1 byte = * = buffer type
//
// string = message
// 1 byte = \n = newline delimiting the message
//
// string = command that first encountered the message
// 1 byte = \n = newline delimiting the command
//
// string = printed number of times the message was encountered
// 1 byte = \n = newline delimiting the number
//
// 1 byte = 0 = null-terminator for the buffer
//
bool ErrMsgDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
bool bInvalidPkgRetCode = true;
// Parse the err msg packet
char *szMsgLine = pBuf->data + 1;
char *szCommand = FindNext( szMsgLine, "\n" );
if ( !*szCommand )
return bInvalidPkgRetCode;
*( szCommand ++ ) = 0;
char *szNumTimesReported = FindNext( szCommand, "\n" );
if ( !*szNumTimesReported )
return bInvalidPkgRetCode;
*( szNumTimesReported ++ ) = 0;
char *szTerminator = FindNext( szNumTimesReported, "\n" );
if ( !*szTerminator )
return bInvalidPkgRetCode;
*( szTerminator ++ ) = 0;
// Set the msg info
g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand, atoi( szNumTimesReported ) );
return true;
}
CDispatchReg g_ErrMsgReg( ERRMSG_PACKETID, ErrMsgDispatch );
void ShaderHadErrorDispatchInt( char const *szShader )
{
g_Master_ShaderHadError[ szShader ] = true;
}
//
// BUFFER:
// 1 byte = * = buffer type
//
// string = shader name
// 1 byte = 0 = null-terminator for the name
//
bool ShaderHadErrorDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
ShaderHadErrorDispatchInt( pBuf->data + 1 );
return true;
}
CDispatchReg g_ShaderHadErrorReg( SHADERHADERROR_PACKETID, ShaderHadErrorDispatch );
void DebugOut( const char *pMsg, ... )
{
if (g_bVerbose)
{
char msg[2048];
va_list marker;
va_start( marker, pMsg );
_vsnprintf( msg, sizeof( msg ), pMsg, marker );
va_end( marker );
Msg( "%s", msg );
#ifdef DEBUGFP
fprintf( g_WorkerDebugFp, "%s", msg );
fflush( g_WorkerDebugFp );
#endif
}
}
void Vmpi_Worker_DefaultDisconnectHandler( int procID, const char *pReason )
{
Msg( "Master disconnected.\n ");
DebugOut( "Master disconnected.\n" );
TerminateProcess( GetCurrentProcess(), 1 );
}
typedef void ( * DisconnectHandlerFn_t )( int procID, const char *pReason );
DisconnectHandlerFn_t g_fnDisconnectHandler = Vmpi_Worker_DefaultDisconnectHandler;
// Worker should implement this so it will quit nicely when the master disconnects.
void MyDisconnectHandler( int procID, const char *pReason )
{
// If we're a worker, then it's a fatal error if we lose the connection to the master.
if ( !g_bMPIMaster && g_fnDisconnectHandler )
{
(* g_fnDisconnectHandler)( procID, pReason );
}
}
// new format:
// ver#
// total shader combos
// total dynamic combos
// flags
// centroid mask
// total non-skipped static combos
// [ (sorted by static combo id)
// static combo id
// file offset of packed dynamic combo
// ]
// 0xffffffff (sentinel key)
// end of file offset (so can tell compressed size of last combo)
//
// # of duplicate static combos (if version >= 6 )
// [ (sorted by static combo id)
// static combo id
// id of static bombo which is identical
// ]
//
// each packed dynamic combo for a given static combo is stored as a series of compressed blocks.
// block 1:
// ulong blocksize (high bit set means uncompressed)
// block data
// block2..
// 0xffffffff indicates no more blocks for this combo
//
// each block, when uncompressed, holds one or more dynamic combos:
// dynamic combo id (full id if v<6, dynamic combo id only id >=6)
// size of shader
// ..
// there is no terminator - the size of the uncompressed shader tells you when to stop
// this record is then bzip2'd.
// qsort driver function
// returns negative number if idA is less than idB, positive when idA is greater than idB
// and zero if the ids are equal
static int __cdecl CompareDupComboIndices( const StaticComboAliasRecord_t *pA, const StaticComboAliasRecord_t *pB )
{
if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
return -1;
if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
return 1;
return 0;
}
static void FlushCombos( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, MessageBuffer *pBuf )
{
if ( !pDynamicComboBuffer->TellPut() )
// Nothing to do here
return;
size_t nCompressedSize;
uint8 *pCompressedShader = LZMA_OpportunisticCompress( reinterpret_cast<uint8 *> ( pDynamicComboBuffer->Base() ),
pDynamicComboBuffer->TellPut(),
&nCompressedSize );
// high 2 bits of length =
// 00 = bzip2 compressed
// 10 = uncompressed
// 01 = lzma compressed
// 11 = unused
if ( ! pCompressedShader )
{
// it grew
long lFlagSize = AsTargetLong( 0x80000000 | pDynamicComboBuffer->TellPut() );
pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
pBuf->write( pDynamicComboBuffer->Base(), pDynamicComboBuffer->TellPut() );
*pnTotalFlushedSize += sizeof( lFlagSize ) + pDynamicComboBuffer->TellPut();
}
else
{
long lFlagSize = AsTargetLong( 0x40000000 | nCompressedSize );
pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
pBuf->write( pCompressedShader, nCompressedSize );
delete[] pCompressedShader;
*pnTotalFlushedSize += sizeof( lFlagSize ) + nCompressedSize;
}
pDynamicComboBuffer->Clear(); // start over
}
static void OutputDynamicCombo( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
MessageBuffer *pBuf, uint64 nComboID, int nComboSize,
uint8 *pComboCode )
{
if ( pDynamicComboBuffer->TellPut() + nComboSize+16 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
{
FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
}
pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) );
pDynamicComboBuffer->PutInt( nComboSize );
// pDynamicComboBuffer->PutInt( CRC32_ProcessSingleBuffer( pComboCode, nComboSize ) );
pDynamicComboBuffer->Put( pComboCode, nComboSize );
}
static void OutputDynamicComboDup( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
MessageBuffer *pBuf, uint64 nComboID, uint64 nBaseCombo )
{
if ( pDynamicComboBuffer->TellPut() + 8 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
{
FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
}
pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) | 0x80000000 );
pDynamicComboBuffer->PutInt( uint64_as_uint32( nBaseCombo ) );
}
void GetVCSFilenames( char *pszMainOutFileName, ShaderInfo_t const &si )
{
sprintf( pszMainOutFileName, "%s\\shaders\\fxc", g_pShaderPath );
struct _stat buf;
if( _stat( pszMainOutFileName, &buf ) == -1 )
{
printf( "mkdir %s\n", pszMainOutFileName );
// doh. . need to make the directory that the vcs file is going to go into.
_mkdir( pszMainOutFileName );
}
strcat( pszMainOutFileName, "\\" );
strcat( pszMainOutFileName, si.m_pShaderName );
if ( g_bIsX360 )
{
strcat( pszMainOutFileName, ".360" );
}
strcat( pszMainOutFileName, ".vcs" ); // Different extensions for main output file
// Check status of vcs file...
if( _stat( pszMainOutFileName, &buf ) != -1 )
{
// The file exists, let's see if it's writable.
if( !( buf.st_mode & _S_IWRITE ) )
{
// It isn't writable. . we'd better change its permissions (or check it out possibly)
printf( "Warning: making %s writable!\n", pszMainOutFileName );
_chmod( pszMainOutFileName, _S_IREAD | _S_IWRITE );
}
}
}
// WriteShaderFiles
//
// should be called either on the main thread or
// on the async writing thread.
//
// So the function WriteShaderFiles should not be reentrant, however the
// data that it uses might be updated by the main thread when built pieces
// are received from the workers.
//
#define STATIC_COMBO_HASH_SIZE 73
struct StaticComboAuxInfo_t : StaticComboRecord_t
{
uint32 m_nCRC32; // CRC32 of packed data
struct CStaticCombo *m_pByteCode;
};
static int __cdecl CompareComboIds( const StaticComboAuxInfo_t *pA, const StaticComboAuxInfo_t *pB )
{
if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
return -1;
if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
return 1;
return 0;
}
static void WriteShaderFiles( const char *pShaderName )
{
if ( !g_Master_ShaderWrittenToDisk.Defined( pShaderName ) )
g_Master_ShaderWrittenToDisk[ pShaderName ] = true;
else
return;
bool bShaderFailed = g_Master_ShaderHadError.Defined( pShaderName );
char const *szShaderFileOperation = bShaderFailed ? "Removing failed" : "Writing";
//
// Progress indication
//
if ( g_numCommandsCompleted < g_numCompileCommands )
{
static char chProgress[] = { '/', '-', '\\', '|' };
static int iProgressSymbol = 0;
Msg( "\b%c", chProgress[ ( ++ iProgressSymbol ) % 4 ] );
}
else
{
char chShaderName[33];
Q_snprintf( chShaderName, 29, "%s...", pShaderName );
sprintf( chShaderName + sizeof( chShaderName ) - 5, "..." );
Msg( "\r%s %s \r", szShaderFileOperation, chShaderName );
}
//
// Retrieve the data we are going to operate on
// from global variables under lock.
//
GLOBAL_DATA_MTX_LOCK();
StaticComboNodeHash_t *pByteCodeArray;
{
StaticComboNodeHash_t *&rp = g_ShaderByteCode[pShaderName]; // Get a static combo pointer, reset it as well
pByteCodeArray = rp;
rp = NULL;
/*
Assert( pByteCodeArray );
if ( !pByteCodeArray )
ShaderHadErrorDispatchInt( pShaderName );
*/
}
ShaderInfo_t shaderInfo = g_ShaderToShaderInfo[pShaderName];
if ( !shaderInfo.m_pShaderName )
{
for ( CfgProcessor::CfgEntryInfo const *pAnalyze = g_arrCompileEntries.Get() ;
pAnalyze->m_szName ;
++ pAnalyze )
{
if ( !strcmp( pAnalyze->m_szName, pShaderName ) )
{
Shader_ParseShaderInfoFromCompileCommands( pAnalyze, shaderInfo );
g_ShaderToShaderInfo[ pShaderName ] = shaderInfo;
break;
}
}
}
GLOBAL_DATA_MTX_UNLOCK();
if ( !shaderInfo.m_pShaderName )
return;
//
// Shader vcs file name
//
char szVCSfilename[MAX_PATH];
GetVCSFilenames( szVCSfilename, shaderInfo );
if ( bShaderFailed )
{
DebugOut( "Removing failed shader file \"%s\".\n", szVCSfilename );
unlink( szVCSfilename );
return;
}
if ( !pByteCodeArray )
return;
DebugOut( "%s : %I64u combos centroid mask: 0x%x numDynamicCombos: %I64u flags: 0x%x\n",
pShaderName, shaderInfo.m_nTotalShaderCombos,
shaderInfo.m_CentroidMask, shaderInfo.m_nDynamicCombos, shaderInfo.m_Flags );
//
// Static combo headers
//
CUtlVector< StaticComboAuxInfo_t > StaticComboHeaders;
StaticComboHeaders.EnsureCapacity( 1 + pByteCodeArray->Count() ); // we know how much ram we need
CUtlVector< int > comboIndicesHashedByCRC32[STATIC_COMBO_HASH_SIZE];
CUtlVector< StaticComboAliasRecord_t > duplicateCombos;
// now, lets fill in our combo headers, sort, and write
for( int nChain = 0 ; nChain < NELEMS( pByteCodeArray->m_HashChains) ; nChain++ )
{
for( CStaticCombo *pStatic = pByteCodeArray->m_HashChains[ nChain ].m_pHead;
pStatic;
pStatic = pStatic->m_pNext )
{
if ( pStatic->m_abPackedCode.GetLength() )
{
StaticComboAuxInfo_t Hdr;
Hdr.m_nStaticComboID = uint64_as_uint32( pStatic->m_nStaticComboID );
Hdr.m_nFileOffset = 0; // fill in later
Hdr.m_nCRC32 = CRC32_ProcessSingleBuffer( pStatic->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetLength() );
int nHashIdx = Hdr.m_nCRC32 % STATIC_COMBO_HASH_SIZE;
Hdr.m_pByteCode = pStatic;
// now, see if we have an identical static combo
bool bIsDuplicate = false;
for( int i = 0; i < comboIndicesHashedByCRC32[nHashIdx].Count() ; i++ )
{
StaticComboAuxInfo_t const &check = StaticComboHeaders[comboIndicesHashedByCRC32[nHashIdx][i]];
if (
( check.m_nCRC32 == Hdr.m_nCRC32 ) &&
( check.m_pByteCode->m_abPackedCode.GetLength() == pStatic->m_abPackedCode.GetLength() ) &&
( memcmp( check.m_pByteCode->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetData(), check.m_pByteCode->m_abPackedCode.GetLength() ) == 0 )
)
{
// this static combo is the same as another one!!
StaticComboAliasRecord_t aliasHdr;
aliasHdr.m_nStaticComboID = Hdr.m_nStaticComboID;
aliasHdr.m_nSourceStaticCombo = check.m_nStaticComboID;
duplicateCombos.AddToTail( aliasHdr );
bIsDuplicate = true;
break;
}
}
if ( ! bIsDuplicate )
{
StaticComboHeaders.AddToTail( Hdr );
comboIndicesHashedByCRC32[nHashIdx].AddToTail( StaticComboHeaders.Count() - 1 );
}
}
}
}
// add sentinel key
StaticComboAuxInfo_t Hdr;
Hdr.m_nStaticComboID = 0xffffffff;
Hdr.m_nFileOffset = 0;
StaticComboHeaders.AddToTail( Hdr );
// now, sort. sentinel key will end up at end
StaticComboHeaders.Sort( CompareComboIds );
// Set the CRC to zero for now. . will patch in copyshaders.pl with the correct CRC.
unsigned int crc32 = 0;
//
// Shader file stream buffer
//
CUtlStreamBuffer ShaderFile( szVCSfilename, NULL ); // Streaming buffer for vcs file (since this can blow memory)
ShaderFile.SetBigEndian( g_bIsX360 ); // Swap the header bytes to X360 format
// ------ Header --------------
ShaderFile.PutInt( SHADER_VCS_VERSION_NUMBER ); // Version
ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nTotalShaderCombos ) );
ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nDynamicCombos ) );
ShaderFile.PutUnsignedInt( shaderInfo.m_Flags );
ShaderFile.PutUnsignedInt( shaderInfo.m_CentroidMask );
ShaderFile.PutUnsignedInt( StaticComboHeaders.Count() );
ShaderFile.PutUnsignedInt( crc32 );
// static combo dictionary
int nDictionaryOffset= ShaderFile.TellPut();
// we will re write this one we know the offsets
ShaderFile.Put( StaticComboHeaders.Base(), sizeof( StaticComboRecord_t ) * StaticComboHeaders.Count() ); // dummy write, 8 bytes per static combo
ShaderFile.PutUnsignedInt( duplicateCombos.Count() );
// now, write out all duplicate header records
// sort duplicate combo records for binary search
duplicateCombos.Sort( CompareDupComboIndices );
for( int i = 0; i < duplicateCombos.Count(); i++ )
{
ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nStaticComboID );
ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nSourceStaticCombo );
}
// now, write out all static combos
for( int i=0 ; i<StaticComboHeaders.Count(); i++ )
{
StaticComboRecord_t &SRec = StaticComboHeaders[i];
SRec.m_nFileOffset = ShaderFile.TellPut();
if ( SRec.m_nStaticComboID != 0xffffffff ) // sentinel key?
{
CStaticCombo *pStatic=pByteCodeArray->FindByKey( SRec.m_nStaticComboID );
Assert( pStatic );
// Put the packed chunk of code for this static combo
if ( size_t nPackedLen = pStatic->m_abPackedCode.GetLength() )
ShaderFile.Put( pStatic->m_abPackedCode.GetData(), nPackedLen );
ShaderFile.PutInt( 0xffffffff ); // end of dynamic combos
}
if ( g_bIsX360 )
{
SRec.m_nFileOffset = BigLong( SRec.m_nFileOffset );
SRec.m_nStaticComboID = BigLong( SRec.m_nStaticComboID );
}
}
ShaderFile.Close();
//
// Re-writing the combo header
//
{
FILE *Handle=fopen( szVCSfilename, "rb+" );
if (! Handle )
printf(" failed to re-open %s\n",szVCSfilename );
fseek( Handle, nDictionaryOffset, SEEK_SET );
// now, rewrite header. data is already byte-swapped appropriately
for( int i = 0; i < StaticComboHeaders.Count(); i++ )
{
fwrite( &( StaticComboHeaders[i].m_nStaticComboID ), 4, 1, Handle );
fwrite( &( StaticComboHeaders[i].m_nFileOffset ), 4, 1, Handle );
}
fclose( Handle );
}
// Finalize, free memory
delete pByteCodeArray;
if ( g_numCommandsCompleted >= g_numCompileCommands )
{
Msg( "\r \r" );
}
}
// pBuf is ready to read the results written to the buffer in ProcessWorkUnitFn.
// work is done. .master gets it back this way.
// compiled code in pBuf
void Master_ReceiveWorkUnitFn( uint64 iWorkUnit, MessageBuffer *pBuf, int iWorker )
{
GLOBAL_DATA_MTX_LOCK_AUTO;
uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
comboEnd = min( g_numStaticCombos, comboEnd );
char const *chLastShaderName = "";
ShaderInfo_t siLastShaderInfo;
memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );
siLastShaderInfo.m_pShaderName = chLastShaderName;
uint64 nComboOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;
for( uint64 iCombo = comboStart; iCombo ++ < comboEnd;
( ( ! nComboOfTheEntry -- ) ? ( ++ pEntry, nComboOfTheEntry = pEntry->m_numStaticCombos - 1 ) : 0 ) )
{
Assert( nComboOfTheEntry < pEntry->m_numStaticCombos );
// Read length
int len;
pBuf->read( &len, sizeof( len ) );
// Length can indicate the number of skips to make
if ( len <= 0 )
{
// remember how many static combos get skipped
g_numSkippedStaticCombos += -len;
// then we skip as instructed
for ( int64 numSkips = - len - 1;
numSkips > 0; )
{
if ( numSkips <= nComboOfTheEntry )
{
nComboOfTheEntry -= numSkips;
iCombo += numSkips;
numSkips = 0;
}
else
{
numSkips -= nComboOfTheEntry + 1;
iCombo += nComboOfTheEntry + 1;
++ pEntry;
nComboOfTheEntry = pEntry->m_numStaticCombos - 1;
}
}
if ( iCombo < comboEnd )
continue;
else
break;
}
// Shader code arrived
char const *chShaderName = pEntry->m_szName;
// If starting new shader remember shader info
if ( chLastShaderName != chShaderName )
{
Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );
chLastShaderName = chShaderName;
g_ShaderToShaderInfo[ chLastShaderName ] = siLastShaderInfo;
}
// Read buffer
uint8 *pCodeBuffer = StaticComboFromDictAdd( chShaderName, nComboOfTheEntry )->AllocPackedCodeBlock( len );
if ( pCodeBuffer )
pBuf->read( pCodeBuffer, len );
}
}
//
// A function that will wait for right Ctrl+Alt+Shift to be held down simultaneously.
// This is useful for debugging short-lived processes and gives time for debugger to
// get attached.
//
void DebugSafeWaitPoint( bool bForceWait = false )
{
static bool s_bDebuggerAttached = ( CommandLine()->FindParm( "-debugwait" ) == 0 );
if ( bForceWait )
{
s_bDebuggerAttached = false;
}
if ( !s_bDebuggerAttached )
{
Msg( "Waiting for right Ctrl+Alt+Shift to continue..." );
while ( !s_bDebuggerAttached )
{
Msg( "." );
Sleep(1000);
if ( short( GetAsyncKeyState( VK_RCONTROL ) ) < 0 &&
short( GetAsyncKeyState( VK_RSHIFT ) ) < 0 &&
short( GetAsyncKeyState( VK_RMENU ) ) < 0 )
{
s_bDebuggerAttached = true;
}
}
Msg( " ok.\n" );
}
}
// same as "system", but doesn't pop up a window
void MySystem( char const * const pCommand, CmdSink::IResponse **ppResponse )
{
// Trap the command in InterceptFxc
if ( InterceptFxc::TryExecuteCommand( pCommand, ppResponse ) )
{
Sleep( 0 );
return;
}
unlink( "shader.o" );
FILE *batFp = fopen( "temp.bat", "w" );
fprintf( batFp, "%s\n", pCommand );
fclose( batFp );
STARTUPINFO si;
PROCESS_INFORMATION pi;
ZeroMemory( &si, sizeof(si) );
si.cb = sizeof(si);
ZeroMemory( &pi, sizeof(pi) );
// Start the child process.
if( !CreateProcess( NULL, // No module name (use command line).
"temp.bat", // Command line.
NULL, // Process handle not inheritable.
NULL, // Thread handle not inheritable.
FALSE, // Set handle inheritance to FALSE.
IDLE_PRIORITY_CLASS | CREATE_NO_WINDOW, // No creation flags.
NULL, // Use parent's environment block.
g_WorkerTempPath, // Use parent's starting directory.
&si, // Pointer to STARTUPINFO structure.
&pi ) // Pointer to PROCESS_INFORMATION structure.
)
{
Error( "CreateProcess failed." );
Assert( 0 );
}
// Wait until child process exits.
WaitForSingleObject( pi.hProcess, INFINITE );
// Close process and thread handles.
CloseHandle( pi.hProcess );
CloseHandle( pi.hThread );
}
// Assemble a reply package to the master from the compiled bytecode
// return the length of the package.
size_t AssembleWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
MessageBuffer *pBuf )
{
GLOBAL_DATA_MTX_LOCK();
CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ];
GLOBAL_DATA_MTX_UNLOCK();
size_t nBytesWritten = 0;
if ( pStComboRec && pStComboRec->m_DynamicCombos.Count() )
{
CUtlBuffer ubDynamicComboBuffer;
ubDynamicComboBuffer.SetBigEndian( g_bIsX360 );
pStComboRec->SortDynamicCombos();
// iterate over all dynamic combos.
for(int i = 0 ; i < pStComboRec->m_DynamicCombos.Count(); i++ )
{
CByteCodeBlock *pCode = pStComboRec->m_DynamicCombos[i];
// check if we have already output an identical combo
bool bDup = false;
#if 0
// check for duplicate bytecode. actually doesn't save much because bzip does a good
// job compressing dupes.
for( int j = 0; j < i; j++ )
{
if (
( pCode->m_nCRC32 == pStComboRec->m_DynamicCombos[j]->m_nCRC32 ) &&
( pCode->m_nCodeSize == pStComboRec->m_DynamicCombos[j]->m_nCodeSize ) &&
( memcmp( pCode->m_ByteCode, pStComboRec->m_DynamicCombos[i]->m_ByteCode, pCode->m_nCodeSize ) == 0 )
) // identical bytecode?
{
bDup = true;
OutputDynamicComboDup( &nBytesWritten, &ubDynamicComboBuffer,
pBuf, pCode->m_nComboID,
pStComboRec->m_DynamicCombos[j]->m_nComboID );
}
}
#endif
if ( ! bDup )
OutputDynamicCombo( &nBytesWritten, &ubDynamicComboBuffer,
pBuf, pCode->m_nComboID,
pCode->m_nCodeSize, pCode->m_ByteCode );
}
FlushCombos( &nBytesWritten, &ubDynamicComboBuffer, pBuf );
}
// Time to limit amount of prints
static float s_fLastInfoTime = 0;
float fCurTime = ( float ) Plat_FloatTime();
GLOBAL_DATA_MTX_LOCK();
if ( pStComboRec )
pByteCodeArray->DeleteByKey( nComboOfEntry );
if( fabs( fCurTime - s_fLastInfoTime ) > 1.f )
{
Msg( "\rCompiling %s [ %2llu remaining ] ... \r",
pEntry->m_szName, nComboOfEntry );
s_fLastInfoTime = fCurTime;
}
GLOBAL_DATA_MTX_UNLOCK();
return nBytesWritten;
}
// Copy a reply package to the master from the compiled bytecode
// return the length of the data copied.
size_t CopyWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
MessageBuffer *pBuf, int nSkipsSoFar )
{
GLOBAL_DATA_MTX_LOCK();
CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ]; // Get a static combo pointer
GLOBAL_DATA_MTX_UNLOCK();
int len = pStComboRec ? pStComboRec->m_abPackedCode.GetLength() : NULL;
if ( len )
{
if ( nSkipsSoFar )
{
pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
}
pBuf->write( &len, sizeof( len ) );
if ( len )
pBuf->write( pStComboRec->m_abPackedCode.GetData(), len );
}
if ( pStComboRec )
{
GLOBAL_DATA_MTX_LOCK();
pByteCodeArray->DeleteByKey( nComboOfEntry );
GLOBAL_DATA_MTX_UNLOCK();
}
return size_t( len );
}
template < typename TMutexType >
class CWorkerAccumState : public CParallelProcessorBase < CWorkerAccumState < TMutexType > >
{
friend ThisParallelProcessorBase_t;
private:
static bool & DisconnectState() { static bool sb = false; return sb; }
static void Special_DisconnectHandler( int procID, const char *pReason ) { DisconnectState() = true; }
public:
explicit CWorkerAccumState( TMutexType *pMutex ) :
m_pMutex( pMutex ), m_iFirstCommand( 0 ), m_iNextCommand( 0 ),
m_iEndCommand( 0 ), m_iLastFinished( 0 ),
m_hCombo( NULL ),
m_fnOldDisconnectHandler( g_fnDisconnectHandler ),
m_autoRestoreDisconnectHandler( g_fnDisconnectHandler )
{
DisconnectState() = false;
}
~CWorkerAccumState() { QuitSubs(); }
void RangeBegin( uint64 iFirstCommand, uint64 iEndCommand );
void RangeFinished( void );
void ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo );
void ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo );
void HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse );
public:
using ThisParallelProcessorBase_t::Run;
public:
bool OnProcess();
bool OnProcessST();
protected:
TMutexType *m_pMutex;
protected:
struct SubProcess
{
DWORD dwIndex;
DWORD dwSvcThreadId;
uint64 iRunningCommand;
PROCESS_INFORMATION pi;
SubProcessKernelObjects *pCommObjs;
};
CThreadLocal < SubProcess * > m_lpSubProcessInfo;
CUtlVector < SubProcess * > m_arrSubProcessInfos;
uint64 m_iFirstCommand;
uint64 m_iNextCommand;
uint64 m_iEndCommand;
uint64 m_iLastFinished;
CfgProcessor::ComboHandle m_hCombo;
DisconnectHandlerFn_t m_fnOldDisconnectHandler;
CAutoPushPop< DisconnectHandlerFn_t > m_autoRestoreDisconnectHandler;
void QuitSubs( void );
void TryToPackageData( uint64 iCommandNumber );
void PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs );
};
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeBegin( uint64 iFirstCommand, uint64 iEndCommand )
{
m_iFirstCommand = iFirstCommand;
m_iNextCommand = iFirstCommand;
m_iEndCommand = iEndCommand;
m_iLastFinished = iFirstCommand;
m_hCombo = NULL;
CfgProcessor::Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
g_fnDisconnectHandler = Special_DisconnectHandler;
// Notify all connected sub-processes that the master is still alive
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
if ( void *pvMemory = shrmem.Lock() )
{
strcpy( ( char * ) pvMemory, "keepalive" );
shrmem.Unlock();
}
}
}
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeFinished( void )
{
if( !DisconnectState() )
{
// Finish packaging data
TryToPackageData( m_iEndCommand - 1 );
}
else
{
// Master disconnected
QuitSubs();
}
g_fnDisconnectHandler = m_fnOldDisconnectHandler;
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::QuitSubs( void )
{
CUtlVector < HANDLE > m_arrWait;
m_arrWait.EnsureCapacity( m_arrSubProcessInfos.Count() );
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
if ( void *pvMemory = shrmem.Lock() )
{
strcpy( ( char * ) pvMemory, "quit" );
shrmem.Unlock();
}
m_arrWait.AddToTail( pSp->pi.hProcess );
}
}
if ( m_arrWait.Count() )
{
DWORD dwWait = WaitForMultipleObjects( m_arrWait.Count(), m_arrWait.Base(), TRUE, 2 * 1000 );
if ( WAIT_TIMEOUT == dwWait )
{
Warning( "Timed out while waiting for sub-processes to shut down!\n" );
}
}
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
CloseHandle( pSp->pi.hThread );
CloseHandle( pSp->pi.hProcess );
delete pSp->pCommObjs;
delete pSp;
}
}
if ( DisconnectState() )
Vmpi_Worker_DefaultDisconnectHandler( 0, "Master disconnected during compilation." );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs )
{
SubProcess *pSp = m_lpSubProcessInfo.Get();
SubProcessKernelObjects *pCommObjs = NULL;
if ( pSp )
{
pCommObjs = pSp->pCommObjs;
}
else
{
pSp = new SubProcess;
m_lpSubProcessInfo.Set( pSp );
pSp->dwSvcThreadId = ThreadGetCurrentId();
char chBaseNameBuffer[0x30];
sprintf( chBaseNameBuffer, "SHCMPL_SUB_%08X_%08llX_%08X", pSp->dwSvcThreadId, (long long)time( NULL ), GetCurrentProcessId() );
pCommObjs = pSp->pCommObjs = new SubProcessKernelObjects_Create( chBaseNameBuffer );
ZeroMemory( &pSp->pi, sizeof( pSp->pi ) );
STARTUPINFO si;
ZeroMemory( &si, sizeof( si ) );
si.cb = sizeof( si );
char chCommandLine[0x100];
sprintf( chCommandLine, "\"%s\\shadercompile.exe\" -subprocess %s", g_WorkerTempPath, chBaseNameBuffer );
#ifdef _DEBUG
V_strncat( chCommandLine, " -allowdebug", sizeof( chCommandLine ) );
#endif
BOOL bCreateResult = CreateProcess( NULL, chCommandLine, NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, g_WorkerTempPath, &si, &pSp->pi );
( void ) bCreateResult;
Assert( bCreateResult && "CreateProcess failed?" );
m_pMutex->Lock();
pSp->dwIndex = m_arrSubProcessInfos.AddToTail( pSp );
m_pMutex->Unlock();
}
if ( ppSp ) *ppSp = pSp;
if ( ppCommObjs ) *ppCommObjs = pCommObjs;
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo )
{
// DebugOut( "threaded: running: \"%s\"\n", szCommand );
SubProcessKernelObjects *pCommObjs = NULL;
PrepareSubProcess( NULL, &pCommObjs );
// Execute the command
SubProcessKernelObjects_Memory shrmem( pCommObjs );
{
void *pvMemory = shrmem.Lock();
Assert( pvMemory );
Combo_FormatCommand( hCombo, ( char * ) pvMemory );
shrmem.Unlock();
}
// Obtain the command response
{
void const *pvMemory = shrmem.Lock();
Assert( pvMemory );
// TODO: Vitaliy :: TEMP fix:
// Usually what happens if we fail to lock here is
// when our subprocess dies and to recover we will
// attempt to restart on another worker.
if ( !pvMemory )
// ::RaiseException( GetLastError(), EXCEPTION_NONCONTINUABLE, 0, NULL );
TerminateProcess( GetCurrentProcess(), 1 );
CmdSink::IResponse *pResponse;
if ( pvMemory )
pResponse = new CSubProcessResponse( pvMemory );
else
pResponse = new CmdSink::CResponseError;
HandleCommandResponse( hCombo, pResponse );
shrmem.Unlock();
}
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo )
{
CmdSink::IResponse *pResponse = NULL;
{
char chBuffer[ 4096 ];
Combo_FormatCommand( hCombo, chBuffer );
DebugOut( "running: \"%s\"\n", chBuffer );
MySystem( chBuffer, &pResponse );
}
HandleCommandResponse( hCombo, pResponse );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse )
{
VMPI_HandleSocketErrors();
if ( !pResponse )
pResponse = new CmdSink::CResponseFiles( "shader.o", "output.txt" );
// Command info
CfgProcessor::CfgEntryInfo const *pEntryInfo = Combo_GetEntryInfo( hCombo );
uint64 iComboIndex = Combo_GetComboNum( hCombo );
uint64 iCommandNumber = Combo_GetCommandNum( hCombo );
if ( pResponse->Succeeded() )
{
GLOBAL_DATA_MTX_LOCK();
uint64 nStComboIdx = iComboIndex / pEntryInfo->m_numDynamicCombos;
uint64 nDyComboIdx = iComboIndex - ( nStComboIdx * pEntryInfo->m_numDynamicCombos );
StaticComboFromDictAdd( pEntryInfo->m_szName, nStComboIdx )->AddDynamicCombo( nDyComboIdx , pResponse->GetResultBuffer(), pResponse->GetResultBufferLen() );
GLOBAL_DATA_MTX_UNLOCK();
}
// Tell the master that this shader failed
if ( !pResponse->Succeeded() )
{
GLOBAL_DATA_MTX_LOCK();
ShaderHadErrorDispatchInt( pEntryInfo->m_szName );
GLOBAL_DATA_MTX_UNLOCK();
}
// Process listing even if the shader succeeds for warnings
char const *szListing = pResponse->GetListing();
if ( ( !g_bSuppressWarnings && szListing ) || !pResponse->Succeeded() )
{
char chCommandNumber[50];
sprintf( chCommandNumber, "%I64u", iCommandNumber );
char chUnreportedListing[0xFF];
if ( !szListing )
{
sprintf( chUnreportedListing, "(0): error 0000: Compiler failed without error description, latest version of fxc.exe might give a description." );
szListing = chUnreportedListing;
}
// Send the listing for dispatch
CUtlBinaryBlock errMsg;
errMsg.SetLength(
strlen( chCommandNumber ) + 1 + // command + newline
strlen( szListing ) + 1 + // listing + newline
1 // null-terminator
);
sprintf( ( char * ) errMsg.Get(), "%s\n%s\n", chCommandNumber, szListing );
GLOBAL_DATA_MTX_LOCK();
ErrMsgDispatchInt( ( char * ) errMsg.Get(), pEntryInfo->m_szShaderFileName );
GLOBAL_DATA_MTX_UNLOCK();
}
// Maybe zip things up
TryToPackageData( iCommandNumber );
}
template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::TryToPackageData( uint64 iCommandNumber )
{
m_pMutex->Lock();
uint64 iFinishedByNow = iCommandNumber + 1;
// Check if somebody is running an earlier command
for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
{
if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
{
if ( pSp->iRunningCommand < iCommandNumber )
{
iFinishedByNow = 0;
break;
}
}
}
uint64 iLastFinished = m_iLastFinished;
if ( iFinishedByNow > m_iLastFinished )
{
m_iLastFinished = iFinishedByNow;
m_pMutex->Unlock();
}
else
{
m_pMutex->Unlock();
return;
}
CfgProcessor::ComboHandle hChBegin = CfgProcessor::Combo_GetCombo( iLastFinished );
CfgProcessor::ComboHandle hChEnd = CfgProcessor::Combo_GetCombo( iFinishedByNow );
Assert( hChBegin && hChEnd );
CfgProcessor::CfgEntryInfo const *pInfoBegin = Combo_GetEntryInfo( hChBegin );
CfgProcessor::CfgEntryInfo const *pInfoEnd = Combo_GetEntryInfo( hChEnd );
uint64 nComboBegin = Combo_GetComboNum( hChBegin ) / pInfoBegin->m_numDynamicCombos;
uint64 nComboEnd = Combo_GetComboNum( hChEnd ) / pInfoEnd->m_numDynamicCombos;
for ( ; pInfoBegin && (
( pInfoBegin->m_iCommandStart < pInfoEnd->m_iCommandStart ) ||
( nComboBegin > nComboEnd ) ); )
{
// Zip this combo
MessageBuffer mbPacked;
size_t nPackedLength = AssembleWorkerReplyPackage( pInfoBegin, nComboBegin, &mbPacked );
if ( nPackedLength )
{
// Packed buffer
GLOBAL_DATA_MTX_LOCK();
uint8 *pCodeBuffer = StaticComboFromDictAdd( pInfoBegin->m_szName,
nComboBegin )->AllocPackedCodeBlock( nPackedLength );
GLOBAL_DATA_MTX_UNLOCK();
if ( pCodeBuffer )
mbPacked.read( pCodeBuffer, nPackedLength );
}
// Next iteration
if ( ! nComboBegin -- )
{
Combo_Free( hChBegin );
if ( ( hChBegin = CfgProcessor::Combo_GetCombo( pInfoBegin->m_iCommandEnd ) ) != NULL )
{
pInfoBegin = Combo_GetEntryInfo( hChBegin );
nComboBegin = pInfoBegin->m_numStaticCombos - 1;
}
}
}
Combo_Free( hChBegin );
Combo_Free( hChEnd );
}
template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcess()
{
m_pMutex->Lock();
CfgProcessor::ComboHandle hThreadCombo = m_hCombo ? Combo_Alloc( m_hCombo ) : NULL;
m_pMutex->Unlock();
uint64 iThreadCommand = ~uint64(0);
SubProcess *pSp = NULL;
PrepareSubProcess( &pSp, NULL );
for ( ; ; )
{
m_pMutex->Lock();
if ( DisconnectState() )
Combo_Free( m_hCombo );
if ( m_hCombo )
{
Combo_Assign( hThreadCombo, m_hCombo );
pSp->iRunningCommand = Combo_GetCommandNum( hThreadCombo );
Combo_GetNext( iThreadCommand, m_hCombo, m_iEndCommand );
}
else
{
Combo_Free( hThreadCombo );
iThreadCommand = ~uint64(0);
pSp->iRunningCommand = ~uint64(0);
}
m_pMutex->Unlock();
if ( hThreadCombo )
{
ExecuteCompileCommandThreaded( hThreadCombo );
}
else
break;
}
Combo_Free( hThreadCombo );
return false;
}
template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcessST()
{
while ( m_hCombo )
{
ExecuteCompileCommand( m_hCombo );
Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
}
return false;
}
//
// Worker_ProcessCommandRange_Singleton
//
class Worker_ProcessCommandRange_Singleton
{
public:
static Worker_ProcessCommandRange_Singleton *& Instance() { static Worker_ProcessCommandRange_Singleton *s_ptr = NULL; return s_ptr; }
static Worker_ProcessCommandRange_Singleton * GetInstance() { Worker_ProcessCommandRange_Singleton *p = Instance(); Assert( p ); return p; }
public:
Worker_ProcessCommandRange_Singleton() { Assert( !Instance() ); Instance() = this; Startup(); }
~Worker_ProcessCommandRange_Singleton() { Assert( Instance() == this ); Instance() = NULL; Shutdown(); }
public:
void ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd );
protected:
void Startup( void );
void Shutdown( void );
//
// Multi-threaded section
protected:
struct MT {
MT() : pWorkerObj( NULL ), pThreadPool( NULL ) {}
typedef CThreadFastMutex MultiThreadMutex_t;
MultiThreadMutex_t mtx;
typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
WorkerClass_t *pWorkerObj;
IThreadPool *pThreadPool;
ThreadPoolStartParams_t tpsp;
} m_MT;
//
// Single-threaded section
protected:
struct ST {
ST() : pWorkerObj( NULL ) {}
typedef CThreadNullMutex MultiThreadMutex_t;
MultiThreadMutex_t mtx;
typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
WorkerClass_t *pWorkerObj;
} m_ST;
};
void Worker_ProcessCommandRange_Singleton::Startup( void )
{
bool bInitializedThreadPool = false;
CPUInformation const &cpu = *GetCPUInformation();
if ( cpu.m_nLogicalProcessors > 1 )
{
// Attempt to initialize thread pool
m_MT.pThreadPool = g_pThreadPool;
if ( m_MT.pThreadPool )
{
m_MT.tpsp.bIOThreads = false;
m_MT.tpsp.nThreads = cpu.m_nLogicalProcessors - 1;
if ( m_MT.pThreadPool->Start( m_MT.tpsp ) )
{
if ( m_MT.pThreadPool->NumThreads() >= 1 )
{
// Make sure that our mutex is in multi-threaded mode
Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );
m_MT.pWorkerObj = new MT::WorkerClass_t( &m_MT.mtx );
bInitializedThreadPool = true;
}
else
{
m_MT.pThreadPool->Stop();
}
}
if ( !bInitializedThreadPool )
m_MT.pThreadPool = NULL;
}
}
// Otherwise initialize single-threaded mode
if ( !bInitializedThreadPool )
{
m_ST.pWorkerObj = new ST::WorkerClass_t( &m_ST.mtx );
}
}
void Worker_ProcessCommandRange_Singleton::Shutdown( void )
{
if ( m_MT.pThreadPool )
{
if( m_MT.pWorkerObj )
delete m_MT.pWorkerObj;
m_MT.pThreadPool->Stop();
m_MT.pThreadPool = NULL;
}
else
{
if ( m_ST.pWorkerObj )
delete m_ST.pWorkerObj;
}
}
void Worker_ProcessCommandRange_Singleton::ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
if ( m_MT.pThreadPool )
{
MT::WorkerClass_t *pWorkerObj = m_MT.pWorkerObj;
pWorkerObj->RangeBegin( shaderStart, shaderEnd );
pWorkerObj->Run();
pWorkerObj->RangeFinished();
}
else
{
ST::WorkerClass_t *pWorkerObj = m_ST.pWorkerObj;
pWorkerObj->RangeBegin( shaderStart, shaderEnd );
pWorkerObj->OnProcessST();
pWorkerObj->RangeFinished();
}
}
// You must process the work unit range.
void Worker_ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
Worker_ProcessCommandRange_Singleton::GetInstance()->ProcessCommandRange( shaderStart, shaderEnd );
}
// You must append data to pBuf with the work unit results.
void Worker_ProcessWorkUnitFn( int iThread, uint64 iWorkUnit, MessageBuffer *pBuf )
{
uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
comboEnd = min( g_numStaticCombos, comboEnd );
// Determine the commands required to be executed:
uint64 nComboOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = NULL;
pEntry = GetEntryByStaticComboNum( comboEnd, &nComboOfTheEntry );
uint64 commandEnd = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;
Assert( commandEnd <= g_numCompileCommands );
pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
uint64 commandStart = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;
// Compile all the shader combos
Worker_ProcessCommandRange( commandStart, commandEnd );
nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;
// Copy off the reply packages
int nSkipsSoFar = 0;
for ( uint64 kCombo = comboStart; kCombo < comboEnd; ++ kCombo )
{
size_t nCpBytes = CopyWorkerReplyPackage( pEntry, nComboOfTheEntry, pBuf, nSkipsSoFar );
if ( nCpBytes )
nSkipsSoFar = 0;
else
-- nSkipsSoFar;
if ( nComboOfTheEntry == 0 )
{
++pEntry;
nComboOfTheEntry = pEntry->m_numStaticCombos;
}
nComboOfTheEntry--;
}
if ( nSkipsSoFar )
{
pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
}
//////////////////////////////////////////////////////////////////////////
//
// Now deliver all our accumulated spew to the master
//
//////////////////////////////////////////////////////////////////////////
// Failed shaders
for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
{
char const *szShaderName = g_Master_ShaderHadError.String( k );
if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
continue;
int const len = strlen( szShaderName );
CUtlBinaryBlock bb;
bb.SetLength( 1 + len + 1 );
sprintf( ( char * ) bb.Get(), "%c%s", SHADERHADERROR_PACKETID, szShaderName );
VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
VMPI_HandleSocketErrors();
}
// Compiler spew
for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
{
char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];
char const * const szFirstCmd = cmi.GetFirstCommand();
int const numReported = cmi.GetNumTimesReported();
char chNumReported[0x40];
sprintf( chNumReported, "%d", numReported );
CUtlBinaryBlock bb;
bb.SetLength( 1 + strlen(szMsg) + 1 + strlen( szFirstCmd ) + 1 + strlen( chNumReported ) + 1 + 1 );
sprintf( ( char * ) bb.Get(), "%c%s\n%s\n%s\n", ERRMSG_PACKETID, szMsg, szFirstCmd, chNumReported );
VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
VMPI_HandleSocketErrors();
}
// Clean all reported msgs
g_Master_CompilerMsgInfo.Purge();
}
void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo )
{
if ( CfgProcessor::ComboHandle hCombo = CfgProcessor::Combo_GetCombo( pEntry->m_iCommandStart ) )
{
char cmd[ 4096 ];
Combo_FormatCommand( hCombo, cmd );
{
memset( &shaderInfo, 0, sizeof( ShaderInfo_t ) );
const char *pCentroidMask = strstr( cmd, "/DCENTROIDMASK=" );
const char *pFlags = strstr( cmd, "/DFLAGS=0x" );
const char *pShaderModel = strstr( cmd, "/DSHADER_MODEL_" );
if( !pCentroidMask || !pFlags || !pShaderModel )
{
Assert( !"!pCentroidMask || !pFlags || !pShaderModel" );
return;
}
sscanf( pCentroidMask + strlen( "/DCENTROIDMASK=" ), "%u", &shaderInfo.m_CentroidMask );
sscanf( pFlags + strlen( "/DFLAGS=0x" ), "%x", &shaderInfo.m_Flags );
// Copy shader model
pShaderModel += strlen( "/DSHADER_MODEL_" );
for ( char *pszSm = shaderInfo.m_szShaderModel, * const pszEnd = pszSm + sizeof( shaderInfo.m_szShaderModel ) - 1;
pszSm < pszEnd ; ++ pszSm )
{
char &rchLastChar = (*pszSm = *pShaderModel ++);
if ( !rchLastChar ||
V_isspace( rchLastChar ) ||
'=' == rchLastChar )
{
rchLastChar = 0;
break;
}
}
shaderInfo.m_nShaderCombo = 0;
shaderInfo.m_nTotalShaderCombos = pEntry->m_numCombos;
shaderInfo.m_nDynamicCombos = pEntry->m_numDynamicCombos;
shaderInfo.m_nStaticCombo = 0;
shaderInfo.m_pShaderName = pEntry->m_szName;
shaderInfo.m_pShaderSrc = pEntry->m_szShaderFileName;
}
Combo_Free( hCombo );
}
}
void Worker_GetLocalCopyOfShaders( void )
{
// Create virtual files for all of the stuff that we need to compile the shader
// make sure and prefix the file name so that it doesn't find it locally.
char filename[1024];
sprintf( filename, "%s\\uniquefilestocopy.txt", g_pShaderPath );
CUtlInplaceBuffer bffr( 0, 0, CUtlBuffer::TEXT_BUFFER );
if( !g_pFileSystem->ReadFile( filename, NULL, bffr ) )
{
fprintf( stderr, "Can't open uniquefilestocopy.txt!\n" );
exit( -1 );
}
while( char *pszLineToCopy = bffr.InplaceGetLinePtr() )
{
V_MakeAbsolutePath( filename, sizeof( filename ), pszLineToCopy, g_pShaderPath );
if ( g_bVerbose )
printf( "getting local copy of shader: \"%s\" (\"%s\")\n", pszLineToCopy, filename );
CUtlBuffer fileBuf;
if ( !g_pFileSystem->ReadFile( filename, NULL, fileBuf ) )
{
Warning( "Can't find \"%s\"\n", filename );
continue;
}
// Grab just the filename.
char justFilename[MAX_PATH];
char *pLastSlash = max( strrchr( pszLineToCopy, '/' ), strrchr( pszLineToCopy, '\\' ) );
if ( pLastSlash )
Q_strncpy( justFilename, pLastSlash + 1, sizeof( justFilename ) );
else
Q_strncpy( justFilename, pszLineToCopy, sizeof( justFilename ) );
sprintf( filename, "%s%s", g_WorkerTempPath, justFilename );
if ( g_bVerbose )
printf( "creating \"%s\"\n", filename );
FILE *fp3 = fopen( filename, "wb" );
if ( !fp3 )
{
Error( "Can't open '%s' for writing.", pszLineToCopy );
continue;
}
fwrite( fileBuf.Base(), 1, fileBuf.GetBytesRemaining(), fp3 );
fclose( fp3 );
// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
// while we're running since they're in the temp path.
static CUtlVector< FILE * > s_arrHackedFiles;
static struct X_s_arrHackedFiles { ~X_s_arrHackedFiles() {
for ( int k = 0; k < s_arrHackedFiles.Count(); ++ k )
fclose( s_arrHackedFiles[k] );
} } s_autoCloseHackedFiles;
/* THIS IS THE EVIL LINE ----> */ FILE *fHack = fopen( filename, "r" );
s_arrHackedFiles.AddToTail( fHack );
// -- END of EVIL
}
}
void Worker_GetLocalCopyOfBinary( const char *pFilename )
{
CUtlBuffer fileBuf;
char tmpFilename[MAX_PATH];
sprintf( tmpFilename, "%s\\%s", g_ExeDir, pFilename );
if ( g_bVerbose )
printf( "trying to open: %s\n", tmpFilename );
FILE *fp = fopen( tmpFilename, "rb" );
if( !fp )
{
Assert( 0 );
fprintf( stderr, "Can't open %s!\n", pFilename );
exit( -1 );
}
fseek( fp, 0, SEEK_END );
int fileLen = ftell( fp );
fseek( fp, 0, SEEK_SET );
fileBuf.EnsureCapacity( fileLen );
int nBytesRead = fread( fileBuf.Base(), 1, fileLen, fp );
fclose( fp );
fileBuf.SeekPut( CUtlBuffer::SEEK_HEAD, nBytesRead );
char newFilename[MAX_PATH];
sprintf( newFilename, "%s%s", g_WorkerTempPath, pFilename );
FILE *fp2 = fopen( newFilename, "wb" );
if( !fp2 )
{
Assert( 0 );
fprintf( stderr, "Can't open %s!\n", newFilename );
exit( -1 );
}
fwrite( fileBuf.Base(), 1, fileLen, fp2 );
fclose( fp2 );
// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
// while we're running since they're in the temp path.
fopen( newFilename, "r" );
}
void Worker_GetLocalCopyOfBinaries( void )
{
Worker_GetLocalCopyOfBinary( "mysql_wrapper.dll" ); // This is necessary so VMPI doesn't run in SDK mode.
Worker_GetLocalCopyOfBinary( "vstdlib.dll" );
Worker_GetLocalCopyOfBinary( "tier0.dll" );
}
void Shared_ParseListOfCompileCommands( void )
{
// double tt_start = Plat_FloatTime();
char fileListFileName[1024];
sprintf( fileListFileName, "%s\\filelist.txt", g_pShaderPath );
CUtlInplaceBuffer bffr( 0, 0, CUtlInplaceBuffer::TEXT_BUFFER );
if( !g_pFileSystem->ReadFile( fileListFileName, NULL, bffr) )
{
DebugOut( "Can't open %s!\n", fileListFileName );
fprintf( stderr, "Can't open %s!\n", fileListFileName );
exit( -1 );
}
CfgProcessor::ReadConfiguration( &bffr );
CfgProcessor::DescribeConfiguration( g_arrCompileEntries );
for ( CfgProcessor::CfgEntryInfo const *pInfo = g_arrCompileEntries.Get();
pInfo && pInfo->m_szName; ++ pInfo )
{
++ g_numShaders;
g_numStaticCombos += pInfo->m_numStaticCombos;
g_numCompileCommands = pInfo->m_iCommandEnd;
}
// double tt_end = Plat_FloatTime();
Msg( "\rCompiling %s commands. \r", PrettyPrintNumber( g_numCompileCommands ) );
}
void SetupExeDir( int argc, char **argv )
{
strcpy( g_ExeDir, argv[0] );
Q_StripFilename( g_ExeDir );
if ( g_ExeDir[0] == 0 )
{
Q_strncpy( g_ExeDir, ".\\", sizeof( g_ExeDir ) );
}
Q_FixSlashes( g_ExeDir );
}
void SetupPaths( int argc, char **argv )
{
GetTempPath( sizeof( g_WorkerTempPath ), g_WorkerTempPath );
strcat( g_WorkerTempPath, "shadercompiletemp\\" );
char tmp[MAX_PATH];
sprintf( tmp, "rd /s /q \"%s\"", g_WorkerTempPath );
system( tmp );
_mkdir( g_WorkerTempPath );
// printf( "g_WorkerTempPath: \"%s\"\n", g_WorkerTempPath );
CommandLine()->CreateCmdLine( argc, argv );
g_pShaderPath = CommandLine()->ParmValue( "-shaderpath", "" );
g_bVerbose = CommandLine()->FindParm("-verbose") != 0;
}
void SetupDebugFile( void )
{
#ifdef DEBUGFP
const char *pComputerName = getenv( "COMPUTERNAME" );
char filename[MAX_PATH];
sprintf( filename, "\\\\fileserver\\user\\gary\\debug\\%s.txt", pComputerName );
g_WorkerDebugFp = fopen( filename, "w" );
Assert( g_WorkerDebugFp );
DebugOut( "opened debug file\n" );
#endif
}
void CompileShaders_NoVMPI()
{
Worker_ProcessCommandRange_Singleton pcr;
//
// We will iterate on the cfg entries and process them
//
for ( CfgProcessor::CfgEntryInfo const *pEntry = g_arrCompileEntries.Get();
pEntry && pEntry->m_szName; ++ pEntry )
{
//
// Stick the shader info
//
ShaderInfo_t siLastShaderInfo;
memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );
Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );
g_ShaderToShaderInfo[ pEntry->m_szName ] = siLastShaderInfo;
//
// Compile stuff
//
Worker_ProcessCommandRange( pEntry->m_iCommandStart, pEntry->m_iCommandEnd );
//
// Now when the whole shader is finished we can write it
//
char const *szShaderToWrite = pEntry->m_szName;
g_numCommandsCompleted = g_numCompileCommands;
WriteShaderFiles( szShaderToWrite );
g_numCommandsCompleted = pEntry->m_iCommandEnd;
}
Msg( "\r \r" );
}
class CDistributeShaderCompileMaster : public IWorkUnitDistributorCallbacks
{
public:
CDistributeShaderCompileMaster( void );
~CDistributeShaderCompileMaster( void );
public:
virtual void OnWorkUnitsCompleted( uint64 numWorkUnits );
private:
void ThreadProc( void );
friend DWORD WINAPI CDistributeShaderCompileMaster::ThreadProcAdapter( LPVOID pvArg );
static DWORD WINAPI ThreadProcAdapter( LPVOID pvArg ) { reinterpret_cast< CDistributeShaderCompileMaster * >( pvArg )->ThreadProc(); return 0; }
private:
HANDLE m_hThread;
HANDLE m_hEvent;
CThreadFastMutex m_mtx;
BOOL m_bRunning;
private:
CfgProcessor::CfgEntryInfo const *m_pAnalyzeShaders;
CUtlVector< char const * > m_arrShaderNamesToWrite;
};
CDistributeShaderCompileMaster::CDistributeShaderCompileMaster( void ) :
m_hThread( NULL ),
m_hEvent( NULL ),
m_bRunning( TRUE )
{
m_hEvent = CreateEvent( NULL, FALSE, FALSE, NULL );
m_hThread = CreateThread( NULL, 0, ThreadProcAdapter, reinterpret_cast< LPVOID >(this), 0, NULL );
m_pAnalyzeShaders = g_arrCompileEntries.Get();
}
CDistributeShaderCompileMaster::~CDistributeShaderCompileMaster( void )
{
m_bRunning = FALSE;
SetEvent( m_hEvent );
WaitForSingleObject( m_hThread, INFINITE );
CloseHandle( m_hThread );
CloseHandle( m_hEvent );
}
void CDistributeShaderCompileMaster::OnWorkUnitsCompleted( uint64 numWorkUnits )
{
// Make sure that our mutex is in multi-threaded mode
Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );
// Figure out how many commands have completed based on work units
g_numCompletedStaticCombos = numWorkUnits * g_nStaticCombosPerWorkUnit;
uint64 numStaticCombosOfTheEntry = 0;
CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( g_numCompletedStaticCombos, &numStaticCombosOfTheEntry );
g_numCommandsCompleted = pEntry->m_iCommandStart + numStaticCombosOfTheEntry * pEntry->m_numDynamicCombos;
// Iterate over the shaders yet to be written and see if we can queue them
for ( ; m_pAnalyzeShaders->m_szName &&
m_pAnalyzeShaders->m_iCommandEnd <= g_numCommandsCompleted;
++ m_pAnalyzeShaders
)
{
m_mtx.Lock();
m_arrShaderNamesToWrite.AddToTail( m_pAnalyzeShaders->m_szName );
SetEvent( m_hEvent );
m_mtx.Unlock();
}
}
void CDistributeShaderCompileMaster::ThreadProc( void )
{
for ( ; m_bRunning; )
{
WaitForSingleObject( m_hEvent, INFINITE );
// Do a pump of shaders to write
for ( int numShadersWritten = 0; /* forever */ ; ++ numShadersWritten )
{
m_mtx.Lock();
char const * szShaderToWrite = NULL;
if ( m_arrShaderNamesToWrite.Count() > numShadersWritten )
szShaderToWrite = m_arrShaderNamesToWrite[ numShadersWritten ];
else
m_arrShaderNamesToWrite.RemoveAll();
m_mtx.Unlock();
if ( !szShaderToWrite )
break;
// We have the shader to write asynchronously
WriteShaderFiles( szShaderToWrite );
}
}
}
int ShaderCompile_Main( int argc, char* argv[] )
{
InstallSpewFunction();
g_bSuppressPrintfOutput = false;
g_flStartTime = Plat_FloatTime();
SetupDebugFile();
numthreads = 1; // managed specifically in Worker_ProcessCommandRange_Singleton::Startup
/*
Special section of code implementing "-subprocess" flag
*/
if ( int iSubprocess = CommandLine()->FindParm( "-subprocess" ) )
{
char const *szSubProcessData = CommandLine()->GetParm( 1 + iSubprocess );
return ShaderCompile_Subprocess_Main( szSubProcessData );
}
// This needs to get called before VMPI is setup because in SDK mode, VMPI will change the args around.
SetupExeDir( argc, argv );
g_bIsX360 = CommandLine()->FindParm( "-x360" ) != 0;
// g_bSuppressWarnings = g_bIsX360;
bool bShouldUseVMPI = ( CommandLine()->FindParm( "-nompi" ) == 0 );
if ( bShouldUseVMPI )
{
// Master, start accepting connections.
// Worker, make a connection.
DebugOut( "Before VMPI_Init\n" );
g_bSuppressPrintfOutput = true;
VMPIRunMode mode = VMPI_RUN_NETWORKED;
if ( !VMPI_Init( argc, argv, "dependency_info_shadercompile.txt", MyDisconnectHandler, mode ) )
{
g_bSuppressPrintfOutput = false;
DebugOut( "MPI_Init failed.\n" );
Error( "MPI_Init failed." );
}
extern void VMPI_SetWorkUnitsPartitionSize( int numWusToDeal );
VMPI_SetWorkUnitsPartitionSize( 32 );
}
SetupPaths( argc, argv );
g_bSuppressPrintfOutput = false;
DebugOut( "After VMPI_Init\n" );
// Setting up the minidump handlers
if ( bShouldUseVMPI && !g_bMPIMaster )
SetupToolsMinidumpHandler( VMPI_ExceptionFilter );
else
SetupDefaultToolsMinidumpHandler();
if ( CommandLine()->FindParm( "-game" ) == 0 )
{
// Used with filesystem_stdio.dll
FileSystem_Init( NULL, 0, FS_INIT_COMPATIBILITY_MODE );
}
else
{
// SDK uses this since it only has filesystem_steam.dll.
FileSystem_Init( NULL, 0, FS_INIT_FULL );
}
DebugOut( "After VMPI_FileSystem_Init\n" );
Shared_ParseListOfCompileCommands();
DebugOut( "After Shared_ParseListOfCompileCommands\n" );
if ( bShouldUseVMPI )
{
// Partition combos
g_nStaticCombosPerWorkUnit = 0;
if ( g_numStaticCombos )
{
if ( g_numStaticCombos <= 1024 )
g_nStaticCombosPerWorkUnit = 1;
else if ( g_numStaticCombos > 1024 * 10 )
g_nStaticCombosPerWorkUnit = 10;
else
g_nStaticCombosPerWorkUnit = g_numStaticCombos / 1024;
}
uint64 nWorkUnits;
if( g_nStaticCombosPerWorkUnit == 0 )
{
nWorkUnits = 1;
g_nStaticCombosPerWorkUnit = g_numStaticCombos;
}
else
{
nWorkUnits = g_numStaticCombos / g_nStaticCombosPerWorkUnit + 1;
}
DebugOut( "Before conditional\n" );
if ( g_bMPIMaster )
{
// Send all of the workers the complete list of work to do.
DebugOut( "Before STARTWORK_PACKETID\n" );
char packetID = STARTWORK_PACKETID;
VMPI_SendData( &packetID, sizeof( packetID ), VMPI_PERSISTENT );
// Compile master distribution tracker
CDistributeShaderCompileMaster dscm;
g_pDistributeWorkCallbacks = &dscm;
{
char chCommands[50], chStaticCombos[50], chNumWorkUnits[50];
sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
sprintf( chNumWorkUnits, "%s", PrettyPrintNumber( nWorkUnits ) );
Msg( "\rCompiling %s commands in %s work units.\n", chCommands, chNumWorkUnits );
}
// nWorkUnits is how many work units. . .1000 is good.
// The work unit number impies which combo to do.
DebugOut( "Before DistributeWork\n" );
DistributeWork( nWorkUnits, WORKUNIT_PACKETID, NULL, Master_ReceiveWorkUnitFn );
g_pDistributeWorkCallbacks = NULL;
}
else
{
// wait until we get a packet from the master to start doing stuff.
MessageBuffer buf;
DebugOut( "Before VMPI_DispatchUntil\n" );
while ( !g_bGotStartWorkPacket )
{
VMPI_DispatchNextMessage();
}
DebugOut( "after VMPI_DispatchUntil\n" );
DebugOut( "Before Worker_GetLocalCopyOfShaders\n" );
Worker_GetLocalCopyOfShaders();
DebugOut( "Before Worker_GetLocalCopyOfBinaries\n" );
Worker_GetLocalCopyOfBinaries();
DebugOut( "Before _chdir\n" );
_chdir( g_WorkerTempPath );
// nWorkUnits is how many work units. . .1000 is good.
// The work unit number impies which combo to do.
DebugOut( "Before DistributeWork\n" );
// Allows calling into ProcessCommandRange inside the worker function
{
Worker_ProcessCommandRange_Singleton pcr;
DistributeWork( nWorkUnits, WORKUNIT_PACKETID, Worker_ProcessWorkUnitFn, NULL );
}
}
g_bSuppressPrintfOutput = true;
g_bSuppressPrintfOutput = false;
}
else // no VMPI
{
Worker_GetLocalCopyOfShaders();
Worker_GetLocalCopyOfBinaries();
_chdir( g_WorkerTempPath );
{
char chCommands[50], chStaticCombos[50];
sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
Msg( "\rCompiling %s commands in %s static combos.\n", chCommands, chStaticCombos );
}
CompileShaders_NoVMPI();
}
Msg( "\r \r" );
if ( g_bMPIMaster || !bShouldUseVMPI )
{
char str[ 4096 ];
// Write everything that succeeded
int nStrings = g_ShaderByteCode.GetNumStrings();
for( int i = 0; i < nStrings; i++ )
{
WriteShaderFiles( g_ShaderByteCode.String(i) );
}
// Write all the errors
//////////////////////////////////////////////////////////////////////////
//
// Now deliver all our accumulated spew to the output
//
//////////////////////////////////////////////////////////////////////////
bool bValveVerboseComboErrors = ( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) &&
atoi( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) ) ) ? true : false;
// Compiler spew
for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
{
char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];
char const * const szFirstCmd = cmi.GetFirstCommand();
int const numReported = cmi.GetNumTimesReported();
uint64 iFirstCommand = _strtoui64( szFirstCmd, NULL, 10 );
CfgProcessor::ComboHandle hCombo = NULL;
CfgProcessor::CfgEntryInfo const *pComboEntryInfo = NULL;
if ( CfgProcessor::Combo_GetNext( iFirstCommand, hCombo, g_numCompileCommands ) )
{
Combo_FormatCommand( hCombo, str );
pComboEntryInfo = Combo_GetEntryInfo( hCombo );
Combo_Free( hCombo );
}
else
{
sprintf( str, "cmd # %s", szFirstCmd );
}
Msg( "\n%s\n", szMsg );
Msg( " Reported %d time(s), example command:\n", numReported);
if ( bValveVerboseComboErrors )
{
Msg( " Verbose Description:\n" );
if ( pComboEntryInfo )
{
Msg( " Src File: %s\n", pComboEntryInfo->m_szShaderFileName );
Msg( " Tgt File: %s\n", pComboEntryInfo->m_szName );
}
// Between /DSHADERCOMBO= and /Dmain
char const *pBegin = strstr( str, "/DSHADERCOMBO=" );
char const *pEnd = strstr( str, "/Dmain" );
if ( pBegin )
{
pBegin += strlen( "/DSHADERCOMBO=" ) ;
char const *pSpace = strchr( pBegin, ' ' );
if ( pSpace )
Msg( " Combo # : %.*s\n", ( pSpace - pBegin ), pBegin );
}
if ( !pEnd )
pEnd = str + strlen( str );
while ( pBegin && *pBegin && !V_isspace( *pBegin ) )
++ pBegin;
while ( pBegin && *pBegin && V_isspace( *pBegin ) )
++ pBegin;
// Now parse all combo defines in [pBegin, pEnd]
while ( pBegin && *pBegin && ( pBegin < pEnd ) )
{
char const *pDefine = strstr( pBegin, "/D" );
if ( !pDefine || pDefine >= pEnd )
break;
char const *pEqSign = strchr( pDefine, '=' );
if ( !pEqSign || pEqSign >= pEnd )
break;
char const *pSpace = strchr( pEqSign, ' ' );
if ( !pSpace || pSpace >= pEnd )
pSpace = pEnd;
pBegin = pSpace;
Msg( " %.*s %.*s\n",
( pSpace - pEqSign - 1 ), pEqSign + 1,
( pEqSign - pDefine - 2 ), pDefine + 2 );
}
}
Msg( " %s\n", str );
}
// Failed shaders summary
for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
{
char const *szShaderName = g_Master_ShaderHadError.String( k );
if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
continue;
Msg( "FAILED: %s\n", szShaderName );
}
//
// End
//
double end = Plat_FloatTime();
GetHourMinuteSecondsString( (int)( end - g_flStartTime ), str, sizeof( str ) );
DebugOut( "%s elapsed\n", str );
DebugOut( "Precise timing = %.5f\n", ( end - g_flStartTime ) );
if ( bShouldUseVMPI )
{
VMPI_FileSystem_Term();
DebugOut( "Before VMPI_Finalize\n" );
VMPI_Finalize();
}
}
return g_Master_ShaderHadError.GetNumStrings();
}
class CShaderCompileDLL : public IShaderCompileDLL
{
int main( int argc, char **argv );
};
int CShaderCompileDLL::main( int argc, char **argv )
{
return ShaderCompile_Main( argc, argv );
}
EXPOSE_SINGLE_INTERFACE( CShaderCompileDLL, IShaderCompileDLL, SHADER_COMPILE_INTERFACE_VERSION );
class CLaunchableDLL : public ILaunchableDLL
{
int main( int argc, char **argv )
{
return ShaderCompile_Main( argc, argv );
}
};
EXPOSE_SINGLE_INTERFACE( CLaunchableDLL, ILaunchableDLL, LAUNCHABLE_DLL_INTERFACE_VERSION );