Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

1904 lines
48 KiB

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: Memory allocation!
//
// $NoKeywords: $
//=============================================================================//
#include "pch_tier0.h"
#if !defined(STEAM) && !defined(NO_MALLOC_OVERRIDE)
#if defined( _WIN32 ) && !defined( _X360 )
#define WIN_32_LEAN_AND_MEAN
#include <windows.h>
#define VA_COMMIT_FLAGS MEM_COMMIT
#define VA_RESERVE_FLAGS MEM_RESERVE
#elif defined( _X360 )
#undef Verify
#define VA_COMMIT_FLAGS (MEM_COMMIT|MEM_NOZERO|MEM_LARGE_PAGES)
#define VA_RESERVE_FLAGS (MEM_RESERVE|MEM_LARGE_PAGES)
#endif
#ifdef OSX
#include <malloc/malloc.h>
#else
#include <malloc.h>
#endif
#include "tier0/valve_minmax_off.h" // GCC 4.2.2 headers screw up our min/max defs.
#include <algorithm>
#include "tier0/valve_minmax_on.h" // GCC 4.2.2 headers screw up our min/max defs.
#include "tier0/dbg.h"
#include "tier0/memalloc.h"
#include "tier0/threadtools.h"
#include "mem_helpers.h"
#include "memstd.h"
#ifdef _X360
#include "xbox/xbox_console.h"
#endif
// Force on redirecting all allocations to the process heap on Win64,
// which currently means the GC. This is to make AppVerifier more effective
// at catching memory stomps.
#if defined( _WIN64 )
#define FORCE_PROCESS_HEAP
#elif defined( _WIN32 )
// Define this to force using the OS Heap* functions for allocations. This is useful
// in conjunction with AppVerifier/PageHeap in order to find memory problems, and
// also allows ETW/xperf tracing to be used to record allocations.
// Normally the command-line option -processheap can be used instead.
//#define FORCE_PROCESS_HEAP
#define ALLOW_PROCESS_HEAP
#endif
// Track this to decide how to handle out-of-memory.
static bool s_bPageHeapEnabled = false;
#ifdef TIME_ALLOC
CAverageCycleCounter g_MallocCounter;
CAverageCycleCounter g_ReallocCounter;
CAverageCycleCounter g_FreeCounter;
#define PrintOne( name ) \
Msg("%-48s: %6.4f avg (%8.1f total, %7.3f peak, %5d iters)\n", \
#name, \
g_##name##Counter.GetAverageMilliseconds(), \
g_##name##Counter.GetTotalMilliseconds(), \
g_##name##Counter.GetPeakMilliseconds(), \
g_##name##Counter.GetIters() ); \
memset( &g_##name##Counter, 0, sizeof(g_##name##Counter) )
void PrintAllocTimes()
{
PrintOne( Malloc );
PrintOne( Realloc );
PrintOne( Free );
}
#define PROFILE_ALLOC(name) CAverageTimeMarker name##_ATM( &g_##name##Counter )
#else
#define PROFILE_ALLOC( name ) ((void)0)
#define PrintAllocTimes() ((void)0)
#endif
#if _MSC_VER < 1400 && defined( MSVC ) && !defined(_STATIC_LINKED) && (defined(_DEBUG) || defined(USE_MEM_DEBUG))
void *operator new( unsigned int nSize, int nBlockUse, const char *pFileName, int nLine )
{
return ::operator new( nSize );
}
void *operator new[] ( unsigned int nSize, int nBlockUse, const char *pFileName, int nLine )
{
return ::operator new[]( nSize );
}
#endif
#if (!defined(_DEBUG) && !defined(USE_MEM_DEBUG))
// Support for CHeapMemAlloc for easy switching to using the process heap.
#ifdef ALLOW_PROCESS_HEAP
// Round a size up to a multiple of 4 KB to aid in calculating how much
// memory is required if full pageheap is enabled.
static size_t RoundUpToPage( size_t nSize )
{
nSize += 0xFFF;
nSize &= ~0xFFF;
return nSize;
}
// Convenience function to deal with the necessary type-casting
static void InterlockedAddSizeT( size_t volatile *Addend, size_t Value )
{
#ifdef PLATFORM_WINDOWS_PC32
COMPILE_TIME_ASSERT( sizeof( size_t ) == sizeof( int32 ) );
InterlockedExchangeAdd( ( LONG* )Addend, LONG( Value ) );
#else
InterlockedExchangeAdd64( ( LONGLONG* )Addend, LONGLONG( Value ) );
#endif
}
class CHeapMemAlloc : public IMemAlloc
{
public:
CHeapMemAlloc()
{
// Make sure that we return 64-bit addresses in 64-bit builds.
ReserveBottomMemory();
// Do all allocations with the shared process heap so that we can still
// allocate from one DLL and free in another.
m_heap = GetProcessHeap();
}
void Init( bool bZeroMemory )
{
m_HeapFlags = bZeroMemory ? HEAP_ZERO_MEMORY : 0;
// Can't use Msg here because it isn't necessarily initialized yet.
if ( s_bPageHeapEnabled )
{
OutputDebugStringA("PageHeap is on. Memory use will be larger than normal.\n" );
}
else
{
OutputDebugStringA("PageHeap is off. Memory use will be normal.\n" );
}
if( bZeroMemory )
{
OutputDebugStringA( " HEAP_ZERO_MEMORY is specified.\n" );
}
}
// Release versions
virtual void *Alloc( size_t nSize )
{
// Ensure that the constructor has run already. Poorly defined
// order of construction can result in the allocator being used
// before it is constructed. Which could be bad.
if ( !m_heap )
__debugbreak();
void* pMem = HeapAlloc( m_heap, m_HeapFlags, nSize );
if ( pMem )
{
InterlockedAddSizeT( &m_nOutstandingBytes, nSize );
InterlockedAddSizeT( &m_nOutstandingPageHeapBytes, RoundUpToPage( nSize ) );
InterlockedIncrement( &m_nOutstandingAllocations );
InterlockedIncrement( &m_nLifetimeAllocations );
}
else if ( nSize )
{
// Having PageHeap enabled leads to lots of allocation failures. These
// then lead to crashes. In order to avoid confusion about the cause of
// these crashes, halt immediately on allocation failures.
__debugbreak();
InterlockedIncrement( &m_nAllocFailures );
}
return pMem;
}
virtual void *Realloc( void *pMem, size_t nSize )
{
// If you pass zero to HeapReAlloc then it fails (with GetLastError() saying S_OK!)
// so only call HeapReAlloc if pMem is non-zero.
if ( pMem )
{
if ( !nSize )
{
// Call the regular free function.
Free( pMem );
return 0;
}
size_t nOldSize = HeapSize( m_heap, 0, pMem );
void* pNewMem = HeapReAlloc( m_heap, m_HeapFlags, pMem, nSize );
// If we successfully allocated the requested memory (zero counts as
// success if we requested zero bytes) then update the counters for the
// change.
if ( pNewMem )
{
InterlockedAddSizeT( &m_nOutstandingBytes, nSize - nOldSize );
InterlockedAddSizeT( &m_nOutstandingPageHeapBytes, RoundUpToPage( nSize ) );
InterlockedAddSizeT( &m_nOutstandingPageHeapBytes, 0 - RoundUpToPage( nOldSize ) );
// Outstanding allocation count isn't affected by Realloc, but
// lifetime allocation count is.
InterlockedIncrement( &m_nLifetimeAllocations );
}
else
{
// Having PageHeap enabled leads to lots of allocation failures. These
// then lead to crashes. In order to avoid confusion about the cause of
// these crashes, halt immediately on allocation failures.
__debugbreak();
InterlockedIncrement( &m_nAllocFailures );
}
return pNewMem;
}
// Call the regular alloc function.
return Alloc( nSize );
}
virtual void Free( void *pMem )
{
if ( pMem )
{
size_t nOldSize = HeapSize( m_heap, 0, pMem );
InterlockedAddSizeT( &m_nOutstandingBytes, 0 - nOldSize );
InterlockedAddSizeT( &m_nOutstandingPageHeapBytes, 0 - RoundUpToPage( nOldSize ) );
InterlockedDecrement( &m_nOutstandingAllocations );
HeapFree( m_heap, 0, pMem );
}
}
virtual void *Expand_NoLongerSupported( void *pMem, size_t nSize ) { return 0; }
// Debug versions
virtual void *Alloc( size_t nSize, const char *pFileName, int nLine ) { return Alloc( nSize ); }
virtual void *Realloc( void *pMem, size_t nSize, const char *pFileName, int nLine ) { return Realloc(pMem, nSize); }
virtual void Free( void *pMem, const char *pFileName, int nLine ) { Free( pMem ); }
virtual void *Expand_NoLongerSupported( void *pMem, size_t nSize, const char *pFileName, int nLine ) { return 0; }
#ifdef MEMALLOC_SUPPORTS_ALIGNED_ALLOCATIONS
// Not currently implemented
#error
#endif
virtual void *RegionAlloc( int region, size_t nSize ) { __debugbreak(); return 0; }
virtual void *RegionAlloc( int region, size_t nSize, const char *pFileName, int nLine ) { __debugbreak(); return 0; }
// Returns size of a particular allocation
// If zero is returned then return the total size of allocated memory.
virtual size_t GetSize( void *pMem )
{
if ( !pMem )
{
return m_nOutstandingBytes;
}
return HeapSize( m_heap, 0, pMem );
}
// Force file + line information for an allocation
virtual void PushAllocDbgInfo( const char *pFileName, int nLine ) {}
virtual void PopAllocDbgInfo() {}
virtual long CrtSetBreakAlloc( long lNewBreakAlloc ) { return 0; }
virtual int CrtSetReportMode( int nReportType, int nReportMode ) { return 0; }
virtual int CrtIsValidHeapPointer( const void *pMem ) { return 0; }
virtual int CrtIsValidPointer( const void *pMem, unsigned int size, int access ) { return 0; }
virtual int CrtCheckMemory( void ) { return 0; }
virtual int CrtSetDbgFlag( int nNewFlag ) { return 0; }
virtual void CrtMemCheckpoint( _CrtMemState *pState ) {}
virtual void* CrtSetReportFile( int nRptType, void* hFile ) { return 0; }
virtual void* CrtSetReportHook( void* pfnNewHook ) { return 0; }
virtual int CrtDbgReport( int nRptType, const char * szFile,
int nLine, const char * szModule, const char * pMsg ) { return 0; }
virtual int heapchk() { return -2/*_HEAPOK*/; }
virtual void DumpStats()
{
const size_t MB = 1024 * 1024;
Msg( "Sorry -- no stats saved to file memstats.txt when the heap allocator is enabled.\n" );
// Print requested memory.
Msg( "%u MB allocated.\n", ( unsigned )( m_nOutstandingBytes / MB ) );
// Print memory after rounding up to pages.
Msg( "%u MB assuming maximum PageHeap overhead.\n", ( unsigned )( m_nOutstandingPageHeapBytes / MB ));
// Print memory after adding in reserved page after every allocation. Do 64-bit calculations
// because the pageHeap required memory can easily go over 4 GB.
__int64 pageHeapBytes = m_nOutstandingPageHeapBytes + m_nOutstandingAllocations * 4096LL;
Msg( "%u MB address space used assuming maximum PageHeap overhead.\n", ( unsigned )( pageHeapBytes / MB ));
Msg( "%u outstanding allocations (%d delta).\n", ( unsigned )m_nOutstandingAllocations, ( int )( m_nOutstandingAllocations - m_nOldOutstandingAllocations ) );
Msg( "%u lifetime allocations (%u delta).\n", ( unsigned )m_nLifetimeAllocations, ( unsigned )( m_nLifetimeAllocations - m_nOldLifetimeAllocations ) );
Msg( "%u allocation failures.\n", ( unsigned )m_nAllocFailures );
// Update the numbers on outstanding and lifetime allocation counts so
// that we can print out deltas.
m_nOldOutstandingAllocations = m_nOutstandingAllocations;
m_nOldLifetimeAllocations = m_nLifetimeAllocations;
}
virtual void DumpStatsFileBase( char const *pchFileBase ) {}
virtual size_t ComputeMemoryUsedBy( char const *pchSubStr ) { return 0; }
virtual void GlobalMemoryStatus( size_t *pUsedMemory, size_t *pFreeMemory ) {}
virtual bool IsDebugHeap() { return false; }
virtual uint32 GetDebugInfoSize() { return 0; }
virtual void SaveDebugInfo( void *pvDebugInfo ) { }
virtual void RestoreDebugInfo( const void *pvDebugInfo ) {}
virtual void InitDebugInfo( void *pvDebugInfo, const char *pchRootFileName, int nLine ) {}
virtual void GetActualDbgInfo( const char *&pFileName, int &nLine ) {}
virtual void RegisterAllocation( const char *pFileName, int nLine, int nLogicalSize, int nActualSize, unsigned nTime ) {}
virtual void RegisterDeallocation( const char *pFileName, int nLine, int nLogicalSize, int nActualSize, unsigned nTime ) {}
virtual int GetVersion() { return MEMALLOC_VERSION; }
virtual void OutOfMemory( size_t nBytesAttempted = 0 ) {}
virtual void CompactHeap() {}
virtual void CompactIncremental() {}
virtual MemAllocFailHandler_t SetAllocFailHandler( MemAllocFailHandler_t pfnMemAllocFailHandler ) { return 0; }
void DumpBlockStats( void *p ) {}
#if defined( _MEMTEST )
// Not currently implemented
#error
#endif
virtual size_t MemoryAllocFailed() { return 0; }
private:
// Handle to the process heap.
HANDLE m_heap;
uint32 m_HeapFlags;
// Total outstanding bytes allocated.
volatile size_t m_nOutstandingBytes;
// Total outstanding committed bytes assuming that all allocations are
// put on individual 4-KB pages (true when using full PageHeap from
// App Verifier).
volatile size_t m_nOutstandingPageHeapBytes;
// Total outstanding allocations. With PageHeap enabled each allocation
// requires an extra 4-KB page of address space.
volatile LONG m_nOutstandingAllocations;
LONG m_nOldOutstandingAllocations;
// Total allocations without subtracting freed memory.
volatile LONG m_nLifetimeAllocations;
LONG m_nOldLifetimeAllocations;
// Total number of allocation failures.
volatile LONG m_nAllocFailures;
};
#endif //ALLOW_PROCESS_HEAP
//-----------------------------------------------------------------------------
// Singletons...
//-----------------------------------------------------------------------------
#pragma warning( disable:4074 ) // warning C4074: initializers put in compiler reserved initialization area
#pragma init_seg( compiler )
static CStdMemAlloc s_StdMemAlloc CONSTRUCT_EARLY;
#ifndef TIER0_VALIDATE_HEAP
IMemAlloc *g_pMemAlloc = &s_StdMemAlloc;
#else
IMemAlloc *g_pActualAlloc = &s_StdMemAlloc;
#endif
#if defined(ALLOW_PROCESS_HEAP) && !defined(TIER0_VALIDATE_HEAP)
void EnableHeapMemAlloc( bool bZeroMemory )
{
// Place this here to guarantee it is constructed
// before we call Init.
static CHeapMemAlloc s_HeapMemAlloc;
static bool s_initCalled = false;
if ( !s_initCalled )
{
s_HeapMemAlloc.Init( bZeroMemory );
g_pMemAlloc = &s_HeapMemAlloc;
s_initCalled = true;
}
}
// Check whether PageHeap (part of App Verifier) has been enabled for this process.
// It specifically checks whether it was enabled by the EnableAppVerifier.bat
// batch file. This can be used to automatically enable -processheap when
// App Verifier is in use.
static bool IsPageHeapEnabled( bool& bETWHeapEnabled )
{
// Assume false.
bool result = false;
bETWHeapEnabled = false;
// First we get the application's name so we can look in the registry
// for App Verifier settings.
HMODULE exeHandle = GetModuleHandle( 0 );
if ( exeHandle )
{
char appName[ MAX_PATH ];
if ( GetModuleFileNameA( exeHandle, appName, ARRAYSIZE( appName ) ) )
{
// Guarantee null-termination -- not guaranteed on Windows XP!
appName[ ARRAYSIZE( appName ) - 1 ] = 0;
// Find the file part of the name.
const char* pFilePart = strrchr( appName, '\\' );
if ( pFilePart )
{
++pFilePart;
size_t len = strlen( pFilePart );
if ( len > 0 && pFilePart[ len - 1 ] == ' ' )
{
OutputDebugStringA( "Trailing space on executable name! This will cause Application Verifier and ETW Heap tracing to fail!\n" );
DebuggerBreakIfDebugging();
}
// Generate the key name for App Verifier settings for this process.
char regPathName[ MAX_PATH ];
_snprintf( regPathName, ARRAYSIZE( regPathName ),
"Software\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options\\%s",
pFilePart );
regPathName[ ARRAYSIZE( regPathName ) - 1 ] = 0;
HKEY key;
LONG regResult = RegOpenKeyA( HKEY_LOCAL_MACHINE,
regPathName,
&key );
if ( regResult == ERROR_SUCCESS )
{
// If PageHeapFlags exists then that means that App Verifier is enabled
// for this application. The StackTraceDatabaseSizeInMB is only
// set by Valve's enabling batch file so this indicates that
// a developer at Valve is using App Verifier.
if ( RegQueryValueExA( key, "StackTraceDatabaseSizeInMB", 0, NULL, NULL, NULL ) == ERROR_SUCCESS &&
RegQueryValueExA( key, "PageHeapFlags", 0, NULL, NULL, NULL) == ERROR_SUCCESS )
{
result = true;
}
if ( RegQueryValueExA( key, "TracingFlags", 0, NULL, NULL, NULL) == ERROR_SUCCESS )
bETWHeapEnabled = true;
RegCloseKey( key );
}
}
}
}
return result;
}
// Check for various allocator overrides such as -processheap and -reservelowmem.
// Returns true if -processheap is enabled, by a command line switch or other method.
bool CheckWindowsAllocSettings( const char* upperCommandLine )
{
// Are we doing ETW heap profiling?
bool bETWHeapEnabled = false;
s_bPageHeapEnabled = IsPageHeapEnabled( bETWHeapEnabled );
// Should we reserve the bottom 4 GB of RAM in order to flush out pointer
// truncation bugs? This helps ensure 64-bit compatibility.
// However this needs to be off by default to avoid causing compatibility problems,
// with Steam detours and other systems. It should also be disabled when PageHeap
// is on because for some reason the combination turns into 4 GB of working set, which
// can easily cause problems.
if ( strstr( upperCommandLine, "-RESERVELOWMEM" ) && !s_bPageHeapEnabled )
ReserveBottomMemory();
// Uninitialized data, including pointers, is often set to 0xFFEEFFEE.
// If we reserve that block of memory then we can turn these pointer
// dereferences into crashes a little bit earlier and more reliably.
// We don't really care whether this allocation succeeds, but it's
// worth trying. Note that we do this in all cases -- whether we are using
// -processheap or not.
VirtualAlloc( (void*)0xFFEEFFEE, 1, MEM_RESERVE, PAGE_NOACCESS );
// Enable application termination (breakpoint) on heap corruption. This is
// better than trying to patch it up and continue, both from a security and
// a bug-finding point of view. Do this always on Windows since the heap is
// used by video drivers and other in-proc components.
//HeapSetInformation( NULL, HeapEnableTerminationOnCorruption, NULL, 0 );
// The HeapEnableTerminationOnCorruption requires a recent platform SDK,
// so fake it up.
#if defined(PLATFORM_WINDOWS_PC)
HeapSetInformation( NULL, (HEAP_INFORMATION_CLASS)1, NULL, 0 );
#endif
bool bZeroMemory = false;
bool bProcessHeap = false;
// Should we force using the process heap? This is handy for gathering memory
// statistics with ETW/xperf. When using App Verifier -processheap is automatically
// turned on.
if ( strstr( upperCommandLine, "-PROCESSHEAP" ) )
{
bProcessHeap = true;
bZeroMemory = !!strstr( upperCommandLine, "-PROCESSHEAPZEROMEM" );
}
// Unless specifically disabled, turn on -processheap if pageheap or ETWHeap tracing
// are enabled.
if ( !strstr( upperCommandLine, "-NOPROCESSHEAP" ) && ( s_bPageHeapEnabled || bETWHeapEnabled ) )
bProcessHeap = true;
if ( bProcessHeap )
{
// Now all allocations will go through the system heap.
EnableHeapMemAlloc( bZeroMemory );
}
return bProcessHeap;
}
class CInitGlobalMemAllocPtr
{
public:
CInitGlobalMemAllocPtr()
{
char *pStr = (char*)Plat_GetCommandLineA();
if ( pStr )
{
char tempStr[512];
strncpy( tempStr, pStr, sizeof( tempStr ) - 1 );
tempStr[ sizeof( tempStr ) - 1 ] = 0;
_strupr( tempStr );
CheckWindowsAllocSettings( tempStr );
}
#if defined(FORCE_PROCESS_HEAP)
// This may cause EnableHeapMemAlloc to be called twice, but that's okay.
EnableHeapMemAlloc( false );
#endif
}
};
CInitGlobalMemAllocPtr sg_InitGlobalMemAllocPtr;
#endif
#ifdef _WIN32
//-----------------------------------------------------------------------------
// Small block heap (multi-pool)
//-----------------------------------------------------------------------------
#ifndef NO_SBH
#ifdef ALLOW_NOSBH
static bool g_UsingSBH = true;
#define UsingSBH() g_UsingSBH
#else
#define UsingSBH() true
#endif
#else
#define UsingSBH() false
#endif
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
template <typename T>
inline T MemAlign( T val, size_t alignment )
{
return (T)( ( (size_t)val + alignment - 1 ) & ~( alignment - 1 ) );
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
void CSmallBlockPool::Init( unsigned nBlockSize, byte *pBase, unsigned initialCommit )
{
if ( !( nBlockSize % MIN_SBH_ALIGN == 0 && nBlockSize >= MIN_SBH_BLOCK && nBlockSize >= sizeof(TSLNodeBase_t) ) )
DebuggerBreak();
m_nBlockSize = nBlockSize;
m_pCommitLimit = m_pNextAlloc = m_pBase = pBase;
m_pAllocLimit = m_pBase + MAX_POOL_REGION;
if ( initialCommit )
{
initialCommit = MemAlign( initialCommit, SBH_PAGE_SIZE );
if ( !VirtualAlloc( m_pCommitLimit, initialCommit, VA_COMMIT_FLAGS, PAGE_READWRITE ) )
{
Assert( 0 );
return;
}
m_pCommitLimit += initialCommit;
}
}
size_t CSmallBlockPool::GetBlockSize()
{
return m_nBlockSize;
}
bool CSmallBlockPool::IsOwner( void *p )
{
return ( p >= m_pBase && p < m_pAllocLimit );
}
void *CSmallBlockPool::Alloc()
{
void *pResult = m_FreeList.Pop();
if ( !pResult )
{
int nBlockSize = m_nBlockSize;
byte *pCommitLimit;
byte *pNextAlloc;
for (;;)
{
pCommitLimit = m_pCommitLimit;
pNextAlloc = m_pNextAlloc;
if ( pNextAlloc + nBlockSize <= pCommitLimit )
{
if ( m_pNextAlloc.AssignIf( pNextAlloc, pNextAlloc + m_nBlockSize ) )
{
pResult = pNextAlloc;
break;
}
}
else
{
AUTO_LOCK( m_CommitMutex );
if ( pCommitLimit == m_pCommitLimit )
{
if ( pCommitLimit + COMMIT_SIZE <= m_pAllocLimit )
{
if ( !VirtualAlloc( pCommitLimit, COMMIT_SIZE, VA_COMMIT_FLAGS, PAGE_READWRITE ) )
{
Assert( 0 );
return NULL;
}
m_pCommitLimit = pCommitLimit + COMMIT_SIZE;
}
else
{
return NULL;
}
}
}
}
}
return pResult;
}
void CSmallBlockPool::Free( void *p )
{
Assert( IsOwner( p ) );
m_FreeList.Push( p );
}
// Count the free blocks.
int CSmallBlockPool::CountFreeBlocks()
{
return m_FreeList.Count();
}
// Size of committed memory managed by this heap:
int CSmallBlockPool::GetCommittedSize()
{
unsigned totalSize = (unsigned)m_pCommitLimit - (unsigned)m_pBase;
Assert( 0 != m_nBlockSize );
return totalSize;
}
// Return the total blocks memory is committed for in the heap
int CSmallBlockPool::CountCommittedBlocks()
{
return GetCommittedSize() / GetBlockSize();
}
// Count the number of allocated blocks in the heap:
int CSmallBlockPool::CountAllocatedBlocks()
{
return CountCommittedBlocks( ) - ( CountFreeBlocks( ) + ( m_pCommitLimit - (byte *)m_pNextAlloc ) / GetBlockSize() );
}
int CSmallBlockPool::Compact()
{
int nBytesFreed = 0;
if ( m_FreeList.Count() )
{
int i;
int nFree = CountFreeBlocks();
FreeBlock_t **pSortArray = (FreeBlock_t **)malloc( nFree * sizeof(FreeBlock_t *) ); // can't use new because will reenter
if ( !pSortArray )
{
return 0;
}
i = 0;
while ( i < nFree )
{
pSortArray[i++] = m_FreeList.Pop();
}
std::sort( pSortArray, pSortArray + nFree );
byte *pOldNextAlloc = m_pNextAlloc;
for ( i = nFree - 1; i >= 0; i-- )
{
if ( (byte *)pSortArray[i] == m_pNextAlloc - m_nBlockSize )
{
pSortArray[i] = NULL;
m_pNextAlloc -= m_nBlockSize;
}
else
{
break;
}
}
if ( pOldNextAlloc != m_pNextAlloc )
{
byte *pNewCommitLimit = MemAlign( (byte *)m_pNextAlloc, SBH_PAGE_SIZE );
if ( pNewCommitLimit < m_pCommitLimit )
{
nBytesFreed = m_pCommitLimit - pNewCommitLimit;
VirtualFree( pNewCommitLimit, nBytesFreed, MEM_DECOMMIT );
m_pCommitLimit = pNewCommitLimit;
}
}
if ( pSortArray[0] )
{
for ( i = 0; i < nFree ; i++ )
{
if ( !pSortArray[i] )
{
break;
}
m_FreeList.Push( pSortArray[i] );
}
}
free( pSortArray );
}
return nBytesFreed;
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
#define GetInitialCommitForPool( i ) 0
CSmallBlockHeap::CSmallBlockHeap()
{
// Make sure that we return 64-bit addresses in 64-bit builds.
ReserveBottomMemory();
if ( !UsingSBH() )
{
return;
}
m_pBase = (byte *)VirtualAlloc( NULL, NUM_POOLS * MAX_POOL_REGION, VA_RESERVE_FLAGS, PAGE_NOACCESS );
m_pLimit = m_pBase + NUM_POOLS * MAX_POOL_REGION;
// Build a lookup table used to find the correct pool based on size
const int MAX_TABLE = MAX_SBH_BLOCK >> 2;
int i = 0;
int nBytesElement = 0;
byte *pCurBase = m_pBase;
CSmallBlockPool *pCurPool = NULL;
int iCurPool = 0;
#if _M_X64
// Blocks sized 0 - 256 are in pools in increments of 16
for ( ; i < 64 && i < MAX_TABLE; i++ )
{
if ( (i + 1) % 4 == 1)
{
nBytesElement += 16;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
#else
// Blocks sized 0 - 128 are in pools in increments of 8
for ( ; i < 32; i++ )
{
if ( (i + 1) % 2 == 1)
{
nBytesElement += 8;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 129 - 256 are in pools in increments of 16
for ( ; i < 64; i++ )
{
if ( (i + 1) % 4 == 1)
{
nBytesElement += 16;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
#endif
// Blocks sized 257 - 512 are in pools in increments of 32
for ( ; i < 128; i++ )
{
if ( (i + 1) % 8 == 1)
{
nBytesElement += 32;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 513 - 768 are in pools in increments of 64
for ( ; i < 192; i++ )
{
if ( (i + 1) % 16 == 1)
{
nBytesElement += 64;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 769 - 1024 are in pools in increments of 128
for ( ; i < 256; i++ )
{
if ( (i + 1) % 32 == 1)
{
nBytesElement += 128;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 1025 - 2048 are in pools in increments of 256
for ( ; i < MAX_TABLE; i++ )
{
if ( (i + 1) % 64 == 1)
{
nBytesElement += 256;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement, pCurBase, GetInitialCommitForPool(iCurPool) );
iCurPool++;
m_PoolLookup[i] = pCurPool;
pCurBase += MAX_POOL_REGION;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
Assert( iCurPool == NUM_POOLS );
}
bool CSmallBlockHeap::ShouldUse( size_t nBytes )
{
return ( UsingSBH() && nBytes <= MAX_SBH_BLOCK );
}
bool CSmallBlockHeap::IsOwner( void * p )
{
return ( UsingSBH() && p >= m_pBase && p < m_pLimit );
}
void *CSmallBlockHeap::Alloc( size_t nBytes )
{
if ( nBytes == 0)
{
nBytes = 1;
}
Assert( ShouldUse( nBytes ) );
CSmallBlockPool *pPool = FindPool( nBytes );
void *p = pPool->Alloc();
if ( p )
{
return p;
}
if ( s_StdMemAlloc.CallAllocFailHandler( nBytes ) >= nBytes )
{
p = pPool->Alloc();
if ( p )
{
return p;
}
}
void *pRet = malloc( nBytes );
if ( !pRet )
{
s_StdMemAlloc.SetCRTAllocFailed( nBytes );
}
return pRet;
}
void *CSmallBlockHeap::Realloc( void *p, size_t nBytes )
{
if ( nBytes == 0)
{
nBytes = 1;
}
CSmallBlockPool *pOldPool = FindPool( p );
CSmallBlockPool *pNewPool = ( ShouldUse( nBytes ) ) ? FindPool( nBytes ) : NULL;
if ( pOldPool == pNewPool )
{
return p;
}
void *pNewBlock = NULL;
if ( pNewPool )
{
pNewBlock = pNewPool->Alloc();
if ( !pNewBlock )
{
if ( s_StdMemAlloc.CallAllocFailHandler( nBytes ) >= nBytes )
{
pNewBlock = pNewPool->Alloc();
}
}
}
if ( !pNewBlock )
{
pNewBlock = malloc( nBytes );
if ( !pNewBlock )
{
s_StdMemAlloc.SetCRTAllocFailed( nBytes );
}
}
if ( pNewBlock )
{
int nBytesCopy = min( nBytes, pOldPool->GetBlockSize() );
memcpy( pNewBlock, p, nBytesCopy );
}
pOldPool->Free( p );
return pNewBlock;
}
void CSmallBlockHeap::Free( void *p )
{
CSmallBlockPool *pPool = FindPool( p );
pPool->Free( p );
}
size_t CSmallBlockHeap::GetSize( void *p )
{
CSmallBlockPool *pPool = FindPool( p );
return pPool->GetBlockSize();
}
void CSmallBlockHeap::DumpStats( FILE *pFile )
{
bool bSpew = true;
if ( pFile )
{
for ( int i = 0; i < NUM_POOLS; i++ )
{
// output for vxconsole parsing
fprintf( pFile, "Pool %i: Size: %llu Allocated: %i Free: %i Committed: %i CommittedSize: %i\n",
i,
(uint64)m_Pools[i].GetBlockSize(),
m_Pools[i].CountAllocatedBlocks(),
m_Pools[i].CountFreeBlocks(),
m_Pools[i].CountCommittedBlocks(),
m_Pools[i].GetCommittedSize() );
}
bSpew = false;
}
if ( bSpew )
{
unsigned bytesCommitted = 0;
unsigned bytesAllocated = 0;
for ( int i = 0; i < NUM_POOLS; i++ )
{
Msg( "Pool %i: (size: %llu) blocks: allocated:%i free:%i committed:%i (committed size:%u kb)\n",i, (uint64)m_Pools[i].GetBlockSize(),m_Pools[i].CountAllocatedBlocks(), m_Pools[i].CountFreeBlocks(),m_Pools[i].CountCommittedBlocks(), m_Pools[i].GetCommittedSize() / 1024);
bytesCommitted += m_Pools[i].GetCommittedSize();
bytesAllocated += ( m_Pools[i].CountAllocatedBlocks() * m_Pools[i].GetBlockSize() );
}
Msg( "Totals: Committed:%u kb Allocated:%u kb\n", bytesCommitted / 1024, bytesAllocated / 1024 );
}
}
int CSmallBlockHeap::Compact()
{
int nBytesFreed = 0;
for( int i = 0; i < NUM_POOLS; i++ )
{
nBytesFreed += m_Pools[i].Compact();
}
return nBytesFreed;
}
CSmallBlockPool *CSmallBlockHeap::FindPool( size_t nBytes )
{
return m_PoolLookup[(nBytes - 1) >> 2];
}
CSmallBlockPool *CSmallBlockHeap::FindPool( void *p )
{
size_t i = ((byte *)p - m_pBase) / MAX_POOL_REGION;
return &m_Pools[i];
}
#endif
#if USE_PHYSICAL_SMALL_BLOCK_HEAP
CX360SmallBlockPool *CX360SmallBlockPool::gm_AddressToPool[BYTES_X360_SBH/PAGESIZE_X360_SBH];
byte *CX360SmallBlockPool::gm_pPhysicalBlock;
byte *CX360SmallBlockPool::gm_pPhysicalBase;
byte *CX360SmallBlockPool::gm_pPhysicalLimit;
void CX360SmallBlockPool::Init( unsigned nBlockSize )
{
if ( !gm_pPhysicalBlock )
{
gm_pPhysicalBase = (byte *)XPhysicalAlloc( BYTES_X360_SBH, MAXULONG_PTR, 4096, PAGE_READWRITE | MEM_16MB_PAGES );
gm_pPhysicalLimit = gm_pPhysicalBase + BYTES_X360_SBH;
gm_pPhysicalBlock = gm_pPhysicalBase;
}
if ( !( nBlockSize % MIN_SBH_ALIGN == 0 && nBlockSize >= MIN_SBH_BLOCK && nBlockSize >= sizeof(TSLNodeBase_t) ) )
DebuggerBreak();
m_nBlockSize = nBlockSize;
m_pCurBlockEnd = m_pNextAlloc = NULL;
m_CommittedSize = 0;
}
size_t CX360SmallBlockPool::GetBlockSize()
{
return m_nBlockSize;
}
bool CX360SmallBlockPool::IsOwner( void *p )
{
return ( FindPool( p ) == this );
}
void *CX360SmallBlockPool::Alloc()
{
void *pResult = m_FreeList.Pop();
if ( !pResult )
{
if ( !m_pNextAlloc && gm_pPhysicalBlock >= gm_pPhysicalLimit )
{
return NULL;
}
int nBlockSize = m_nBlockSize;
byte *pCurBlockEnd;
byte *pNextAlloc;
for (;;)
{
pCurBlockEnd = m_pCurBlockEnd;
pNextAlloc = m_pNextAlloc;
if ( pNextAlloc + nBlockSize <= pCurBlockEnd )
{
if ( m_pNextAlloc.AssignIf( pNextAlloc, pNextAlloc + m_nBlockSize ) )
{
pResult = pNextAlloc;
break;
}
}
else
{
AUTO_LOCK( m_CommitMutex );
if ( pCurBlockEnd == m_pCurBlockEnd )
{
for (;;)
{
if ( gm_pPhysicalBlock >= gm_pPhysicalLimit )
{
m_pCurBlockEnd = m_pNextAlloc = NULL;
return NULL;
}
byte *pPhysicalBlock = gm_pPhysicalBlock;
if ( ThreadInterlockedAssignPointerIf( (void **)&gm_pPhysicalBlock, (void *)(pPhysicalBlock + PAGESIZE_X360_SBH), (void *)pPhysicalBlock ) )
{
int index = (size_t)((byte *)pPhysicalBlock - gm_pPhysicalBase) / PAGESIZE_X360_SBH;
gm_AddressToPool[index] = this;
m_pNextAlloc = pPhysicalBlock;
m_CommittedSize += PAGESIZE_X360_SBH;
__sync();
m_pCurBlockEnd = pPhysicalBlock + PAGESIZE_X360_SBH;
break;
}
}
}
}
}
}
return pResult;
}
void CX360SmallBlockPool::Free( void *p )
{
Assert( IsOwner( p ) );
m_FreeList.Push( p );
}
// Count the free blocks.
int CX360SmallBlockPool::CountFreeBlocks()
{
return m_FreeList.Count();
}
// Size of committed memory managed by this heap:
int CX360SmallBlockPool::GetCommittedSize()
{
return m_CommittedSize;
}
// Return the total blocks memory is committed for in the heap
int CX360SmallBlockPool::CountCommittedBlocks()
{
return GetCommittedSize() / GetBlockSize();
}
// Count the number of allocated blocks in the heap:
int CX360SmallBlockPool::CountAllocatedBlocks()
{
int nBytesPossible = ( m_pNextAlloc ) ? ( m_pCurBlockEnd - (byte *)m_pNextAlloc ) : 0;
return CountCommittedBlocks( ) - ( CountFreeBlocks( ) + nBytesPossible / GetBlockSize() );
}
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
#define GetInitialCommitForPool( i ) 0
CX360SmallBlockHeap::CX360SmallBlockHeap()
{
if ( !UsingSBH() )
{
return;
}
// Build a lookup table used to find the correct pool based on size
const int MAX_TABLE = MAX_SBH_BLOCK >> 2;
int i = 0;
int nBytesElement = 0;
CX360SmallBlockPool *pCurPool = NULL;
int iCurPool = 0;
// Blocks sized 0 - 128 are in pools in increments of 8
for ( ; i < 32; i++ )
{
if ( (i + 1) % 2 == 1)
{
nBytesElement += 8;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 129 - 256 are in pools in increments of 16
for ( ; i < 64; i++ )
{
if ( (i + 1) % 4 == 1)
{
nBytesElement += 16;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 257 - 512 are in pools in increments of 32
for ( ; i < 128; i++ )
{
if ( (i + 1) % 8 == 1)
{
nBytesElement += 32;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 513 - 768 are in pools in increments of 64
for ( ; i < 192; i++ )
{
if ( (i + 1) % 16 == 1)
{
nBytesElement += 64;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 769 - 1024 are in pools in increments of 128
for ( ; i < 256; i++ )
{
if ( (i + 1) % 32 == 1)
{
nBytesElement += 128;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
// Blocks sized 1025 - 2048 are in pools in increments of 256
for ( ; i < MAX_TABLE; i++ )
{
if ( (i + 1) % 64 == 1)
{
nBytesElement += 256;
pCurPool = &m_Pools[iCurPool];
pCurPool->Init( nBytesElement );
iCurPool++;
m_PoolLookup[i] = pCurPool;
}
else
{
m_PoolLookup[i] = pCurPool;
}
}
Assert( iCurPool == NUM_POOLS );
}
bool CX360SmallBlockHeap::ShouldUse( size_t nBytes )
{
return ( UsingSBH() && nBytes <= MAX_SBH_BLOCK );
}
bool CX360SmallBlockHeap::IsOwner( void * p )
{
int index = (size_t)((byte *)p - CX360SmallBlockPool::gm_pPhysicalBase) / PAGESIZE_X360_SBH;
return ( UsingSBH() && ( index >= 0 && index < ARRAYSIZE(CX360SmallBlockPool::gm_AddressToPool) ) );
}
void *CX360SmallBlockHeap::Alloc( size_t nBytes )
{
if ( nBytes == 0)
{
nBytes = 1;
}
Assert( ShouldUse( nBytes ) );
CX360SmallBlockPool *pPool = FindPool( nBytes );
void *p = pPool->Alloc();
if ( p )
{
return p;
}
return GetStandardSBH()->Alloc( nBytes );
}
void *CX360SmallBlockHeap::Realloc( void *p, size_t nBytes )
{
if ( nBytes == 0)
{
nBytes = 1;
}
CX360SmallBlockPool *pOldPool = FindPool( p );
CX360SmallBlockPool *pNewPool = ( ShouldUse( nBytes ) ) ? FindPool( nBytes ) : NULL;
if ( pOldPool == pNewPool )
{
return p;
}
void *pNewBlock = NULL;
if ( pNewPool )
{
pNewBlock = pNewPool->Alloc();
if ( !pNewBlock )
{
pNewBlock = GetStandardSBH()->Alloc( nBytes );
}
}
if ( !pNewBlock )
{
pNewBlock = malloc( nBytes );
}
if ( pNewBlock )
{
int nBytesCopy = min( nBytes, pOldPool->GetBlockSize() );
memcpy( pNewBlock, p, nBytesCopy );
}
pOldPool->Free( p );
return pNewBlock;
}
void CX360SmallBlockHeap::Free( void *p )
{
CX360SmallBlockPool *pPool = FindPool( p );
pPool->Free( p );
}
size_t CX360SmallBlockHeap::GetSize( void *p )
{
CX360SmallBlockPool *pPool = FindPool( p );
return pPool->GetBlockSize();
}
void CX360SmallBlockHeap::DumpStats( FILE *pFile )
{
bool bSpew = true;
if ( pFile )
{
for( int i = 0; i < NUM_POOLS; i++ )
{
// output for vxconsole parsing
fprintf( pFile, "Pool %i: Size: %u Allocated: %i Free: %i Committed: %i CommittedSize: %i\n",
i,
m_Pools[i].GetBlockSize(),
m_Pools[i].CountAllocatedBlocks(),
m_Pools[i].CountFreeBlocks(),
m_Pools[i].CountCommittedBlocks(),
m_Pools[i].GetCommittedSize() );
}
bSpew = false;
}
if ( bSpew )
{
unsigned bytesCommitted = 0;
unsigned bytesAllocated = 0;
for( int i = 0; i < NUM_POOLS; i++ )
{
bytesCommitted += m_Pools[i].GetCommittedSize();
bytesAllocated += ( m_Pools[i].CountAllocatedBlocks() * m_Pools[i].GetBlockSize() );
}
Msg( "Totals: Committed:%u kb Allocated:%u kb\n", bytesCommitted / 1024, bytesAllocated / 1024 );
}
}
CSmallBlockHeap *CX360SmallBlockHeap::GetStandardSBH()
{
return &(GET_OUTER( CStdMemAlloc, m_LargePageSmallBlockHeap )->m_SmallBlockHeap);
}
CX360SmallBlockPool *CX360SmallBlockHeap::FindPool( size_t nBytes )
{
return m_PoolLookup[(nBytes - 1) >> 2];
}
CX360SmallBlockPool *CX360SmallBlockHeap::FindPool( void *p )
{
return CX360SmallBlockPool::FindPool( p );
}
#endif
//-----------------------------------------------------------------------------
// Release versions
//-----------------------------------------------------------------------------
void *CStdMemAlloc::Alloc( size_t nSize )
{
PROFILE_ALLOC(Malloc);
void *pMem;
#ifdef _WIN32
#ifdef USE_PHYSICAL_SMALL_BLOCK_HEAP
if ( m_LargePageSmallBlockHeap.ShouldUse( nSize ) )
{
pMem = m_LargePageSmallBlockHeap.Alloc( nSize );
ApplyMemoryInitializations( pMem, nSize );
return pMem;
}
#endif
if ( m_SmallBlockHeap.ShouldUse( nSize ) )
{
pMem = m_SmallBlockHeap.Alloc( nSize );
ApplyMemoryInitializations( pMem, nSize );
return pMem;
}
#endif
pMem = malloc( nSize );
ApplyMemoryInitializations( pMem, nSize );
if ( !pMem )
{
SetCRTAllocFailed( nSize );
}
return pMem;
}
void *CStdMemAlloc::Realloc( void *pMem, size_t nSize )
{
if ( !pMem )
{
return Alloc( nSize );
}
PROFILE_ALLOC(Realloc);
#ifdef MEM_SBH_ENABLED
#ifdef USE_PHYSICAL_SMALL_BLOCK_HEAP
if ( m_LargePageSmallBlockHeap.IsOwner( pMem ) )
{
return m_LargePageSmallBlockHeap.Realloc( pMem, nSize );
}
#endif
if ( m_SmallBlockHeap.IsOwner( pMem ) )
{
return m_SmallBlockHeap.Realloc( pMem, nSize );
}
#endif
void *pRet = realloc( pMem, nSize );
if ( !pRet )
{
SetCRTAllocFailed( nSize );
}
return pRet;
}
void CStdMemAlloc::Free( void *pMem )
{
if ( !pMem )
{
return;
}
PROFILE_ALLOC(Free);
#ifdef MEM_SBH_ENABLED
#ifdef USE_PHYSICAL_SMALL_BLOCK_HEAP
if ( m_LargePageSmallBlockHeap.IsOwner( pMem ) )
{
m_LargePageSmallBlockHeap.Free( pMem );
return;
}
#endif
if ( m_SmallBlockHeap.IsOwner( pMem ) )
{
m_SmallBlockHeap.Free( pMem );
return;
}
#endif
free( pMem );
}
void *CStdMemAlloc::Expand_NoLongerSupported( void *pMem, size_t nSize )
{
return NULL;
}
//-----------------------------------------------------------------------------
// Debug versions
//-----------------------------------------------------------------------------
void *CStdMemAlloc::Alloc( size_t nSize, const char *pFileName, int nLine )
{
return CStdMemAlloc::Alloc( nSize );
}
void *CStdMemAlloc::Realloc( void *pMem, size_t nSize, const char *pFileName, int nLine )
{
return CStdMemAlloc::Realloc( pMem, nSize );
}
void CStdMemAlloc::Free( void *pMem, const char *pFileName, int nLine )
{
CStdMemAlloc::Free( pMem );
}
void *CStdMemAlloc::Expand_NoLongerSupported( void *pMem, size_t nSize, const char *pFileName, int nLine )
{
return NULL;
}
#if defined (LINUX)
#include <malloc.h>
#elif defined (OSX)
#define malloc_usable_size( ptr ) malloc_size( ptr )
extern "C" {
extern size_t malloc_size( const void *ptr );
}
#endif
//-----------------------------------------------------------------------------
// Returns size of a particular allocation
//-----------------------------------------------------------------------------
size_t CStdMemAlloc::GetSize( void *pMem )
{
#ifdef MEM_SBH_ENABLED
if ( !pMem )
return CalcHeapUsed();
else
{
#ifdef USE_PHYSICAL_SMALL_BLOCK_HEAP
if ( m_LargePageSmallBlockHeap.IsOwner( pMem ) )
{
return m_LargePageSmallBlockHeap.GetSize( pMem );
}
#endif
if ( m_SmallBlockHeap.IsOwner( pMem ) )
{
return m_SmallBlockHeap.GetSize( pMem );
}
return _msize( pMem );
}
#else
return malloc_usable_size( pMem );
#endif
}
//-----------------------------------------------------------------------------
// Force file + line information for an allocation
//-----------------------------------------------------------------------------
void CStdMemAlloc::PushAllocDbgInfo( const char *pFileName, int nLine )
{
}
void CStdMemAlloc::PopAllocDbgInfo()
{
}
//-----------------------------------------------------------------------------
// FIXME: Remove when we make our own heap! Crt stuff we're currently using
//-----------------------------------------------------------------------------
long CStdMemAlloc::CrtSetBreakAlloc( long lNewBreakAlloc )
{
return 0;
}
int CStdMemAlloc::CrtSetReportMode( int nReportType, int nReportMode )
{
return 0;
}
int CStdMemAlloc::CrtIsValidHeapPointer( const void *pMem )
{
return 1;
}
int CStdMemAlloc::CrtIsValidPointer( const void *pMem, unsigned int size, int access )
{
return 1;
}
int CStdMemAlloc::CrtCheckMemory( void )
{
return 1;
}
int CStdMemAlloc::CrtSetDbgFlag( int nNewFlag )
{
return 0;
}
void CStdMemAlloc::CrtMemCheckpoint( _CrtMemState *pState )
{
}
// FIXME: Remove when we have our own allocator
void* CStdMemAlloc::CrtSetReportFile( int nRptType, void* hFile )
{
return 0;
}
void* CStdMemAlloc::CrtSetReportHook( void* pfnNewHook )
{
return 0;
}
int CStdMemAlloc::CrtDbgReport( int nRptType, const char * szFile,
int nLine, const char * szModule, const char * pMsg )
{
return 0;
}
int CStdMemAlloc::heapchk()
{
#ifdef _WIN32
return _HEAPOK;
#else
return 1;
#endif
}
void CStdMemAlloc::DumpStats()
{
DumpStatsFileBase( "memstats" );
}
void CStdMemAlloc::DumpStatsFileBase( char const *pchFileBase )
{
#ifdef _WIN32
char filename[ 512 ];
_snprintf( filename, sizeof( filename ) - 1, ( IsX360() ) ? "D:\\%s.txt" : "%s.txt", pchFileBase );
filename[ sizeof( filename ) - 1 ] = 0;
FILE *pFile = fopen( filename, "wt" );
#ifdef USE_PHYSICAL_SMALL_BLOCK_HEAP
fprintf( pFile, "X360 Large Page SBH:\n" );
m_LargePageSmallBlockHeap.DumpStats(pFile);
#endif
fprintf( pFile, "\nSBH:\n" );
m_SmallBlockHeap.DumpStats(pFile); // Dump statistics to small block heap
#if defined( _X360 ) && !defined( _RETAIL )
XBX_rMemDump( filename );
#endif
fclose( pFile );
#endif
}
void CStdMemAlloc::GlobalMemoryStatus( size_t *pUsedMemory, size_t *pFreeMemory )
{
if ( !pUsedMemory || !pFreeMemory )
return;
#if defined ( _X360 )
// GlobalMemoryStatus tells us how much physical memory is free
MEMORYSTATUS stat;
::GlobalMemoryStatus( &stat );
*pFreeMemory = stat.dwAvailPhys;
// NOTE: we do not count free memory inside our small block heaps, as this could be misleading
// (even with lots of SBH memory free, a single allocation over 2kb can still fail)
#if defined( USE_DLMALLOC )
// Account for free memory contained within DLMalloc
for ( int i = 0; i < ARRAYSIZE( g_AllocRegions ); i++ )
{
mallinfo info = mspace_mallinfo( g_AllocRegions[ i ] );
*pFreeMemory += info.fordblks;
}
#endif
// Used is total minus free (discount the 32MB system reservation)
*pUsedMemory = ( stat.dwTotalPhys - 32*1024*1024 ) - *pFreeMemory;
#else
// no data
*pFreeMemory = 0;
*pUsedMemory = 0;
#endif
}
void CStdMemAlloc::CompactHeap()
{
#if !defined( NO_SBH ) && defined( _WIN32 )
int nBytesRecovered = m_SmallBlockHeap.Compact();
Msg( "Compact freed %d bytes\n", nBytesRecovered );
#endif
}
MemAllocFailHandler_t CStdMemAlloc::SetAllocFailHandler( MemAllocFailHandler_t pfnMemAllocFailHandler )
{
MemAllocFailHandler_t pfnPrevious = m_pfnFailHandler;
m_pfnFailHandler = pfnMemAllocFailHandler;
return pfnPrevious;
}
size_t CStdMemAlloc::DefaultFailHandler( size_t nBytes )
{
if ( IsX360() && !IsRetail() )
{
#ifdef _X360
ExecuteOnce(
{
char buffer[256];
_snprintf( buffer, sizeof( buffer ), "***** Memory pool overflow, attempted allocation size: %u ****\n", nBytes );
XBX_OutputDebugString( buffer );
}
);
#endif
}
return 0;
}
#if defined( _MEMTEST )
void CStdMemAlloc::void SetStatsExtraInfo( const char *pMapName, const char *pComment )
{
}
#endif
void CStdMemAlloc::SetCRTAllocFailed( size_t nSize )
{
m_sMemoryAllocFailed = nSize;
MemAllocOOMError( nSize );
}
size_t CStdMemAlloc::MemoryAllocFailed()
{
return m_sMemoryAllocFailed;
}
#endif
void ReserveBottomMemory()
{
// If we are running a 64-bit build then reserve all addresses below the
// 4 GB line to push as many pointers as possible above the line.
#ifdef PLATFORM_WINDOWS_PC64
// Avoid the cost of calling this multiple times.
static bool s_initialized = false;
if ( s_initialized )
return;
s_initialized = true;
// If AppVerifier is enabled then memory reservations get turned into committed
// memory in the working set. This means that ReserveBottomMemory() can end
// up adding almost 4 GB to the working set, which is a significant problem if
// you run many processes in parallel. Therefore, if vfbasics.dll (part of AppVerifier)
// is loaded, don't do the reservation.
HMODULE vfBasicsDLL = GetModuleHandle( "vfbasics.dll" );
if ( vfBasicsDLL )
return;
// Start by reserving large blocks of memory. When those reservations
// have exhausted the bottom 4 GB then halve the size and try again.
// The granularity for reserving address space is 64 KB so if we wanted
// to reserve every single page we would need to continue down to 64 KB.
// However stopping at 1 MB is sufficient because it prevents the Windows
// heap (and dlmalloc and the small block heap) from grabbing address space
// from the bottom 4 GB, while still allowing Steam to allocate a few pages
// for setting up detours.
const size_t LOW_MEM_LINE = 0x100000000LL;
size_t totalReservation = 0;
size_t numVAllocs = 0;
size_t numHeapAllocs = 0;
for ( size_t blockSize = 256 * 1024 * 1024; blockSize >= 1024 * 1024; blockSize /= 2 )
{
for (;;)
{
void* p = VirtualAlloc( 0, blockSize, MEM_RESERVE, PAGE_NOACCESS );
if ( !p )
break;
if ( (size_t)p >= LOW_MEM_LINE )
{
// We don't need this memory, so release it completely.
VirtualFree( p, 0, MEM_RELEASE );
break;
}
totalReservation += blockSize;
++numVAllocs;
}
}
// Now repeat the same process but making heap allocations, to use up the
// already committed heap blocks that are below the 4 GB line. Now we start
// with 64-KB allocations and proceed down to 16-byte allocations.
HANDLE heap = GetProcessHeap();
for ( size_t blockSize = 64 * 1024; blockSize >= 16; blockSize /= 2 )
{
for (;;)
{
void* p = HeapAlloc( heap, 0, blockSize );
if ( !p )
break;
if ( (size_t)p >= LOW_MEM_LINE )
{
// We don't need this memory, so release it completely.
HeapFree( heap, 0, p );
break;
}
totalReservation += blockSize;
++numHeapAllocs;
}
}
// Print diagnostics showing how many allocations we had to make in order to
// reserve all of low memory. In one test run it took 55 virtual allocs and
// 85 heap allocs. Note that since the process may have multiple heaps (each
// CRT seems to have its own) there is likely to be a few MB of address space
// that was previously reserved and is available to be handed out by some allocators.
//char buffer[1000];
//sprintf_s( buffer, "Reserved %1.3f MB (%d vallocs, %d heap allocs) to keep allocations out of low-memory.\n",
// totalReservation / (1024 * 1024.0), (int)numVAllocs, (int)numHeapAllocs );
// Can't use Msg here because it isn't necessarily initialized yet.
//OutputDebugString( buffer );
#endif
}
#endif // STEAM