Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

453 lines
12 KiB

5 years ago
//-----------------------------------------------------------------------------
// NOTE! This should never be called directly from leaf code
// Just use new,delete,malloc,free etc. They will call into this eventually
//-----------------------------------------------------------------------------
#include "pch_tier0.h"
#if IS_WINDOWS_PC
#define WIN_32_LEAN_AND_MEAN
#include <windows.h>
#define VA_COMMIT_FLAGS MEM_COMMIT
#define VA_RESERVE_FLAGS MEM_RESERVE
#elif defined( _X360 )
#undef Verify
#define _XBOX
#include <xtl.h>
#undef _XBOX
#include "xbox/xbox_win32stubs.h"
#define VA_COMMIT_FLAGS (MEM_COMMIT|MEM_NOZERO|MEM_LARGE_PAGES)
#define VA_RESERVE_FLAGS (MEM_RESERVE|MEM_LARGE_PAGES)
#elif defined( _PS3 )
#include "sys/memory.h"
#include "sys/mempool.h"
#include "sys/process.h"
#include <sys/vm.h>
#endif
//#include <malloc.h>
#include <algorithm>
#include "tier0/dbg.h"
#include "tier0/memalloc.h"
#include "tier0/threadtools.h"
#include "tier0/tslist.h"
#include "mem_helpers.h"
#ifndef _PS3
#pragma pack(4)
#endif
#define MIN_SBH_BLOCK 8
#define MIN_SBH_ALIGN 8
#define MAX_SBH_BLOCK 2048
#define MAX_POOL_REGION (4*1024*1024)
#define NUM_POOLS 42
#if defined( _WIN32 ) || defined( _PS3 )
// FIXME: Disable small block heap on win64 for now; it's busted because
// it's expecting SLIST_HEADER to look different than it does on win64
#if !defined( PLATFORM_WINDOWS_PC64 )
#define MEM_SBH_ENABLED 1
#endif
#endif
#if !defined(_CERT) && ( defined(_X360) || defined(_PS3) )
#define TRACK_SBH_COUNTS
#endif
#if defined(_X360)
// 360 uses a 48MB primary (physical) SBH and 10MB secondary (virtual) SBH, with no fallback
#define MBYTES_PRIMARY_SBH 48
#define MEMALLOC_USE_SECONDARY_SBH
#define MBYTES_SECONDARY_SBH 10
#define MEMALLOC_NO_FALLBACK
#elif defined(_PS3)
// PS3 uses just a 32MB SBH - this was enough to avoid overflow when Portal 2 shipped.
// NOTE: when Steam uses the game's tier0 allocator (see memalloc.h), we increase the size
// of the SBH and MBH (see memstd.cpp) to accommodate those extra allocations.
#define MBYTES_PRIMARY_SBH ( 32 + MBYTES_STEAM_SBH_USAGE )
#define MEMALLOC_NO_FALLBACK
#else // _X360 | _PS3
// Other platforms use a 48MB primary SBH and a (32MB) fallback SBH
#define MBYTES_PRIMARY_SBH 48
#endif // _X360 | _PS3
#define MEMSTD_COMPILE_TIME_ASSERT( pred ) switch(0){case 0:case pred:;}
//-----------------------------------------------------------------------------
// Small block pool
//-----------------------------------------------------------------------------
class CFreeList : public CTSListBase
{
public:
void Push( void *p ) { CTSListBase::Push( (TSLNodeBase_t *)p ); }
byte *Pop() { return (byte *)CTSListBase::Pop(); }
};
template <typename CAllocator>
class CSmallBlockHeap;
template <typename CAllocator>
class CSmallBlockPool
{
public:
CSmallBlockPool()
{
m_nBlockSize = 0;
m_nCommittedPages = 0;
m_pFirstPage = NULL;
}
void Init( unsigned nBlockSize );
size_t GetBlockSize();
void *Alloc();
void Free( void *p );
int CountFreeBlocks();
int GetCommittedSize();
int CountCommittedBlocks();
int CountAllocatedBlocks();
size_t Compact( bool bIncremental );
bool Validate();
enum
{
BYTES_PAGE = CAllocator::BYTES_PAGE,
NOT_COMMITTED = -1
};
private:
typedef CSmallBlockHeap<CAllocator> CHeap;
friend class CSmallBlockHeap<CAllocator>;
struct PageStatus_t : public TSLNodeBase_t
{
PageStatus_t()
{
m_pPool = NULL;
m_nAllocated = NOT_COMMITTED;
m_pNextPageInPool = NULL;
}
CSmallBlockPool<CAllocator> * m_pPool;
PageStatus_t * m_pNextPageInPool;
CInterlockedInt m_nAllocated;
CTSListBase m_SortList;
};
struct SharedData_t
{
CAllocator m_Allocator;
CTSListBase m_FreePages;
CThreadSpinRWLock m_Lock;
PageStatus_t m_PageStatus[CAllocator::TOTAL_BYTES/CAllocator::BYTES_PAGE];
byte * m_pNextBlock;
byte * m_pBase;
byte * m_pLimit;
};
static int PageSort( const void *p1, const void *p2 ) ;
bool RemovePagesFromFreeList( byte **pPages, int nPages, bool bSortList );
void ValidateFreelist( SharedData_t *pSharedData );
CFreeList m_FreeList;
CInterlockedPtr<byte> m_pNextAlloc;
PageStatus_t * m_pFirstPage;
unsigned m_nBlockSize;
unsigned m_nCommittedPages;
CThreadFastMutex m_CommitMutex;
#ifdef TRACK_SBH_COUNTS
CInterlockedInt m_nFreeBlocks;
#endif
static SharedData_t *GetSharedData()
{
return &gm_SharedData;
}
static SharedData_t gm_SharedData;
};
//-----------------------------------------------------------------------------
// Small block heap (multi-pool)
//-----------------------------------------------------------------------------
template <typename CAllocator>
class CSmallBlockHeap
{
public:
CSmallBlockHeap();
bool ShouldUse( size_t nBytes );
bool IsOwner( void * p );
void *Alloc( size_t nBytes );
void *Realloc( void *p, size_t nBytes );
void Free( void *p );
size_t GetSize( void *p );
void DumpStats( const char *pszTag, FILE *pFile = NULL );
void Usage( size_t &bytesCommitted, size_t &bytesAllocated );
size_t Compact( bool bIncremental );
bool Validate();
enum
{
BYTES_PAGE = CAllocator::BYTES_PAGE
};
private:
typedef CSmallBlockPool<CAllocator> CPool;
typedef struct CSmallBlockPool<CAllocator>::SharedData_t SharedData_t;
CPool *FindPool( size_t nBytes );
CPool *FindPool( void *p );
// Map size to a pool address to a pool
CPool *m_PoolLookup[MAX_SBH_BLOCK >> 2];
CPool m_Pools[NUM_POOLS];
SharedData_t *m_pSharedData;
};
//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
class CStdMemAlloc : public IMemAlloc
{
public:
CStdMemAlloc();
// Internal versions
void *InternalAlloc( int region, size_t nSize );
#ifdef MEMALLOC_SUPPORTS_ALIGNED_ALLOCATIONS
void *InternalAllocAligned( int region, size_t nSize, size_t align );
#endif
void *InternalAllocFromPools( size_t nSize );
void *InternalRealloc( void *pMem, size_t nSize );
#ifdef MEMALLOC_SUPPORTS_ALIGNED_ALLOCATIONS
void *InternalReallocAligned( void *pMem, size_t nSize, size_t align );
#endif
void InternalFree( void *pMem );
void CompactOnFail();
// Release versions
virtual void *Alloc( size_t nSize );
virtual void *Realloc( void *pMem, size_t nSize );
virtual void Free( void *pMem );
virtual void *Expand_NoLongerSupported( void *pMem, size_t nSize );
// Debug versions
virtual void *Alloc( size_t nSize, const char *pFileName, int nLine );
virtual void *Realloc( void *pMem, size_t nSize, const char *pFileName, int nLine );
virtual void Free( void *pMem, const char *pFileName, int nLine );
virtual void *Expand_NoLongerSupported( void *pMem, size_t nSize, const char *pFileName, int nLine );
#ifdef MEMALLOC_SUPPORTS_ALIGNED_ALLOCATIONS
virtual void *AllocAlign( size_t nSize, size_t align );
virtual void *AllocAlign( size_t nSize, size_t align, const char *pFileName, int nLine );
virtual void *ReallocAlign( void *pMem, size_t nSize, size_t align );
virtual void *ReallocAlign( void *pMem, size_t nSize, size_t align, const char *pFileName, int nLine );
#endif
virtual void *RegionAlloc( int region, size_t nSize );
virtual void *RegionAlloc( int region, size_t nSize, const char *pFileName, int nLine );
// Returns size of a particular allocation
virtual size_t GetSize( void *pMem );
// Force file + line information for an allocation
virtual void PushAllocDbgInfo( const char *pFileName, int nLine );
virtual void PopAllocDbgInfo();
virtual int32 CrtSetBreakAlloc( int32 lNewBreakAlloc );
virtual int CrtSetReportMode( int nReportType, int nReportMode );
virtual int CrtIsValidHeapPointer( const void *pMem );
virtual int CrtIsValidPointer( const void *pMem, unsigned int size, int access );
virtual int CrtCheckMemory( void );
virtual int CrtSetDbgFlag( int nNewFlag );
virtual void CrtMemCheckpoint( _CrtMemState *pState );
void* CrtSetReportFile( int nRptType, void* hFile );
void* CrtSetReportHook( void* pfnNewHook );
int CrtDbgReport( int nRptType, const char * szFile,
int nLine, const char * szModule, const char * pMsg );
virtual int heapchk();
virtual void DumpStats();
virtual void DumpStatsFileBase( char const *pchFileBase );
virtual size_t ComputeMemoryUsedBy( char const *pchSubStr );
virtual void GlobalMemoryStatus( size_t *pUsedMemory, size_t *pFreeMemory );
virtual bool IsDebugHeap() { return false; }
virtual void GetActualDbgInfo( const char *&pFileName, int &nLine ) {}
virtual void RegisterAllocation( const char *pFileName, int nLine, size_t nLogicalSize, size_t nActualSize, unsigned nTime ) {}
virtual void RegisterDeallocation( const char *pFileName, int nLine, size_t nLogicalSize, size_t nActualSize, unsigned nTime ) {}
virtual int GetVersion() { return MEMALLOC_VERSION; }
virtual void OutOfMemory( size_t nBytesAttempted = 0 ) { SetCRTAllocFailed( nBytesAttempted ); }
virtual IVirtualMemorySection * AllocateVirtualMemorySection( size_t numMaxBytes );
virtual int GetGenericMemoryStats( GenericMemoryStat_t **ppMemoryStats );
virtual void CompactHeap();
virtual void CompactIncremental();
virtual MemAllocFailHandler_t SetAllocFailHandler( MemAllocFailHandler_t pfnMemAllocFailHandler );
size_t CallAllocFailHandler( size_t nBytes ) { return (*m_pfnFailHandler)( nBytes); }
virtual uint32 GetDebugInfoSize() { return 0; }
virtual void SaveDebugInfo( void *pvDebugInfo ) { }
virtual void RestoreDebugInfo( const void *pvDebugInfo ) {}
virtual void InitDebugInfo( void *pvDebugInfo, const char *pchRootFileName, int nLine ) {}
static size_t DefaultFailHandler( size_t );
void DumpBlockStats( void *p ) {}
#if MEM_SBH_ENABLED
class CVirtualAllocator
{
public:
enum
{
BYTES_PAGE = (64*1024),
TOTAL_BYTES = (32*1024*1024),
MIN_RESERVE_PAGES = 4,
};
byte *AllocatePoolMemory()
{
#ifdef _WIN32
return (byte *)VirtualAlloc( NULL, TOTAL_BYTES, VA_RESERVE_FLAGS, PAGE_NOACCESS );
#elif defined( _PS3 )
Error( "" );
return NULL;
#else
#error
#endif
}
bool IsVirtual()
{
return true;
}
bool Decommit( void *pPage )
{
#ifdef _WIN32
return ( VirtualFree( pPage, BYTES_PAGE, MEM_DECOMMIT ) != 0 );
#elif defined( _PS3 )
return false;
#else
#error
#endif
}
bool Commit( void *pPage )
{
#ifdef _WIN32
return ( VirtualAlloc( pPage, BYTES_PAGE, VA_COMMIT_FLAGS, PAGE_READWRITE ) != NULL );
#elif defined( _PS3 )
return false;
#else
#error
#endif
}
};
typedef CSmallBlockHeap<CVirtualAllocator> CVirtualSmallBlockHeap;
template <size_t SIZE_MB, bool bPhysical>
class CFixedAllocator
{
public:
enum
{
BYTES_PAGE = (16*1024),
TOTAL_BYTES = (SIZE_MB*1024*1024),
MIN_RESERVE_PAGES = TOTAL_BYTES/BYTES_PAGE,
};
byte *AllocatePoolMemory()
{
#ifdef _WIN32
#ifdef _X360
if ( bPhysical )
return (byte *)XPhysicalAlloc( TOTAL_BYTES, MAXULONG_PTR, 4096, PAGE_READWRITE | MEM_16MB_PAGES );
#endif
return (byte *)VirtualAlloc( NULL, TOTAL_BYTES, VA_COMMIT_FLAGS, PAGE_READWRITE );
#elif defined( _PS3 )
// TODO: release this section on shutdown (use GetMemorySectionForAddress)
extern IVirtualMemorySection * VirtualMemoryManager_AllocateVirtualMemorySection( size_t numMaxBytes );
IVirtualMemorySection *pSection = VirtualMemoryManager_AllocateVirtualMemorySection( TOTAL_BYTES );
if ( !pSection )
Error( "CFixedAllocator::AllocatePoolMemory() failed in VirtualMemoryManager_AllocateVirtualMemorySection\n" );
if ( !pSection->CommitPages( pSection->GetBaseAddress(), TOTAL_BYTES ) )
Error( "CFixedAllocator::AllocatePoolMemory() failed in IVirtualMemorySection::CommitPages\n" );
return reinterpret_cast<byte *>( pSection->GetBaseAddress() );
#else
#error
#endif
}
bool IsVirtual()
{
return false;
}
bool Decommit( void *pPage )
{
return false;
}
bool Commit( void *pPage )
{
return false;
}
};
typedef CSmallBlockHeap<CFixedAllocator< MBYTES_PRIMARY_SBH, true> > CFixedSmallBlockHeap;
#ifdef MEMALLOC_USE_SECONDARY_SBH
typedef CSmallBlockHeap<CFixedAllocator< MBYTES_SECONDARY_SBH, false> > CFixedVirtualSmallBlockHeap; // @TODO: move back into above heap if number stays at 16 [7/15/2009 tom]
#endif
CFixedSmallBlockHeap m_PrimarySBH;
#ifdef MEMALLOC_USE_SECONDARY_SBH
CFixedVirtualSmallBlockHeap m_SecondarySBH;
#endif
#ifndef MEMALLOC_NO_FALLBACK
CVirtualSmallBlockHeap m_FallbackSBH;
#endif
#endif // MEM_SBH_ENABLED
virtual void SetStatsExtraInfo( const char *pMapName, const char *pComment );
virtual size_t MemoryAllocFailed();
void SetCRTAllocFailed( size_t nMemSize );
MemAllocFailHandler_t m_pfnFailHandler;
size_t m_sMemoryAllocFailed;
CThreadFastMutex m_CompactMutex;
bool m_bInCompact;
};
#ifndef _PS3
#pragma pack()
#endif