Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

462 lines
13 KiB

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
//===========================================================================//
#ifndef PACKEDSTORE_H
#define PACKEDSTORE_H
#ifdef _WIN32
#pragma once
#endif
#include <tier0/platform.h>
#include <tier0/threadtools.h>
#include <tier0/tslist.h>
#include <tier2/tier2.h>
#include "filesystem.h"
#include "tier1/utlintrusivelist.h"
#include "tier1/utlvector.h"
#include "tier1/utllinkedlist.h"
#include "tier1/UtlSortVector.h"
#include "tier1/utlmap.h"
#include "tier1/checksum_md5.h"
#define VPK_ENABLE_SIGNING
const int k_nVPKDefaultChunkSize = 200 * 1024 * 1024;
class CPackedStore;
struct ChunkHashFraction_t
{
int m_nPackFileNumber;
int m_nFileFraction;
int m_cbChunkLen;
MD5Value_t m_md5contents;
};
class ChunkHashFractionLess_t
{
public:
bool Less( const ChunkHashFraction_t& lhs, const ChunkHashFraction_t& rhs, void *pContext )
{
if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber )
return true;
if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber )
return false;
if ( lhs.m_nFileFraction < rhs.m_nFileFraction )
return true;
if ( lhs.m_nFileFraction > rhs.m_nFileFraction )
return false;
return false;
}
};
class CPackedStoreFileHandle
{
public:
int m_nFileNumber;
int m_nFileOffset;
int m_nFileSize;
int m_nCurrentFileOffset;
void const *m_pMetaData;
uint16 m_nMetaDataSize;
CPackedStore *m_pOwner;
struct CFileHeaderFixedData *m_pHeaderData;
uint8 *m_pDirFileNamePtr; // pointer to basename in dir block
FORCEINLINE operator bool( void ) const
{
return ( m_nFileNumber != -1 );
}
FORCEINLINE int Read( void *pOutData, int nNumBytes );
CPackedStoreFileHandle( void )
{
m_nFileNumber = -1;
}
int Seek( int nOffset, int nWhence )
{
switch( nWhence )
{
case SEEK_CUR:
nOffset = m_nFileOffset + nOffset ;
break;
case SEEK_END:
nOffset = m_nFileSize + nOffset;
break;
}
m_nCurrentFileOffset = MAX( 0, MIN( m_nFileSize, nOffset ) );
return m_nCurrentFileOffset;
}
int Tell( void ) const
{
return m_nCurrentFileOffset;
}
uint32 GetFileCRCFromHeaderData() const
{
uint32 *pCRC = (uint32 *)m_pHeaderData;
return *pCRC;
}
FORCEINLINE void GetPackFileName( char *pchFileNameOut, int cchFileNameOut );
};
#define MAX_ARCHIVE_FILES_TO_KEEP_OPEN_AT_ONCE 512
#define PACKEDFILE_EXT_HASH_SIZE 15
#ifdef _WIN32
typedef HANDLE PackDataFileHandle_t;
#else
typedef FileHandle_t PackDataFileHandle_t;
#endif
struct FileHandleTracker_t
{
int m_nFileNumber;
PackDataFileHandle_t m_hFileHandle;
int m_nCurOfs;
CThreadFastMutex m_Mutex;
FileHandleTracker_t( void )
{
m_nFileNumber = -1;
}
};
enum ePackedStoreAddResultCode
{
EPADD_NEWFILE, // the file was added and is new
EPADD_ADDSAMEFILE, // the file was already present, and the contents are the same as what you passed.
EPADD_UPDATEFILE, // the file was alreayd present and its contents have been updated
EPADD_ERROR, // some error has resulted
};
// Describe a file inside of a VPK file. Is not memory efficient; only used for interface
// purposes and during file building
struct VPKContentFileInfo_t
{
CUtlString m_sName;
int m_idxChunk;
uint32 m_iTotalSize;
uint32 m_iOffsetInChunk;
uint32 m_iPreloadSize;
const void *m_pPreloadData;
//MD5Value_t m_md5Source; // source content before munging & release optimization. Used for incremental builds
uint32 m_crc; // CRC of actual file contents
/// Size of the data in the chunk file. (Excludes the preload data size)
uint32 GetSizeInChunkFile() const
{
Assert( m_iTotalSize >= m_iPreloadSize );
return m_iTotalSize - m_iPreloadSize;
}
VPKContentFileInfo_t()
{
m_idxChunk = -1;
m_iTotalSize = 0;
m_iOffsetInChunk = 0;
m_iPreloadSize = 0;
m_crc = 0;
m_pPreloadData = NULL;
//memset( m_md5Source.bits, 0, sizeof( m_md5Source.bits ) );
}
};
// a 1MB chunk of cached VPK data
// For CPackedStoreReadCache
struct CachedVPKRead_t
{
CachedVPKRead_t()
{
m_nPackFileNumber = 0;
m_nFileFraction = 0;
m_pubBuffer = NULL;
m_cubBuffer = 0;
m_idxLRU = -1;
m_hMD5RequestHandle= 0;
m_cFailedHashes = 0;
}
int m_nPackFileNumber; // identifier
int m_nFileFraction; // identifier
uint8 *m_pubBuffer; // data
int m_cubBuffer; // data
int m_idxLRU; // bookkeeping
int m_hMD5RequestHandle;// bookkeeping
int m_cFailedHashes; // did the MD5 match what it was supposed to?
MD5Value_t m_md5Value;
static bool Less( const CachedVPKRead_t& lhs, const CachedVPKRead_t& rhs )
{
if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber )
return true;
if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber )
return false;
if ( lhs.m_nFileFraction < rhs.m_nFileFraction )
return true;
if ( lhs.m_nFileFraction > rhs.m_nFileFraction )
return false;
return false;
}
};
// Read the VPK file in 1MB chunks
// and we hang on to those chunks so we can serve other reads out of the cache
// This sounds great, but is only of secondary importance.
// The primary reason we do this is so that the FileTracker can calculate the
// MD5 of the 1MB chunks asynchronously in another thread - while we hold
// the chunk in cache - making the MD5 calculation "free"
class CPackedStoreReadCache
{
public:
CPackedStoreReadCache( IBaseFileSystem *pFS );
bool ReadCacheLine( FileHandleTracker_t &fHandle, CachedVPKRead_t &cachedVPKRead );
bool BCanSatisfyFromReadCache( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead );
bool BCanSatisfyFromReadCacheInternal( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead );
bool CheckMd5Result( CachedVPKRead_t &cachedVPKRead );
int FindBufferToUse();
void RetryBadCacheLine( CachedVPKRead_t &cachedVPKRead );
void RetryAllBadCacheLines();
// cache 64 MB total
static const int k_nCacheBuffersToKeep = 4;
static const int k_cubCacheBufferSize = 0x00100000; // 1MB
static const int k_nCacheBufferMask = 0x7FF00000;
CThreadRWLock m_rwlock;
CUtlRBTree<CachedVPKRead_t> m_treeCachedVPKRead; // all the reads we have done
CTSQueue<CachedVPKRead_t> m_queueCachedVPKReadsRetry; // all the reads that have failed
CUtlLinkedList<CachedVPKRead_t> m_listCachedVPKReadsFailed; // all the reads that have failed
// current items in the cache
int m_cItemsInCache;
int m_rgCurrentCacheIndex[k_nCacheBuffersToKeep];
CInterlockedUInt m_rgLastUsedTime[k_nCacheBuffersToKeep];
CPackedStore *m_pPackedStore;
IBaseFileSystem *m_pFileSystem;
IThreadedFileMD5Processor *m_pFileTracker;
// stats
int m_cubReadFromCache;
int m_cReadFromCache;
int m_cDiscardsFromCache;
int m_cAddedToCache;
int m_cCacheMiss;
int m_cubCacheMiss;
int m_cFileErrors;
int m_cFileErrorsCorrected;
int m_cFileResultsDifferent;
};
class CPackedStore
{
public:
CPackedStore( char const *pFileBasename, char *pszFName, IBaseFileSystem *pFS, bool bOpenForWrite = false );
void RegisterFileTracker( IThreadedFileMD5Processor *pFileTracker ) { m_pFileTracker = pFileTracker; m_PackedStoreReadCache.m_pFileTracker = pFileTracker; }
CPackedStoreFileHandle OpenFile( char const *pFile );
CPackedStoreFileHandle GetHandleForHashingFiles();
/// Add/update the given file to the directory. Does not write any chunk files
void AddFileToDirectory( const VPKContentFileInfo_t &info );
/// Remove the specified file from the directory. Returns true if removed, false if not found
bool RemoveFileFromDirectory( const char *pszName );
/// Add file, writing file data to the end
/// of the current chunk
ePackedStoreAddResultCode AddFile( char const *pFile, uint16 nMetaDataSize, const void *pFileData, uint32 nFullFileSize, bool bMultiChunk, uint32 const *pCrcToUse = NULL );
// write out the file directory
void Write( void );
int ReadData( CPackedStoreFileHandle &handle, void *pOutData, int nNumBytes );
~CPackedStore( void );
FORCEINLINE void *DirectoryData( void )
{
return m_DirectoryData.Base();
}
// Get a list of all the files in the zip You are responsible for freeing the contents of
// outFilenames (call outFilenames.PurgeAndDeleteElements).
int GetFileList( CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput );
// Get a list of all files that match the given wildcard string
int GetFileList( const char *pWildCard, CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput );
/// Get a list of all files that match the given wildcard string, fetching all the details
/// at once
void GetFileList( const char *pWildcard, CUtlVector<VPKContentFileInfo_t> &outVecResults );
// Get a list of all directories of the given wildcard
int GetFileAndDirLists( const char *pWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput );
int GetFileAndDirLists( CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput );
bool IsEmpty( void ) const;
/// Hash metadata and chunk files
void HashEverything();
/// Hash all chunk files. Don't forget to rehash the metadata afterwords!
void HashAllChunkFiles();
/// Hash all the metadata. (Everything that's not in the chunk files)
void HashMetadata();
/// Re-hash a single chunk file. Don't forget to rehash the metadata afterwords!
void HashChunkFile( int iChunkFileIndex );
bool HashEntirePackFile( CPackedStoreFileHandle &handle, int64 &nFileSize, int nFileFraction, int nFractionSize, FileHash_t &fileHash );
void ComputeDirectoryHash( MD5Value_t &md5Directory );
void ComputeChunkHash( MD5Value_t &md5ChunkHashes );
MD5Value_t &GetDirFileMD5Value() { return m_TotalFileMD5; }
bool BTestDirectoryHash();
bool BTestMasterChunkHash();
CUtlSortVector<ChunkHashFraction_t, ChunkHashFractionLess_t > &AccessPackFileHashes() { return m_vecChunkHashFraction; }
bool FindFileHashFraction( int nPackFileNumber, int nFileFraction, ChunkHashFraction_t &chunkFileHashFraction );
void GetPackFileLoadErrorSummary( CUtlString &sErrors );
void GetPackFileName( CPackedStoreFileHandle &handle, char *pchFileNameOut, int cchFileNameOut ) const;
void GetDataFileName( char *pchFileNameOut, int cchFileNameOut, int nFileNumber ) const;
char const *BaseName( void )
{
return m_pszFileBaseName;
}
char const *FullPathName( void )
{
return m_pszFullPathName;
}
void SetWriteChunkSize( int nWriteChunkSize )
{
m_nWriteChunkSize = nWriteChunkSize;
}
int GetWriteChunkSize() const { return m_nWriteChunkSize; }
int GetHighestChunkFileIndex() { return m_nHighestChunkFileIndex; }
void DiscardChunkHashes( int iChunkFileIndex );
const CUtlVector<uint8> &GetSignaturePublicKey() const { return m_SignaturePublicKey; }
const CUtlVector<uint8> &GetSignature() const { return m_Signature; }
#ifdef VPK_ENABLE_SIGNING
enum ESignatureCheckResult
{
eSignatureCheckResult_NotSigned,
eSignatureCheckResult_WrongKey,
eSignatureCheckResult_Failed, // IO error, etc
eSignatureCheckResult_InvalidSignature,
eSignatureCheckResult_ValidSignature,
};
ESignatureCheckResult CheckSignature( int nSignatureSize, const void *pSignature ) const;
void SetKeysForSigning( int nPrivateKeySize, const void *pPrivateKeyData, int nPublicKeySize, const void *pPublicKeyData );
#endif
void SetUseDirFile() { m_bUseDirFile = true; }
int m_PackFileID;
private:
char m_pszFileBaseName[MAX_PATH];
char m_pszFullPathName[MAX_PATH];
int m_nDirectoryDataSize;
int m_nWriteChunkSize;
bool m_bUseDirFile;
IBaseFileSystem *m_pFileSystem;
IThreadedFileMD5Processor *m_pFileTracker;
CThreadFastMutex m_Mutex;
CPackedStoreReadCache m_PackedStoreReadCache;
CUtlIntrusiveList<class CFileExtensionData> m_pExtensionData[PACKEDFILE_EXT_HASH_SIZE];
CUtlVector<uint8> m_DirectoryData;
CUtlBlockVector<uint8> m_EmbeddedChunkData;
CUtlSortVector<ChunkHashFraction_t, ChunkHashFractionLess_t > m_vecChunkHashFraction;
bool BFileContainedHashes() { return m_vecChunkHashFraction.Count() > 0; }
// these are valid if BFileContainedHashes() is true
MD5Value_t m_DirectoryMD5;
MD5Value_t m_ChunkHashesMD5;
MD5Value_t m_TotalFileMD5;
int m_nHighestChunkFileIndex;
/// The private key that will be used to sign the directory file.
/// This will be empty for unsigned VPK's, or if we don't know the
/// private key.
CUtlVector<uint8> m_SignaturePrivateKey;
/// The public key in the VPK.
CUtlVector<uint8> m_SignaturePublicKey;
/// The signature that was read / computed
CUtlVector<uint8> m_Signature;
/// The number of bytes in the dir file that were signed
uint32 m_nSizeOfSignedData;
FileHandleTracker_t m_FileHandles[MAX_ARCHIVE_FILES_TO_KEEP_OPEN_AT_ONCE];
void Init( void );
struct CFileHeaderFixedData *FindFileEntry(
char const *pDirname, char const *pBaseName, char const *pExtension,
uint8 **pExtBaseOut = NULL, uint8 **pNameBaseOut = NULL );
void BuildHashTables( void );
FileHandleTracker_t &GetFileHandle( int nFileNumber );
void CloseWriteHandle( void );
// For cache-ing directory and contents data
CUtlStringList m_directoryList; // The index of this list of directories...
CUtlMap<int, CUtlStringList*> m_dirContents; // ...is the key to this map of filenames
void BuildFindFirstCache();
bool InternalRemoveFileFromDirectory( const char *pszName );
friend class CPackedStoreReadCache;
};
FORCEINLINE int CPackedStoreFileHandle::Read( void *pOutData, int nNumBytes )
{
return m_pOwner->ReadData( *this, pOutData, nNumBytes );
}
FORCEINLINE void CPackedStoreFileHandle::GetPackFileName( char *pchFileNameOut, int cchFileNameOut )
{
m_pOwner->GetPackFileName( *this, pchFileNameOut, cchFileNameOut );
}
#endif // packedtsore_h