d4708
/
source-engine
mirror of https://github.com/nillerusr/source-engine.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
142 Commits
25 Branches
6 Tags
221 MiB
Tree: 0772ca1063
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '0772ca1063'
${ noResults }
source-engine/gcsdk/gcjob.cpp

1591 lines
59 KiB
Raw Normal View History Unescape

1
5 years ago
//========= Copyright Valve Corporation, All rights reserved. ============//
#include "stdafx.h"
#include "msgprotobuf.h"
#include "smartptr.h"
#include "rtime.h"
#include "gcsdk/gcreportprinter.h"
#include <winsock.h>
// memdbgon must be the last include file in a .cpp file!!!
#include "tier0/memdbgon.h"
namespace GCSDK
{
GCConVar cv_webapi_result_size_limit( "webapi_result_size_limit", "50000000" );
GCConVar cv_webapi_serialize_threads( "webapi_serialize_threads", "1", 0, "Switch for serializing webAPI responses on threads" );
static GCConVar webapi_account_tracking( "webapi_account_tracking", "1", "Controls whether or not account tracking stats are collected for web api usage" );
static GCConVar webapi_kill_switch( "webapi_kill_switch", "0", "When set to zero this will block no web api calls, when set to 1 this will block all web api except those sent by steam. To block steam, use the webapi_enable_steam_<priority> controls" );
static GCConVar webapi_kill_switch_error_response( "webapi_kill_switch_error_response", "1", "Determines if a response should be sent when the kill switch kills a web api request" );
static GCConVar webapi_rate_limit_calls_per_min( "webapi_rate_limit_calls_per_min", "600", "Determines how many messages can be sent from an account via the web api per minute. <0 disables this limiting" );
static GCConVar webapi_rate_limit_mb_per_min( "webapi_rate_limit_mb_per_min", "20", "Determines how many megabytes of data can be sent to an account via the web api per minute. <0 disables this limiting" );
static GCConVar webapi_elevated_rate_limit_calls_per_min( "webapi_elevated_rate_limit_calls_per_min", "-1", "Determines how many messages can be sent from an account via the web api per minute for elevated accounts. <0 disables this limiting" );
static GCConVar webapi_elevated_rate_limit_mb_per_min( "webapi_elevated_rate_limit_mb_per_min", "-1", "Determines how many megabytes of data can be sent to an account via the web api per minute for elevated accounts. <0 disables this limiting" );
static GCConVar webapi_ip_rate_limit( "webapi_ip_rate_limit", "1", "Controls whether or not we rate limit based upon IPs or just accounts" );
//-----------------------------------------------------------------------------
// CWebAPIAccountTracker
//
// Utility that tracks web api calls based upon accesses made by various users
//-----------------------------------------------------------------------------
class CWebAPIAccountTracker
{
public:
//called when a web api request is made to track the call. A return value of true indicates that it should be allowed,
//false indicates it should be blocked
bool TrackUser( AccountID_t nID, uint32 nIP );
//called once the size of the response is known and will track bandwidth and caller attributed to the provided function
void TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize );
//called to reset all permissions to default
void ResetAccountPermissions();
//called to associate a permission level with an account
void SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel );
//completely resets accumulated stats
void ResetStats();
//resets just the profile stats
void ResetProfileStats();
//different caller report filters that can be used
enum EDumpCaller
{
eDumpCaller_All,
eDumpCaller_Blocked,
eDumpCaller_Status,
eDumpCaller_Calls,
};
//different reports that can be provided
void DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const;
void DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter = NULL ) const;
void DumpCaller( AccountID_t nID ) const;
void DumpIP( uint32 nIP ) const;
void DumpFunctions() const;
void DumpProfile( bool bAllTime ) const;
void DumpSteamServers() const;
//given a steam server name, this will return the identifier of that server
uint32 GetSteamServerID( const char* pszServer );
//for steam level requests, we have a priority provided, and to track stats separately we break them up to different account IDs instead of just zero
static const uint32 k_nSteamIP_High = 2;
static const uint32 k_nSteamIP_Normal = 1;
static const uint32 k_nSteamIP_Low = 0;
private:
//called to get the starting time of this rate interval
RTime32 GetRateIntervalStart( AccountID_t nCaller ) const;
struct SCallerStats
{
SCallerStats() :
m_nBlockedCalls( 0 ),
m_eLevel( eWebAPIAccountLevel_RateLimited )
{
ResetRateInterval( 0 );
}
void ResetRateInterval( RTime32 nStart )
{
m_nRateIntervalStartTime = nStart;
m_nRateIntervalCalls = 0;
m_nRateIntervalBytes = 0;
}
//the most recent rate interval that we received a message from the user (used to expire old counts)
RTime32 m_nRateIntervalStartTime;
//how many messages have been sent within this rate interval (used for rate limiting)
uint32 m_nRateIntervalCalls;
//how many bytes have been sent for this account during this interval
uint32 m_nRateIntervalBytes;
//total number of blocked calls
uint32 m_nBlockedCalls;
//flags associated with this caller, used to block/whitelist, etc
EWebAPIAccountLevel m_eLevel;
};
struct SFunctionStats
{
//track the number of calls and bandwidth. The profile versions are separate and used for displaying profiles over a window
uint32 m_nTotalCalls;
uint32 m_nProfileCalls;
uint32 m_nMaxBytes;
uint32 m_nProfileMaxBytes;
uint64 m_nTotalBytes;
uint64 m_nProfileBytes;
//a map of who has called us, which consists of the caller account and IP as the key
struct SCaller
{
bool operator==( const SCaller& rhs ) const { return m_nAccountID == rhs.m_nAccountID && m_nIP == rhs.m_nIP; }
AccountID_t m_nAccountID;
uint32 m_nIP;
};
struct SCalls
{
uint32 m_nCalls;
uint64 m_nBytes;
};
CUtlHashMapLarge< SCaller, SCalls > m_Callers;
};
//a structure used to simplify reporting so a vector can just be built of these, and then provided to the report function which will handle sorting it and displaying
struct SReportRow
{
SReportRow( const char* pszFunction, uint32 nCalls, uint64 nSize ) :
m_pszFunction( pszFunction ),
m_nCalls( nCalls ),
m_nSize( nSize )
{}
const char* m_pszFunction;
uint32 m_nCalls;
uint64 m_nSize;
};
struct SSteamServer
{
CUtlString m_sName;
uint32 m_nID;
};
//called to find an existing user, or create one if not in the list already
SCallerStats* CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart );
SCallerStats* CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart );
//called to print a report of the provided report rows as either an ID list or a function list. This will re-sort the provided vector
static void PrintReport( const CUtlVector< SReportRow >& vec );
//how many seconds are in a rate interval
static const uint32 knRateIntervalTimeS = 60;
CUtlHashMapLarge< AccountID_t, SCallerStats > m_AccountCallers;
CUtlHashMapLarge< uint32, SCallerStats > m_IPCallers;
CUtlHashMapLarge< uintp, SFunctionStats* > m_Functions;
CUtlHashMapLarge< const char*, SSteamServer*, CaseSensitiveStrEquals, MurmurHash3ConstCharPtr > m_SteamServers;
CJobTime m_ProfileTime;
};
//our global profiler
static CWebAPIAccountTracker g_WebAPIAccountTracker;
void WebAPIAccount_ResetAllPermissions()
{
g_WebAPIAccountTracker.ResetAccountPermissions();
}
void WebAPIAccount_SetPermission( AccountID_t nID, EWebAPIAccountLevel eLevel )
{
g_WebAPIAccountTracker.SetAccountPermission( nID, eLevel );
}
bool WebAPIAccount_BTrackUserAndValidate( AccountID_t nID, uint32 unIP )
{
return g_WebAPIAccountTracker.TrackUser( nID, unIP );
}
RTime32 CWebAPIAccountTracker::GetRateIntervalStart( AccountID_t nCaller ) const
{
//we shift the time by the account ID so that all users don't wrap at the same time which can cause a temporary surge in web API requests
RTime32 curTime = CRTime::RTime32TimeCur() + ( nCaller % knRateIntervalTimeS );
return curTime - ( curTime % knRateIntervalTimeS );
}
CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateAccountUser( AccountID_t nID, RTime32 nRateIntervalStart )
{
int nIndex = m_AccountCallers.Find( nID );
if( nIndex == m_AccountCallers.InvalidIndex() )
{
nIndex = m_AccountCallers.Insert( nID );
SCallerStats& caller = m_AccountCallers[ nIndex ];
caller.m_nRateIntervalStartTime = nRateIntervalStart;
//account ID is always unrestricted!
if( nID == 0 )
caller.m_eLevel = eWebAPIAccountLevel_Unlimited;
}
return &m_AccountCallers[ nIndex ];
}
CWebAPIAccountTracker::SCallerStats* CWebAPIAccountTracker::CreateIPUser( uint32 nIP, RTime32 nRateIntervalStart )
{
int nIndex = m_IPCallers.Find( nIP );
if( nIndex == m_IPCallers.InvalidIndex() )
{
nIndex = m_IPCallers.Insert( nIP );
SCallerStats& caller = m_IPCallers[ nIndex ];
caller.m_nRateIntervalStartTime = nRateIntervalStart;
}
return &m_IPCallers[ nIndex ];
}
uint32 CWebAPIAccountTracker::GetSteamServerID( const char* pszServer )
{
int nIndex = m_SteamServers.Find( pszServer );
if( nIndex == m_SteamServers.InvalidIndex() )
{
SSteamServer* pServer = new SSteamServer;
pServer->m_sName = pszServer;
pServer->m_nID = m_SteamServers.Count();
m_SteamServers.Insert( pServer->m_sName, pServer );
return pServer->m_nID;
}
return m_SteamServers[ nIndex ]->m_nID;
}
void CWebAPIAccountTracker::DumpSteamServers() const
{
CGCReportPrinter rp;
rp.AddStringColumn( "ID" );
rp.AddStringColumn( "Server" );
FOR_EACH_MAP_FAST( m_SteamServers, nServer )
{
const uint32 nID = m_SteamServers[ nServer ]->m_nID;
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nID << 8 ) ) );
rp.StrValue( m_SteamServers[ nServer ]->m_sName );
rp.CommitRow();
}
rp.SortReport( "ID", false );
rp.PrintReport( SPEW_CONSOLE );
}
//determines what the resulting account level access should be based upon the access rights of the IP address and the account
static EWebAPIAccountLevel DetermineAccessLevel( EWebAPIAccountLevel eAccount, EWebAPIAccountLevel eIP )
{
//unrestricted users should always be allowed, regardless of the IP range that they are making requests from, same with unlimited IP addresses
if( ( eAccount == eWebAPIAccountLevel_Unlimited ) || ( eIP == eWebAPIAccountLevel_Unlimited ) )
return eWebAPIAccountLevel_Unlimited;
//otherwise, if either is blocked, then block
if( ( eAccount == eWebAPIAccountLevel_Blocked ) || ( eIP == eWebAPIAccountLevel_Blocked ) )
return eWebAPIAccountLevel_Blocked;
//now we are dealing with default case versus elevated. Elevated wins over default
if( ( eAccount == eWebAPIAccountLevel_Elevated ) || ( eIP == eWebAPIAccountLevel_Elevated ) )
return eWebAPIAccountLevel_Elevated;
//default
return eWebAPIAccountLevel_RateLimited;
}
bool CWebAPIAccountTracker::TrackUser( AccountID_t nID, uint32 nIP )
{
if( !webapi_account_tracking.GetBool() )
return true;
//first off update their aggregate caller stats
{
//what is our current time, and at what time did this rate interval start
const RTime32 rateIntervalStart = GetRateIntervalStart( nID );
//see if this account is completely blocked
SCallerStats* pAccountCaller = CreateAccountUser( nID, rateIntervalStart );
SCallerStats* pIPCaller = CreateIPUser( nIP, rateIntervalStart );
//determine what our policy should be based upon the access level of the IP and the user
EWebAPIAccountLevel eAccessLevel = DetermineAccessLevel( pAccountCaller->m_eLevel, pIPCaller->m_eLevel );
//if we are blocked, just bail now
if( eAccessLevel == eWebAPIAccountLevel_Blocked )
{
pAccountCaller->m_nBlockedCalls++;
pIPCaller->m_nBlockedCalls++;
return false;
}
//reset the rate interval tracking
if( pAccountCaller->m_nRateIntervalStartTime < rateIntervalStart )
pAccountCaller->ResetRateInterval( rateIntervalStart );
if( pIPCaller->m_nRateIntervalStartTime < rateIntervalStart )
pIPCaller->ResetRateInterval( rateIntervalStart );
//now handle rate limiting
if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) || ( eAccessLevel == eWebAPIAccountLevel_Elevated ) )
{
//determine the rate we want to limit
int32 nCallsPerMin = ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_calls_per_min.GetInt() : webapi_elevated_rate_limit_calls_per_min.GetInt();
int32 nBytesPerMin = ( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) ? webapi_rate_limit_mb_per_min.GetInt() : webapi_elevated_rate_limit_mb_per_min.GetInt() ) * 1024 * 1024;
//see if this account is rate limited
if( ( eAccessLevel == eWebAPIAccountLevel_RateLimited ) )
{
bool bAllow = true;
//see if we are being limited based upon call rate limiting (tracking based upon ip and account) Note that
//we don't return until we've dones stat tracking for both so the reports are accurate and capture it at both levels
if( ( nCallsPerMin >= 0 && pAccountCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) ||
( nBytesPerMin >= 0 && pAccountCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) )
{
pAccountCaller->m_nBlockedCalls++;
bAllow = false;
}
if( webapi_ip_rate_limit.GetBool() )
{
if( ( nCallsPerMin >= 0 && pIPCaller->m_nRateIntervalCalls >= ( uint32 )nCallsPerMin ) ||
( nBytesPerMin >= 0 && pIPCaller->m_nRateIntervalBytes >= ( uint32 )nBytesPerMin ) )
{
pIPCaller->m_nBlockedCalls++;
bAllow = false;
}
}
if( !bAllow )
return false;
}
}
}
return true;
}
void CWebAPIAccountTracker::TrackFunction( AccountID_t nID, uint32 nIP, const char* pszFunction, uint32 nResponseSize )
{
if( !webapi_account_tracking.GetBool() )
return;
//update the bytes for that user
{
int nCallerIndex = m_AccountCallers.Find( nID );
if( nCallerIndex != m_AccountCallers.InvalidIndex() )
{
SCallerStats& caller = m_AccountCallers[ nCallerIndex ];
caller.m_nRateIntervalBytes += nResponseSize;
caller.m_nRateIntervalCalls++;
}
}
//update the bytes for that user and for their IP
{
int nCallerIndex = m_IPCallers.Find( nIP );
if( nCallerIndex != m_IPCallers.InvalidIndex() )
{
SCallerStats& caller = m_IPCallers[ nCallerIndex ];
caller.m_nRateIntervalBytes += nResponseSize;
caller.m_nRateIntervalCalls++;
}
}
//now update the function specific stats
{
int nFunctionIndex = m_Functions.Find( ( uintp )pszFunction );
if( nFunctionIndex == m_Functions.InvalidIndex() )
{
SFunctionStats* pNewStats = new SFunctionStats;
pNewStats->m_nTotalCalls = 0;
pNewStats->m_nProfileCalls = 0;
pNewStats->m_nTotalBytes = 0;
pNewStats->m_nProfileBytes = 0;
pNewStats->m_nMaxBytes = 0;
pNewStats->m_nProfileMaxBytes = 0;
nFunctionIndex = m_Functions.Insert( ( uintp )pszFunction, pNewStats );
}
//update our stats
SFunctionStats& function = *m_Functions[ nFunctionIndex ];
function.m_nTotalCalls++;
function.m_nProfileCalls++;
function.m_nTotalBytes += nResponseSize;
function.m_nProfileBytes += nResponseSize;
function.m_nMaxBytes = MAX( function.m_nMaxBytes, nResponseSize );
function.m_nProfileMaxBytes = MAX( function.m_nProfileMaxBytes, nResponseSize );
//update caller stats
{
struct SFunctionStats::SCaller caller;
caller.m_nAccountID = nID;
caller.m_nIP = nIP;
int nCallerIndex = function.m_Callers.Find( caller );
if( nCallerIndex == function.m_Callers.InvalidIndex() )
{
nCallerIndex = function.m_Callers.Insert( caller );
function.m_Callers[ nCallerIndex ].m_nCalls = 1;
function.m_Callers[ nCallerIndex ].m_nBytes = nResponseSize;
}
else
{
function.m_Callers[ nCallerIndex ].m_nCalls++;
function.m_Callers[ nCallerIndex ].m_nBytes += nResponseSize;
}
}
}
}
void CWebAPIAccountTracker::SetAccountPermission( AccountID_t nID, EWebAPIAccountLevel eLevel )
{
SCallerStats* pCaller = CreateAccountUser( nID, GetRateIntervalStart( nID ) );
pCaller->m_eLevel = eLevel;
}
void CWebAPIAccountTracker::ResetAccountPermissions()
{
FOR_EACH_MAP_FAST( m_AccountCallers, nCaller )
{
m_AccountCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited;
}
FOR_EACH_MAP_FAST( m_IPCallers, nCaller )
{
m_IPCallers[ nCaller ].m_eLevel = eWebAPIAccountLevel_RateLimited;
}
}
void CWebAPIAccountTracker::ResetStats()
{
FOR_EACH_MAP_FAST( m_AccountCallers, nCaller )
{
m_AccountCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_AccountCallers.Key( nCaller ) ) );
m_AccountCallers[ nCaller ].m_nBlockedCalls = 0;
}
FOR_EACH_MAP_FAST( m_IPCallers, nCaller )
{
m_IPCallers[ nCaller ].ResetRateInterval( GetRateIntervalStart( m_IPCallers.Key( nCaller ) ) );
m_IPCallers[ nCaller ].m_nBlockedCalls = 0;
}
m_Functions.PurgeAndDeleteElements();
}
void CWebAPIAccountTracker::ResetProfileStats()
{
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
m_Functions[ nFunction ]->m_nProfileCalls = 0;
m_Functions[ nFunction ]->m_nProfileBytes = 0;
m_Functions[ nFunction ]->m_nProfileMaxBytes = 0;
}
m_ProfileTime.SetToJobTime();
}
static const int k_cSteamIDRenderedMaxLen = 36;
//-----------------------------------------------------------------------------
// Purpose: Renders the steam ID to a buffer with an admin console link. NOTE: for convenience of
// calling code, this code returns a pointer to a static buffer and is NOT thread-safe.
// Output: buffer with rendered Steam ID
//-----------------------------------------------------------------------------
static const char * CSteamID_RenderLink( const CSteamID & steamID )
{
// longest length of returned string is k_cBufLen
// <link cmd="steamid64 %llu"></link> => 30 + 20 == 50
// 50 + k_cSteamIDRenderedMaxLen + 1
const int k_cBufLen = 50 + k_cSteamIDRenderedMaxLen + 1;
const int k_cBufs = 4; // # of static bufs to use (so people can compose output with multiple calls to RenderLink() )
static char rgchBuf[k_cBufs][k_cBufLen];
static int nBuf = 0;
char * pchBuf = rgchBuf[nBuf]; // get pointer to current static buf
nBuf++; // use next buffer for next call to this method
nBuf %= k_cBufs;
Q_snprintf( pchBuf, k_cBufLen, "<link cmd=\"steamid64 %llu\">%s</link>", steamID.ConvertToUint64(), steamID.Render() );
return pchBuf;
}
//-----------------------------------------------------------------------------
// Purpose: Renders the passed-in steam ID to a buffer with admin console link. NOTE: for convenience
// of calling code, this code returns a pointer to a static buffer and is NOT thread-safe.
// Input: 64-bit representation of Steam ID to render
// Output: buffer with rendered Steam ID link
//-----------------------------------------------------------------------------
static const char * CSteamID_RenderLink( uint64 ulSteamID )
{
CSteamID steamID( ulSteamID );
return CSteamID_RenderLink( steamID );
}
void CWebAPIAccountTracker::DumpCaller( AccountID_t nID ) const
{
const CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( nID );
//cache the account name here so we don't yield while we have indices
CUtlString sPersona = GGCBase()->YieldingGetPersonaName( steamID, "[Unknown]" );
//dump high level user stats
int nCallerIndex = m_AccountCallers.Find( nID );
if( nCallerIndex == m_AccountCallers.InvalidIndex() )
{
EG_MSG( SPEW_CONSOLE, "User %u not found in any web api calls\n", nID );
return;
}
//a map of IP addresses that have been used by this account
CUtlHashMapLarge< uint32, SFunctionStats::SCalls > ipCalls;
//now each function they called
uint64 nTotalBytes = 0;
uint32 nTotalCalls = 0;
CUtlVector< SReportRow > vFuncs;
FOR_EACH_MAP_FAST( m_Functions, nFunc )
{
//add up how many calls they made to this function across all IPs
uint64 nFnBytes = 0;
uint32 nFnCalls = 0;
FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller )
{
const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller );
if( caller.m_nAccountID == nID )
{
const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ];
nFnBytes += calls.m_nBytes;
nFnCalls += calls.m_nCalls;
int nIPIndex = ipCalls.Find( caller.m_nIP );
if( nIPIndex == ipCalls.InvalidIndex() )
{
SFunctionStats::SCalls toAdd;
toAdd.m_nBytes = calls.m_nBytes;
toAdd.m_nCalls = calls.m_nCalls;
ipCalls.Insert( caller.m_nIP, toAdd );
}
else
{
ipCalls[ nIPIndex ].m_nBytes += calls.m_nBytes;
ipCalls[ nIPIndex ].m_nBytes += calls.m_nCalls;
}
}
}
if( nFnCalls > 0 )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) );
}
nTotalBytes += nFnBytes;
nTotalCalls += nFnCalls;
}
const SCallerStats& caller = m_AccountCallers[ nCallerIndex ];
EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" );
EG_MSG( SPEW_CONSOLE, "User %s: \"%s\"\n", CSteamID_RenderLink( steamID ), sPersona.String() );
double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 );
double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 );
double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 );
EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour );
//don't let someone accidentally change Steam's access!
if( nID != 0 )
{
if( caller.m_eLevel == eWebAPIAccountLevel_RateLimited )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Block Account]</link>", nID, eWebAPIAccountLevel_Blocked );
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Elevate Account]</link>\n", nID, eWebAPIAccountLevel_Elevated );
}
else if( caller.m_eLevel == eWebAPIAccountLevel_Blocked )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Unblock Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited );
}
else if( caller.m_eLevel == eWebAPIAccountLevel_Elevated )
{
EG_MSG( SPEW_CONSOLE, "\t<link cmd=\"webapi_account_set_access %u %d\">[Demote Account]</link>\n", nID, eWebAPIAccountLevel_RateLimited );
}
}
//print a report of the IP addresses that they are calling from
{
CGCReportPrinter rp;
rp.AddStringColumn( "IP" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
FOR_EACH_MAP_FAST( ipCalls, nIP )
{
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( ipCalls.Key( nIP ) ) ), CFmtStr( "webapi_account_dump_ip %u", ipCalls.Key( nIP ) ) );
rp.IntValue( ipCalls[ nIP ].m_nCalls );
rp.IntValue( ipCalls[ nIP ].m_nBytes );
rp.CommitRow();
}
rp.SortReport( "MB" );
rp.PrintReport( SPEW_CONSOLE );
}
//and print a report of all the functions that they've called
PrintReport( vFuncs );
}
void CWebAPIAccountTracker::DumpIP( uint32 nIP ) const
{
//dump high level user stats
int nCallerIndex = m_IPCallers.Find( nIP );
if( nCallerIndex == m_IPCallers.InvalidIndex() )
{
EG_MSG( SPEW_CONSOLE, "IP %u not found in any web api calls\n", nIP );
return;
}
//a map of IP addresses that have been used by this account
CUtlHashMapLarge< AccountID_t, SFunctionStats::SCalls > accountCalls;
//now each function they called
uint64 nTotalBytes = 0;
uint32 nTotalCalls = 0;
CUtlVector< SReportRow > vFuncs;
FOR_EACH_MAP_FAST( m_Functions, nFunc )
{
//add up how many calls they made to this function across all IPs
uint64 nFnBytes = 0;
uint32 nFnCalls = 0;
FOR_EACH_MAP_FAST( m_Functions[ nFunc ]->m_Callers, nCaller )
{
const CWebAPIAccountTracker::SFunctionStats::SCaller& caller = m_Functions[ nFunc ]->m_Callers.Key( nCaller );
if( caller.m_nIP == nIP )
{
const CWebAPIAccountTracker::SFunctionStats::SCalls& calls = m_Functions[ nFunc ]->m_Callers[ nCaller ];
nFnBytes += calls.m_nBytes;
nFnCalls += calls.m_nCalls;
int nAccountIndex = accountCalls.Find( caller.m_nAccountID );
if( nAccountIndex == accountCalls.InvalidIndex() )
{
SFunctionStats::SCalls toAdd;
toAdd.m_nBytes = calls.m_nBytes;
toAdd.m_nCalls = calls.m_nCalls;
accountCalls.Insert( caller.m_nAccountID, toAdd );
GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( caller.m_nAccountID ) );
}
else
{
accountCalls[ nAccountIndex ].m_nBytes += calls.m_nBytes;
accountCalls[ nAccountIndex ].m_nBytes += calls.m_nCalls;
}
}
}
if( nFnCalls > 0 )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( nFunc ), nFnCalls, nFnBytes ) );
}
nTotalBytes += nFnBytes;
nTotalCalls += nFnCalls;
}
const SCallerStats& caller = m_IPCallers[ nCallerIndex ];
EG_MSG( SPEW_CONSOLE, "---------------------------------------------------\n" );
EG_MSG( SPEW_CONSOLE, "IP %u.%u.%u.%u\n", iptod( nIP ) );
double fTotalMB = nTotalBytes / ( 1024.0 * 1024.0 );
double fMBPerHour = fTotalMB / ( GGCBase()->GetGCUpTime() / 3600.0 );
double fCallsPerHour = nTotalCalls / ( GGCBase()->GetGCUpTime() / 3600.0 );
EG_MSG( SPEW_CONSOLE, "\tAccess: %u, Total Calls: %u, Blocked calls: %u, Total: %.2fMB, MB/h: %.2f, Calls/h: %.0f\n", caller.m_eLevel, nTotalCalls, caller.m_nBlockedCalls, fTotalMB, fMBPerHour, fCallsPerHour );
//print a report of the accounts that they are calling from
{
CGCReportPrinter rp;
rp.AddSteamIDColumn( "Account" );
rp.AddStringColumn( "Persona" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
FOR_EACH_MAP_FAST( accountCalls, nAccount )
{
CSteamID steamID = GGCInterface()->ConstructSteamIDForClient( accountCalls.Key( nAccount ) );
rp.SteamIDValue( steamID, CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) );
rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ), CFmtStr( "webapi_account_dump_caller %u", steamID.GetAccountID() ) );
rp.IntValue( accountCalls[ nAccount ].m_nCalls );
rp.IntValue( accountCalls[ nAccount ].m_nBytes );
rp.CommitRow();
}
rp.SortReport( "MB" );
rp.PrintReport( SPEW_CONSOLE );
}
//and print a report of all the functions that they've called
PrintReport( vFuncs );
}
struct SCallerReportStats
{
uint32 m_nFunctions;
uint32 m_nCalls;
uint64 m_nBytes;
};
void CWebAPIAccountTracker::DumpTotalCallers( EDumpCaller eFilter, const char* pszFunctionFilter ) const
{
//accumulate stats for each unique caller
CUtlHashMapLarge< AccountID_t, SCallerReportStats > mapCallers;
FOR_EACH_MAP_FAST( m_Functions, nCurrFunction )
{
//handle filtering out functions we don't care about
const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction );
if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) )
continue;
const SFunctionStats& function = *m_Functions[ nCurrFunction ];
FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller )
{
const AccountID_t key = function.m_Callers.Key( nCurrCaller ).m_nAccountID;
//add this account
int nStatIndex = mapCallers.Find( key );
if( nStatIndex == mapCallers.InvalidIndex() )
{
SCallerReportStats stats;
stats.m_nFunctions = 0;
stats.m_nCalls = 0;
stats.m_nBytes = 0;
nStatIndex = mapCallers.Insert( key, stats );
GGCBase()->PreloadPersonaName( GGCInterface()->ConstructSteamIDForClient( key ) );
}
mapCallers[ nStatIndex ].m_nFunctions += 1;
mapCallers[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls;
mapCallers[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes;
}
}
CGCReportPrinter rp;
rp.AddStringColumn( "Account" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None );
rp.AddStringColumn( "UserName" );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
//now show the report
FOR_EACH_MAP_FAST( m_AccountCallers, nCurrCaller )
{
//apply filters to our results
if( ( eFilter == eDumpCaller_Blocked ) && ( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) )
continue;
if( ( eFilter == eDumpCaller_Status ) && ( m_AccountCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) )
continue;
const AccountID_t accountID = m_AccountCallers.Key( nCurrCaller );
const SCallerReportStats* pStats = NULL;
int nCollectedStats = mapCallers.Find( accountID );
if( nCollectedStats != mapCallers.InvalidIndex() )
{
pStats = &mapCallers[ nCollectedStats ];
}
//filter out users that didn't make any calls if appropriate
if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) )
continue;
const CSteamID steamID( GGCInterface()->ConstructSteamIDForClient( accountID ) );
rp.StrValue( steamID.Render() , CFmtStr( "webapi_account_dump_caller %u", accountID ) );
rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 );
rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 );
rp.IntValue( m_AccountCallers[ nCurrCaller ].m_nBlockedCalls );
rp.IntValue( m_AccountCallers[ nCurrCaller ].m_eLevel );
rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 );
rp.StrValue( GGCBase()->YieldingGetPersonaName( steamID, "[unknown]" ) );
rp.CommitRow();
}
const char* pszSort = "MB/h";
if( eFilter == eDumpCaller_Blocked )
pszSort = "Blocked";
else if( eFilter == eDumpCaller_Status )
pszSort = "Access";
rp.SortReport( pszSort );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::DumpTotalIPs( EDumpCaller eFilter, const char* pszFunctionFilter ) const
{
//accumulate stats for each unique caller
CUtlHashMapLarge< uint32, SCallerReportStats > mapIPs;
FOR_EACH_MAP_FAST( m_Functions, nCurrFunction )
{
//handle filtering out functions we don't care about
const char* pszFunctionName = ( const char* )m_Functions.Key( nCurrFunction );
if( pszFunctionFilter && ( V_stristr( pszFunctionName, pszFunctionFilter ) == NULL ) )
continue;
const SFunctionStats& function = *m_Functions[ nCurrFunction ];
FOR_EACH_MAP_FAST( function.m_Callers, nCurrCaller )
{
const uint32 key = function.m_Callers.Key( nCurrCaller ).m_nIP;
//add this account
int nStatIndex = mapIPs.Find( key );
if( nStatIndex == mapIPs.InvalidIndex() )
{
SCallerReportStats stats;
stats.m_nFunctions = 0;
stats.m_nCalls = 0;
stats.m_nBytes = 0;
nStatIndex = mapIPs.Insert( key, stats );
}
mapIPs[ nStatIndex ].m_nFunctions += 1;
mapIPs[ nStatIndex ].m_nCalls += function.m_Callers[ nCurrCaller ].m_nCalls;
mapIPs[ nStatIndex ].m_nBytes += function.m_Callers[ nCurrCaller ].m_nBytes;
}
}
CGCReportPrinter rp;
rp.AddStringColumn( "IP" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Blocked", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "Access", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "APIs", CGCReportPrinter::eSummary_None );
rp.AddIntColumn( "MB/h", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddIntColumn( "Calls/h", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "KB/c", CGCReportPrinter::eSummary_None );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
//now show the report
FOR_EACH_MAP_FAST( m_IPCallers, nCurrCaller )
{
//apply filters to our results
if( ( eFilter == eDumpCaller_Blocked ) && ( m_IPCallers[ nCurrCaller ].m_nBlockedCalls == 0 ) )
continue;
if( ( eFilter == eDumpCaller_Status ) && ( m_IPCallers[ nCurrCaller ].m_eLevel == eWebAPIAccountLevel_RateLimited ) )
continue;
const uint32 nIP = m_IPCallers.Key( nCurrCaller );
const SCallerReportStats* pStats = NULL;
int nCollectedStats = mapIPs.Find( nIP );
if( nCollectedStats != mapIPs.InvalidIndex() )
{
pStats = &mapIPs[ nCollectedStats ];
}
//filter out users that didn't make any calls if appropriate
if( ( eFilter == eDumpCaller_Calls ) && ( !pStats || ( pStats->m_nCalls == 0 ) ) )
continue;
rp.StrValue( CFmtStr( "%u.%u.%u.%u", iptod( nIP ) ), CFmtStr( "webapi_account_dump_ip %u", nIP ) );
rp.IntValue( ( pStats ) ? pStats->m_nCalls : 0 );
rp.IntValue( ( pStats ) ? pStats->m_nBytes : 0 );
rp.IntValue( m_IPCallers[ nCurrCaller ].m_nBlockedCalls );
rp.IntValue( m_IPCallers[ nCurrCaller ].m_eLevel );
rp.IntValue( ( pStats ) ? pStats->m_nFunctions : 0 );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nBytes : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( int64 )( ( ( pStats ) ? pStats->m_nCalls : 0 ) / fUpTimeHrs ) );
rp.IntValue( ( pStats && pStats->m_nCalls > 0 ) ? ( pStats->m_nBytes / pStats->m_nCalls ) / 1024 : 0 );
rp.CommitRow();
}
const char* pszSort = "MB/h";
if( eFilter == eDumpCaller_Blocked )
pszSort = "Blocked";
else if( eFilter == eDumpCaller_Status )
pszSort = "Access";
rp.SortReport( pszSort );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::DumpFunctions() const
{
CUtlVector< SReportRow > vFuncs;
vFuncs.EnsureCapacity( m_Functions.Count() );
FOR_EACH_MAP_FAST( m_Functions, i )
{
vFuncs.AddToTail( SReportRow( ( const char* )m_Functions.Key( i ), m_Functions[ i ]->m_nTotalCalls, m_Functions[ i ]->m_nTotalBytes ) );
}
PrintReport( vFuncs );
}
void CWebAPIAccountTracker::DumpProfile( bool bAllTime ) const
{
//accumulate totals so we can do percentage
uint32 nTotalCalls = 0;
uint64 nTotalBytes = 0;
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
if( bAllTime )
{
nTotalCalls += m_Functions[ nFunction ]->m_nTotalCalls;
nTotalBytes += m_Functions[ nFunction ]->m_nTotalBytes;
}
else
{
nTotalCalls += m_Functions[ nFunction ]->m_nProfileCalls;
nTotalBytes += m_Functions[ nFunction ]->m_nProfileBytes;
}
}
//determine how much time we are covering, and come up with a scale to normalize the values
uint64 nSampleMicroS = ( bAllTime ) ? ( uint64 )GGCBase()->GetGCUpTime() * 1000000 : m_ProfileTime.CServerMicroSecsPassed();
double fToPS = ( nSampleMicroS > 0 ) ? 1000000.0 / ( double )nSampleMicroS : 1.0;
EmitInfo( SPEW_CONSOLE, SPEW_ALWAYS, LOG_ALWAYS, "Web API Profile: Sampled %.2f seconds\n", nSampleMicroS / ( 1000.0 * 1000.0 ) );
CGCReportPrinter rp;
rp.AddStringColumn( "Web API Name" );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 );
rp.AddFloatColumn( "KBPS", CGCReportPrinter::eSummary_Total, 1 );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddFloatColumn( "%", CGCReportPrinter::eSummary_Total, 1 );
rp.AddFloatColumn( "CallsPS", CGCReportPrinter::eSummary_Total, 1 );
rp.AddIntColumn( "MaxKB", CGCReportPrinter::eSummary_Max );
FOR_EACH_MAP_FAST( m_Functions, nFunction )
{
const SFunctionStats* pFunc = m_Functions[ nFunction ];
uint32 nCalls = ( bAllTime ) ? pFunc->m_nTotalCalls : pFunc->m_nProfileCalls;
uint32 nMax = ( bAllTime ) ? pFunc->m_nMaxBytes : pFunc->m_nProfileMaxBytes;
uint64 nBytes = ( bAllTime ) ? pFunc->m_nTotalBytes : pFunc->m_nProfileBytes;
rp.StrValue( ( const char* )m_Functions.Key( nFunction ) );
rp.IntValue( nBytes );
rp.FloatValue( ( 100.0 * nBytes ) / nTotalBytes );
rp.FloatValue( ( nBytes / 1024.0 ) * fToPS );
rp.IntValue( nCalls );
rp.FloatValue( ( 100.0 * nCalls ) / nTotalCalls );
rp.FloatValue( nCalls * fToPS );
rp.IntValue( nMax / 1024 );
rp.CommitRow();
}
rp.SortReport( "KBPS" );
rp.PrintReport( SPEW_CONSOLE );
}
void CWebAPIAccountTracker::PrintReport( const CUtlVector< SReportRow >& vec )
{
CGCReportPrinter rp;
//now print it out based upon the type
rp.AddStringColumn( "Function" );
rp.AddIntColumn( "Calls", CGCReportPrinter::eSummary_Total );
rp.AddIntColumn( "MB", CGCReportPrinter::eSummary_Total, CGCReportPrinter::eIntDisplay_Memory_MB );
rp.AddFloatColumn( "Calls/h", CGCReportPrinter::eSummary_Total, 0 );
rp.AddFloatColumn( "MB/h", CGCReportPrinter::eSummary_Total );
const double fUpTimeHrs = MAX( GGCBase()->GetGCUpTime() / 3600.0, 0.01 );
for( int i = 0; i < vec.Count(); i++ )
{
rp.StrValue( vec[ i ].m_pszFunction, CFmtStr( "webapi_account_dump_function_callers %s", vec[ i ].m_pszFunction ) );
rp.IntValue( vec[ i ].m_nCalls );
rp.IntValue( vec[ i ].m_nSize );
rp.FloatValue( vec[ i ].m_nCalls / fUpTimeHrs );
rp.FloatValue( ( vec[ i ].m_nSize / ( 1024.0 * 1024.0 ) ) / fUpTimeHrs );
rp.CommitRow();
}
rp.SortReport( "MB/h" );
rp.PrintReport( SPEW_CONSOLE );
}
GC_CON_COMMAND( webapi_account_dump_steam_servers, "Dumps the ID listings of the various steam servers encoded in the IP address of Steam requests" )
{
g_WebAPIAccountTracker.DumpSteamServers();
}
GC_CON_COMMAND( webapi_account_dump_callers, "Dumps the most frequent callers of web api's for the current run of the GC" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls );
}
GC_CON_COMMAND( webapi_account_dump_ips, "Dumps the most frequent ip callers of web api's for the current run of the GC" )
{
g_WebAPIAccountTracker.DumpTotalIPs( CWebAPIAccountTracker::eDumpCaller_Calls );
}
GC_CON_COMMAND( webapi_account_dump_blocked_callers, "Dumps the callers that have been blocked and how many calls have been blocked" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Blocked );
}
GC_CON_COMMAND( webapi_account_dump_caller_access, "Dumps the access rights of any caller that is not the default rate limiting" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Status );
}
GC_CON_COMMAND( webapi_account_dump_functions, "Dumps the most frequently called web api functions" )
{
g_WebAPIAccountTracker.DumpFunctions();
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_function_callers, 1, "<function name> - Dumps the most frequent callers of functions that match the provided substring" )
{
g_WebAPIAccountTracker.DumpTotalCallers( CWebAPIAccountTracker::eDumpCaller_Calls, args[ 1 ] );
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_caller, 1, "<caller account> - Dumps the functions that the provided account ID has been calling the most" )
{
g_WebAPIAccountTracker.DumpCaller( ( AccountID_t )V_atoui64( args[ 1 ] ) );
}
GC_CON_COMMAND_PARAMS( webapi_account_dump_ip, 1, "<ip> - Dumps the functions that the provided ip has been calling the most" )
{
g_WebAPIAccountTracker.DumpIP( ( AccountID_t )V_atoui64( args[ 1 ] ) );
}
GC_CON_COMMAND( webapi_account_reset_stats, "Forces a reset of all stats collected for web api account stats" )
{
g_WebAPIAccountTracker.ResetStats();
}
//utility class for dumping out the profile results after time has expired
static void DumpWebAPIProfile()
{
g_WebAPIAccountTracker.DumpProfile( false );
}
GC_CON_COMMAND_PARAMS( webapi_profile, 1, "<seconds to profile> Turns on web api profiling for N seconds and dumps the results" )
{
float fSeconds = MAX( 1.0f, atof( args[ 1 ] ) );
g_WebAPIAccountTracker.ResetProfileStats();
static CGlobalScheduledFunction s_DumpProfile;
s_DumpProfile.ScheduleMS( DumpWebAPIProfile, fSeconds * 1000.0f );
}
//console commands to control web API profiling
GC_CON_COMMAND( webapi_profile_reset, "Turns on web api profiling" )
{
g_WebAPIAccountTracker.ResetProfileStats();
}
GC_CON_COMMAND( webapi_profile_dump, "Displays stats collected while web api profiling was enabled" )
{
g_WebAPIAccountTracker.DumpProfile( false );
}
GC_CON_COMMAND( webapi_profile_dump_total, "Displays stats collected while web api profiling was enabled" )
{
g_WebAPIAccountTracker.DumpProfile( true );
}
//-----------------------------------------------------------------------------
// Purpose: Sends a message and waits for a response
// Input: steamIDTarget - The entity this message is going to
// msgOut - The message to send
// nTimeoutSec - Number of seconds to wait for a response
// pMsgIn - Pointer to the message that will contain the response
// eMsg - The type of message the response should be
// Returns: True is the response was received, false otherwise. The contents
// of pMsgIn will be valid only if the function returns true.
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, CGCMsgBase *pMsgIn, MsgType_t eMsg )
{
IMsgNetPacket *pNetPacket = NULL;
if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) )
return false;
pMsgIn->SetPacket( pNetPacket );
if ( pMsgIn->Hdr().m_eMsg != eMsg )
return false;
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Sends a message and waits for a response
// Input: steamIDTarget - The entity this message is going to
// msgOut - The message to send
// nTimeoutSec - Number of seconds to wait for a response
// ppNetPackets - Pointer to a IMsgNetPacket pointer which will contain
// the response
// Returns: True is the response was received, false otherwise. *ppNetPacket
// will point to a valid packet only if the function returns true.
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CGCMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket )
{
msgOut.ExpectingReply( GetJobID() );
if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) )
return false;
SetJobTimeout( nTimeoutSec );
return BYieldingWaitForMsg( ppNetPacket );
}
//-----------------------------------------------------------------------------
// Purpose: BYldSendMessageAndGetReply, ProtoBuf edition
//-----------------------------------------------------------------------------
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, CProtoBufMsgBase *pMsgIn, MsgType_t eMsg )
{
IMsgNetPacket *pNetPacket = NULL;
if ( !BYldSendMessageAndGetReply( steamIDTarget, msgOut, nTimeoutSec, &pNetPacket ) )
return false;
pMsgIn->InitFromPacket( pNetPacket );
if ( pMsgIn->GetEMsg() != eMsg )
return false;
return true;
}
bool CGCJob::BYldSendMessageAndGetReply( CSteamID &steamIDTarget, CProtoBufMsgBase &msgOut, uint nTimeoutSec, IMsgNetPacket **ppNetPacket )
{
msgOut.ExpectingReply( GetJobID() );
if ( !m_pGC->BSendGCMsgToClient( steamIDTarget, msgOut ) )
return false;
SetJobTimeout( nTimeoutSec );
return BYieldingWaitForMsg( ppNetPacket );
}
//-----------------------------------------------------------------------------
// Purpose: Constructor
//-----------------------------------------------------------------------------
CGCWGJob::CGCWGJob( CGCBase *pGCBase )
: CGCJob( pGCBase ),
m_steamID( k_steamIDNil ),
m_pWebApiFunc( NULL )
{
}
//-----------------------------------------------------------------------------
// Purpose: Destructor
//-----------------------------------------------------------------------------
CGCWGJob::~CGCWGJob()
{
}
//-----------------------------------------------------------------------------
// Purpose: Receive a k_EMsgWGRequest and manage keyvalues serialization/deserialization
//-----------------------------------------------------------------------------
bool CGCWGJob::BYieldingRunGCJob( IMsgNetPacket * pNetPacket )
{
CGCMsg<MsgGCWGRequest_t> msg( pNetPacket );
CUtlString strRequestName;
KeyValuesAD pkvRequest( "request" );
KeyValuesAD pkvResponse( "response" );
m_steamID = CSteamID( msg.Body().m_ulSteamID );
msg.BReadStr( &strRequestName );
//deserialize KV
m_bufRequest.Clear();
m_bufRequest.Put( msg.PubReadCur(), msg.Body().m_cubKeyValues );
KVPacker packer;
if ( !packer.ReadAsBinary( pkvRequest, m_bufRequest ) )
{
AssertMsg( false, "Failed to deserialize key values from WG request" );
CGCWGJobMgr::SendErrorMessage( msg, "Failed to deserialize key values from WG request", k_EResultInvalidParam );
return false;
}
if( !BVerifyParams( msg, pkvRequest, m_pWebApiFunc ) )
return false;
bool bRet = BYieldingRunJobFromRequest( pkvRequest, pkvResponse );
// request failed, set success for wg
if ( pkvResponse->IsEmpty( "success" ) )
{
pkvResponse->SetInt( "success", bRet ? k_EResultOK : k_EResultFail );
}
// send response msg
CGCWGJobMgr::SendResponse( msg, pkvResponse, bRet );
return true;
}
bool CGCWGJob::BVerifyParams( const CGCMsg<MsgGCWGRequest_t> & msg, KeyValues *pkvRequest, const WebApiFunc_t * pWebApiFunc )
{
// we've found the function; now validate the call
for ( int i = 0; i < Q_ARRAYSIZE( pWebApiFunc->m_rgParams ); i++ )
{
if ( !pWebApiFunc->m_rgParams[i].m_pchParam )
break;
// just simple validation for now; make sure the key exists
if ( !pWebApiFunc->m_rgParams[i].m_bOptional && !pkvRequest->FindKey( pWebApiFunc->m_rgParams[i].m_pchParam ) )
{
CGCWGJobMgr::SendErrorMessage( msg,
CFmtStr( "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName ),
k_EResultInvalidParam );
EmitWarning( SPEW_GC, 2, "Error: Missing parameter '%s' from web request '%s'\n", pWebApiFunc->m_rgParams[i].m_pchParam, pWebApiFunc->m_pchRequestName );
return false;
}
}
return true;
}
void CGCWGJob::SetErrorMessage( KeyValues *pkvErr, const char *pchErrorMsg, int32 nResult )
{
CGCWGJobMgr::SetErrorMessage( pkvErr, pchErrorMsg, nResult );
}
//-----------------------------------------------------------------------------
// CGCJobVerifySession - A job that asks steam if a given user is still connected
// and cleans up the session if the user is gone
//-----------------------------------------------------------------------------
bool CGCJobVerifySession::BYieldingRunGCJob()
{
if ( !m_pGC->BYieldingLockSteamID( m_steamID, __FILE__, __LINE__ ) )
return false;
m_pGC->YieldingRequestSession( m_steamID );
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Constructor
//-----------------------------------------------------------------------------
CWebAPIJob::CWebAPIJob( CGCBase *pGC, EWebAPIOutputFormat eDefaultOutputFormat )
: CGCJob( pGC ), m_eDefaultOutputFormat( eDefaultOutputFormat )
{
}
//-----------------------------------------------------------------------------
// Purpose: Destructor
//-----------------------------------------------------------------------------
CWebAPIJob::~CWebAPIJob()
{
}
//-----------------------------------------------------------------------------
// Called to handle converting a web api response to a serialized result and free the object as well on a background thread
class CEmitWebAPIData
{
public:
CEmitWebAPIData( CMsgHttpRequest* pRequest) : m_bResult( false ), m_Request( pRequest ) {}
//inputs
CHTTPRequest m_Request;
//outputs
CHTTPResponse m_Response;
std::string m_sSerializedResponse;
bool m_bResult;
};
//the worker thread function that is responsible for serializing the response from web api values to a text buffer, freeing the web api value tree, and serializing the message to a protobuf for
//direct sending. If this function fails (a false response value) the message will not be serialized
static void ThreadedEmitFormattedOutputWrapperAndFreeResponse( CWebAPIResponse *pResponse, CEmitWebAPIData* pEmitData, EWebAPIOutputFormat eDefaultFormat, size_t unMaxResultSize )
{
// parse the output type that we want the result to be in
EWebAPIOutputFormat eOutputFormat = eDefaultFormat;
const char *pszParamOutput = pEmitData->m_Request.GetGETParamString( "format", NULL );
if ( !pszParamOutput )
pszParamOutput = pEmitData->m_Request.GetPOSTParamString( "format", NULL );
if ( pszParamOutput )
{
if ( Q_stricmp( pszParamOutput, "xml" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_XML;
else if ( Q_stricmp( pszParamOutput, "vdf" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_VDF;
else if ( Q_stricmp( pszParamOutput, "json" ) == 0 )
eOutputFormat = k_EWebAPIOutputFormat_JSON;
}
pEmitData->m_bResult = pResponse->BEmitFormattedOutput( eOutputFormat, *( pEmitData->m_Response.GetBodyBuffer() ), unMaxResultSize );
delete pResponse;
//update the response code on the output (can probably be done elsewhere), but must be done before we pack the message below
switch( eOutputFormat )
{
case k_EWebAPIOutputFormat_JSON:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "application/json; charset=UTF-8" );
break;
case k_EWebAPIOutputFormat_XML:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/xml; charset=UTF-8" );
break;
case k_EWebAPIOutputFormat_VDF:
pEmitData->m_Response.SetResponseHeaderValue( "content-type", "text/vdf; charset=UTF-8" );
break;
default:
break;
}
//if successful, we can go ahead and convert this all the way into a completely serialized form for sending over the wire
if( pEmitData->m_bResult )
{
CMsgHttpResponse msgResponse;
pEmitData->m_Response.SerializeIntoProtoBuf( msgResponse );
msgResponse.SerializeToString( &pEmitData->m_sSerializedResponse );
}
}
//called to respond to a web api request with the specified response value
static void WebAPIRespondToRequest( const char* pszName, uint32 nSenderIP, const CHTTPResponse& response, const CProtoBufMsg< CMsgWebAPIRequest >& msg )
{
VPROF_BUDGET( "WebAPI - sending result", VPROF_BUDGETGROUP_STEAM );
CProtoBufMsg<CMsgHttpResponse> msgResponse( k_EGCMsgWebAPIJobRequestHttpResponse, msg );
response.SerializeIntoProtoBuf( msgResponse.Body() );
GGCBase()->BReplyToMessage( msgResponse, msg );
//track this message in the web API response
g_WebAPIAccountTracker.TrackFunction( msg.Body().api_key().account_id(), nSenderIP, pszName, msgResponse.Body().body().size() );
}
//responses to the web api message with an error code
static void WebAPIRespondWithError( const char* pszName, uint32 nSenderIP, const CProtoBufMsg< CMsgWebAPIRequest >& msg, EHTTPStatusCode statusCode )
{
CHTTPResponse response;
response.SetStatusCode( statusCode );
WebAPIRespondToRequest( pszName, nSenderIP, response, msg );
}
//parses an IP address out of the provided string. This will be the last IP address in the list
static uint32 ParseIPAddrFromForwardHeader( const char* pszHeader )
{
//find the last comma in the string, our IP address follows that
const char* pszStart = V_strrchr( pszHeader, ',' );
//if no match, then we just have the ip address, otherwise advance past the comma
if( !pszStart )
pszStart = pszHeader;
else
pszStart++;
//skip leading spaces
while( V_isspace( *pszStart ) )
pszStart++;
return ntohl( inet_addr( pszStart ) );
}
//-----------------------------------------------------------------------------
// Purpose: Receive a k_EMsgWebAPIJobRequest and manage serialization/deserialization to
// web request/response objects
//-----------------------------------------------------------------------------
bool CWebAPIJob::BYieldingRunJobFromMsg( IMsgNetPacket * pNetPacket )
{
VPROF_BUDGET( "WebAPI", VPROF_BUDGETGROUP_STEAM );
CProtoBufMsg<CMsgWebAPIRequest> msg( pNetPacket );
//make sure all the required parameters were present
bool bMsgParsedOK = msg.Body().has_api_key()
&& msg.Body().has_interface_name()
&& msg.Body().has_method_name()
&& msg.Body().has_request()
&& msg.Body().has_version();
if( !bMsgParsedOK )
{
WebAPIRespondWithError( GetName(), 0, msg, k_EHTTPStatusCode400BadRequest );
return true;
}
//determine the account that sent this request
const AccountID_t nSenderAccountID = msg.Body().api_key().account_id();
uint32 nSenderIP = 0;
//if this isn't a system request, try and identify the IP address of the sender so we can rate limit accordingly
if( nSenderAccountID != 0 )
{
const int nNumHeaders = msg.Body().request().headers_size();
for( int nHeader = 0; nHeader < nNumHeaders; nHeader++ )
{
if( strcmp( msg.Body().request().headers( nHeader ).name().c_str(), "X-Forwarded-For" ) != 0 )
continue;
//this is our IP address
nSenderIP = ParseIPAddrFromForwardHeader( msg.Body().request().headers( nHeader ).value().c_str() );
}
//see if we have the kill switch turned on
if( webapi_kill_switch.GetBool() )
{
if( webapi_kill_switch_error_response.GetBool() )
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode503ServiceUnavailable );
return true;
}
}
else
{
// !FIXME! DOTAMERGE
// //determine the priority of this steam request
// uint32 nPriority;
// nSenderIP = GetSteamRequestIPAddress( msg.Body().request(), nPriority );
// //and allow for a kill switch based upon this priority
// if( ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Low ) && !webapi_enable_steam_low.GetBool() ) ||
// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_Normal ) && !webapi_enable_steam_normal.GetBool() ) ||
// ( ( nPriority == CWebAPIAccountTracker::k_nSteamIP_High ) && !webapi_enable_steam_high.GetBool() ) )
// {
// if( webapi_kill_switch_error_response.GetBool() )
// WebAPIRespondWithError( GetName(), 0, msg );
// return true;
// }
}
//track stats for this account, and handle rate limiting
if( !g_WebAPIAccountTracker.TrackUser( nSenderAccountID, nSenderIP ) )
{
if( webapi_kill_switch_error_response.GetBool() )
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode429TooManyRequests );
return true;
}
//allocate the data that we'll use to fill out the request and send it to the background thread for work
CPlainAutoPtr< CEmitWebAPIData > pEmitData( new CEmitWebAPIData( const_cast< CMsgHttpRequest* >( &msg.Body().request() ) ) );
CHTTPRequest& request = pEmitData->m_Request;
CHTTPResponse& response = pEmitData->m_Response;
{
VPROF_BUDGET( "WebAPI - Prepare msg", VPROF_BUDGETGROUP_STEAM );
m_webAPIKey.DeserializeFromProtoBuf( msg.Body().api_key() );
}
CPlainAutoPtr< CWebAPIResponse > pwebAPIResponse( new CWebAPIResponse() );
{
VPROF_BUDGET( "WebAPI - Process msg", VPROF_BUDGETGROUP_STEAM );
if( !BYieldingRunJobFromAPIRequest( msg.Body().interface_name().c_str(), msg.Body().method_name().c_str(), msg.Body().version(), &request, &response, pwebAPIResponse.Get() ) )
{
//error executing our job
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError );
return false;
}
}
// !FIXME! DOTAMERGE
// //see if they want to re-route this request
// if( m_nRerouteRequest >= 0 )
// {
// if( ( uint32 )m_nRerouteRequest == m_pGC->GetGCDirIndex() )
// {
// AssertMsg( false, "Error: WebAPI %s attempting to re-route a web api message to itself (%d)", GetName(), m_nRerouteRequest );
// }
// else
// {
// //route to the other GC and discard this message
// CProtoBufMsg< CMsgGCMsgWebAPIJobRequestForwardResponse > msgRoute( k_EGCMsgWebAPIJobRequestForwardResponse );
// msgRoute.Body().set_dir_index( m_nRerouteRequest );
// m_pGC->BReplyToMessage( msgRoute, msg );
// }
// return true;
// }
VPROF_BUDGET( "WebAPI - Emitting result", VPROF_BUDGETGROUP_STEAM );
response.SetStatusCode( pwebAPIResponse->GetStatusCode() );
response.SetExpirationHeaderDeltaFromNow( pwebAPIResponse->GetExpirationSeconds() );
if( pwebAPIResponse->GetLastModified() )
response.SetHeaderTimeValue( "last-modified", pwebAPIResponse->GetLastModified() );
// if we aren't allowed to have a message body on this, simply send the result back now
if( !CHTTPUtil::BStatusCodeAllowsBody( pwebAPIResponse->GetStatusCode() ) )
{
AssertMsg( pwebAPIResponse->GetRootValue() == NULL, "Response HTTP status code %d doesn't allow a body, but one was present", pwebAPIResponse->GetStatusCode() );
//since we didn't have a body to serialize, we need to handle sending just the response
WebAPIRespondToRequest( GetName(), nSenderIP, response, msg );
return true;
}
//let the job convert the formatting and free our response for us (quite costly). Note that we detach the web API response since this job will free it
bool bThreadFuncSucceeded = BYieldingWaitForThreadFunc( CreateFunctor( ThreadedEmitFormattedOutputWrapperAndFreeResponse, pwebAPIResponse.Detach(), pEmitData.Get(), m_eDefaultOutputFormat, (size_t)cv_webapi_result_size_limit.GetInt() ) );
//if we called the function successfully and had a valid result, just send back the preserialized body
if( bThreadFuncSucceeded && pEmitData->m_bResult )
{
m_pGC->BReplyToMessageWithPreSerializedBody( k_EGCMsgWebAPIJobRequestHttpResponse, msg, ( const byte* )pEmitData->m_sSerializedResponse.c_str(), pEmitData->m_sSerializedResponse.size() );
g_WebAPIAccountTracker.TrackFunction( nSenderAccountID, nSenderIP, GetName(), pEmitData->m_sSerializedResponse.size() );
}
else
{
//we failed to generate a response, see if we ran out of space
if( response.GetBodyBuffer()->TellMaxPut() > cv_webapi_result_size_limit.GetInt() )
{
// !FIXME! DOTAMERGE
//CGCAlertInfo alert( "WebAPIResponseSize", "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() );
//
//switch( request.GetEHTTPMethod() )
//{
//case k_EHTTPMethodGET:
// {
// const uint32 nNumParams = request.GetGETParamCount();
// for( uint32 nParam = 0; nParam < nNumParams; nParam++ )
// {
// alert.AddExtendedInfoLine( "%s=%s", request.GetGETParamName( nParam ), request.GetGETParamValue( nParam ) );
// }
// }
// break;
//case k_EHTTPMethodPOST:
// {
// const uint32 nNumParams = request.GetPOSTParamCount();
// for( uint32 nParam = 0; nParam < nNumParams; nParam++ )
// {
// alert.AddExtendedInfoLine( "%s=%s", request.GetPOSTParamName( nParam ), request.GetPOSTParamValue( nParam ) );
// }
// }
// break;
//}
//
//GGCBase()->PostAlert( alert );
GGCBase()->PostAlert( k_EAlertTypeInfo, false, CFmtStr( "WebAPI request %s failed to emit because it exceeded %d characters", request.GetURL(), cv_webapi_result_size_limit.GetInt() ) );
}
else
{
EG_WARNING( SPEW_GC, "WebAPI %s - request %s failed to emit for unknown reason\n", GetName(), request.GetURL() );
}
//make sure that they get an error code back
WebAPIRespondWithError( GetName(), nSenderIP, msg, k_EHTTPStatusCode500InternalServerError );
}
return true;
}
void CWebAPIJob::AddLocalizedString( CWebAPIValues *pOutDefn, const char *pchFieldName, const char *pchKeyName, ELanguage eLang, bool bReturnTokenIfNotFound )
{
// NULL keys we just skip
if( !pchKeyName )
return;
const char *pchValue;
if( eLang == k_Lang_None )
{
pchValue = pchKeyName;
}
else
{
pchValue = GGCBase()->LocalizeToken( pchKeyName, eLang, bReturnTokenIfNotFound );
}
if( pchValue )
{
pOutDefn->CreateChildObject( pchFieldName )->SetStringValue( pchValue );
}
}
//-----------------------------------------------------------------------------
// Purpose: A wrapper to call BEmitFormattedOutput and pass out a return value
// since functors don't make return values available.
//-----------------------------------------------------------------------------
/*static*/ void CWebAPIJob::ThreadedEmitFormattedOutputWrapper( CWebAPIResponse *pResponse, EWebAPIOutputFormat eFormat, CUtlBuffer *poutputBuffer, size_t unMaxResultSize, bool *pbResult )
{
*pbResult = pResponse->BEmitFormattedOutput( eFormat, *poutputBuffer, unMaxResultSize );
}
} // namespace GCSDK