source-engine/utils/source_builder/source_build.py

#====== Copyright 1996-2005, Valve Corporation, All rights reserved. =======
#
# Purpose: Syncs, builds and runs tests on the steam code in
#          the currently active p4 clientspec
#
#=============================================================================

# INSTALLATION INSTRUCTIONS:
#
# 1. add the vc compiler solution compiler devenv.exe to the path (defaults to being in "C:\\Program Files\\Microsoft Visual Studio .NET 2003\\Common7\\IDE\\devenv.exe")
# 2. create a P4 clientspec that contains the steam code, eg. //steam/main/...
# 3. make that the default clientspec
# 4. sync to it
# 5. run this build script from inside that directory


import sys, os, string, re, time, smtplib, getopt, P4, SystemHelpers

# config options
g_bTest = 0                                     # 1: disables reporting to SRCDEV; only reports to admin
g_bRunTests = 0                                 # 1: enables testing after build is complete
g_bSync = 0                                     # 1: enables syncing to perforce 
g_bLockMutex = 0                                # 1: require mutex to be free before proceeding to build
g_bBuild = 0                                    # 1: build binaries
g_bDebug = 0                                    # 1: enables debug output
g_bDev = 0                                      # 1: script builds immediately upon start, regardless of presence of perforce state
g_bShaders = 0
g_bXBOX = 0

g_szEmailAlias = ""     # email alias of users interested in build failures
g_szAdministrator = ""   # address to auto-email if the script fails for some reason, test output
g_szMailhost = "" 	# set to hostname of machine running an SMTP server
g_szSenderEmail = ""     # email address that the failure mails come from; TODO: Get buildmachine email 

g_szBuildExe = ""                   # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild
#g_szBuildExe = "devenv"                        # executable for compilation; devenv: .Net, BuildConsole: IncrediBuild
g_szBuildType = ""                      # build type; /rebuild or /build
g_szBuildFlags = ""                        # should be empty for devenv, /all for buildconsole

g_szTestingFile = ""               # name of testing file, located in .\Build Machine Tests\
g_szLocalLogFile = ""               # name of generated log file; currently not used
g_szPublishedErrorsDir = ""  # place to copy log file to reference in failure email
g_szTestDirectory = ""     # directory where the testing files are located

g_szBaseDir = os.getcwd()                       # directory from which the script is launched

g_szP4SrcFilesToWatch = ""     # path to src files to watch
g_szP4ForceFilesToWatch = ""    # path to bin files to watch
g_szP4SyncFilesToWatch = ""
g_szP4Mutex = ""                        # name of perforce mutex to wait for
g_szP4ChangeCounter = ""		# perforce counter containg the changelist number we're verifying
g_szP4VerifiedCounter = ""		# perforce counter the last changelist number we have verified

g_nP4MostRecentCheckin = 0                      # used to keep track of current checkin
g_nP4LastVerifiedCheckin = 0                    # used to keep track of last verified checkin

g_nSleepTimeBetweenChecks = 10                  # number of seconds to wait before checking for a free mutex

g_szBuildName = ""


# mails off a success checkin
def SendSuccessEmail( _szEmail, _szChangeNumber ):
    cMailport = smtplib.SMTP(g_szMailhost)
    szTestString = ""
    if g_bTest:
        szTestString = "TEST"
    print "sending success email to: " + _szEmail

    szMessage = 'From: ' + g_szBuildName + ' Builder ' + ' <' + g_szSenderEmail + '>\n' +\
              'To: ' + szTestString + _szEmail + '\n' +\
              'Subject: [ ' + g_szBuildName + ' build successful ]' +\
              '\n' +\
              '\n' +\
              'Your checkin:\n\n' +\
              _szChangeNumber +\
              '\n\n' +\
              'has been successfully built and verified against all tests.\n'

    if ( g_bTest | g_bDev ):
        cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage )
    else:       
        cMailport.sendmail( g_szSenderEmail, _szEmail, szMessage )
    cMailport.quit()

# print of debugging messages
def dPrint( _szText ):
    if g_bDebug:
        print _szText

# mails off the current error string
def SendFailureEmail( _szErr ):
    cMailport = smtplib.SMTP( g_szMailhost )
    szTestString = ""
    if g_bTest:
        szTestString = "TEST"
    print "sending failure email to: " + g_szEmailAlias
    print "failure reason: " + _szErr

    szMessage = 'From: ' + g_szBuildName + ' Builder' + ' <' + g_szSenderEmail + '>\n' +\
              'To: ' + szTestString + g_szEmailAlias + '\n' +\
              'Subject: [ ' + g_szBuildName + ' branch broken ]' +\
              '\n' +\
              '\n' +\
              _szErr +\
              '\n' +\
              P4.GetUnverifiedCheckins( g_szP4SrcFilesToWatch, g_szP4VerifiedCounter, g_szP4ChangeCounter )              

    if g_bTest | g_bDev:
        cMailport.sendmail( g_szSenderEmail, g_szAdministrator, szMessage )
    else:
        cMailport.sendmail( g_szSenderEmail, szTestString + g_szEmailAlias, szMessage )
    cMailport.quit()

# use to email the admin about any unexpected errors that occur
def ComplainToAdmin( _szErr ):
    cMailport = smtplib.SMTP(g_szMailhost)
    print "sending script failure email to: " + g_szAdministrator
    print "failure reason: |" + _szErr + "|"

    szMessage = 'From: ' + g_szBuildName + " Builder" + ' <' + g_szSenderEmail + '>\n' +\
              'To: ' + g_szAdministrator + '\n' +\
              'Subject: [ ' + g_szBuildName + ' builder build script error ]' +\
              '\n' +\
              '\n' +\
              'error reason:\n' +\
              _szErr +\
              '\n'
              
    cMailport.sendmail(g_szSenderEmail, g_szAdministrator, szMessage)
    cMailport.quit()

# parses out the reason for failure from a test
def GetTestFailureReason( _nBuild ):

    # start our error message
    szMsg = "Test log file:\n"

    # make a directory to copy the error files into, based on the build config and the p4 changelist number
    nBuildNum = P4.GetCounter(g_szP4ChangeCounter)
    szErrDir = g_szPublishedErrorsDir + nBuildNum + "\\" + _nBuild
    os.popen("mkdir " + szErrDir, "r").read()
    
    # copy all the files to the errors dir
    os.popen("copy " + _nBuild + "\\*.*" + " " + szErrDir + " /Y", "r").read()
    # add the error log to the failure message
    szMsg += "    " + szErrDir + "\\" + g_szLocalLogFile + "\n"

    # add the minidumps to the failure message
    rgMinidumps = string.split( os.popen("dir " + _nBuild + "\\*.mdmp /b", "r").read(), "\n" )
    if len(rgMinidumps) > 1:
        szMsg += "\nCrash dump:\n"
        for szMinidump in rgMinidumps:
            if len(szMinidump) > 16:
                szMsg += "    " + szErrDir + "\\" + szMinidump + "\n"
        szMsg += "To view the minidump, click the link above then click \"open\" on the next dialog.\nHit \"F5\" in the now open debugger.  When it asks for the source code, change the start of the suggested path from \"c:\\\" to \"\\\\steam3builder\\\".\n"

    # delete the minidumps, so they don't confuse us next time
    os.popen("del " + _nBuild + "\\*.mdmp", "r").read()
        

    # parse out the error from the logfile 
    szLog = os.popen( "type " + _nBuild + "\\" + szLocalLogFile, "r").read()
    szLogLines = []
    szLogLines = string.split( szLog, "\n" )
    bFoundErr = 0
    for szLine in szLogLines:
        aszToken = string.split(szLine, ' ')
        if len( aszToken ) > 3:
            if aszToken[0] == "***" and aszToken[1] == "TEST" and aszToken[2] == "FAILED":
                # probably an error line, add to the list
                szMsg += szLine
                szMsg += "\n"
                bFoundErr = 1

    # if we haven't found any error lines, use the last line of the file
    if bFoundErr == 0 and len(szLogLines) > 0:
        szMsg += szLogLines[len(szLogLines) - 1]

    return szMsg


# performs a single test run of the specified exe with the specified test parameters
def RunTest( _szCommand ):

    aszParms = string.split( _szCommand, " ", 1)
    print "running " + _szCommand
    print os.popen( _szCommand, "r" ).read()

    # return success
    return aszParms[0]

def RunCompare( _szCommand ):
    szResult = os.popen( _szCommand, "r" ).read()
    return szResult

def SearchFileForErrors( _szFile, _szError ):
    print ("Searching " + _szFile + " for occurences of " + _szError + ".\n")
    bIncludeNext = 0
    szErrorResults = ""
    aszFileLines = string.split( os.popen( "type " + _szFile, "r").read(), "\n" )

    for szLine in aszFileLines:
        if ( bIncludeNext == 1 ):
            szErrorResults += szLine + "\n"
            bIncludeNext = 0
        aszParms = string.split(szLine, " ", 1)
        if (aszParms[0] == _szError):
            #there is a UnitTest error, warning, or assert
            szErrorResults += "\n" + szLine + "\n"
            bIncludeNext = 1;
    return szErrorResults
 
# runs a set of tests as defined by the test script file
def RunTestScript( ):
    print ("Enter Testing")
    # make sure we're in the right dir
    SystemHelpers.ChangeDir("..")
    # load the script
    szReturn = os.popen( "RunTestScripts.py" ).read()
    return szReturn

# runs a single build, and runs the tests on the build if specified
def RunBuild( _szBuild ):
    # launch devstudio to build the solution
    szCmd = _szBuild
    aszToken = string.split(szCmd, " ", 3)

    if aszToken[0] == "devenv":
        #we are building this. Find out what it is.
        if aszToken[2] == "/project":
            #building a project.  Grab it.
            aszToken = string.split(szCmd, " ", 5)
            szCmd = g_szBuildExe + " " + aszToken[1] + " /project " + aszToken[3] + " " + g_szBuildType + " " + aszToken[5] 
        else:
            #not build a specific project.  Probably everything under lostcoast.
            szCmd = g_szBuildExe + " " + aszToken[1] + " " + g_szBuildType + " " + aszToken[3] + " " + g_szBuildFlags
    else:
        #didn't see the devenv line; not building this but it isn't an error either.
        return ""
                
    print "building: " + szCmd
    szOutput = os.popen(szCmd, "r").read()
    # parse the output for any errors
    aszOutputLines = string.split(szOutput, "\n")
    bSuccess = 1    
    szBuildErr = "\n\n\nError building configuration " + _szBuild + ":\n"

    for szLine in aszOutputLines:
        if g_bDebug:
            print szLine;
        aszTokens = string.split(string.lstrip(szLine), ' ', 4)
        if len(aszTokens) > 1:
            if aszTokens[0] == "--------------------Configuration:":
                bPrintedProject = 0
                szProject = aszTokens[1]
            if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] != "PRJ0019":
                # probably an error line, add to the list
                count = string.count( szBuildErr, aszTokens[4])
                if ( count == 0 ):
                    if ( bPrintedProject == 0 ):
                        szBuildErr += "Project: " + szProject + "\n"
                        bPrintedProject = 1                    
                    szBuildErr += szLine + "\n"
                #err2 += szLine + "\n"
                bSuccess = 0
            if aszTokens[1] == ":" and aszTokens[2] == "error" and aszTokens[3] == "PRJ0019":
                # the delete error line, add to the list
                szBuildErr += "IGNORE: "
                szBuildErr += szLine + "\n"
                #err2 += szLine + "\n"
        aszTokens = string.split(string.lstrip(szLine), ' ')        
        if len(aszTokens) > 4:
            # can't do this: the weird delete error will trigger this and we need to ignore that
            # check that the "Rebuild All: x succeeded, y failed, z skipped line says no failures
            #if szT[0] == "Rebuild" and szT[3] rrorMessages + "The " + szTestName + " test failed\nThe error is " + szCompareLine
            #if szErrorMessages== "succeeded," and szT[5] == "failed," and not szT[4] == "0":
                    # failure
            #       bSuccess = 0
            # check for linker errors
            if aszTokens[0] == "LINK" and aszTokens[1] == ":" and aszTokens[2] == "fatal" and aszTokens[3] == "error":
                if ( bPrintedProject == 0 ):
                    szBuildErr += "Project: " + szProject + "\n"
                    bPrintedProject = 1 
                # linker error line, add to the list
                szBuildErr += szLine + "\n"
                #err2 += szLine + "\n"
                bSuccess = 0
            # can't do this either:  Delete file problem
            # check the standard error dealie.
            # if szT[1] == '-' and szT[3] == "error(s)," and szT[2] != '0':
                #we have a build error here
            #    err += szLine
            #    err += "\n"
            #check for fatal error
            if aszTokens[2] == "fatal" and aszTokens[3] == "error":
                if ( bPrintedProject == 0 ):
                    szBuildErr += "Project: " + szProject + "\n"
                    bPrintedProject = 1 
                #fatal error
                szBuildErr += szLine + "\n"
                #err2 += szLine + "\n"
                bSuccess = 0
    
    # return immediately if failed    
    if not bSuccess:
        print szBuildErr
        szBuildErr + "\n\n"
        #ComplainToAdmin(err2)
        return szBuildErr

    return ""

def RunBuildBatch():    
    bSuccess = 1
    #aszBatchBuildLines = string.split( os.popen( "type " + szBatchFile, "r").read(), "\n" )
    #bIsDevLine = 0
    szBuildErrs = ""
    szBuildResult = RunBuild( "devenv everything.sln /build \"debug|win32\" " )

    szBuildResult += RunBuild( "devenv everything.sln /build \"release|win32\" " )
    szTestResult = RunTestScript()
    if szBuildResult != "":
        szBuildErrs += szBuildResult
        bSuccess = 0
    if szTestResult != "\n":
        szBuildErrs += szTestResult
        bSuccess = 0
    if not bSuccess:
        SendFailureEmail(szBuildErrs)
    return bSuccess   
    
    
# runs all the builds
def RunAllBuilds():    
    bSuccess = 1
    szBuildErrs = ""
    SystemHelpers.ChangeDir("\game")
    if g_bBuild:
        SystemHelpers.ChangeDir("\src")
                
        # build the shadercompiler and all shaders
        if g_bShaders:
            print( "Compiling shadercompile" )
            os.system( "devenv shadercompile.sln /rebuild release > silence" )
            # should check here to make sure the shadercompile worked
            SystemHelpers.ChangeDir("\\src\\materialsystem\\stdshaders")
            print( "Building shaders" )
            child_stdin, child_stdout, child_stderr = os.popen3( "buildallshaders.bat" )
            print( child_stdout.read() )
            szSomeShaderErr = child_stderr.read()
            szShaderErrors = ""
            aszShaderLines = string.split( szSomeShaderErr, "\n" )
            for szLine in aszShaderLines:
                dPrint( szLine )
                nCount = string.count( szLine, 'U1073:' )
                if nCount > 0:
                    aszToken = string.split( szLine, " " )
                    szShaderName = aszToken[9]
                    szShaderErrors = szShaderErrors + "The shader file " + szShaderName + " is missing and failed during buildallshaders.bat.\n"
                    if szShaderErrors:
                        SendFailureEmail( szShaderErrors )

        SystemHelpers.ChangeDir("\\src")
        bSuccess = bSuccess & RunBuildBatch();

#XBOX Section
        if g_bXBOX:            
            szXBoxOutput = RunBuild( "devenv source_x360.sln /rebuild \"release|xbox 360\"" )
            if "\n\n\nError building configuration " + "devenv source_x360.sln /rebuild release" + ":\n" != szXBoxOutput:
                bSuccess = 0
                SendFailureEmail( szXBoxOutput )
     #delete tier0.dll 
        
        
    if g_bRunTests:
        szTestErrors = RunTestScript()
        if szTestErrors != "\n":
            # success = 0
            ComplainToAdmin( szTestErrors )
    return bSuccess

# builds from a local branch and runs a subset of tests
def PerformSourceBuild():

    # build and test in each configuration
    if not RunAllBuilds():
        print "Source build failed"
        return 0

    print "Source build: SUCCESS"
    return 1
    

# syncs, builds, runs tests
def PerformDailyBuild():
    print "  changes detected, starting daily build"
    # update the counter to be what we're verifying
    change = P4.SubmittedChangelist( g_szP4SrcFilesToWatch )
    g_nP4MostRecentCheckin = change
    g_nP4LastVerifiedCheckin = P4.GetCounter(g_szP4VerifiedCounter)
    if g_nP4MostRecentCheckin and g_nP4LastVerifiedCheckin:
        print "Most recent checkin is " + g_nP4MostRecentCheckin + "\n"
        print "Last verified checkin is " + g_nP4LastVerifiedCheckin + "\n"
    # the p4 command can occasionally fail to deliver a valid changelist number, unclear why
    # can't update the counter, it just means we'll run twice
    if change:
        P4.SetCounter(g_szP4ChangeCounter, change)
    # sync to the new files
    if ( g_bSync ):
        SystemHelpers.ChangeDir("\\src")
        print( "Cleaning\n" )
        os.system("cleanalltargets.bat > silence")
        SystemHelpers.ChangeDir("\\")        
        print "Synching force files."
        P4.Sync( g_szP4ForceFilesToWatch, 1 )
        print "Synching other files."
        P4.Sync( g_szP4SyncFilesToWatch, 0 )
        print( "Setting up VPC" )
        os.system("setupVPC.bat")
        
    #P4.UnlockMutex(g_szP4Mutex)
    # build and test in each configuration
    if not RunAllBuilds():
        print "Daily build failed"
        return
        
    # send a success email, from past the last successful checkin to the current
    if change:
        szVerifiedOrig = P4.GetCounter(g_szP4VerifiedCounter)
        if szVerifiedOrig:
            szVerifiedPlusOne = str( int( szVerifiedOrig ) + 1 )
            changes = P4.GetChangelistRange(szVerifiedPlusOne, change, g_szP4SrcFilesToWatch );
            for ch in changes:
                if len(ch) > 1:
                    szEmail = P4.GetEmailFromChangeLine(ch)
                    SendSuccessEmail(szEmail, ch)
                    #SendSuccessEmail("jason", ch)
            # remember this change that we've verified
            P4.SetCounter(g_szP4VerifiedCounter, change)

    print "Daily build: AN UNEQUIVOCAL SUCCESS"

def PrintConfig():
    print("Configuration:")
    print("test = " + str(g_bTest))
    print("run_tests = " + str(g_bRunTests))
    print("lock_mutex = " + str(g_bLockMutex))
    print("build = " + str(g_bBuild))
    print("debug = " + str(g_bDebug))
    print("dev = " + str(g_bDev))
    print("shaders = " + str(g_bShaders))
    print("sync = " + str(g_bSync))
    print("email_alias = " + g_szEmailAlias)
    print("admin_email = " + g_szAdministrator)
    print("mail_host = " + g_szMailhost)
    print("sender_email = " + g_szSenderEmail)
    print("build_exe = " + g_szBuildExe)
    print("build_type = " + g_szBuildType)
    print("build_flags = " + g_szBuildFlags)
    print("test_file = " + g_szTestingFile)
    print("log_file = " + g_szLocalLogFile)
    print("error_dir = " + g_szPublishedErrorsDir)
    print("test_dir = " + g_szTestDirectory)
    print("src_files = " + g_szP4SrcFilesToWatch)
    print("force_files = " + g_szP4ForceFilesToWatch)
    print("sync_files = " + g_szP4SyncFilesToWatch)
    print("mutex = " + g_szP4Mutex)
    print("change_counter = " + g_szP4ChangeCounter)
    print("verify_counter = " + g_szP4VerifiedCounter)
    print("build_name = " + g_szBuildName)

def ParseConfigFile(configFileName):
     aszBatchBuildLines = string.split( os.popen( "type " + configFileName, "r").read(), "\n" )
     global g_bTest
     global g_bRunTests
     global g_bLockMutex
     global g_bBuild
     global g_bDebug
     global g_bDev
     global g_bShaders
     global g_bSync
     global g_szEmailAlias
     global g_szAdministrator
     global g_szMailhost
     global g_szSenderEmail
     global g_szBuildExe
     global g_szBuildType
     global g_szBuildFlags
     global g_szTestingFile
     global g_szLocalLogFile
     global g_szPublishedErrorsDir
     global g_szTestDirectory
     global g_szP4SrcFilesToWatch
     global g_szP4ForceFilesToWatch
     global g_szP4SyncFilesToWatch
     global g_szP4Mutex
     global g_szP4ChangeCounter
     global g_szP4VerifiedCounter
     global g_szBuildName
     for szLine in aszBatchBuildLines:
         aszTokens = string.split(string.lstrip(szLine), ' ', 2)
         firstToken = aszTokens[0]
         if firstToken == '#' or firstToken == '':
             continue
         secondToken = aszTokens[1]
         if firstToken == "test":
             g_bTest = int(secondToken)
         elif firstToken == "run_tests":
             g_bRunTests = int(secondToken)
         elif firstToken == "lock_mutex":
             g_bLockMutex = int(secondToken)
         elif firstToken == "build":
             g_bBuild = int(secondToken)
         elif firstToken == "debug":
             g_bDebug = int(secondToken)
         elif firstToken == "dev":
             g_bDev = int(secondToken)
         elif firstToken == "shaders":
             g_bShaders = int(secondToken)
         elif firstToken == "sync":
             g_bSync = int(secondToken)
         elif firstToken == "email_alias":
             g_szEmailAlias = secondToken
         elif firstToken == "admin_email":
             g_szAdministrator = secondToken
         elif firstToken == "mail_host":
             g_szMailhost = secondToken
         elif firstToken == "sender_email":
             g_szSenderEmail = secondToken
         elif firstToken == "build_exe":
             g_szBuildExe = secondToken
         elif firstToken == "build_type":
             g_szBuildType = secondToken
         elif firstToken == "build_flags":
             g_szBuildFlags = secondToken
         elif firstToken == "test_file":
             g_szTestingFile = secondToken
         elif firstToken == "log_file":
             g_szLocalLogFile = secondToken
         elif firstToken == "error_dir":
             g_szPublishedErrorsDir = secondToken
         elif firstToken == "test_dir":
             g_szTestDirectory = secondToken
         elif firstToken == "src_files":
             g_szP4SrcFilesToWatch += secondToken 
         elif firstToken == "force_files":
             g_szP4ForceFilesToWatch += secondToken + ";"
         elif firstToken == "sync_files":
             g_szP4SyncFilesToWatch += secondToken + ";"
         elif firstToken == "mutex":
             g_szP4Mutex = secondToken
         elif firstToken == "change_counter":
             g_szP4ChangeCounter = secondToken
         elif firstToken == "verify_counter":
             g_szP4VerifiedCounter = secondToken
         elif firstToken == "build_name":
             g_szBuildName = secondToken
     PrintConfig()

#-----------------------------------------------------------------------------
# Main 
#-----------------------------------------------------------------------------
if __name__ == '__main__':
    try:
                print "----------------------------------------------------"
                print g_szBuildName + " BUILD SCRIPT STARTED"
                ParseConfigFile(sys.argv[1])

                while 1:
                    if (g_bDev | P4.AnyNewCheckins( g_szP4ChangeCounter, g_szP4SrcFilesToWatch )):
                        print "Changes Detected.\n"
                        if ( (g_bTest & ~g_bLockMutex) | ~g_bLockMutex | P4.Query(g_szP4Mutex) ):
                            PerformDailyBuild()
                            g_bDev = 0
                            print ""
                            print "------------------------------------------"
                            print "waiting for changes to be detected..."
                        else:
                            time.sleep( g_nSleepTimeBetweenChecks - ( g_bTest * g_nSleepTimeBetweenChecks ))
                    else:
                        time.sleep( g_nSleepTimeBetweenChecks )
    except RuntimeError, e:
        ComplainToAdmin(e)