|
|
|
@ -5326,8 +5326,10 @@ static void *watchdog_thread(void *userdata)
@@ -5326,8 +5326,10 @@ static void *watchdog_thread(void *userdata)
|
|
|
|
|
} else if (now.tv_sec - thr->last.tv_sec > 600 && gpus[i].status == LIFE_SICK) { |
|
|
|
|
gpus[gpu].status = LIFE_DEAD; |
|
|
|
|
applog(LOG_ERR, "Thread %d not responding for more than 10 minutes, GPU %d declared DEAD!", i, gpu); |
|
|
|
|
} else if (now.tv_sec - thr->sick.tv_sec > 60 && gpus[i].status == LIFE_SICK) { |
|
|
|
|
/* Attempt to restart a GPU once every minute */ |
|
|
|
|
gettimeofday(&thr->sick, NULL); |
|
|
|
|
} else if (now.tv_sec - thr->sick.tv_sec > 60 && |
|
|
|
|
(gpus[i].status == LIFE_SICK || gpus[i].status == LIFE_DEAD)) { |
|
|
|
|
/* Attempt to restart a GPU that's sick or dead once every minute */ |
|
|
|
|
gettimeofday(&thr->sick, NULL); |
|
|
|
|
#ifdef HAVE_ADL |
|
|
|
|
if (adl_active && gpus[gpu].has_adl && gpu_activity(gpu) > 50) { |
|
|
|
|