Merge pull request #382 from kanoi/hotplug

USB tidy up requests + Hotplug

API-README

@@ -1357,9 +1357,12 @@ You can only see fields listed in 'group' and 'calc'
 
 A 'calc' is formatted as: 'Field' => 'function'
 The current list of operations available for 'calc' are:
-'sum', 'avg', 'min', 'max', 'lo', 'hi', 'any'
+'sum', 'avg', 'min', 'max', 'lo', 'hi', 'coount', 'any'
 The first 4 are as expected - the numerical sum, average, minimum or maximum
 'lo' is the first string of the list, sorted ignoring case
 'hi' is the last string of the list, sorted ignoring case
+'count' is the number of rows in the section specified in the calc e.g.
+ ('DEVS.Name' => 'count') would be the number of DEVS selected in the 'where'
+ of course any valid 'DEVS.Xyz' would give the same 'count' value
 'any' is effectively random: the field value in the first row of the grouped data
 An unrecognised 'function' uses 'any'

adl.c

@@ -1392,7 +1392,7 @@ updated:
 		clear_logwin();
 		return;
 	}
-	sleep(1);
+	nmsleep(1000);
 	goto updated;
 }
 #endif

api.c

@@ -1148,6 +1148,7 @@ static int numpgas()
 	int count = 0;
 	int i;
 
+	mutex_lock(&devices_lock);
 	for (i = 0; i < total_devices; i++) {
 #ifdef USE_BITFORCE
 		if (devices[i]->drv->drv_id == DRIVER_BITFORCE)
@@ -1166,6 +1167,7 @@ static int numpgas()
 			count++;
 #endif
 	}
+	mutex_unlock(&devices_lock);
 	return count;
 }
 
@@ -1174,6 +1176,7 @@ static int pgadevice(int pgaid)
 	int count = 0;
 	int i;
 
+	mutex_lock(&devices_lock);
 	for (i = 0; i < total_devices; i++) {
 #ifdef USE_BITFORCE
 		if (devices[i]->drv->drv_id == DRIVER_BITFORCE)
@@ -1192,9 +1195,16 @@ static int pgadevice(int pgaid)
 			count++;
 #endif
 		if (count == (pgaid + 1))
-			return i;
+			goto foundit;
 	}
+
+	mutex_unlock(&devices_lock);
 	return -1;
+
+foundit:
+
+	mutex_unlock(&devices_lock);
+	return i;
 }
 #endif
 
@@ -1527,7 +1537,7 @@ static void pgastatus(struct io_data *io_data, int pga, bool isjson, bool precom
 		if (dev < 0) // Should never happen
 			return;
 
-		struct cgpu_info *cgpu = devices[dev];
+		struct cgpu_info *cgpu = get_devices(dev);
 		double frequency = 0;
 		float temp = cgpu->temp;
 
@@ -1747,6 +1757,7 @@ static void pgadev(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char *p
 
 static void pgaenable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char *param, bool isjson, __maybe_unused char group)
 {
+	struct cgpu_info *cgpu;
 	int numpga = numpgas();
 	struct thr_info *thr;
 	int pga;
@@ -1775,7 +1786,7 @@ static void pgaenable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char
 		return;
 	}
 
-	struct cgpu_info *cgpu = devices[dev];
+	cgpu = get_devices(dev);
 
 	applog(LOG_DEBUG, "API: request to pgaenable pgaid %d device %d %s%u",
 			id, dev, cgpu->drv->name, cgpu->device_id);
@@ -1814,6 +1825,7 @@ static void pgaenable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char
 
 static void pgadisable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char *param, bool isjson, __maybe_unused char group)
 {
+	struct cgpu_info *cgpu;
 	int numpga = numpgas();
 	int id;
 
@@ -1839,7 +1851,7 @@ static void pgadisable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, cha
 		return;
 	}
 
-	struct cgpu_info *cgpu = devices[dev];
+	cgpu = get_devices(dev);
 
 	applog(LOG_DEBUG, "API: request to pgadisable pgaid %d device %d %s%u",
 			id, dev, cgpu->drv->name, cgpu->device_id);
@@ -1856,6 +1868,8 @@ static void pgadisable(struct io_data *io_data, __maybe_unused SOCKETTYPE c, cha
 
 static void pgaidentify(struct io_data *io_data, __maybe_unused SOCKETTYPE c, char *param, bool isjson, __maybe_unused char group)
 {
+	struct cgpu_info *cgpu;
+	struct device_drv *drv;
 	int numpga = numpgas();
 	int id;
 
@@ -1881,8 +1895,8 @@ static void pgaidentify(struct io_data *io_data, __maybe_unused SOCKETTYPE c, ch
 		return;
 	}
 
-	struct cgpu_info *cgpu = devices[dev];
-	struct device_drv *drv = cgpu->drv;
+	cgpu = get_devices(dev);
+	drv = cgpu->drv;
 
 	if (!drv->identify_device)
 		message(io_data, MSG_PGANOID, id, NULL, isjson);
@@ -2794,6 +2808,7 @@ void notifystatus(struct io_data *io_data, int device, struct cgpu_info *cgpu, b
 
 static void notify(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __maybe_unused char *param, bool isjson, char group)
 {
+	struct cgpu_info *cgpu;
 	bool io_open = false;
 	int i;
 
@@ -2807,8 +2822,10 @@ static void notify(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __maybe
 	if (isjson)
 		io_open = io_add(io_data, COMSTR JSON_NOTIFY);
 
-	for (i = 0; i < total_devices; i++)
-		notifystatus(io_data, i, devices[i], isjson, group);
+	for (i = 0; i < total_devices; i++) {
+		cgpu = get_devices(i);
+		notifystatus(io_data, i, cgpu, isjson, group);
+	}
 
 	if (isjson && io_open)
 		io_close(io_data);
@@ -2833,7 +2850,7 @@ static void devdetails(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __m
 		io_open = io_add(io_data, COMSTR JSON_DEVDETAILS);
 
 	for (i = 0; i < total_devices; i++) {
-		cgpu = devices[i];
+		cgpu = get_devices(i);
 
 		root = api_add_int(root, "DEVDETAILS", &i, false);
 		root = api_add_string(root, "Name", cgpu->drv->name, false);
@@ -2930,6 +2947,7 @@ static int itemstats(struct io_data *io_data, int i, char *id, struct cgminer_st
 
 static void minerstats(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __maybe_unused char *param, bool isjson, __maybe_unused char group)
 {
+	struct cgpu_info *cgpu;
 	bool io_open = false;
 	struct api_data *extra;
 	char id[20];
@@ -2942,7 +2960,7 @@ static void minerstats(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __m
 
 	i = 0;
 	for (j = 0; j < total_devices; j++) {
-		struct cgpu_info *cgpu = devices[j];
+		cgpu = get_devices(j);
 
 		if (cgpu && cgpu->drv) {
 			if (cgpu->drv->get_api_stats)
@@ -3182,6 +3200,8 @@ static void usbstats(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __may
 #ifdef HAVE_AN_FPGA
 static void pgaset(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __maybe_unused char *param, bool isjson, __maybe_unused char group)
 {
+	struct cgpu_info *cgpu;
+	struct device_drv *drv;
 	char buf[TMPBUFSIZ];
 	int numpga = numpgas();
 
@@ -3215,8 +3235,8 @@ static void pgaset(struct io_data *io_data, __maybe_unused SOCKETTYPE c, __maybe
 		return;
 	}
 
-	struct cgpu_info *cgpu = devices[dev];
-	struct device_drv *drv = cgpu->drv;
+	cgpu = get_devices(dev);
+	drv = cgpu->drv;
 
 	char *set = strchr(opt, ',');
 	if (set)
@@ -3772,7 +3792,7 @@ void api(int api_thr_id)
 
 	/* This should be done before curl in needed
 	 * to ensure curl has already called WSAStartup() in windows */
-	sleep(opt_log_interval);
+	nmsleep(opt_log_interval*1000);
 
 	sock = socket(AF_INET, SOCK_STREAM, 0);
 	if (sock == INVSOCK) {
@@ -3818,7 +3838,7 @@ void api(int api_thr_id)
 				break;
 			else {
 				applog(LOG_WARNING, "API bind to port %d failed - trying again in 30sec", port);
-				sleep(30);
+				nmsleep(30000);
 			}
 		} else
 			bound = 1;

cgminer.c

@@ -157,7 +157,14 @@ static int input_thr_id;
 #endif
 int gpur_thr_id;
 static int api_thr_id;
+#if defined(USE_MODMINER) || defined(USE_BITFORCE)
+static int hotplug_thr_id;
+#endif
 static int total_control_threads;
+bool hotplug_mode;
+static int new_devices;
+static int new_threads;
+static int start_devices;
 
 #ifdef HAVE_LIBUSB
 pthread_mutex_t cgusb_lock;
@@ -173,6 +180,7 @@ static pthread_mutex_t sshare_lock;
 
 pthread_rwlock_t netacc_lock;
 pthread_mutex_t mining_thr_lock;
+pthread_mutex_t devices_lock;
 
 static pthread_mutex_t lp_lock;
 static pthread_cond_t lp_cond;
@@ -382,6 +390,16 @@ static struct cgpu_info *get_thr_cgpu(int thr_id)
 	return thr->cgpu;
 }
 
+struct cgpu_info *get_devices(int id)
+{
+	struct cgpu_info *cgpu;
+
+	mutex_lock(&devices_lock);
+	cgpu = devices[id];
+	mutex_unlock(&devices_lock);
+	return cgpu;
+}
+
 static void sharelog(const char*disposition, const struct work*work)
 {
 	char *target, *hash, *data;
@@ -742,18 +760,24 @@ static void load_temp_cutoffs()
 			if (val < 0 || val > 200)
 				quit(1, "Invalid value passed to set temp cutoff");
 
+			mutex_lock(&devices_lock);
 			devices[device]->cutofftemp = val;
+			mutex_unlock(&devices_lock);
 		}
 	} else {
+		mutex_lock(&devices_lock);
 		for (i = device; i < total_devices; ++i) {
 			if (!devices[i]->cutofftemp)
 				devices[i]->cutofftemp = opt_cutofftemp;
 		}
+		mutex_unlock(&devices_lock);
 		return;
 	}
 	if (device <= 1) {
+		mutex_lock(&devices_lock);
 		for (i = device; i < total_devices; ++i)
 			devices[i]->cutofftemp = val;
+		mutex_unlock(&devices_lock);
 	}
 }
 
@@ -1986,9 +2010,12 @@ static void curses_print_devstatus(int thr_id)
 	char displayed_hashes[16], displayed_rolling[16];
 	uint64_t dh64, dr64;
 
+	if (opt_compact)
+		return;
+
 	cgpu = get_thr_cgpu(thr_id);
 
-	if (devcursor + cgpu->cgminer_id > LINES - 2 || opt_compact)
+	if (cgpu->cgminer_id >= start_devices || devcursor + cgpu->cgminer_id > LINES - 2)
 		return;
 
 	cgpu->utility = cgpu->accepted / total_secs * 60;
@@ -2004,6 +2031,11 @@ static void curses_print_devstatus(int thr_id)
 	suffix_string(dh64, displayed_hashes, 4);
 	suffix_string(dr64, displayed_rolling, 4);
 
+#if defined(USE_MODMINER) || defined(USE_BITFORCE)
+	if (cgpu->usbinfo.nodev)
+		wprintw(statuswin, "ZOMBIE");
+	else
+#endif
 	if (cgpu->status == LIFE_DEAD)
 		wprintw(statuswin, "DEAD  ");
 	else if (cgpu->status == LIFE_SICK)
@@ -2460,7 +2492,7 @@ static bool submit_upstream_work(struct work *work, CURL *curl, bool resubmit)
 			pool->remotefail_occasions++;
 			applog(LOG_WARNING, "Pool %d communication failure, caching submissions", pool->pool_no);
 		}
-		sleep(5);
+		nmsleep(5000);
 		goto out;
 	} else if (pool_tclear(pool, &pool->submit_fail))
 		applog(LOG_WARNING, "Pool %d communication resumed, submitting work", pool->pool_no);
@@ -2767,6 +2799,16 @@ static void __kill_work(void)
 
 	applog(LOG_INFO, "Received kill message");
 
+#if defined(USE_MODMINER) || defined(USE_BITFORCE)
+	/* Best to get rid of it first so it doesn't
+	 * try to create any new devices */
+	if (!opt_scrypt) {
+		applog(LOG_DEBUG, "Killing off HotPlug thread");
+		thr = &control_thr[hotplug_thr_id];
+		thr_info_cancel(thr);
+	}
+#endif
+
 	applog(LOG_DEBUG, "Killing off watchpool thread");
 	/* Kill the watchpool thread */
 	thr = &control_thr[watchpool_thr_id];
@@ -2785,7 +2827,7 @@ static void __kill_work(void)
 		thr->pause = true;
 	}
 
-	sleep(1);
+	nmsleep(1000);
 
 	applog(LOG_DEBUG, "Killing off mining threads");
 	/* Kill the mining threads*/
@@ -4000,10 +4042,10 @@ void zero_stats(void)
 
 	zero_bestshare();
 
-	mutex_lock(&hash_lock);
 	for (i = 0; i < total_devices; ++i) {
-		struct cgpu_info *cgpu = devices[i];
+		struct cgpu_info *cgpu = get_devices(i);
 
+		mutex_lock(&hash_lock);
 		cgpu->total_mhashes = 0;
 		cgpu->accepted = 0;
 		cgpu->rejected = 0;
@@ -4014,8 +4056,8 @@ void zero_stats(void)
 		cgpu->diff_accepted = 0;
 		cgpu->diff_rejected = 0;
 		cgpu->last_share_diff = 0;
+		mutex_unlock(&hash_lock);
 	}
-	mutex_unlock(&hash_lock);
 }
 
 #ifdef HAVE_CURSES
@@ -4728,7 +4770,7 @@ static void *stratum_thread(void *userdata)
 				while (!initiate_stratum(pool) || !auth_stratum(pool)) {
 					if (pool->removed)
 						goto out;
-					sleep(30);
+					nmsleep(30000);
 				}
 			}
 		}
@@ -4766,7 +4808,7 @@ static void *stratum_thread(void *userdata)
 			while (!initiate_stratum(pool) || !auth_stratum(pool)) {
 				if (pool->removed)
 					goto out;
-				sleep(30);
+				nmsleep(30000);
 			}
 			stratum_resumed(pool);
 			continue;
@@ -5656,7 +5698,7 @@ retry_pool:
 	if (!pool) {
 		applog(LOG_WARNING, "No suitable long-poll found for %s", cp->rpc_url);
 		while (!pool) {
-			sleep(60);
+			nmsleep(60000);
 			pool = select_longpoll_pool(cp);
 		}
 	}
@@ -5731,7 +5773,7 @@ retry_pool:
 				continue;
 			if (failures == 1)
 				applog(LOG_WARNING, "longpoll failed for %s, retrying every 30s", lp_url);
-			sleep(30);
+			nmsleep(30000);
 		}
 
 		if (pool != cp) {
@@ -5835,7 +5877,7 @@ static void *watchpool_thread(void __maybe_unused *userdata)
 			switch_pools(NULL);
 		}
 
-		sleep(30);
+		nmsleep(30000);
 			
 	}
 	return NULL;
@@ -5925,7 +5967,7 @@ static void *watchdog_thread(void __maybe_unused *userdata)
 		}
 
 		for (i = 0; i < total_devices; ++i) {
-			struct cgpu_info *cgpu = devices[i];
+			struct cgpu_info *cgpu = get_devices(i);
 			struct thr_info *thr = cgpu->thr[0];
 			enum dev_enable *denable;
 			char dev_str[8];
@@ -6092,8 +6134,11 @@ void print_summary(void)
 	}
 
 	applog(LOG_WARNING, "Summary of per device statistics:\n");
-	for (i = 0; i < total_devices; ++i)
-		log_print_status(devices[i]);
+	for (i = 0; i < total_devices; ++i) {
+		struct cgpu_info *cgpu = get_devices(i);
+
+		log_print_status(cgpu);
+	}
 
 	if (opt_shares)
 		applog(LOG_WARNING, "Mined %d accepted shares of %d requested\n", total_accepted, opt_shares);
@@ -6343,7 +6388,7 @@ void enable_curses(void) {
 }
 #endif
 
-/* TODO: fix need a dummy CPU device_api even if no support for CPU mining */
+/* TODO: fix need a dummy CPU device_drv even if no support for CPU mining */
 #ifndef WANT_CPUMINE
 struct device_drv cpu_drv;
 struct device_drv cpu_drv = {
@@ -6454,11 +6499,18 @@ void fill_device_api(struct cgpu_info *cgpu)
 void enable_device(struct cgpu_info *cgpu)
 {
 	cgpu->deven = DEV_ENABLED;
+	mutex_lock(&devices_lock);
 	devices[cgpu->cgminer_id = cgminer_id_count++] = cgpu;
-	mining_threads += cgpu->threads;
+	mutex_unlock(&devices_lock);
+	if (hotplug_mode) {
+		new_threads += cgpu->threads;
+		adj_width(mining_threads + new_threads, &dev_width);
+	} else {
+		mining_threads += cgpu->threads;
 #ifdef HAVE_CURSES
-	adj_width(mining_threads, &dev_width);
+		adj_width(mining_threads, &dev_width);
 #endif
+	}
 #ifdef HAVE_OPENCL
 	if (cgpu->drv->drv_id == DRIVER_OPENCL) {
 		gpu_threads += cgpu->threads;
@@ -6487,8 +6539,13 @@ bool add_cgpu(struct cgpu_info*cgpu)
 		cgpu->device_id = d->lastid = 0;
 		HASH_ADD_STR(devids, name, d);
 	}
-	devices = realloc(devices, sizeof(struct cgpu_info *) * (total_devices + 2));
-	devices[total_devices++] = cgpu;
+	mutex_lock(&devices_lock);
+	devices = realloc(devices, sizeof(struct cgpu_info *) * (total_devices + new_devices + 2));
+	mutex_unlock(&devices_lock);
+	if (hotplug_mode)
+		devices[total_devices + new_devices++] = cgpu;
+	else
+		devices[total_devices++] = cgpu;
 	return true;
 }
 
@@ -6507,6 +6564,103 @@ struct device_drv *copy_drv(struct device_drv *drv)
 	return copy;
 }
 
+#if defined(USE_MODMINER) || defined(USE_BITFORCE)
+static void hotplug_process()
+{
+	struct thr_info *thr;
+	int i, j;
+
+	for (i = 0; i < new_devices; i++) {
+		struct cgpu_info *cgpu = devices[total_devices + i];
+		enable_device(cgpu);
+		cgpu->cgminer_stats.getwork_wait_min.tv_sec = MIN_SEC_UNSET;
+		cgpu->rolling = cgpu->total_mhashes = 0;
+	}
+
+	mutex_lock(&mining_thr_lock);
+	mining_thr = realloc(mining_thr, sizeof(thr) * (mining_threads + new_threads + 1));
+	mutex_unlock(&mining_thr_lock);
+	if (!mining_thr)
+		quit(1, "Failed to hotplug realloc mining_thr");
+	for (i = 0; i < new_threads; i++) {
+		mining_thr[mining_threads + i] = calloc(1, sizeof(*thr));
+		if (!mining_thr[mining_threads + i])
+			quit(1, "Failed to hotplug calloc mining_thr[%d]", i);
+	}
+
+	// Start threads
+	for (i = 0; i < new_devices; ++i) {
+		struct cgpu_info *cgpu = devices[total_devices];
+		cgpu->thr = malloc(sizeof(*cgpu->thr) * (cgpu->threads+1));
+		cgpu->thr[cgpu->threads] = NULL;
+		cgpu->status = LIFE_INIT;
+
+		for (j = 0; j < cgpu->threads; ++j) {
+			thr = get_thread(mining_threads);
+			thr->id = mining_threads;
+			thr->cgpu = cgpu;
+			thr->device_thread = j;
+
+			thr->q = tq_new();
+			if (!thr->q)
+				quit(1, "tq_new hotplug failed in starting %s%d mining thread (#%d)", cgpu->drv->name, cgpu->device_id, total_devices);
+
+			/* Enable threads for devices set not to mine but disable
+			 * their queue in case we wish to enable them later */
+			if (cgpu->deven != DEV_DISABLED) {
+				applog(LOG_DEBUG, "Pushing hotplug ping to thread %d", thr->id);
+				tq_push(thr->q, &ping);
+			}
+
+			if (cgpu->drv->thread_prepare && !cgpu->drv->thread_prepare(thr))
+				continue;
+
+			thread_reportout(thr);
+
+			if (unlikely(thr_info_create(thr, NULL, miner_thread, thr)))
+				quit(1, "hotplug thread %d create failed", thr->id);
+
+			cgpu->thr[j] = thr;
+
+			mining_threads++;
+		}
+		total_devices++;
+		applog(LOG_WARNING, "Hotplug: %s added %s %i", cgpu->drv->dname, cgpu->drv->name, cgpu->device_id);
+	}
+}
+
+static void *hotplug_thread(void __maybe_unused *userdata)
+{
+	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+
+	RenameThread("hotplug");
+
+	hotplug_mode = true;
+
+	while (0x2a) {
+		nmsleep(5000);
+
+// Version 0.1 just add the devices on - worry about using nodev later
+
+		new_devices = 0;
+		new_threads = 0;
+
+#ifdef USE_BITFORCE
+		bitforce_drv.drv_detect();
+#endif
+
+#ifdef USE_MODMINER
+		modminer_drv.drv_detect();
+#endif
+
+		if (new_devices)
+			hotplug_process();
+	}
+
+	return NULL;
+}
+#endif
+
 int main(int argc, char *argv[])
 {
 	bool pools_active = false;
@@ -6548,6 +6702,7 @@ int main(int argc, char *argv[])
 	rwlock_init(&blk_lock);
 	rwlock_init(&netacc_lock);
 	mutex_init(&mining_thr_lock);
+	mutex_init(&devices_lock);
 
 	mutex_init(&lp_lock);
 	if (unlikely(pthread_cond_init(&lp_cond, NULL)))
@@ -6783,6 +6938,8 @@ int main(int argc, char *argv[])
 	if (!total_devices)
 		quit(1, "All devices disabled, cannot mine!");
 
+	start_devices = total_devices;
+
 	load_temp_cutoffs();
 
 	for (i = 0; i < total_devices; ++i)
@@ -6841,7 +6998,7 @@ int main(int argc, char *argv[])
 			quit(1, "Failed to calloc mining_thr[%d]", i);
 	}
 
-	total_control_threads = 7;
+	total_control_threads = 8;
 	control_thr = calloc(total_control_threads, sizeof(*thr));
 	if (!control_thr)
 		quit(1, "Failed to calloc control_thr");
@@ -7016,11 +7173,21 @@ begin_bench:
 	if (thr_info_create(thr, NULL, api_thread, thr))
 		quit(1, "API thread create failed");
 
+#if defined(USE_MODMINER) || defined(USE_BITFORCE)
+	if (!opt_scrypt) {
+		hotplug_thr_id = 6;
+		thr = &control_thr[hotplug_thr_id];
+		if (thr_info_create(thr, NULL, hotplug_thread, thr))
+			quit(1, "hotplug thread create failed");
+		pthread_detach(thr->pth);
+	}
+#endif
+
 #ifdef HAVE_CURSES
 	/* Create curses input thread for keyboard input. Create this last so
 	 * that we know all threads are created since this can call kill_work
-	 * to try and shut down ll previous threads. */
-	input_thr_id = 6;
+	 * to try and shut down all previous threads. */
+	input_thr_id = 7;
 	thr = &control_thr[input_thr_id];
 	if (thr_info_create(thr, NULL, input_thread, thr))
 		quit(1, "input thread create failed");
@@ -7028,8 +7195,8 @@ begin_bench:
 #endif
 
 	/* Just to be sure */
-	if (total_control_threads != 7)
-		quit(1, "incorrect total_control_threads (%d) should be 7", total_control_threads);
+	if (total_control_threads != 8)
+		quit(1, "incorrect total_control_threads (%d) should be 8", total_control_threads);
 
 	/* Once everything is set up, main() becomes the getwork scheduler */
 	while (42) {
@@ -7075,7 +7242,7 @@ retry:
 			while (!pool->stratum_active || !pool->stratum_notify) {
 				struct pool *altpool = select_pool(true);
 
-				sleep(5);
+				nmsleep(5000);
 				if (altpool != pool) {
 					pool = altpool;
 					goto retry;
@@ -7091,7 +7258,7 @@ retry:
 			while (pool->idle) {
 				struct pool *altpool = select_pool(true);
 
-				sleep(5);
+				nmsleep(5000);
 				if (altpool != pool) {
 					pool = altpool;
 					goto retry;
@@ -7125,7 +7292,7 @@ retry:
 			 * requests but is up as we'll keep hammering it */
 			if (++pool->seq_getfails > mining_threads + opt_queue)
 				pool_died(pool);
-			sleep(5);
+			nmsleep(5000);
 			push_curl_entry(ce, pool);
 			pool = select_pool(!opt_fail_only);
 			goto retry;

driver-bitforce.c

@@ -343,7 +343,7 @@ static void bitforce_flash_led(struct cgpu_info *bitforce)
 	} else {
 		/* However, this stops anything else getting a reply
 		 * So best to delay any other access to the BFL */
-		sleep(4);
+		nmsleep(4000);
 	}
 
 	/* Once we've tried - don't do it until told to again */

driver-ztex.c

@@ -389,8 +389,11 @@ static void ztex_shutdown(struct thr_info *thr)
 
 static void ztex_disable(struct thr_info *thr)
 {
+	struct cgpu_info *cgpu;
+
 	applog(LOG_ERR, "%s: Disabling!", thr->cgpu->device_ztex->repr);
-	devices[thr->cgpu->device_id]->deven = DEV_DISABLED;
+	cgpu = get_devices(thr->cgpu->device_id);
+	cgpu->deven = DEV_DISABLED;
 	ztex_shutdown(thr);
 }
 

miner.h

@@ -740,6 +740,7 @@ extern pthread_mutex_t hash_lock;
 extern pthread_mutex_t console_lock;
 extern pthread_mutex_t ch_lock;
 extern pthread_mutex_t mining_thr_lock;
+extern pthread_mutex_t devices_lock;
 
 extern pthread_mutex_t restart_lock;
 extern pthread_cond_t restart_cond;
@@ -780,6 +781,7 @@ extern void add_pool_details(struct pool *pool, bool live, char *url, char *user
 #define _MAX_INTENSITY_STR "14"
 #endif
 
+extern bool hotplug_mode;
 extern struct list_head scan_devices;
 extern int nDevs;
 extern int opt_n_threads;
@@ -1110,6 +1112,7 @@ extern void free_work(struct work *work);
 extern void __copy_work(struct work *work, struct work *base_work);
 extern struct work *copy_work(struct work *base_work);
 extern struct thr_info *get_thread(int thr_id);
+extern struct cgpu_info *get_devices(int id);
 
 enum api_data_type {
 	API_ESCAPE,

miner.php

@@ -2076,6 +2076,8 @@ function docalc($func, $data)
 			if (strcasecmp($val, $ans) > 0)
 				$ans = $val;
 	return $ans;
+ case 'count':
+	return count($data);
  case 'any':
  default:
 	return $data[0];

usbutils.c

@@ -579,8 +579,7 @@ static void cgusb_check_init()
 	mutex_unlock(&cgusb_lock);
 }
 
-#ifdef WIN32
-#else
+#ifndef WIN32
 #include <errno.h>
 #include <unistd.h>
 #include <sys/types.h>
@@ -624,7 +623,7 @@ static bool cgminer_usb_lock_bd(struct device_drv *drv, uint8_t bus_number, uint
 		case WAIT_ABANDONED:
 			// Am I using it already?
 			for (i = 0; i < total_devices; i++) {
-				cgpu = devices[i];
+				cgpu = get_devices(i);
 				if (cgpu->usbinfo.bus_number == bus_number &&
 				    cgpu->usbinfo.device_address == device_address &&
 				    cgpu->usbinfo.nodev == false) {
@@ -642,9 +641,10 @@ static bool cgminer_usb_lock_bd(struct device_drv *drv, uint8_t bus_number, uint
 			}
 			return true;
 		case WAIT_TIMEOUT:
-			applog(LOG_WARNING,
-				"MTX: %s USB failed to get '%s' - device in use",
-				drv->dname, name);
+			if (!hotplug_mode)
+				applog(LOG_WARNING,
+					"MTX: %s USB failed to get '%s' - device in use",
+					drv->dname, name);
 			goto fail;
 		case WAIT_FAILED:
 			applog(LOG_ERR,
@@ -726,9 +726,10 @@ fail:
 
 	if (semop(sem, sops, 2)) {
 		if (errno == EAGAIN) {
-			applog(LOG_WARNING,
-				"SEM: %s USB failed to get (%d) '%s' - device in use",
-				drv->dname, sem, name);
+			if (!hotplug_mode)
+				applog(LOG_WARNING,
+					"SEM: %s USB failed to get (%d) '%s' - device in use",
+					drv->dname, sem, name);
 		} else {
 			applog(LOG_DEBUG,
 				"SEM: %s USB failed to get (%d) '%s' err (%d) %s",
@@ -841,14 +842,19 @@ static struct cg_usb_device *free_cgusb(struct cg_usb_device *cgusb)
 
 void usb_uninit(struct cgpu_info *cgpu)
 {
+	// May have happened already during a failed initialisation
+	//  if release_cgpu() was called due to a USB NODEV(err)
+	if (!cgpu->usbdev)
+		return;
 	libusb_release_interface(cgpu->usbdev->handle, cgpu->usbdev->found->interface);
 	libusb_close(cgpu->usbdev->handle);
 	cgpu->usbdev = free_cgusb(cgpu->usbdev);
 }
 
-void release_cgpu(struct cgpu_info *cgpu)
+static void release_cgpu(struct cgpu_info *cgpu)
 {
 	struct cg_usb_device *cgusb = cgpu->usbdev;
+	struct cgpu_info *lookcgpu;
 	int i;
 
 	cgpu->usbinfo.nodev = true;
@@ -857,14 +863,16 @@ void release_cgpu(struct cgpu_info *cgpu)
 
 	// Any devices sharing the same USB device should be marked also
 	// Currently only MMQ shares a USB device
-	for (i = 0; i < total_devices; i++)
-		if (devices[i] != cgpu && devices[i]->usbdev == cgusb) {
-			devices[i]->usbinfo.nodev = true;
-			devices[i]->usbinfo.nodev_count++;
-			memcpy(&(devices[i]->usbinfo.last_nodev),
+	for (i = 0; i < total_devices; i++) {
+		lookcgpu = get_devices(i);
+		if (lookcgpu != cgpu && lookcgpu->usbdev == cgusb) {
+			lookcgpu->usbinfo.nodev = true;
+			lookcgpu->usbinfo.nodev_count++;
+			memcpy(&(lookcgpu->usbinfo.last_nodev),
 				&(cgpu->usbinfo.last_nodev), sizeof(struct timeval));
-			devices[i]->usbdev = NULL;
+			lookcgpu->usbdev = NULL;
 		}
+	}
 
 	usb_uninit(cgpu);
 

usbutils.h

@@ -129,7 +129,6 @@ struct device_drv;
 struct cgpu_info;
 
 void usb_uninit(struct cgpu_info *cgpu);
-void release_cgpu(struct cgpu_info *cgpu);
 bool usb_init(struct cgpu_info *cgpu, struct libusb_device *dev, struct usb_find_devices *found);
 void usb_detect(struct device_drv *drv, bool (*device_detect)(struct libusb_device *, struct usb_find_devices *));
 struct api_data *api_usb_stats(int *count);