uWSGI 1.4.3

Avoid call storming of pidfile check that leads to multiple daemons
running instead of one. Fix #65.

core/daemons.c

@@ -30,6 +30,16 @@ extern struct uwsgi_server uwsgi;
 */
 
 void uwsgi_daemons_smart_check() {
+	static time_t last_run = 0;
+
+	time_t now = uwsgi_now();
+
+	if (now - last_run <= 0) {
+		return;
+	}
+
+	last_run = now;
+
 	struct uwsgi_daemon *ud = uwsgi.daemons;
 	while (ud) {
 		if (ud->pidfile) {
@@ -41,14 +51,14 @@ void uwsgi_daemons_smart_check() {
 				}
 				else {
 					ud->pidfile_checks++;
-					if (ud->pidfile_checks >= (uint64_t) ud->freq) {
+					if (ud->pidfile_checks >= (unsigned int) ud->freq) {
 						uwsgi_log("[uwsgi-daemons] found changed pidfile for \"%s\" (old_pid: %d new_pid: %d)\n", ud->command, (int) ud->pid, (int) checked_pid);
 						uwsgi_spawn_daemon(ud);
 					}
 				}
 			}
 			else if (checked_pid != ud->pid) {
-				uwsgi_log("[uwsgi-daemons] found changed pidfile for \"%s\" (old_pid: %d new_pid: %d)\n", ud->command, (int) ud->pid, (int) checked_pid);
+				uwsgi_log("[uwsgi-daemons] found changed pid for \"%s\" (old_pid: %d new_pid: %d)\n", ud->command, (int) ud->pid, (int) checked_pid);
 				ud->pid = checked_pid;
 			}
 			// all ok, pidfile and process found
@@ -183,6 +193,7 @@ void uwsgi_spawn_daemon(struct uwsgi_daemon *ud) {
 	else {
 		// close uwsgi sockets
 		uwsgi_close_all_sockets();
+		uwsgi_close_all_fds();
 
 		if (ud->daemonize) {
 			/* refork... */

core/emperor.c

@@ -784,6 +784,8 @@ void uwsgi_imperial_monitor_directory_init(struct uwsgi_emperor_scanner *ues) {
 		exit(1);
 	}
 
+	ues->arg = uwsgi.emperor_absolute_dir;
+
 }
 
 struct uwsgi_imperial_monitor *imperial_monitor_get_by_id(char *scheme) {

core/protocol.c

@@ -784,7 +784,8 @@ int uwsgi_parse_vars(struct wsgi_request *wsgi_req) {
 							wsgi_req->method = ptrbuf;
 							wsgi_req->method_len = strsize;
 						}
-						else if (!uwsgi.log_x_forwarded_for && !uwsgi_strncmp("REMOTE_ADDR", 11, wsgi_req->hvec[wsgi_req->var_cnt].iov_base, wsgi_req->hvec[wsgi_req->var_cnt].iov_len)) {
+						else if ((!uwsgi.log_x_forwarded_for || uwsgi_strncmp("HTTP_X_FORWARDED_FOR", 20, wsgi_req->hvec[wsgi_req->var_cnt].iov_base, wsgi_req->hvec[wsgi_req->var_cnt].iov_len))
+							&& !uwsgi_strncmp("REMOTE_ADDR", 11, wsgi_req->hvec[wsgi_req->var_cnt].iov_base, wsgi_req->hvec[wsgi_req->var_cnt].iov_len)) {
 							wsgi_req->remote_addr = ptrbuf;
 							wsgi_req->remote_addr_len = strsize;
 						}

core/utils.c

@@ -3274,10 +3274,46 @@ pid_t uwsgi_run_command(char *command, int *stdin_fd, int stdout_fd) {
 	}
 
 	uwsgi_close_all_sockets();
+	//uwsgi_close_all_fds();
+	int i;
+	for (i = 3; i < (int) uwsgi.max_fd; i++) {
+		if (stdin_fd) {
+			if (i == stdin_fd[0] || i == stdin_fd[1]) {
+				continue;
+			} 
+		}
+		if (stdout_fd > -1) {
+			if (i == stdout_fd) {
+				continue;
+			}
+		}
+#ifdef __APPLE__
+                fcntl(i, F_SETFD, FD_CLOEXEC);
+#else
+                close(i);
+#endif
+        }
+
+	
 
 	if (stdin_fd) {
 		close(stdin_fd[1]);
 	}
+	else {
+		if (!uwsgi_valid_fd(0)) {
+			int in_fd = open("/dev/null", O_RDONLY);
+			if (in_fd < 0) {
+				uwsgi_error_open("/dev/null");
+			}
+			else {
+				if (in_fd != 0) {
+					if (dup2(in_fd, 0)) {
+						uwsgi_error("dup2()");
+					}
+				}
+			}
+		}
+	}
 
 	if (stdout_fd > -1 && stdout_fd != 1) {
 		if (dup2(stdout_fd, 1) < 0) {
@@ -5039,3 +5075,23 @@ end:
 	free(symbol_name);
 	return ret;
 }
+
+int uwsgi_valid_fd(int fd) {
+	int ret = fcntl(fd, F_GETFL);
+	if (ret == 0) {
+		return 1;
+	}
+	return 0;
+}
+
+void uwsgi_close_all_fds(void) {
+	int i;
+	for (i = 3; i < (int) uwsgi.max_fd; i++) {
+#ifdef __APPLE__
+        	fcntl(i, F_SETFD, FD_CLOEXEC);
+#else
+                close(i);
+#endif
+	}
+}
+

core/uwsgi.c

@@ -471,6 +471,7 @@ static struct uwsgi_option uwsgi_base_options[] = {
 #endif
 
 	{"offload-threads", required_argument, 0, "set the number of offload threads to spawn (per-worker, default 0)", uwsgi_opt_set_int, &uwsgi.offload_threads, 0},
+	{"offload-thread", required_argument, 0, "set the number of offload threads to spawn (per-worker, default 0)", uwsgi_opt_set_int, &uwsgi.offload_threads, 0},
 
 	{"file-serve-mode", required_argument, 0, "set static file serving mode", uwsgi_opt_fileserve_mode, NULL, UWSGI_OPT_MIME},
 	{"fileserve-mode", required_argument, 0, "set static file serving mode", uwsgi_opt_fileserve_mode, NULL, UWSGI_OPT_MIME},

plugins/carbon/carbon.c

@@ -82,7 +82,7 @@ void carbon_post_init() {
 	// set next update to now()+retry_delay, this way we will have first flush just after start
 	u_carbon.last_update = uwsgi_now() - u_carbon.freq + u_carbon.retry_delay;
 
-	uwsgi_log("[carbon] carbon plugin started, %is frequency, %is timeout, max retries %i, retry delay %is",
+	uwsgi_log("[carbon] carbon plugin started, %is frequency, %is timeout, max retries %i, retry delay %is\n",
 		u_carbon.freq, u_carbon.timeout, u_carbon.max_retries, u_carbon.retry_delay);
 }
 

plugins/cheaper_busyness/cheaper_busyness.c

@@ -78,7 +78,7 @@ void set_next_cheap_time(void) {
 		// to have quicker recovery from big but short load spikes
 		// otherwise we might wait a lot before cheaping all emergency workers
 		if (uwsgi_cheaper_busyness_global.verbose)
-			uwsgi_log("[busyness] %d emergency worker(s) running, using %d seconds cheaper timer\n",
+			uwsgi_log("[busyness] %d emergency worker(s) running, using %llu seconds cheaper timer\n",
 				uwsgi_cheaper_busyness_global.emergency_workers, uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.backlog_multi);
 		uwsgi_cheaper_busyness_global.next_cheap = now + uwsgi.cheaper_overload*uwsgi_cheaper_busyness_global.backlog_multi*1000000;
 	} else {
@@ -95,7 +95,7 @@ void decrease_multi(void) {
 	// will decrease multiplier but only down to initial value
 	if (uwsgi_cheaper_busyness_global.cheap_multi > uwsgi_cheaper_busyness_global.min_multi) {
 		uwsgi_cheaper_busyness_global.cheap_multi--;
-		uwsgi_log("[busyness] decreasing cheaper multiplier to %d\n", uwsgi_cheaper_busyness_global.cheap_multi);
+		uwsgi_log("[busyness] decreasing cheaper multiplier to %llu\n", uwsgi_cheaper_busyness_global.cheap_multi);
 	}
 }
 
@@ -157,7 +157,7 @@ int cheaper_busyness_algo(void) {
 		// store initial multiplier so we don't loose its initial value
 		uwsgi_cheaper_busyness_global.min_multi = uwsgi_cheaper_busyness_global.cheap_multi;
 		// since this is first run we will print current values
-		uwsgi_log("[busyness] settings: min=%d%%, max=%d%%, overload=%d, multiplier=%d, respawn penalty=%d\n",
+		uwsgi_log("[busyness] settings: min=%llu%%, max=%llu%%, overload=%llu, multiplier=%llu, respawn penalty=%llu\n",
 			uwsgi_cheaper_busyness_global.busyness_min, uwsgi_cheaper_busyness_global.busyness_max,
 			uwsgi.cheaper_overload, uwsgi_cheaper_busyness_global.cheap_multi, uwsgi_cheaper_busyness_global.penalty);
 #ifdef __linux__
@@ -171,7 +171,7 @@ int cheaper_busyness_algo(void) {
 
 	if (uwsgi_cheaper_busyness_global.next_cheap == 0) set_next_cheap_time();
 
-	int64_t active_workers = 0;
+	int active_workers = 0;
 	uint64_t total_busyness = 0;
 	uint64_t avg_busyness = 0;
 
@@ -195,7 +195,7 @@ int cheaper_busyness_algo(void) {
 				if (percent > 100) percent = 100;
 				total_busyness += percent;
 				if (uwsgi_cheaper_busyness_global.verbose && active_workers > 1)
-					uwsgi_log("[busyness] worker nr %d %ds average busyness is at %d%%\n",
+					uwsgi_log("[busyness] worker nr %d %llus average busyness is at %llu%%\n",
 						i+1, uwsgi.cheaper_overload, percent);
 			}
 			uwsgi_cheaper_busyness_global.last_values[i] = uwsgi.workers[i+1].running_time;
@@ -204,7 +204,7 @@ int cheaper_busyness_algo(void) {
 		avg_busyness = (active_workers ? total_busyness / active_workers : 0);
 		if (uwsgi_cheaper_busyness_global.verbose)
 			uwsgi_log("[busyness] %ds average busyness of %d worker(s) is at %d%%\n",
-				uwsgi.cheaper_overload, active_workers, avg_busyness);
+				(int) uwsgi.cheaper_overload, (int) active_workers, (int) avg_busyness);
 
 		if (avg_busyness > uwsgi_cheaper_busyness_global.busyness_max) {
 
@@ -229,7 +229,7 @@ int cheaper_busyness_algo(void) {
 					// worker was cheaped and then spawned back in less than current multiplier*cheaper_overload seconds
 					// we will increase the multiplier so that next time worker will need to wait longer before being cheaped
 					uwsgi_cheaper_busyness_global.cheap_multi += uwsgi_cheaper_busyness_global.penalty;
-					uwsgi_log("[busyness] worker(s) respawned to fast, increasing chpeaper multiplier to %d (+%d)\n",
+					uwsgi_log("[busyness] worker(s) respawned to fast, increasing chpeaper multiplier to %llu (+%llu)\n",
 						uwsgi_cheaper_busyness_global.cheap_multi, uwsgi_cheaper_busyness_global.penalty);
 				} else {
 					decrease_multi();
@@ -237,10 +237,10 @@ int cheaper_busyness_algo(void) {
 
 				set_next_cheap_time();
 
-				uwsgi_log("[busyness] %ds average busyness is at %d%%, will spawn %d new worker(s)\n",
+				uwsgi_log("[busyness] %llus average busyness is at %llu%%, will spawn %d new worker(s)\n",
 					uwsgi.cheaper_overload, avg_busyness, decheaped);
 			} else {
-				uwsgi_log("[busyness] %ds average busyness is at %d%% but we already started maximum number of workers (%d)\n",
+				uwsgi_log("[busyness] %llus average busyness is at %llu%% but we already started maximum number of workers (%d)\n",
 					uwsgi.cheaper_overload, avg_busyness, uwsgi.numproc);
 			}
 
@@ -267,8 +267,8 @@ int cheaper_busyness_algo(void) {
 					if (uwsgi_cheaper_busyness_global.last_action == 2) decrease_multi();
 					set_next_cheap_time();
 
-					uwsgi_log("[busyness] %ds average busyness is at %d%%, cheap one of %d running workers\n",
-						uwsgi.cheaper_overload, avg_busyness, active_workers);
+					uwsgi_log("[busyness] %llus average busyness is at %llu%%, cheap one of %d running workers\n",
+						uwsgi.cheaper_overload, avg_busyness, (int) active_workers);
 					// store timestamp
 					uwsgi_cheaper_busyness_global.last_cheaped = uwsgi_micros();
 
@@ -280,7 +280,7 @@ int cheaper_busyness_algo(void) {
 
 					return -1;
 				} else if (uwsgi_cheaper_busyness_global.verbose)
-					uwsgi_log("[busyness] need to wait %d more second(s) to cheap worker\n", (uwsgi_cheaper_busyness_global.next_cheap - now)/1000000);
+					uwsgi_log("[busyness] need to wait %llu more second(s) to cheap worker\n", (uwsgi_cheaper_busyness_global.next_cheap - now)/1000000);
 			}
 
 		} else {
@@ -301,14 +301,14 @@ int cheaper_busyness_algo(void) {
 				// time needed to cheap them, than a lot min<busy<max when we do not reset timer
 				// and then another idle cycle than would trigger cheaping
 				if (uwsgi_cheaper_busyness_global.verbose)
-					uwsgi_log("[busyness] %ds average busyness is at %d%%, %d non-idle cycle(s), reseting cheaper timer\n",
+					uwsgi_log("[busyness] %llus average busyness is at %llu%%, %llu non-idle cycle(s), reseting cheaper timer\n",
 						uwsgi.cheaper_overload, avg_busyness, uwsgi_cheaper_busyness_global.tolerance_counter);
 				set_next_cheap_time();
 			} else {
 				// we had < 3 idle cycles in a row so we won't reset idle timer yet since this might be just short load spike
 				// but we need to add cheaper-overload seconds to the cheaper timer so this cycle isn't counted as idle
 				if (uwsgi_cheaper_busyness_global.verbose)
-					uwsgi_log("[busyness] %ds average busyness is at %d%%, %d non-idle cycle(s), adjusting cheaper timer\n",
+					uwsgi_log("[busyness] %llus average busyness is at %llu%%, %llu non-idle cycle(s), adjusting cheaper timer\n",
 						uwsgi.cheaper_overload, avg_busyness, uwsgi_cheaper_busyness_global.tolerance_counter);
 				uwsgi_cheaper_busyness_global.next_cheap += uwsgi.cheaper_overload*1000000;
 			}

uwsgi.h

@@ -37,7 +37,7 @@ extern "C" {
 #define thunder_lock if (uwsgi.threads > 1 && !uwsgi.is_et) {pthread_mutex_lock(&uwsgi.thunder_mutex);}
 #define thunder_unlock if (uwsgi.threads > 1 && !uwsgi.is_et) {pthread_mutex_unlock(&uwsgi.thunder_mutex);}
 
-#define uwsgi_check_scheme(file) (!uwsgi_startswith(file, "http://", 7) || !uwsgi_startswith(file, "data://", 7) || !uwsgi_startswith(file, "sym://", 6) || !uwsgi_startswith(file, "fd://", 5) || !uwsgi_startswith(file, "exec://", 7) || !uwsgi_startswith(file, "section://", 10))
+#define uwsgi_check_scheme(file) (!uwsgi_startswith(file, "emperor://", 10) || !uwsgi_startswith(file, "http://", 7) || !uwsgi_startswith(file, "data://", 7) || !uwsgi_startswith(file, "sym://", 6) || !uwsgi_startswith(file, "fd://", 5) || !uwsgi_startswith(file, "exec://", 7) || !uwsgi_startswith(file, "section://", 10))
 
 #define ushared uwsgi.shared
 
@@ -3479,6 +3479,8 @@ void uwsgi_user_lock(int);
 void uwsgi_user_unlock(int);
 
 void simple_loop_run_int(int);
+int uwsgi_valid_fd(int);
+void uwsgi_close_all_fds(void);
 
 void uwsgi_check_emperor(void);
 #ifdef UWSGI_AS_SHARED_LIBRARY

uwsgiconfig.py

@@ -1,6 +1,6 @@
 # uWSGI build system
 
-uwsgi_version = '1.4.2'
+uwsgi_version = '1.4.3'
 
 import os
 import re
@@ -33,15 +33,20 @@ if not GCC:
 
 CPP = os.environ.get('CPP', 'cpp')
 
-CPUCOUNT = 1
 try:
-    import multiprocessing
-    CPUCOUNT = multiprocessing.cpu_count()
+    CPUCOUNT = int(os.environ.get('CPUCOUNT', -1))
 except:
+    CPUCOUNT = -1
+
+if CPUCOUNT < 1:
     try:
-        CPUCOUNT = int(os.sysconf('SC_NPROCESSORS_ONLN'))
+        import multiprocessing
+        CPUCOUNT = multiprocessing.cpu_count()
     except:
-        pass
+        try:
+            CPUCOUNT = int(os.sysconf('SC_NPROCESSORS_ONLN'))
+        except:
+            CPUCOUNT = 1
 
 binary_list = []