fixed http protocol post optimization

.hgtags

@@ -41,3 +41,5 @@ a2b8f25fd17ed7424d1470a287d0b3144d2a12c8 1.0-rc6
 a3ca2e8e27c5c12c44ba0cacb4de4cb51377f184 1.0-rc7
 365e2514cc19f0366365318f12a1d63113b02924 1.0-rc8
 5da6fe43a8c29056bc9442c684615e11a3699795 1.0-rc9
+af490b8c30f83bc5fb6f3837a760981474647755 1.0-rc10
+3c70927335856883283fe4db448e092cecc0ccdb 1.0

ChangeLog

@@ -1,3 +1,28 @@
+*** december 2011 ***
+
+ * 1.0 [20111230]
+
+- adaptive process spawning (--cheaper option)
+- new control options --stop, --reload, --suspend, --resume, --pause
+- stats framework
+- new subscription system
+- support for mime types in static file serving
+- posix capablities support
+- uWSGI mules and farms
+- support for multiple --touch-reload options
+- new logging subsystem
+- new custom locking subsystem
+- process name handling 
+- better (and faster) HTTP parser
+- perl and python threading improvements
+- new plugins: rrdtool, carbon, cgi, php
+- various improvements in the Rack/ruby plugin
+- Linux KSM support
+- refactored ping and nagios plugins
+- support for signals in lazy mode
+- optimized static-file serving
+- support for linux unshare()
+
 *** august 2011 ***
 
  * 0.9.9 [20110829]

contrib/uwsgi-cache-monitor.py

@@ -0,0 +1,123 @@
+import mmap
+import os
+import struct
+import array
+import time
+import sys
+from optparse import OptionParser
+
+class Cache:
+
+    def __init__(self, filename, cache_slots, block_size=65536, sample_sleep=1):
+
+        self.block_size = block_size
+        self.cache_slots = cache_slots
+
+        self.key_size = 2048
+        self.item_size = 2 + 2 + 4 + 8 + 8 + 8 + 8 + 8 + self.key_size
+        self.block_size_start = self.item_size * self.cache_slots
+
+        fd = os.open(filename, os.O_RDONLY)
+        self.cache_store = mmap.mmap(fd, 0, mmap.MAP_SHARED, mmap.PROT_READ)
+
+        self.sample_sleep = sample_sleep
+        self.samples = 0
+        self.history = []
+        self.cache_full = 0
+        self.cache_empty = 0
+        self.cache_items = 0
+        self.block_sizes = 0
+
+    def store_read_item_block(self, position):
+        pos = self.cache_store.tell()
+        # uwsgi cache stores cache entries first and then the blocks
+        self.cache_store.seek(self.block_size_start + (position * self.block_size))
+        buf = self.cache_store.read(self.block_size)
+        self.cache_store.seek(pos)
+        return buf
+
+    def store_read_item(self, position):
+        buf = self.cache_store.read(self.item_size)
+        fields = struct.unpack_from('@HHIQQQQQ2048c', buf)
+        key = array.array('c', fields[8:self.key_size+8]).tostring().rstrip('\x00')
+
+        if [x for x in key if x!= '\x00']:
+            buf = self.store_read_item_block(position)
+            value = array.array('c', buf).tostring().rstrip('\x00')
+        else:
+            value = ''
+        return (position, key, value, len(value))
+
+    def read(self):
+        data = [self.store_read_item(i) for i in range(self.cache_slots)]
+        self.cache_store.seek(0)
+        self.update_stats(data)
+        if self.sample_sleep:
+            time.sleep(self.sample_sleep)
+        return data
+ 
+    def update_stats(self, data):
+        # data is a list of (position, key, value, len(value)) tuples
+        items = len([1 for x in data if x[3] > 0])
+        self.cache_items += items
+        full, empty = items == self.cache_slots, items == 0
+        if full:
+            self.cache_full += 1
+        if empty:
+            self.cache_empty += 1
+        self.samples += 1
+        block_sizes = sum([x[3] for x in data])
+        self.block_sizes += block_sizes
+        self.history.append({'full': full, 'empty': empty, 'data': data, \
+            'items': items, 'block_sizes': block_sizes})
+
+    def dump(self):
+        return {
+            'samples': self.samples,
+            'history': self.history,
+            'cache_slots': self.cache_slots,
+            'sample_sleep': self.sample_sleep, 
+            'cache_empty': self.cache_empty,
+            'cache_full': self.cache_full,
+            'cache_items': self.cache_items,
+            'block_sizes': self.block_sizes,
+         }
+
+    def show_dump(self):
+        d = self.dump()
+        print
+        print "Recorded %d samples (%d second(s) sleep between samples)" % \
+            (d['samples'], d['sample_sleep'])
+        print "Cache empty %d times, full %d times, %.2f items on average" % \
+            (d['cache_empty'], d['cache_full'], d['cache_items'] / d['samples'])
+        print "Block size average size: %d bytes" % \
+            (d['block_sizes'] / d['cache_items'] * 8)
+        print "Data in cache average: %d bytes" % \
+            (d['block_sizes'] / d['samples'] * 8)
+
+def main(options):
+    cache = Cache(options.cache_store, options.cache_slots, options.block_size,
+        options.sleep_time)
+    print "Recording..."
+    while True:
+        try:
+            data = cache.read()
+        except KeyboardInterrupt:
+            cache.show_dump()
+            sys.exit(0)
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option("-s", "--cache-slots", dest="cache_slots", type="int",
+        help="Slots available in the cache, uwsgi cache option")
+    parser.add_option("-c", "--cache-store", dest="cache_store", default="uwsgi.cache",
+        help="The filename of the cache store, uwsgi cache-store option. Default: uwsgi.cache")
+    parser.add_option("-b", "--block-size", dest="block_size", default=65536, type="int",
+        help="The size of the cache block, uwsgi cache-blocksize option. Default: 65536")
+    parser.add_option("-t", "--sleep-time", dest="sleep_time", default=1, type="int",
+        help="The time to sleep between each sample. Default: 1")
+
+    (options, args) = parser.parse_args()
+    if not options.cache_slots:
+        parser.error('Option -s / --cache-slots is mandatory')
+    main(options)

master.c

@@ -96,6 +96,8 @@ void *logger_thread_loop(void *noarg) {
 			master_log();
 		}
 	}
+
+	return NULL;
 }
 
 void uwsgi_subscribe(char *subscription, uint8_t cmd) {
@@ -633,7 +635,7 @@ int master_loop(char **argv, char **environ) {
 		}
 
 		// cheaper management
-		if (uwsgi.cheaper) {
+		if (uwsgi.cheaper && !uwsgi.cheap && !uwsgi.to_heaven && !uwsgi.to_hell) {
 			for (i = 1; i <= uwsgi.numproc; i++) {
 				if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
 					if (uwsgi.workers[i].busy == 0) {
@@ -648,15 +650,18 @@ int master_loop(char **argv, char **environ) {
 		}
 
 	      healthy:
-		if (uwsgi.cheaper) {
+		if (uwsgi.cheaper && !uwsgi.cheap && !uwsgi.to_heaven && !uwsgi.to_hell) {
 			if (overload_count > 3) {
 				// activate the first available worker
+				int decheaped = 0;
 				for (i = 1; i <= uwsgi.numproc; i++) {
 					if (uwsgi.workers[i].cheaped == 1 && uwsgi.workers[i].pid == 0) {
 						if (uwsgi_respawn_worker(i))
 							return 0;
 						overload_count = 0;
-						break;
+						decheaped++;
+						if (decheaped >= uwsgi.cheaper_step)
+							break;
 					}
 				}
 			}
@@ -1001,7 +1006,11 @@ int master_loop(char **argv, char **environ) {
 							found = 1;
 							uwsgi.cheap = 0;
 							uwsgi_del_sockets_from_queue(uwsgi.master_queue);
-							for (i = 1; i <= uwsgi.numproc; i++) {
+							int needed = uwsgi.numproc;
+							if (uwsgi.cheaper) {
+								needed = uwsgi.cheaper_count;
+							}
+							for (i = 1; i <= needed; i++) {
 								if (uwsgi_respawn_worker(i))
 									return 0;
 							}
@@ -1282,8 +1291,9 @@ int master_loop(char **argv, char **environ) {
 					last_request_timecheck = uwsgi.current_time;
 					last_request_count = uwsgi.workers[0].requests;
 				}
-				else if (uwsgi.current_time - last_request_timecheck > uwsgi.idle) {
-					uwsgi_log("workers have been inactive for more than %d seconds\n", uwsgi.idle);
+				// a bit of over-engeneering to avoid clock skews
+				else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle)) {
+					uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck);
 					uwsgi.cheap = 1;
 					master_has_children = 0;
 					if (uwsgi.die_on_idle) {

plugins/fiber/fiber.c

@@ -58,14 +58,13 @@ void fiber_loop() {
         uwsgi.schedule_to_main = fiber_schedule_to_main;
         uwsgi.schedule_to_req = fiber_schedule_to_req;
 
-	// must run all the rack/ruby plugins without protection
 	ur.unprotected = 1;
 
         rb_protect(protected_async_loop, 0, &error);
 
         if (error) {
                 uwsgi_ruby_exception();
-                exit(1);
+		exit(1);
         }
 
         // never here

plugins/fiber/uwsgiplugin.py

@@ -9,6 +9,7 @@ NAME='fiber'
 CFLAGS = os.popen(RUBYPATH + " -e \"require 'rbconfig';print Config::CONFIG['CFLAGS']\"").read().rstrip().split()
 
 CFLAGS.append('-Wno-unused-parameter')
+CFLAGS.append('-DRUBY19')
 
 includedir = os.popen(RUBYPATH + " -e \"require 'rbconfig';print Config::CONFIG['rubyhdrdir']\"").read().rstrip()
 if includedir == 'nil':

plugins/rack/rack_plugin.c

@@ -590,7 +590,6 @@ int uwsgi_rack_request(struct wsgi_request *wsgi_req) {
 
 	rb_hash_aset(env, rb_str_new2("rack.errors"), rb_funcall( rb_const_get(rb_cObject, rb_intern("IO")), rb_intern("new"), 2, INT2NUM(2), rb_str_new("w",1) ));
 
-
 	ret = rb_protect( call_dispatch, env, &error);
 	if (error) {
 		uwsgi_ruby_exception();
@@ -674,9 +673,14 @@ int uwsgi_rack_request(struct wsgi_request *wsgi_req) {
 			}
 		}
 		else if (rb_respond_to( body, rb_intern("each") )) {
-			rb_protect( iterate_body, body, &error);
-			if (error) {
-				uwsgi_ruby_exception();
+			if (ur.unprotected) {
+				iterate_body(body);
+			}
+			else {
+				rb_protect( iterate_body, body, &error);
+				if (error) {
+					uwsgi_ruby_exception();
+				}
 			}
 		}
 

plugins/rack/uwsgi_rack.h

@@ -69,6 +69,8 @@ struct uwsgi_rack {
 	int rb_shell;
 	int app_id;
 
+	int unprotected;
+
 	struct uwsgi_string_list *rbrequire;
 
 };

proto/http.c

@@ -256,6 +256,7 @@ int uwsgi_proto_http_parser(struct wsgi_request *wsgi_req) {
 	ssize_t remains;
 	// make this buffer configurable
 	char post_buf[8192];
+	char *post_tail = NULL;
 
 	// first round ? this memory area will be freed by async_loop
 	if (!wsgi_req->proto_parser_buf) {
@@ -317,6 +318,10 @@ int uwsgi_proto_http_parser(struct wsgi_request *wsgi_req) {
 		else if (*ptr == '\n' && wsgi_req->proto_parser_status == 3) {
 			ptr++;
 			remains = len - (j + 1);
+			if (remains > 0) {
+				post_tail = uwsgi_malloc(remains);
+				memcpy(post_tail, ptr, remains);
+			}
 			http_parse(wsgi_req, ptr);
 			//is there a Content_Length ?
 			if (wsgi_req->post_cl > 0) {
@@ -328,13 +333,15 @@ int uwsgi_proto_http_parser(struct wsgi_request *wsgi_req) {
 				}
 				wsgi_req->proto_parser_pos = 0;
 				remains = UMIN((size_t) remains, wsgi_req->post_cl);
-				if (remains) {
-					if (!fwrite(ptr, remains, 1, wsgi_req->async_post)) {
+				if (remains && post_tail) {
+					if (!fwrite(post_tail, remains, 1, wsgi_req->async_post)) {
+						free(post_tail);
 						free(wsgi_req->proto_parser_buf);
 						uwsgi_error("fwrite()");
 						fclose(wsgi_req->async_post);
 						return -1;
 					}
+					free(post_tail);
 					wsgi_req->proto_parser_pos += remains;
 					if (wsgi_req->proto_parser_pos >= wsgi_req->post_cl) {
 						free(wsgi_req->proto_parser_buf);

protocol.c

@@ -1631,18 +1631,9 @@ int uwsgi_file_serve(struct wsgi_request *wsgi_req, char *document_root, uint16_
 			// nginx
 			if (uwsgi.file_serve_mode == 1) {
 				headers_vec[0].iov_base = "X-Accel-Redirect: "; headers_vec[0].iov_len = 18 ;
-				headers_vec[1].iov_base = document_root; headers_vec[1].iov_len = document_root_len;
-				if (document_root[document_root_len-1] != '/') {
-					headers_vec[2].iov_base = "/"; headers_vec[2].iov_len = 1;
-					headers_vec[3].iov_base = path_info; headers_vec[3].iov_len = path_info_len;
-					headers_vec[4].iov_base = "\r\n"; headers_vec[4].iov_len = 2;
-					wsgi_req->headers_size += wsgi_req->socket->proto_writev_header(wsgi_req, headers_vec, 5);
-				}
-				else {
-					headers_vec[2].iov_base = path_info; headers_vec[2].iov_len = path_info_len;
-					headers_vec[3].iov_base = "\r\n"; headers_vec[3].iov_len = 2;
-					wsgi_req->headers_size += wsgi_req->socket->proto_writev_header(wsgi_req, headers_vec, 4);
-				}
+				headers_vec[1].iov_base = real_filename; headers_vec[1].iov_len = real_filename_len;
+				headers_vec[2].iov_base = "\r\n"; headers_vec[2].iov_len = 2;
+				wsgi_req->headers_size += wsgi_req->socket->proto_writev_header(wsgi_req, headers_vec, 3);
 				// this is the final header (\r\n added)
                         	set_http_date(st.st_mtime, http_last_modified);
                         	wsgi_req->headers_size += wsgi_req->socket->proto_write_header(wsgi_req, http_last_modified, 48);

utils.c

@@ -1099,7 +1099,7 @@ void parse_sys_envs(char **envs) {
 	char *earg, *eq_pos;
 
 	while (*uenvs) {
-		if (!strncmp(*uenvs, "UWSGI_", 6) && strncmp(*uenvs, "UWSGI_RELOADS=", 14)) {
+		if (!strncmp(*uenvs, "UWSGI_", 6) && strncmp(*uenvs, "UWSGI_RELOADS=", 14) && strncmp(*uenvs, "UWSGI_ORIGINAL_PROC_NAME=", 25)) {
 			earg = uwsgi_malloc(strlen(*uenvs + 6) + 1);
 			env_to_arg(*uenvs + 6, earg);
 			eq_pos = strchr(earg, '=');

uwsgi.c

@@ -258,6 +258,7 @@ static struct option long_base_options[] = {
 	{"lazy", no_argument, &uwsgi.lazy, 1},
 	{"cheap", no_argument, &uwsgi.cheap, 1},
 	{"cheaper", required_argument, 0, LONG_ARGS_CHEAPER},
+	{"cheaper-step", required_argument, 0, LONG_ARGS_CHEAPER_STEP},
 	{"idle", required_argument, 0, LONG_ARGS_IDLE},
 	{"die-on-idle", no_argument, &uwsgi.die_on_idle, 1},
 	{"mount", required_argument, 0, LONG_ARGS_MOUNT},
@@ -3143,6 +3144,9 @@ static int manage_base_opt(int i, char *optarg) {
 		uwsgi.cheaper = 1;
 		uwsgi.cheaper_count = atoi(optarg);
 		return 1;
+	case LONG_ARGS_CHEAPER_STEP:
+		uwsgi.cheaper_step = atoi(optarg);
+		return 1;
 	case LONG_ARGS_CHDIR:
 		uwsgi.chdir = optarg;
 		return 1;

uwsgi.h

@@ -559,6 +559,7 @@ struct uwsgi_opt {
 #define LONG_ARGS_STATIC_SKIP_EXT	17171
 #define LONG_ARGS_LOGGER		17172
 #define LONG_ARGS_STATIC_INDEX		17173
+#define LONG_ARGS_CHEAPER_STEP		17174
 
 
 #define UWSGI_OK	0
@@ -1085,6 +1086,7 @@ struct uwsgi_server {
 	int cheap;
 	// enable cheaper mode
 	int cheaper;
+	int cheaper_step;
 	// minimal number of running workers in cheaper mode
 	int cheaper_count;
 	// enable idle mode

uwsgiconfig.py

@@ -1,6 +1,6 @@
 # uWSGI build system
 
-uwsgi_version = '1.0-dev'
+uwsgi_version = '1.0'
 
 import os
 import re

welcome.py

@@ -30,12 +30,12 @@ def serve_logo(e, sr):
 def serve_options(e, sr):
     sr('200 OK', [('Content-Type', 'text/html')])
     for opt in xrange(0,256):
-        yield "%d = %d<br/>" % (opt, uwsgi.get_option(opt))
+        yield "<b>%d</b> = %d<br/>" % (opt, uwsgi.get_option(opt))
 
 def serve_config(e, sr):
     sr('200 OK', [('Content-Type', 'text/html')])
     for opt in uwsgi.opt.keys():
-        yield "%s = %s<br/>" % (opt, uwsgi.opt[opt])
+        yield "<b>%s</b> = %s<br/>" % (opt, uwsgi.opt[opt])
 
 routes = {}
 routes['/xsendfile'] = xsendfile
@@ -95,7 +95,7 @@ def application(env, start_response):
         """ % (w['id'], w['pid'], w['status'], w['running_time']/1000, w['avg_rt']/1000, w['tx'], apps)
 
     return """
-<img src="/logo"/> version %s<br/>
+<img src="/logo"/> version %s running on %s<br/>
 <hr size="1"/>
 
 Configuration<br/>
@@ -115,7 +115,7 @@ Workers and applications<br/>
 %s
 </table>
 
-    """ % (uwsgi.version, workers)
+    """ % (uwsgi.version, uwsgi.hostname, workers)