clearlinux/uwsgi
2
0
Fork 0
mirror of https://github.com/clearlinux/uwsgi.git synced 2026-08-30 13:50:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

after years we can finally differentiate between stderr and request logging

Browse Source
This commit is contained in:
Unbit
2013-01-27 17:59:23 +01:00
parent 2540e739c7
commit 799ac9310f
6 changed files with 283 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/init.c
+5
View File
@@ -130,6 +130,11 @@ void uwsgi_init_default() {
uwsgi.shared->worker_log_pipe[0] = -1;
uwsgi.shared->worker_log_pipe[1] = -1;
uwsgi.shared->worker_req_log_pipe[0] = -1;
uwsgi.shared->worker_req_log_pipe[1] = -1;
uwsgi.req_log_fd = 2;
#ifdef UWSGI_SSL
// 1 day of tolerance
uwsgi.subscriptions_sign_check_tolerance = 3600 * 24;
core/logging.c
+209 -50
View File
@@ -144,6 +144,21 @@ void create_logpipe(void) {
exit(1);
}
if (uwsgi.req_log_master) {
#if defined(SOCK_SEQPACKET) && defined(__linux__)
if (socketpair(AF_UNIX, SOCK_SEQPACKET, 0, uwsgi.shared->worker_req_log_pipe)) {
#else
if (socketpair(AF_UNIX, SOCK_DGRAM, 0, uwsgi.shared->worker_req_log_pipe)) {
#endif
uwsgi_error("socketpair()\n");
exit(1);
}
uwsgi_socket_nb(uwsgi.shared->worker_req_log_pipe[0]);
uwsgi_socket_nb(uwsgi.shared->worker_req_log_pipe[1]);
uwsgi.req_log_fd = uwsgi.shared->worker_req_log_pipe[1];
}
}
#ifdef UWSGI_ZEROMQ
@@ -294,62 +309,71 @@ void uwsgi_setup_log() {
}
static struct uwsgi_logger *setup_choosen_logger(struct uwsgi_string_list *usl) {
char *id = NULL;
char *name = usl->value;
char *space = strchr(name, ' ');
if (space) {
int is_id = 1;
int i;
for (i = 0; i < (space - name); i++) {
if (!isalnum(name[i])) {
is_id = 0;
break;
}
}
if (is_id) {
id = uwsgi_concat2n(name, space - name, "", 0);
name = space + 1;
}
}
char *colon = strchr(name, ':');
if (colon) {
*colon = 0;
}
struct uwsgi_logger *choosen_logger = uwsgi_get_logger(name);
if (!choosen_logger) {
uwsgi_log("unable to find logger %s\n", name);
exit(1);
}
// make a copy of the logger
struct uwsgi_logger *copy_of_choosen_logger = uwsgi_malloc(sizeof(struct uwsgi_logger));
memcpy(copy_of_choosen_logger, choosen_logger, sizeof(struct uwsgi_logger));
choosen_logger = copy_of_choosen_logger;
choosen_logger->id = id;
choosen_logger->next = NULL;
if (colon) {
choosen_logger->arg = colon + 1;
// check for empty string
if (*choosen_logger->arg == 0) {
choosen_logger->arg = NULL;
}
*colon = ':';
}
return choosen_logger;
}
void uwsgi_setup_log_master(void) {
struct uwsgi_string_list *usl = uwsgi.requested_logger;
while (usl) {
char *id = NULL;
char *name = usl->value;
char *space = strchr(name, ' ');
if (space) {
int is_id = 1;
int i;
for (i = 0; i < (space - name); i++) {
if (!isalnum(name[i])) {
is_id = 0;
break;
}
}
if (is_id) {
id = uwsgi_concat2n(name, space - name, "", 0);
name = space + 1;
}
}
char *colon = strchr(name, ':');
if (colon) {
*colon = 0;
}
struct uwsgi_logger *choosen_logger = uwsgi_get_logger(name);
if (!choosen_logger) {
uwsgi_log("unable to find logger %s\n", name);
exit(1);
}
// make a copy of the logger
struct uwsgi_logger *copy_of_choosen_logger = uwsgi_malloc(sizeof(struct uwsgi_logger));
memcpy(copy_of_choosen_logger, choosen_logger, sizeof(struct uwsgi_logger));
choosen_logger = copy_of_choosen_logger;
choosen_logger->id = id;
choosen_logger->next = NULL;
if (colon) {
choosen_logger->arg = colon + 1;
// check for empty string
if (*choosen_logger->arg == 0) {
choosen_logger->arg = NULL;
}
*colon = ':';
}
struct uwsgi_logger *choosen_logger = setup_choosen_logger(usl);
uwsgi_append_logger(choosen_logger);
usl = usl->next;
}
usl = uwsgi.requested_req_logger;
while (usl) {
struct uwsgi_logger *choosen_logger = setup_choosen_logger(usl);
uwsgi_append_req_logger(choosen_logger);
usl = usl->next;
}
#ifdef UWSGI_PCRE
// set logger by its id
struct uwsgi_regexp_list *url = uwsgi.log_route;
@@ -357,6 +381,11 @@ void uwsgi_setup_log_master(void) {
url->custom_ptr = uwsgi_get_logger_from_id(url->custom_str);
url = url->next;
}
url = uwsgi.log_req_route;
while (url) {
url->custom_ptr = uwsgi_get_logger_from_id(url->custom_str);
url = url->next;
}
#endif
uwsgi.original_log_fd = dup(1);
@@ -637,7 +666,7 @@ void uwsgi_logit_simple(struct wsgi_request *wsgi_req) {
logvec[logvecpos].iov_len = rlen;
// do not check for errors
rlen = writev(2, logvec, logvecpos + 1);
rlen = writev(uwsgi.req_log_fd, logvec, logvecpos + 1);
}
void get_memusage(uint64_t * rss, uint64_t * vsz) {
@@ -786,6 +815,24 @@ void uwsgi_append_logger(struct uwsgi_logger *ul) {
}
}
void uwsgi_append_req_logger(struct uwsgi_logger *ul) {
if (!uwsgi.choosen_req_logger) {
uwsgi.choosen_req_logger = ul;
return;
}
struct uwsgi_logger *ucl = uwsgi.choosen_req_logger;
while (ucl) {
if (!ucl->next) {
ucl->next = ul;
return;
}
ucl = ucl->next;
}
}
struct uwsgi_logger *uwsgi_get_logger(char *name) {
struct uwsgi_logger *ul = uwsgi.loggers;
@@ -856,7 +903,7 @@ void uwsgi_logit_lf(struct wsgi_request *wsgi_req) {
}
// do not check for errors
rlen = writev(2, uwsgi.logvectors[wsgi_req->async_id], uwsgi.logformat_vectors);
rlen = writev(uwsgi.req_log_fd, uwsgi.logvectors[wsgi_req->async_id], uwsgi.logformat_vectors);
// free allocated memory
logchunk = uwsgi.logchunks;
@@ -1107,3 +1154,115 @@ void uwsgi_add_logchunk(int variable, int pos, char *ptr, size_t len) {
}
}
}
int uwsgi_master_log(void) {
ssize_t rlen = read(uwsgi.shared->worker_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize);
if (rlen > 0) {
#ifdef UWSGI_ALARM
uwsgi_alarm_log_check(uwsgi.log_master_buf, rlen);
#endif
#ifdef UWSGI_PCRE
struct uwsgi_regexp_list *url = uwsgi.log_drain_rules;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
return 0;
}
url = url->next;
}
if (uwsgi.log_filter_rules) {
int show = 0;
url = uwsgi.log_filter_rules;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
show = 1;
break;
}
url = url->next;
}
if (!show)
return 0;
}
url = uwsgi.log_route;
int finish = 0;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr;
if (ul_route) {
ul_route->func(ul_route, uwsgi.log_master_buf, rlen);
finish = 1;
}
}
url = url->next;
}
if (finish)
return 0;
#endif
int raw_log = 1;
struct uwsgi_logger *ul = uwsgi.choosen_logger;
while (ul) {
// check for named logger
if (ul->id) {
goto next;
}
ul->func(ul, uwsgi.log_master_buf, rlen);
raw_log = 0;
next:
ul = ul->next;
}
if (raw_log) {
rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen);
}
return 0;
}
return -1;
}
int uwsgi_master_req_log(void) {
ssize_t rlen = read(uwsgi.shared->worker_req_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize);
if (rlen > 0) {
#ifdef UWSGI_PCRE
url = uwsgi.log_req_route;
int finish = 0;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr;
if (ul_route) {
ul_route->func(ul_route, uwsgi.log_master_buf, rlen);
finish = 1;
}
}
url = url->next;
}
if (finish)
return 0;
#endif
int raw_log = 1;
struct uwsgi_logger *ul = uwsgi.choosen_req_logger;
while (ul) {
// check for named logger
if (ul->id) {
goto next;
}
ul->func(ul, uwsgi.log_master_buf, rlen);
raw_log = 0;
next:
ul = ul->next;
}
if (raw_log) {
rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen);
}
return 0;
}
return -1;
}
core/master.c
+34 -75
View File
@@ -255,91 +255,40 @@ void expire_rb_timeouts(struct rb_root *root) {
}
}
int uwsgi_master_log(void) {
ssize_t rlen = read(uwsgi.shared->worker_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize);
if (rlen > 0) {
#ifdef UWSGI_ALARM
uwsgi_alarm_log_check(uwsgi.log_master_buf, rlen);
#endif
#ifdef UWSGI_PCRE
struct uwsgi_regexp_list *url = uwsgi.log_drain_rules;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
return 0;
}
url = url->next;
}
if (uwsgi.log_filter_rules) {
int show = 0;
url = uwsgi.log_filter_rules;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
show = 1;
break;
}
url = url->next;
}
if (!show)
return 0;
}
url = uwsgi.log_route;
int finish = 0;
while (url) {
if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) {
struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr;
if (ul_route) {
ul_route->func(ul_route, uwsgi.log_master_buf, rlen);
finish = 1;
}
}
url = url->next;
}
if (finish)
return 0;
#endif
int raw_log = 1;
struct uwsgi_logger *ul = uwsgi.choosen_logger;
while (ul) {
// check for named logger
if (ul->id) {
goto next;
}
ul->func(ul, uwsgi.log_master_buf, rlen);
raw_log = 0;
next:
ul = ul->next;
}
if (raw_log) {
rlen = write(uwsgi.original_log_fd, uwsgi.log_master_buf, rlen);
}
return 0;
}
return -1;
}
void *logger_thread_loop(void *noarg) {
struct pollfd logpoll;
struct pollfd logpoll[2];
// block all signals
sigset_t smask;
sigfillset(&smask);
pthread_sigmask(SIG_BLOCK, &smask, NULL);
logpoll.events = POLLIN;
logpoll.fd = uwsgi.shared->worker_log_pipe[0];
logpoll[0].events = POLLIN;
logpoll[0].fd = uwsgi.shared->worker_log_pipe[0];
int logpolls = 1;
if (uwsgi.req_log_master) {
logpoll[1].events = POLLIN;
logpoll[1].fd = uwsgi.shared->worker_req_log_pipe[0];
}
for (;;) {
int ret = poll(&logpoll, 1, -1);
if (ret > 0 && logpoll.revents & POLLIN) {
pthread_mutex_lock(&uwsgi.threaded_logger_lock);
uwsgi_master_log();
pthread_mutex_unlock(&uwsgi.threaded_logger_lock);
int ret = poll(logpoll, logpolls, -1);
if (ret > 0) {
if (logpoll[0].revents & POLLIN) {
pthread_mutex_lock(&uwsgi.threaded_logger_lock);
uwsgi_master_log();
pthread_mutex_unlock(&uwsgi.threaded_logger_lock);
}
else if (logpolls > 1 && logpoll[1].revents & POLLIN) {
pthread_mutex_lock(&uwsgi.threaded_logger_lock);
uwsgi_master_req_log();
pthread_mutex_unlock(&uwsgi.threaded_logger_lock);
}
}
}
@@ -524,12 +473,18 @@ int master_loop(char **argv, char **environ) {
uwsgi_log("adding %d to master logging\n", uwsgi.shared->worker_log_pipe[0]);
#endif
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]);
if (uwsgi.req_log_master) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]);
}
}
else {
if (pthread_create(&logger_thread, NULL, logger_thread_loop, NULL)) {
uwsgi_error("pthread_create()");
uwsgi_log("falling back to non-threaded logger...\n");
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]);
if (uwsgi.req_log_master) {
event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]);
}
uwsgi.threaded_logger = 0;
}
}
@@ -873,6 +828,10 @@ int master_loop(char **argv, char **environ) {
uwsgi_master_log();
goto health_cycle;
}
if (uwsgi.req_log_master && interesting_fd == uwsgi.shared->worker_req_log_pipe[0]) {
uwsgi_master_req_log();
goto health_cycle;
}
}
if (uwsgi.stats && uwsgi.stats_fd > -1) {
core/utils.c
+5
View File
@@ -1522,6 +1522,11 @@ add:
uwsgi.master_process = 1;
uwsgi.log_master = 1;
}
if (op->flags & UWSGI_OPT_REQ_LOG_MASTER) {
uwsgi.master_process = 1;
uwsgi.log_master = 1;
uwsgi.req_log_master = 1;
}
// requires threads ?
if (op->flags & UWSGI_OPT_THREADS) {
uwsgi.has_threads = 1;
core/uwsgi.c
+17 -1
View File
@@ -399,6 +399,8 @@ static struct uwsgi_option uwsgi_base_options[] = {
{"logfile-chmod", required_argument, 0, "chmod logfiles", uwsgi_opt_logfile_chmod, NULL, 0},
{"log-syslog", optional_argument, 0, "log to syslog", uwsgi_opt_set_logger, "syslog", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"log-socket", required_argument, 0, "send logs to the specified socket", uwsgi_opt_set_logger, "socket", UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"req-logger", required_argument, 0, "set/append a request logger", uwsgi_opt_set_req_logger, NULL, UWSGI_OPT_REQ_LOG_MASTER},
{"logger-req", required_argument, 0, "set/append a request logger", uwsgi_opt_set_req_logger, NULL, UWSGI_OPT_REQ_LOG_MASTER},
{"logger", required_argument, 0, "set/append a logger", uwsgi_opt_set_logger, NULL, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"logger-list", no_argument, 0, "list enabled loggers", uwsgi_opt_true, &uwsgi.loggers_list, 0},
{"loggers-list", no_argument, 0, "list enabled loggers", uwsgi_opt_true, &uwsgi.loggers_list, 0},
@@ -407,6 +409,7 @@ static struct uwsgi_option uwsgi_base_options[] = {
{"log-drain", required_argument, 0, "drain (do not show) log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_drain_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"log-filter", required_argument, 0, "show only log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_filter_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"log-route", required_argument, 0, "log to the specified named logger if regexp applied on logline matches", uwsgi_opt_add_regexp_custom_list, &uwsgi.log_route, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
{"log-req-route", required_argument, 0, "log requests to the specified named logger if regexp applied on logline matches", uwsgi_opt_add_regexp_custom_list, &uwsgi.log_req_route, UWSGI_OPT_REQ_LOG_MASTER},
#endif
#ifdef UWSGI_ALARM
{"alarm", required_argument, 0, "create a new alarm, syntax: <alarm> <plugin:args>", uwsgi_opt_add_string_list, &uwsgi.alarm_list, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER},
@@ -509,7 +512,7 @@ static struct uwsgi_option uwsgi_base_options[] = {
{"static-index", required_argument, 0, "search for specified file if a directory is requested", uwsgi_opt_add_string_list, &uwsgi.static_index, UWSGI_OPT_MIME},
{"static-safe", required_argument, 0, "skip security checks if the file is under the specified path", uwsgi_opt_add_string_list, &uwsgi.static_safe, UWSGI_OPT_MIME},
{"static-cache-paths", required_argument, 0, "put resolved paths in the uWSGI cache for the specified amount of seconds", uwsgi_opt_set_int, &uwsgi.static_cache_paths, UWSGI_OPT_MIME|UWSGI_OPT_MASTER},
{"static-cache-name", required_argument, 0, "use the specified cache for static paths", uwsgi_opt_set_str, &uwsgi.static_cache_paths_name, UWSGI_OPT_MIME|UWSGI_OPT_MASTER},
{"static-cache-paths-name", required_argument, 0, "use the specified cache for static paths", uwsgi_opt_set_str, &uwsgi.static_cache_paths_name, UWSGI_OPT_MIME|UWSGI_OPT_MASTER},
{"mimefile", required_argument, 0, "set mime types file path (default /etc/mime.types)", uwsgi_opt_add_string_list, &uwsgi.mime_file, UWSGI_OPT_MIME},
{"mime-file", required_argument, 0, "set mime types file path (default /etc/mime.types)", uwsgi_opt_add_string_list, &uwsgi.mime_file, UWSGI_OPT_MIME},
@@ -3367,6 +3370,19 @@ void uwsgi_opt_set_logger(char *opt, char *value, void *prefix) {
}
}
void uwsgi_opt_set_req_logger(char *opt, char *value, void *prefix) {
if (!value)
value = "";
if (prefix) {
uwsgi_string_new_list(&uwsgi.requested_req_logger, uwsgi_concat3((char *) prefix, ":", value));
}
else {
uwsgi_string_new_list(&uwsgi.requested_req_logger, uwsgi_str(value));
}
}
void uwsgi_opt_set_str_spaced(char *opt, char *value, void *key) {
char **ptr = (char **) key;
*ptr = uwsgi_concat2((char *) value, " ");
uwsgi.h
+13
View File
@@ -57,6 +57,7 @@ extern "C" {
#define UWSGI_OPT_POST_BUFFERING (1 << 11)
#define UWSGI_OPT_CLUSTER (1 << 12)
#define UWSGI_OPT_MIME (1 << 13)
#define UWSGI_OPT_REQ_LOG_MASTER (1 << 14)
#define MAX_GENERIC_PLUGINS 64
#define MAX_RPC 64
@@ -659,6 +660,9 @@ struct uwsgi_cache {
int thread_server_fd;
struct uwsgi_string_list *nodes;
struct uwsgi_string_list *sync_nodes;
struct uwsgi_lock_item *lock;
struct uwsgi_cache *next;
@@ -1632,6 +1636,7 @@ struct uwsgi_server {
int restore_tc;
// route all of the logs to the master process
int req_log_master;
int log_master;
char *log_master_buf;
size_t log_master_bufsize;
@@ -1642,6 +1647,7 @@ struct uwsgi_server {
char *log_backupname;
int original_log_fd;
int req_log_fd;
// static file serving
int file_serve_mode;
@@ -1666,7 +1672,9 @@ struct uwsgi_server {
struct uwsgi_logger *loggers;
struct uwsgi_logger *choosen_logger;
struct uwsgi_logger *choosen_req_logger;
struct uwsgi_string_list *requested_logger;
struct uwsgi_string_list *requested_req_logger;
#ifdef UWSGI_PCRE
int pcre_jit;
@@ -2268,6 +2276,8 @@ struct uwsgi_shared {
int rpc_count;
int worker_log_pipe[2];
// used for request logging
int worker_req_log_pipe[2];
#if defined(__linux__) || defined(__FreeBSD__)
struct tcp_info ti;
@@ -3204,6 +3214,7 @@ void uwsgi_build_cap(char *);
void uwsgi_register_logger(char *, ssize_t (*func)(struct uwsgi_logger *, char *, size_t));
void uwsgi_append_logger(struct uwsgi_logger *);
void uwsgi_append_req_logger(struct uwsgi_logger *);
struct uwsgi_logger *uwsgi_get_logger(char *);
struct uwsgi_logger *uwsgi_get_logger_from_id(char *);
@@ -3230,6 +3241,7 @@ void uwsgi_opt_print(char *, char *, void *);
void uwsgi_opt_true(char *, char *, void *);
void uwsgi_opt_set_str(char *, char *, void *);
void uwsgi_opt_set_logger(char *, char *, void *);
void uwsgi_opt_set_req_logger(char *, char *, void *);
void uwsgi_opt_set_str_spaced(char *, char *, void *);
void uwsgi_opt_add_string_list(char *, char *, void *);
void uwsgi_opt_add_addr_list(char *, char *, void *);
@@ -3367,6 +3379,7 @@ int uwsgi_cheaper_algo_backlog(void);
int uwsgi_cheaper_algo_backlog2(void);
int uwsgi_master_log(void);
int uwsgi_master_req_log(void);
void uwsgi_flush_logs(void);
void uwsgi_register_cheaper_algo(char *, int(*) (void));