diff --git a/core/logging.c b/core/logging.c index 908e1c05..299994fb 100644 --- a/core/logging.c +++ b/core/logging.c @@ -1289,3 +1289,59 @@ next: return -1; } + +static void *logger_thread_loop(void *noarg) { + struct pollfd logpoll[2]; + + // block all signals + sigset_t smask; + sigfillset(&smask); + pthread_sigmask(SIG_BLOCK, &smask, NULL); + + logpoll[0].events = POLLIN; + logpoll[0].fd = uwsgi.shared->worker_log_pipe[0]; + + int logpolls = 1; + + if (uwsgi.req_log_master) { + logpoll[1].events = POLLIN; + logpoll[1].fd = uwsgi.shared->worker_req_log_pipe[0]; + } + + + for (;;) { + int ret = poll(logpoll, logpolls, -1); + if (ret > 0) { + if (logpoll[0].revents & POLLIN) { + pthread_mutex_lock(&uwsgi.threaded_logger_lock); + uwsgi_master_log(); + pthread_mutex_unlock(&uwsgi.threaded_logger_lock); + } + else if (logpolls > 1 && logpoll[1].revents & POLLIN) { + pthread_mutex_lock(&uwsgi.threaded_logger_lock); + uwsgi_master_req_log(); + pthread_mutex_unlock(&uwsgi.threaded_logger_lock); + } + + } + } + + return NULL; +} + + + +void uwsgi_threaded_logger_spawn() { + pthread_t logger_thread; + + if (pthread_create(&logger_thread, NULL, logger_thread_loop, NULL)) { + uwsgi_error("pthread_create()"); + uwsgi_log("falling back to non-threaded logger...\n"); + event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]); + if (uwsgi.req_log_master) { + event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]); + } + uwsgi.threaded_logger = 0; + } +} + diff --git a/core/master.c b/core/master.c index e0501585..67b7be83 100644 --- a/core/master.c +++ b/core/master.c @@ -31,19 +31,6 @@ void uwsgi_unblock_signal(int signum) { } } -void uwsgi_master_manage_snmp(int snmp_fd) { - struct sockaddr_in udp_client; - socklen_t udp_len = sizeof(udp_client); - ssize_t rlen = recvfrom(snmp_fd, uwsgi.wsgi_req->buffer, uwsgi.buffer_size, 0, (struct sockaddr *) &udp_client, &udp_len); - - if (rlen < 0) { - uwsgi_error("recvfrom()"); - } - else if (rlen > 0) { - manage_snmp(snmp_fd, (uint8_t *) uwsgi.wsgi_req->buffer, rlen, &udp_client); - } -} - void uwsgi_master_manage_udp(int udp_fd) { struct sockaddr_in udp_client; char udp_client_addr[16]; @@ -98,7 +85,7 @@ void uwsgi_master_manage_emperor() { // remove me if (byte == 0) { close(uwsgi.emperor_fd); - if (!uwsgi.to_hell) + if (!uwsgi.status.brutally_reloading) kill_them_all(0); } // reload me @@ -113,7 +100,7 @@ void uwsgi_master_manage_emperor() { else { uwsgi_log("lost connection with my emperor !!!\n"); close(uwsgi.emperor_fd); - if (!uwsgi.to_hell) + if (!uwsgi.status.brutally_reloading) kill_them_all(0); sleep(2); exit(1); @@ -176,45 +163,28 @@ void suspend_resume_them_all(int signum) { } -int uwsgi_master_check_mercy() { +void uwsgi_master_check_mercy() { int i, waitpid_status; - if (uwsgi.master_mercy) { - if (uwsgi.master_mercy < uwsgi_now()) { - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].pid > 0) { - if (uwsgi.lazy && uwsgi.workers[i].destroy == 0) - continue; - uwsgi_log("worker %d (pid: %d) is taking too much time to die...NO MERCY !!!\n", i, uwsgi.workers[i].pid); - if (!kill(uwsgi.workers[i].pid, SIGKILL)) { - if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) { - uwsgi_error("waitpid()"); - } - uwsgi.workers[i].pid = 0; - if (uwsgi.to_hell) { - uwsgi.ready_to_die++; - } - else if (uwsgi.to_heaven) { - uwsgi.ready_to_reload++; - } - else if (uwsgi.to_outworld) { - uwsgi.lazy_respawned++; - if (uwsgi_respawn_worker(i)) - return -1; - } - } - else { - uwsgi_error("kill()"); + for (i = 1; i <= uwsgi.numproc; i++) { + if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cursed_at) { + if (uwsgi_now() > uwsgi.workers[i].no_mercy_at) { + uwsgi_log("worker %d (pid: %d) is taking too much time to die...NO MERCY !!!\n", i, uwsgi.workers[i].pid); + if (!kill(uwsgi.workers[i].pid, SIGKILL)) { + if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) { + uwsgi_error("uwsgi_master_check_mercy()/waitpid()"); } } + else { + uwsgi_error("uwsgi_master_check_mercy()/kill()"); + } + uwsgi.workers[i].pid = 0; + uwsgi.workers[i].cursed_at = 0; + uwsgi.workers[i].no_mercy_at = 0; } - uwsgi.master_mercy = 0; } } - - - return 0; } @@ -248,46 +218,6 @@ void expire_rb_timeouts(struct uwsgi_rbtree *tree) { } } - -void *logger_thread_loop(void *noarg) { - struct pollfd logpoll[2]; - - // block all signals - sigset_t smask; - sigfillset(&smask); - pthread_sigmask(SIG_BLOCK, &smask, NULL); - - logpoll[0].events = POLLIN; - logpoll[0].fd = uwsgi.shared->worker_log_pipe[0]; - - int logpolls = 1; - - if (uwsgi.req_log_master) { - logpoll[1].events = POLLIN; - logpoll[1].fd = uwsgi.shared->worker_req_log_pipe[0]; - } - - - for (;;) { - int ret = poll(logpoll, logpolls, -1); - if (ret > 0) { - if (logpoll[0].revents & POLLIN) { - pthread_mutex_lock(&uwsgi.threaded_logger_lock); - uwsgi_master_log(); - pthread_mutex_unlock(&uwsgi.threaded_logger_lock); - } - else if (logpolls > 1 && logpoll[1].revents & POLLIN) { - pthread_mutex_lock(&uwsgi.threaded_logger_lock); - uwsgi_master_req_log(); - pthread_mutex_unlock(&uwsgi.threaded_logger_lock); - } - - } - } - - return NULL; -} - int uwsgi_get_tcp_info(int fd) { #if defined(__linux__) || defined(__FreeBSD__) @@ -369,25 +299,13 @@ int get_linux_unbit_SIOBKLGQ(int fd) { int master_loop(char **argv, char **environ) { - uint64_t tmp_counter; - struct timeval last_respawn; int last_respawn_rate = 0; - int pid_found = 0; - pid_t diedpid; int waitpid_status; - uint8_t uwsgi_signal; - - time_t last_request_timecheck = 0, now = 0; - uint64_t last_request_count = 0; - - pthread_t logger_thread; - - int udp_fd = -1; - int snmp_fd = -1; + time_t now = 0; int i = 0; int rlen; @@ -457,15 +375,7 @@ int master_loop(char **argv, char **environ) { } } else { - if (pthread_create(&logger_thread, NULL, logger_thread_loop, NULL)) { - uwsgi_error("pthread_create()"); - uwsgi_log("falling back to non-threaded logger...\n"); - event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_log_pipe[0]); - if (uwsgi.req_log_master) { - event_queue_add_fd_read(uwsgi.master_queue, uwsgi.shared->worker_req_log_pipe[0]); - } - uwsgi.threaded_logger = 0; - } + uwsgi_threaded_logger_spawn(); } #ifdef UWSGI_ALARM @@ -519,19 +429,19 @@ int master_loop(char **argv, char **environ) { } if (uwsgi.udp_socket) { - udp_fd = bind_to_udp(uwsgi.udp_socket, 0, 0); - if (udp_fd < 0) { + uwsgi.udp_fd = bind_to_udp(uwsgi.udp_socket, 0, 0); + if (uwsgi.udp_fd < 0) { uwsgi_log("unable to bind to udp socket. SNMP services will be disabled.\n"); } else { uwsgi_log("UDP server enabled.\n"); - event_queue_add_fd_read(uwsgi.master_queue, udp_fd); + event_queue_add_fd_read(uwsgi.master_queue, uwsgi.udp_fd); } } - snmp_fd = uwsgi_setup_snmp(); + uwsgi.snmp_fd = uwsgi_setup_snmp(); - if (uwsgi.cheap) { + if (uwsgi.status.is_cheap) { uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1); for (i = 1; i <= uwsgi.numproc; i++) { uwsgi.workers[i].cheaped = 1; @@ -573,8 +483,10 @@ int master_loop(char **argv, char **environ) { uwsgi_check_touches(uwsgi.touch_reload); uwsgi_check_touches(uwsgi.touch_logrotate); uwsgi_check_touches(uwsgi.touch_logreopen); + uwsgi_check_touches(uwsgi.touch_chain_reload); + uwsgi_check_touches(uwsgi.touch_gracefully_stop); - // setup cheaper algos + // setup cheaper algos (can be stacked) uwsgi.cheaper_algo = uwsgi_cheaper_algo_spare; if (uwsgi.requested_cheaper_algo) { uwsgi.cheaper_algo = NULL; @@ -611,28 +523,27 @@ int master_loop(char **argv, char **environ) { } } - uwsgi_daemons_smart_check(); - - if (uwsgi.to_outworld) { - //uwsgi_log("%d/%d\n", uwsgi.lazy_respawned, uwsgi.numproc); - if (uwsgi.lazy_respawned >= uwsgi.marked_workers || uwsgi.lazy_respawned >= uwsgi.numproc) { - uwsgi.to_outworld = 0; - uwsgi.master_mercy = 0; - uwsgi.lazy_respawned = 0; - } + // check for death + uwsgi_master_check_death(); + // check for realod + if (uwsgi_master_check_reload(argv)) { + return -1; } + // check if some worker is taking too much to die... + uwsgi_master_check_mercy(); - if (uwsgi_master_check_mercy()) - return 0; + // check for daemons (smart and dumb) + uwsgi_daemons_smart_check(); - if (uwsgi.respawn_workers) { - for (i = 1; i <= uwsgi.respawn_workers; i++) { + + if (uwsgi.respawn_snapshots) { + for (i = 1; i <= uwsgi.respawn_snapshots; i++) { if (uwsgi_respawn_worker(i)) return 0; } - uwsgi.respawn_workers = 0; + uwsgi.respawn_snapshots = 0; } if (uwsgi.restore_snapshot) { @@ -641,51 +552,18 @@ int master_loop(char **argv, char **environ) { } // cheaper management - if (uwsgi.cheaper && !uwsgi.cheap && !uwsgi.to_heaven && !uwsgi.to_hell && !uwsgi.to_outworld && !uwsgi.workers[0].suspended) { + if (uwsgi.cheaper && !uwsgi.status.is_cheap && !uwsgi_instance_is_reloading && !uwsgi_instance_is_dying && !uwsgi.workers[0].suspended) { if (!uwsgi_calc_cheaper()) return 0; } - if ((uwsgi.cheap || uwsgi.ready_to_die >= uwsgi.marked_workers || uwsgi.ready_to_die >= uwsgi.numproc) && uwsgi.to_hell) { - // call a series of waitpid to ensure all processes (gateways, mules and daemons) are dead - for (i = 0; i < (ushared->gateways_cnt + uwsgi.daemons_cnt + uwsgi.mules_cnt); i++) { - diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG); - } - - uwsgi_log("goodbye to uWSGI.\n"); - exit(0); - } - - if ((uwsgi.cheap || uwsgi.ready_to_reload >= uwsgi.marked_workers || uwsgi.ready_to_reload >= uwsgi.numproc) && uwsgi.to_heaven) { - uwsgi_reload(argv); - // never here (unless in shared library mode) - return -1; - } - - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].stopped_at > 0 && uwsgi.workers[i].pid > 0 && (uwsgi_now() - uwsgi.workers[i].stopped_at >= uwsgi.worker_reload_mercy)) { - uwsgi_log("worker %d is taking too much time to die (%ds), sending SIGKILL\n",i, (int) (uwsgi_now() - uwsgi.workers[i].stopped_at)); - kill(uwsgi.workers[i].pid, SIGKILL); - } - } + // check if someone is dead diedpid = waitpid(WAIT_ANY, &waitpid_status, WNOHANG); if (diedpid == -1) { if (errno == ECHILD) { // something did not work as expected, just assume all has been cleared - if (uwsgi.to_heaven) { - uwsgi.ready_to_reload = uwsgi.numproc; - continue; - } - else if (uwsgi.to_hell) { - uwsgi.ready_to_die = uwsgi.numproc; - continue; - } - else if (uwsgi.to_outworld) { - uwsgi.lazy_respawned = uwsgi.numproc; - uwsgi_log("*** no workers to reload found ***\n"); - continue; - } + uwsgi_master_commit_status(); diedpid = 0; } else { @@ -697,6 +575,7 @@ int master_loop(char **argv, char **environ) { } } + // no one died just run all of the standard master tasks if (diedpid == 0) { /* all processes ok, doing status scan after N seconds */ @@ -784,181 +663,14 @@ int master_loop(char **argv, char **environ) { uwsgi_unlock(uwsgi.probe_table_lock); } + // some event returned if (rlen > 0) { - - if (uwsgi.log_master && !uwsgi.threaded_logger) { - if (interesting_fd == uwsgi.shared->worker_log_pipe[0]) { - uwsgi_master_log(); - goto health_cycle; - } - if (uwsgi.req_log_master && interesting_fd == uwsgi.shared->worker_req_log_pipe[0]) { - uwsgi_master_req_log(); - goto health_cycle; - } + // if the following function returns -1, a new worker has just spawned + if (uwsgi_master_manage_events(interesting_fd)) { + return 0; } - - if (uwsgi.stats && uwsgi.stats_fd > -1) { - if (interesting_fd == uwsgi.stats_fd) { - uwsgi_send_stats(uwsgi.stats_fd, uwsgi_master_generate_stats); - goto health_cycle; - } - } - - if (uwsgi.zerg_server) { - if (interesting_fd == uwsgi.zerg_server_fd) { - uwsgi_manage_zerg(uwsgi.zerg_server_fd, 0, NULL); - goto health_cycle; - } - } - - if (uwsgi.has_emperor) { - if (interesting_fd == uwsgi.emperor_fd) { - uwsgi_master_manage_emperor(); - goto health_cycle; - } - } - - - if (uwsgi.cheap) { - int found = 0; - struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; - while (uwsgi_sock) { - if (interesting_fd == uwsgi_sock->fd) { - found = 1; - uwsgi.cheap = 0; - uwsgi_del_sockets_from_queue(uwsgi.master_queue); - int needed = uwsgi.numproc; - if (uwsgi.cheaper) { - needed = uwsgi.cheaper_count; - } - for (i = 1; i <= needed; i++) { - if (uwsgi_respawn_worker(i)) - return 0; - } - break; - } - uwsgi_sock = uwsgi_sock->next; - } - // here is better to continue instead going to health_cycle - if (found) - continue; - } - - if (uwsgi.snmp_addr && interesting_fd == snmp_fd) { - uwsgi_master_manage_snmp(snmp_fd); - goto health_cycle; - } - - if (uwsgi.udp_socket && interesting_fd == udp_fd) { - uwsgi_master_manage_udp(udp_fd); - goto health_cycle; - } - - int next_iteration = 0; - - uwsgi_lock(uwsgi.fmon_table_lock); - for (i = 0; i < ushared->files_monitored_cnt; i++) { - if (ushared->files_monitored[i].registered) { - if (interesting_fd == ushared->files_monitored[i].fd) { - struct uwsgi_fmon *uf = event_queue_ack_file_monitor(uwsgi.master_queue, interesting_fd); - // now call the file_monitor handler - if (uf) - uwsgi_route_signal(uf->sig); - break; - } - } - } - - uwsgi_unlock(uwsgi.fmon_table_lock); - if (next_iteration) - goto health_cycle;; - - next_iteration = 0; - - uwsgi_lock(uwsgi.timer_table_lock); - for (i = 0; i < ushared->timers_cnt; i++) { - if (ushared->timers[i].registered) { - if (interesting_fd == ushared->timers[i].fd) { - struct uwsgi_timer *ut = event_queue_ack_timer(interesting_fd); - // now call the file_monitor handler - if (ut) - uwsgi_route_signal(ut->sig); - break; - } - } - } - uwsgi_unlock(uwsgi.timer_table_lock); - if (next_iteration) - goto health_cycle;; - - - // check for worker signal - if (interesting_fd == uwsgi.shared->worker_signal_pipe[0]) { - rlen = read(interesting_fd, &uwsgi_signal, 1); - if (rlen < 0) { - uwsgi_error("read()"); - } - else if (rlen > 0) { -#ifdef UWSGI_DEBUG - uwsgi_log_verbose("received uwsgi signal %d from a worker\n", uwsgi_signal); -#endif - uwsgi_route_signal(uwsgi_signal); - } - else { - uwsgi_log_verbose("lost connection with worker %d\n", i); - close(interesting_fd); - } - goto health_cycle; - } - - // check for spooler signal - if (uwsgi.spoolers) { - if (interesting_fd == uwsgi.shared->spooler_signal_pipe[0]) { - rlen = read(interesting_fd, &uwsgi_signal, 1); - if (rlen < 0) { - uwsgi_error("read()"); - } - else if (rlen > 0) { -#ifdef UWSGI_DEBUG - uwsgi_log_verbose("received uwsgi signal %d from a spooler\n", uwsgi_signal); -#endif - uwsgi_route_signal(uwsgi_signal); - } - else { - uwsgi_log_verbose("lost connection with the spooler\n"); - close(interesting_fd); - } - goto health_cycle; - } - - } - - // check for mules signal - if (uwsgi.mules_cnt > 0) { - if (interesting_fd == uwsgi.shared->mule_signal_pipe[0]) { - rlen = read(interesting_fd, &uwsgi_signal, 1); - if (rlen < 0) { - uwsgi_error("read()"); - } - else if (rlen > 0) { -#ifdef UWSGI_DEBUG - uwsgi_log_verbose("received uwsgi signal %d from a mule\n", uwsgi_signal); -#endif - uwsgi_route_signal(uwsgi_signal); - } - else { - uwsgi_log_verbose("lost connection with a mule\n"); - close(interesting_fd); - } - goto health_cycle; - } - - } - - } -health_cycle: now = uwsgi_now(); if (now - uwsgi.current_time < 1) { continue; @@ -975,66 +687,10 @@ health_cycle: // recalculate requests counter on race conditions risky configurations // a bit of inaccuracy is better than locking;) + uwsgi_master_fix_request_counters(); - if (uwsgi.numproc > 1) { - tmp_counter = 0; - for (i = 1; i < uwsgi.numproc + 1; i++) - tmp_counter += uwsgi.workers[i].requests; - uwsgi.workers[0].requests = tmp_counter; - } - - if (uwsgi.idle > 0 && !uwsgi.cheap) { - uwsgi.current_time = uwsgi_now(); - if (!last_request_timecheck) - last_request_timecheck = uwsgi.current_time; - - int busy_workers = 0; - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { - if (uwsgi.workers[i].busy == 1) { - busy_workers = 1; - break; - } - } - } - - if (last_request_count != uwsgi.workers[0].requests) { - last_request_timecheck = uwsgi.current_time; - last_request_count = uwsgi.workers[0].requests; - } - // a bit of over-engeneering to avoid clock skews - else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle) && !busy_workers) { - uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck); - uwsgi.cheap = 1; - if (uwsgi.die_on_idle) { - if (uwsgi.has_emperor) { - char byte = 22; - if (write(uwsgi.emperor_fd, &byte, 1) != 1) { - uwsgi_error("write()"); - kill_them_all(0); - } - } - else { - kill_them_all(0); - } - continue; - } - for (i = 1; i <= uwsgi.numproc; i++) { - uwsgi.workers[i].cheaped = 1; - if (uwsgi.workers[i].pid == 0) - continue; - kill(uwsgi.workers[i].pid, SIGKILL); - if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) { - if (errno != ECHILD) - uwsgi_error("waitpid()"); - } - } - uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1); - uwsgi_log("cheap mode enabled: waiting for socket connection...\n"); - last_request_timecheck = 0; - continue; - } - } + // check for idle + uwsgi_master_check_idle(); check_interval = uwsgi.shared->options[UWSGI_OPTION_MASTER_INTERVAL]; if (!check_interval) @@ -1057,77 +713,12 @@ health_cycle: uwsgi_sock = uwsgi_sock->next; } - for (i = 1; i <= uwsgi.numproc; i++) { - /* first check for harakiri */ - if (uwsgi.workers[i].harakiri > 0) { - if (uwsgi.workers[i].harakiri < (time_t) uwsgi.current_time) { - trigger_harakiri(i); - } - } - /* then user-defined harakiri */ - if (uwsgi.workers[i].user_harakiri > 0) { - if (uwsgi.workers[i].user_harakiri < (time_t) uwsgi.current_time) { - trigger_harakiri(i); - } - } - // then for evil memory checkers - if (uwsgi.evil_reload_on_as) { - if ((rlim_t) uwsgi.workers[i].vsz_size >= uwsgi.evil_reload_on_as) { - uwsgi_log("*** EVIL RELOAD ON WORKER %d ADDRESS SPACE: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].vsz_size, uwsgi.workers[i].pid); - kill(uwsgi.workers[i].pid, SIGKILL); - uwsgi.workers[i].vsz_size = 0; - } - } - if (uwsgi.evil_reload_on_rss) { - if ((rlim_t) uwsgi.workers[i].rss_size >= uwsgi.evil_reload_on_rss) { - uwsgi_log("*** EVIL RELOAD ON WORKER %d RSS: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].rss_size, uwsgi.workers[i].pid); - kill(uwsgi.workers[i].pid, SIGKILL); - uwsgi.workers[i].rss_size = 0; - } - } - // check if worker was running longer than allowed lifetime - if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.shared->options[UWSGI_OPTION_MAX_WORKER_LIFETIME] > 0) { - uint64_t lifetime = uwsgi_now() - uwsgi.workers[i].last_spawn; - if (lifetime > uwsgi.shared->options[UWSGI_OPTION_MAX_WORKER_LIFETIME] && uwsgi.workers[i].manage_next_request == 1) { - uwsgi_log("worker %d lifetime reached, it was running for %llu second(s)\n", i, (unsigned long long) lifetime); - uwsgi.workers[i].manage_next_request = 0; - kill(uwsgi.workers[i].pid, SIGWINCH); - } - } + // check if some worker has to die (harakiri, evil checks...) + uwsgi_master_check_workers_deadline(); - // need to find a better way - //uwsgi.workers[i].last_running_time = uwsgi.workers[i].running_time; - } - - for (i = 0; i < ushared->gateways_cnt; i++) { - if (ushared->gateways_harakiri[i] > 0) { - if (ushared->gateways_harakiri[i] < (time_t) uwsgi.current_time) { - if (ushared->gateways[i].pid > 0) { - kill(ushared->gateways[i].pid, SIGKILL); - } - ushared->gateways_harakiri[i] = 0; - } - } - } - - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (uwsgi.mules[i].harakiri > 0) { - if (uwsgi.mules[i].harakiri < (time_t) uwsgi.current_time) { - uwsgi_log("*** HARAKIRI ON MULE %d HANDLING SIGNAL %d (pid: %d) ***\n", i + 1, uwsgi.mules[i].signum, uwsgi.mules[i].pid); - kill(uwsgi.mules[i].pid, SIGKILL); - uwsgi.mules[i].harakiri = 0; - } - } - } - struct uwsgi_spooler *uspool = uwsgi.spoolers; - while (uspool) { - if (uspool->harakiri > 0 && uspool->harakiri < (time_t) uwsgi.current_time) { - uwsgi_log("*** HARAKIRI ON THE SPOOLER (pid: %d) ***\n", uspool->pid); - kill(uspool->pid, SIGKILL); - uspool->harakiri = 0; - } - uspool = uspool->next; - } + uwsgi_master_check_gateways_deadline(); + uwsgi_master_check_mules_deadline(); + uwsgi_master_check_spoolers_deadline(); #ifdef __linux__ #ifdef MADV_MERGEABLE @@ -1138,7 +729,7 @@ health_cycle: #endif // resubscribe every 10 cycles by default - if (( (uwsgi.subscriptions || uwsgi.subscriptions2) && ((uwsgi.master_cycles % uwsgi.subscribe_freq) == 0 || uwsgi.master_cycles == 1)) && !uwsgi.to_heaven && !uwsgi.to_hell && !uwsgi.workers[0].suspended) { + if (( (uwsgi.subscriptions || uwsgi.subscriptions2) && ((uwsgi.master_cycles % uwsgi.subscribe_freq) == 0 || uwsgi.master_cycles == 1)) && !uwsgi_instance_is_reloading && !uwsgi_instance_is_dying && !uwsgi.workers[0].suspended) { uwsgi_subscribe_all(0, 0); } @@ -1151,7 +742,7 @@ health_cycle: } // check touch_reload - if (!uwsgi.to_heaven && !uwsgi.to_hell) { + if (!uwsgi_instance_is_reloading && !uwsgi_instance_is_dying) { char *touched = uwsgi_check_touches(uwsgi.touch_reload); if (touched) { uwsgi_log("*** %s has been touched... grace them all !!! ***\n", touched); @@ -1187,64 +778,13 @@ health_cycle: uwsgi_deadlock_check(diedpid); // reload gateways and daemons only on normal workflow (+outworld status) - if (!uwsgi.to_heaven && !uwsgi.to_hell) { - - /* reload the spooler */ - struct uwsgi_spooler *uspool = uwsgi.spoolers; - pid_found = 0; - while (uspool) { - if (uspool->pid > 0 && diedpid == uspool->pid) { - uwsgi_log("OOOPS the spooler is no more...trying respawn...\n"); - uspool->respawned++; - uspool->pid = spooler_start(uspool); - pid_found = 1; - break; - } - uspool = uspool->next; - } - - if (pid_found) - continue; - - if (uwsgi.emperor_pid >= 0) { - uwsgi_log_verbose("!!! Emperor died !!!\n"); - uwsgi_emperor_start(); - continue; - } - - pid_found = 0; - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (uwsgi.mules[i].pid == diedpid) { - uwsgi_log("OOOPS mule %d (pid: %d) crippled...trying respawn...\n", i + 1, uwsgi.mules[i].pid); - uwsgi_mule(i + 1); - pid_found = 1; - break; - } - } - - if (pid_found) - continue; - - - /* reload the gateways */ - pid_found = 0; - for (i = 0; i < ushared->gateways_cnt; i++) { - if (ushared->gateways[i].pid == diedpid) { - gateway_respawn(i); - pid_found = 1; - break; - } - } - - if (pid_found) - continue; - - /* reload the daemons */ - pid_found = uwsgi_daemon_check_pid_reload(diedpid); - - if (pid_found) - continue; + if (!uwsgi_instance_is_reloading && !uwsgi_instance_is_dying) { + if (uwsgi_master_check_emperor_death(diedpid)) continue; + if (uwsgi_master_check_spoolers_death(diedpid)) continue; + if (uwsgi_master_check_mules_death(diedpid)) continue; + if (uwsgi_master_check_gateways_death(diedpid)) continue; + if (uwsgi_master_check_daemons_death(diedpid)) continue; } @@ -1302,27 +842,11 @@ next: // ok a worker died... - if (uwsgi.to_heaven) { - uwsgi.ready_to_reload++; - uwsgi.workers[uwsgi.mywid].pid = 0; - // only to be safe :P - uwsgi.workers[uwsgi.mywid].harakiri = 0; - continue; - } - else if (uwsgi.to_hell) { - uwsgi.ready_to_die++; - uwsgi.workers[uwsgi.mywid].pid = 0; - // only to be safe :P - uwsgi.workers[uwsgi.mywid].harakiri = 0; - continue; - } - else if (uwsgi.to_outworld) { - uwsgi.lazy_respawned++; - uwsgi.workers[uwsgi.mywid].destroy = 0; - uwsgi.workers[uwsgi.mywid].pid = 0; - // only to be safe :P - uwsgi.workers[uwsgi.mywid].harakiri = 0; - } + uwsgi.workers[uwsgi.mywid].pid = 0; + // only to be safe :P + uwsgi.workers[uwsgi.mywid].harakiri = 0; + + // if we are stopping workers, just end here if (WIFEXITED(waitpid_status) && WEXITSTATUS(waitpid_status) == UWSGI_FAILED_APP_CODE) { uwsgi_log("OOPS ! failed loading app in worker %d (pid %d) :( trying again...\n", uwsgi.mywid, (int) diedpid); @@ -1344,7 +868,7 @@ next: uwsgi_log("DAMN ! worker %d (pid: %d) died :( trying respawn ...\n", uwsgi.mywid, (int) diedpid); } } - else if (uwsgi.workers[uwsgi.mywid].stopped_at > 0) { + else if (uwsgi.workers[uwsgi.mywid].cursed_at > 0) { uwsgi_log("worker %d killed successfully (pid: %d)\n", uwsgi.mywid, (int) diedpid); } // manage_next_request is zero, but killed by signal... @@ -1353,11 +877,11 @@ next: } if (uwsgi.workers[uwsgi.mywid].cheaped == 1) { - uwsgi.workers[uwsgi.mywid].pid = 0; uwsgi_log("uWSGI worker %d cheaped.\n", uwsgi.mywid); - uwsgi.workers[uwsgi.mywid].harakiri = 0; continue; } + + // avoid fork bombing gettimeofday(&last_respawn, NULL); if (last_respawn.tv_sec <= uwsgi.respawn_delta + check_interval) { last_respawn_rate++; @@ -1376,6 +900,7 @@ next: gettimeofday(&last_respawn, NULL); uwsgi.respawn_delta = last_respawn.tv_sec; + // respawn the worker (if needed) if (uwsgi_respawn_worker(uwsgi.mywid)) return 0; diff --git a/core/master_checks.c b/core/master_checks.c new file mode 100644 index 00000000..6076ae6c --- /dev/null +++ b/core/master_checks.c @@ -0,0 +1,250 @@ +#include + +extern struct uwsgi_server uwsgi; + +// check if all of the workers are dead and exit uWSGI +void uwsgi_master_check_death() { + if (uwsgi_instance_is_dying) { + int i; + for(i=1;i<=uwsgi.numproc;i++) { + if (uwsgi.workers[i].pid > 0) { + return; + } + } + uwsgi_log("goodbye to uWSGI.\n"); + exit(0); + } +} + +// check if all of the workers are dead, and trigger a reload +int uwsgi_master_check_reload(char **argv) { + if (uwsgi_instance_is_reloading) { + int i; + for(i=1;i<=uwsgi.numproc;i++) { + if (uwsgi.workers[i].pid > 0) { + return 0; + } + } + uwsgi_reload(argv); + // never here (unless in shared library mode) + return -1; + } + return 0; +} + + +// special function for assuming all of the workers are dead +void uwsgi_master_commit_status() { + int i; + for(i=1;i<=uwsgi.numproc;i++) { + uwsgi.workers[i].pid = 0; + } +} + +void uwsgi_master_check_idle() { + + static time_t last_request_timecheck = 0; + static uint64_t last_request_count = 0; + int i; + int waitpid_status; + + if (!uwsgi.idle || uwsgi.status.is_cheap) + return; + + uwsgi.current_time = uwsgi_now(); + if (!last_request_timecheck) + last_request_timecheck = uwsgi.current_time; + + // security check, stop the check if there are busy workers + for (i = 1; i <= uwsgi.numproc; i++) { + if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { + if (uwsgi.workers[i].busy == 1) { + return; + } + } + } + + if (last_request_count != uwsgi.workers[0].requests) { + last_request_timecheck = uwsgi.current_time; + last_request_count = uwsgi.workers[0].requests; + } + // a bit of over-engeneering to avoid clock skews + else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle)) { + uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck); + uwsgi.status.is_cheap = 1; + if (uwsgi.die_on_idle) { + if (uwsgi.has_emperor) { + char byte = 22; + if (write(uwsgi.emperor_fd, &byte, 1) != 1) { + uwsgi_error("write()"); + kill_them_all(0); + } + } + else { + kill_them_all(0); + } + return; + } + for (i = 1; i <= uwsgi.numproc; i++) { + uwsgi.workers[i].cheaped = 1; + if (uwsgi.workers[i].pid == 0) + continue; + kill(uwsgi.workers[i].pid, SIGKILL); + if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) { + if (errno != ECHILD) + uwsgi_error("uwsgi_master_check_idle()/waitpid()"); + } + } + uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1); + uwsgi_log("cheap mode enabled: waiting for socket connection...\n"); + last_request_timecheck = 0; + } + +} + +void uwsgi_master_check_workers_deadline() { + int i; + for (i = 1; i <= uwsgi.numproc; i++) { + /* first check for harakiri */ + if (uwsgi.workers[i].harakiri > 0) { + if (uwsgi.workers[i].harakiri < (time_t) uwsgi.current_time) { + trigger_harakiri(i); + } + } + /* then user-defined harakiri */ + if (uwsgi.workers[i].user_harakiri > 0) { + if (uwsgi.workers[i].user_harakiri < (time_t) uwsgi.current_time) { + trigger_harakiri(i); + } + } + // then for evil memory checkers + if (uwsgi.evil_reload_on_as) { + if ((rlim_t) uwsgi.workers[i].vsz_size >= uwsgi.evil_reload_on_as) { + uwsgi_log("*** EVIL RELOAD ON WORKER %d ADDRESS SPACE: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].vsz_size, uwsgi.workers[i].pid); + kill(uwsgi.workers[i].pid, SIGKILL); + uwsgi.workers[i].vsz_size = 0; + } + } + if (uwsgi.evil_reload_on_rss) { + if ((rlim_t) uwsgi.workers[i].rss_size >= uwsgi.evil_reload_on_rss) { + uwsgi_log("*** EVIL RELOAD ON WORKER %d RSS: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].rss_size, uwsgi.workers[i].pid); + kill(uwsgi.workers[i].pid, SIGKILL); + uwsgi.workers[i].rss_size = 0; + } + } + // check if worker was running longer than allowed lifetime + if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.shared->options[UWSGI_OPTION_MAX_WORKER_LIFETIME] > 0) { + uint64_t lifetime = uwsgi_now() - uwsgi.workers[i].last_spawn; + if (lifetime > uwsgi.shared->options[UWSGI_OPTION_MAX_WORKER_LIFETIME] && uwsgi.workers[i].manage_next_request == 1) { + uwsgi_log("worker %d lifetime reached, it was running for %llu second(s)\n", i, (unsigned long long) lifetime); + uwsgi.workers[i].manage_next_request = 0; + kill(uwsgi.workers[i].pid, SIGWINCH); + } + } + + // need to find a better way + //uwsgi.workers[i].last_running_time = uwsgi.workers[i].running_time; + } + + + +} + + +void uwsgi_master_check_gateways_deadline() { + + int i; + + for (i = 0; i < ushared->gateways_cnt; i++) { + if (ushared->gateways_harakiri[i] > 0) { + if (ushared->gateways_harakiri[i] < (time_t) uwsgi.current_time) { + if (ushared->gateways[i].pid > 0) { + kill(ushared->gateways[i].pid, SIGKILL); + } + ushared->gateways_harakiri[i] = 0; + } + } + } +} + +void uwsgi_master_check_mules_deadline() { + int i; + + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].harakiri > 0) { + if (uwsgi.mules[i].harakiri < (time_t) uwsgi.current_time) { + uwsgi_log("*** HARAKIRI ON MULE %d HANDLING SIGNAL %d (pid: %d) ***\n", i + 1, uwsgi.mules[i].signum, uwsgi.mules[i].pid); + kill(uwsgi.mules[i].pid, SIGKILL); + uwsgi.mules[i].harakiri = 0; + } + } + } +} + +void uwsgi_master_check_spoolers_deadline() { + struct uwsgi_spooler *uspool = uwsgi.spoolers; + while (uspool) { + if (uspool->harakiri > 0 && uspool->harakiri < (time_t) uwsgi.current_time) { + uwsgi_log("*** HARAKIRI ON THE SPOOLER (pid: %d) ***\n", uspool->pid); + kill(uspool->pid, SIGKILL); + uspool->harakiri = 0; + } + uspool = uspool->next; + } +} + + +int uwsgi_master_check_spoolers_death(int diedpid) { + + struct uwsgi_spooler *uspool = uwsgi.spoolers; + while (uspool) { + if (uspool->pid > 0 && diedpid == uspool->pid) { + uwsgi_log("OOOPS the spooler is no more...trying respawn...\n"); + uspool->respawned++; + uspool->pid = spooler_start(uspool); + return -1; + } + uspool = uspool->next; + } + return 0; +} + +int uwsgi_master_check_emperor_death(int diedpid) { + if (uwsgi.emperor_pid >= 0 && diedpid == uwsgi.emperor_pid) { + uwsgi_log_verbose("!!! Emperor died !!!\n"); + uwsgi_emperor_start(); + return -1; + } + return 0; +} + +int uwsgi_master_check_mules_death(int diedpid) { + int i; + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid == diedpid) { + uwsgi_log("OOOPS mule %d (pid: %d) crippled...trying respawn...\n", i + 1, uwsgi.mules[i].pid); + uwsgi_mule(i + 1); + return -1; + } + } + return 0; +} + +int uwsgi_master_check_gateways_death(int diedpid) { + int i; + for (i = 0; i < ushared->gateways_cnt; i++) { + if (ushared->gateways[i].pid == diedpid) { + gateway_respawn(i); + return -1; + } + } + return 0; +} + +int uwsgi_master_check_daemons_death(int diedpid) { + /* reload the daemons */ + if (uwsgi_daemon_check_pid_reload(diedpid)) { + return -1; + } + return 0; +} diff --git a/core/master_events.c b/core/master_events.c new file mode 100644 index 00000000..d11016ae --- /dev/null +++ b/core/master_events.c @@ -0,0 +1,174 @@ +#include + +extern struct uwsgi_server uwsgi; + +int uwsgi_master_manage_events(int interesting_fd) { + + // is a logline ? + if (uwsgi.log_master && !uwsgi.threaded_logger) { + // stderr log ? + if (interesting_fd == uwsgi.shared->worker_log_pipe[0]) { + uwsgi_master_log(); + return 0; + } + // req log ? + if (uwsgi.req_log_master && interesting_fd == uwsgi.shared->worker_req_log_pipe[0]) { + uwsgi_master_req_log(); + return 0; + } + } + + // stats server ? + if (uwsgi.stats && uwsgi.stats_fd > -1) { + if (interesting_fd == uwsgi.stats_fd) { + uwsgi_send_stats(uwsgi.stats_fd, uwsgi_master_generate_stats); + return 0; + } + } + + // a zerg connection ? + if (uwsgi.zerg_server) { + if (interesting_fd == uwsgi.zerg_server_fd) { + uwsgi_manage_zerg(uwsgi.zerg_server_fd, 0, NULL); + return 0; + } + } + + // emperor event ? + if (uwsgi.has_emperor) { + if (interesting_fd == uwsgi.emperor_fd) { + uwsgi_master_manage_emperor(); + return 0; + } + } + + + // wakeup from cheap mode ? + if (uwsgi.status.is_cheap) { + struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; + while (uwsgi_sock) { + if (interesting_fd == uwsgi_sock->fd) { + uwsgi.status.is_cheap = 0; + uwsgi_del_sockets_from_queue(uwsgi.master_queue); + // how many worker we need to respawn ? + int needed = uwsgi.numproc; + // if in cheaper mode, just respawn the minimal amount + if (uwsgi.cheaper) { + needed = uwsgi.cheaper_count; + } + int i; + for (i = 1; i <= needed; i++) { + if (uwsgi_respawn_worker(i)) + return -1; + } + // here we continue instead of returning + break; + } + uwsgi_sock = uwsgi_sock->next; + } + } + + + // an SNMP request ? + if (uwsgi.snmp_addr && interesting_fd == uwsgi.snmp_fd) { + uwsgi_master_manage_snmp(uwsgi.snmp_fd); + return 0; + } + + // a UDP request ? + if (uwsgi.udp_socket && interesting_fd == uwsgi.udp_fd) { + uwsgi_master_manage_udp(uwsgi.udp_fd); + return 0; + } + + + // check if some file monitor is ready + // no need to lock as we are only getting registered items (and only the master can register them) + int i; + for (i = 0; i < ushared->files_monitored_cnt; i++) { + if (ushared->files_monitored[i].registered) { + if (interesting_fd == ushared->files_monitored[i].fd) { + struct uwsgi_fmon *uf = event_queue_ack_file_monitor(uwsgi.master_queue, interesting_fd); + // now call the file_monitor handler + if (uf) + uwsgi_route_signal(uf->sig); + return 0; + } + } + } + + // check if some timer elapsed + // no need to lock again + for (i = 0; i < ushared->timers_cnt; i++) { + if (ushared->timers[i].registered) { + if (interesting_fd == ushared->timers[i].fd) { + struct uwsgi_timer *ut = event_queue_ack_timer(interesting_fd); + // now call the timer handler + if (ut) + uwsgi_route_signal(ut->sig); + return 0; + } + } + } + + uint8_t uwsgi_signal; + // check for worker signal + if (interesting_fd == uwsgi.shared->worker_signal_pipe[0]) { + ssize_t rlen = read(interesting_fd, &uwsgi_signal, 1); + if (rlen < 0) { + uwsgi_error("uwsgi_master_manage_events()/read()"); + } + else if (rlen > 0) { + uwsgi_route_signal(uwsgi_signal); + } + else { + // TODO restart workers here + uwsgi_log_verbose("lost connection with workers !!!\n"); + close(interesting_fd); + } + return 0; + } + + // check for spooler signal + if (uwsgi.spoolers) { + if (interesting_fd == uwsgi.shared->spooler_signal_pipe[0]) { + ssize_t rlen = read(interesting_fd, &uwsgi_signal, 1); + if (rlen < 0) { + uwsgi_error("uwsgi_master_manage_events()/read()"); + } + else if (rlen > 0) { + uwsgi_route_signal(uwsgi_signal); + } + else { + // TODO restart spoolers here + uwsgi_log_verbose("lost connection with spoolers\n"); + close(interesting_fd); + } + return 0; + } + + } + + // check for mules signal + if (uwsgi.mules_cnt > 0) { + if (interesting_fd == uwsgi.shared->mule_signal_pipe[0]) { + ssize_t rlen = read(interesting_fd, &uwsgi_signal, 1); + if (rlen < 0) { + uwsgi_error("uwsgi_master_manage_events()/read()"); + } + else if (rlen > 0) { + uwsgi_route_signal(uwsgi_signal); + } + else { + // TODO respawn mules here + uwsgi_log_verbose("lost connection with mules\n"); + close(interesting_fd); + } + // return 0; + } + + } + + return 0; + +} diff --git a/core/master_utils.c b/core/master_utils.c index 49dc27c7..95bceb1f 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -5,6 +5,60 @@ extern struct uwsgi_server uwsgi; void worker_wakeup() { } +void uwsgi_curse(int wid, int sig) { + uwsgi.workers[wid].cursed_at = uwsgi_now(); + if (uwsgi.reload_mercy) { + uwsgi.workers[wid].no_mercy_at = uwsgi.workers[wid].cursed_at + uwsgi.reload_mercy; + } + else { + uwsgi.workers[wid].no_mercy_at = uwsgi.workers[wid].cursed_at + 5; + } + + if (sig) { + (void) kill(uwsgi.workers[wid].pid, sig); + } +} + +static void uwsgi_signal_spoolers(int signum) { + + struct uwsgi_spooler *uspool = uwsgi.spoolers; + while (uspool) { + if (uspool->pid > 0) { + kill(uspool->pid, SIGKILL); + uwsgi_log("killing the spooler with pid %d\n", uspool->pid); + } + uspool = uspool->next; + } + +} +void uwsgi_destroy_processes() { + + int i; + + uwsgi_signal_spoolers(SIGKILL); + + if (uwsgi.emperor_pid >= 0) { + kill(uwsgi.emperor_pid, SIGKILL); + waitpid(uwsgi.emperor_pid, &i, 0); + uwsgi_log("killing the emperor with pid %d\n", uwsgi.emperor_pid); + } + + + uwsgi_detach_daemons(); + + for (i = 0; i < ushared->gateways_cnt; i++) { + if (ushared->gateways[i].pid > 0) + kill(ushared->gateways[i].pid, SIGKILL); + } + + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0) + kill(uwsgi.mules[i].pid, SIGKILL); + } +} + + + void uwsgi_master_cleanup_hooks(void) { int j; @@ -13,7 +67,7 @@ void uwsgi_master_cleanup_hooks(void) { if (uwsgi.mywid > 0) return; - uwsgi.cleaning = 1; + uwsgi.status.is_cleaning = 1; for (j = 0; j < uwsgi.gp_cnt; j++) { if (uwsgi.gp[j]->master_cleanup) { @@ -78,7 +132,7 @@ int uwsgi_calc_cheaper(void) { #endif uwsgi.workers[oldest_worker].cheaped = 1; uwsgi.workers[oldest_worker].manage_next_request = 0; - uwsgi.workers[oldest_worker].stopped_at = now; + uwsgi.workers[oldest_worker].cursed_at = now; // wakeup task in case of wait (void) kill(uwsgi.workers[oldest_worker].pid, SIGWINCH); } @@ -476,7 +530,8 @@ int uwsgi_respawn_worker(int wid) { uwsgi.workers[wid].rss_size = 0; uwsgi.workers[wid].vsz_size = 0; // ... reset stopped_at - uwsgi.workers[wid].stopped_at = 0; + uwsgi.workers[wid].cursed_at = 0; + uwsgi.workers[wid].no_mercy_at = 0; // internal statuses should be reset too @@ -540,7 +595,7 @@ int uwsgi_respawn_worker(int wid) { uwsgi.my_signal_socket = uwsgi.workers[wid].signal_pipe[1]; if (uwsgi.master_process) { - if ((uwsgi.workers[uwsgi.mywid].respawn_count || uwsgi.cheap)) { + if ((uwsgi.workers[uwsgi.mywid].respawn_count || uwsgi.status.is_cheap)) { for (i = 0; i < 256; i++) { if (uwsgi.p[i]->master_fixup) { uwsgi.p[i]->master_fixup(1); @@ -1406,3 +1461,20 @@ void trigger_harakiri(int i) { // to avoid races } + +void uwsgi_master_fix_request_counters() { + int i; + uint64_t total_counter = 0; + for (i = 1; i <= uwsgi.numproc;i++) { + uint64_t tmp_counter = 0; + int j; + for(j=0;jbuffer, uwsgi.buffer_size, 0, (struct sockaddr *) &udp_client, &udp_len); + + if (rlen < 0) { + uwsgi_error("recvfrom()"); + } + else if (rlen > 0) { + manage_snmp(snmp_fd, (uint8_t *) uwsgi.wsgi_req->buffer, rlen, &udp_client); + } +} + diff --git a/core/uwsgi.c b/core/uwsgi.c index 73fc2d17..929cbe3c 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -441,7 +441,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"chdir2", required_argument, 0, "chdir to specified directory after apps loading", uwsgi_opt_set_str, &uwsgi.chdir2, 0}, {"lazy", no_argument, 0, "set lazy mode (load apps in workers instead of master)", uwsgi_opt_true, &uwsgi.lazy, 0}, {"lazy-apps", no_argument, 0, "load apps in each worker instead of the master", uwsgi_opt_true, &uwsgi.lazy_apps, 0}, - {"cheap", no_argument, 0, "set cheap mode (spawn workers only after the first request)", uwsgi_opt_true, &uwsgi.cheap, UWSGI_OPT_MASTER}, + {"cheap", no_argument, 0, "set cheap mode (spawn workers only after the first request)", uwsgi_opt_true, &uwsgi.status.is_cheap, UWSGI_OPT_MASTER}, {"cheaper", required_argument, 0, "set cheaper mode (adaptive process spawning)", uwsgi_opt_set_int, &uwsgi.cheaper_count, UWSGI_OPT_MASTER | UWSGI_OPT_CHEAPER}, {"cheaper-initial", required_argument, 0, "set the initial number of processes to spawn in cheaper mode", uwsgi_opt_set_int, &uwsgi.cheaper_initial, UWSGI_OPT_MASTER | UWSGI_OPT_CHEAPER}, {"cheaper-algo", required_argument, 0, "choose to algorithm used for adaptive process spawning)", uwsgi_opt_set_str, &uwsgi.requested_cheaper_algo, UWSGI_OPT_MASTER}, @@ -900,7 +900,7 @@ void end_me(int signum) { void simple_goodbye_cruel_world() { - if (uwsgi.threads > 1 && !uwsgi.to_hell) { + if (uwsgi.threads > 1 && !uwsgi_instance_is_dying) { wait_for_threads(); } @@ -910,7 +910,7 @@ void simple_goodbye_cruel_world() { } void goodbye_cruel_world() { - uwsgi.workers[uwsgi.mywid].stopped_at = uwsgi_now(); + uwsgi_curse(uwsgi.mywid, 0); if (!uwsgi.gbcw_hook) { simple_goodbye_cruel_world(); @@ -920,125 +920,39 @@ void goodbye_cruel_world() { } } -static void uwsgi_signal_spoolers(int signum) { - - struct uwsgi_spooler *uspool = uwsgi.spoolers; - while (uspool) { - if (uspool->pid > 0) { - kill(uspool->pid, SIGKILL); - uwsgi_log("killing the spooler with pid %d\n", uspool->pid); - } - uspool = uspool->next; - } - -} - +// gracefully destroy void kill_them_all(int signum) { - int i; - if (uwsgi.to_hell == 1) - return; - - // count the number of active workers - int active_workers = 0; - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { - active_workers++; - } - } - uwsgi.marked_workers = active_workers; - - uwsgi.to_hell = 1; - - if (uwsgi.reload_mercy > 0) { - uwsgi.master_mercy = uwsgi_now() + uwsgi.reload_mercy; - } - else { - uwsgi.master_mercy = uwsgi_now() + 5; - } - - uwsgi_log("SIGINT/SIGQUIT received...killing workers...\n"); + if (uwsgi_instance_is_dying) return; + uwsgi.status.gracefully_destroying = 1; // unsubscribe if needed uwsgi_unsubscribe_all(); + uwsgi_log("SIGINT/SIGQUIT received...killing workers...\n"); + + int i; for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].pid > 0) - kill(uwsgi.workers[i].pid, SIGINT); - } - - uwsgi_signal_spoolers(SIGKILL); - - if (uwsgi.emperor_pid >= 0) { - kill(uwsgi.emperor_pid, SIGKILL); - waitpid(uwsgi.emperor_pid, &i, 0); - uwsgi_log("killing the emperor with pid %d\n", uwsgi.emperor_pid); - } - - - uwsgi_detach_daemons(); - - for (i = 0; i < ushared->gateways_cnt; i++) { - if (ushared->gateways[i].pid > 0) - kill(ushared->gateways[i].pid, SIGKILL); - } - - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (uwsgi.mules[i].pid > 0) - kill(uwsgi.mules[i].pid, SIGKILL); - } + if (uwsgi.workers[i].pid > 0) { + uwsgi_curse(i, SIGINT); + } + } + uwsgi_destroy_processes(); } +// graceful reload void grace_them_all(int signum) { + if (uwsgi_instance_is_reloading || uwsgi_instance_is_dying) + return; + + uwsgi.status.gracefully_reloading = 1; + int i; int waitpid_status; - if (uwsgi.to_heaven == 1 || uwsgi.to_outworld == 1 || uwsgi.lazy_respawned > 0) - return; - - // count the number of active workers - int active_workers = 0; - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { - active_workers++; - } - } - uwsgi.marked_workers = active_workers; - - if (!uwsgi.lazy) - uwsgi.to_heaven = 1; - else - uwsgi.to_outworld = 1; - - if (uwsgi.reload_mercy > 0) { - uwsgi.master_mercy = uwsgi_now() + uwsgi.reload_mercy; - } - else { - // wait max 60 seconds for graceful reload - uwsgi.master_mercy = uwsgi_now() + 60; - } - - uwsgi_signal_spoolers(SIGKILL); - - if (uwsgi.emperor_pid >= 0) { - kill(uwsgi.emperor_pid, SIGKILL); - waitpid(uwsgi.emperor_pid, &i, 0); - uwsgi_log("killing the emperor with pid %d\n", uwsgi.emperor_pid); - } - - uwsgi_detach_daemons(); - - for (i = 0; i < ushared->gateways_cnt; i++) { - if (ushared->gateways[i].pid > 0) - kill(ushared->gateways[i].pid, SIGKILL); - } - - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (uwsgi.mules[i].pid > 0) - kill(uwsgi.mules[i].pid, SIGKILL); - } + uwsgi_destroy_processes(); uwsgi_log("...gracefully killing workers...\n"); @@ -1058,12 +972,11 @@ void grace_them_all(int signum) { if (uwsgi.auto_snapshot > 0 && i > uwsgi.auto_snapshot) { uwsgi.workers[i].snapshot = 0; uwsgi.workers[i].destroy = 1; - kill(uwsgi.workers[i].pid, SIGHUP); + uwsgi_curse(i, SIGHUP); } else { uwsgi.workers[i].snapshot = uwsgi.workers[i].pid; kill(uwsgi.workers[i].pid, SIGURG); - uwsgi.lazy_respawned++; } } } @@ -1075,9 +988,9 @@ void grace_them_all(int signum) { } if (uwsgi.auto_snapshot) { - uwsgi.respawn_workers = uwsgi.numproc - uwsgi.auto_snapshot; - if (!uwsgi.respawn_workers) - uwsgi.respawn_workers = 1; + uwsgi.respawn_snapshots = uwsgi.numproc - uwsgi.auto_snapshot; + if (!uwsgi.respawn_snapshots) + uwsgi.respawn_snapshots = 1; } } @@ -1096,61 +1009,28 @@ void uwsgi_nuclear_blast() { exit(1); } +// brutally reload void reap_them_all(int signum) { - int i; // avoid reace condition in lazy mode - if (uwsgi.to_outworld == 1 || uwsgi.lazy_respawned > 0) + if (uwsgi_instance_is_reloading) return; + uwsgi.status.brutally_reloading = 1; if (!uwsgi.workers) return; - // count the number of active workers - int active_workers = 0; - for (i = 1; i <= uwsgi.numproc; i++) { - if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { - active_workers++; - } - } - uwsgi.marked_workers = active_workers; - - if (!uwsgi.lazy) - uwsgi.to_heaven = 1; - else - uwsgi.to_outworld = 1; - - uwsgi_detach_daemons(); - - for (i = 0; i < ushared->gateways_cnt; i++) { - if (ushared->gateways[i].pid > 0) - kill(ushared->gateways[i].pid, SIGKILL); - } - - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (!uwsgi.mules) - break; - if (uwsgi.mules[i].pid > 0) - kill(uwsgi.mules[i].pid, SIGKILL); - } - - if (uwsgi.emperor_pid >= 0) { - kill(uwsgi.emperor_pid, SIGKILL); - waitpid(uwsgi.emperor_pid, &i, 0); - uwsgi_log("killing the emperor with pid %d\n", uwsgi.emperor_pid); - } - - if (!uwsgi.workers) - return; + uwsgi_destroy_processes(); uwsgi_log("...brutally killing workers...\n"); // unsubscribe if needed uwsgi_unsubscribe_all(); + int i; for (i = 1; i <= uwsgi.numproc; i++) { if (uwsgi.workers[i].pid > 0) - kill(uwsgi.workers[i].pid, SIGTERM); + uwsgi_curse(i, SIGTERM); } } @@ -2403,7 +2283,8 @@ int uwsgi_start(void *v_argv) { if (uwsgi.command_mode) { uwsgi.sockets = NULL; uwsgi.numproc = 1; - uwsgi.to_hell = 1; + // hack to destroy the instance after command exit + uwsgi.status.brutally_destroying = 1; } #ifndef UWSGI_DEBUG @@ -2682,7 +2563,7 @@ next2: uwsgi.current_time = uwsgi_now(); // here we spawn the workers... - if (!uwsgi.cheap) { + if (!uwsgi.status.is_cheap) { if (uwsgi.cheaper && uwsgi.cheaper_count) { int nproc = uwsgi.cheaper_initial; if (!nproc) diff --git a/plugins/carbon/carbon.c b/plugins/carbon/carbon.c index ee126ec6..8303b90c 100644 --- a/plugins/carbon/carbon.c +++ b/plugins/carbon/carbon.c @@ -291,7 +291,7 @@ void carbon_master_cycle() { if (!u_carbon.servers) return; - if (uwsgi.current_time - u_carbon.last_update >= u_carbon.freq || uwsgi.cleaning) { + if (uwsgi.current_time - u_carbon.last_update >= u_carbon.freq || uwsgi.status.is_cleaning) { // update u_carbon.need_retry = 0; carbon_push_stats(0); diff --git a/uwsgi.h b/uwsgi.h index 93d93d37..1fbad783 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -1421,6 +1421,21 @@ struct uwsgi_cache { struct uwsgi_offload_engine; +// these are the possible states of an instance +struct uwsgi_instance_status { + int gracefully_reloading; + int brutally_reloading; + int gracefully_destroying; + int brutally_destroying; + int chain_reloading; + int workers_reloading; + int is_cheap; + int is_cleaning; +}; + +#define uwsgi_instance_is_dying (uwsgi.status.gracefully_destroying || uwsgi.status.brutally_destroying) +#define uwsgi_instance_is_reloading (uwsgi.status.gracefully_reloading || uwsgi.status.brutally_reloading) + struct uwsgi_server { @@ -1450,6 +1465,8 @@ struct uwsgi_server { // quiet startup int no_initial_output; + struct uwsgi_instance_status status; + struct uwsgi_string_list *get_list; // enable threads @@ -1474,13 +1491,13 @@ struct uwsgi_server { struct uwsgi_string_list *whitelist; int snapshot; + int respawn_snapshots; // enable auto-snapshotting int auto_snapshot; pid_t restore_snapshot; - int respawn_workers; unsigned int reloads; // leave master running as root @@ -1504,8 +1521,6 @@ struct uwsgi_server { int lazy; // enable lazy-apps mode int lazy_apps; - // enable cheap mode - int cheap; // enable cheaper mode int cheaper; char *requested_cheaper_algo; @@ -1560,9 +1575,6 @@ struct uwsgi_server { struct uwsgi_string_list *additional_headers; struct uwsgi_string_list *remove_headers; - // maximum time to wait after a reload - time_t master_mercy; - // set cpu affinity int cpu_affinity; @@ -1841,6 +1853,8 @@ struct uwsgi_server { struct uwsgi_string_list *touch_reload; struct uwsgi_string_list *touch_chain_reload; + struct uwsgi_string_list *touch_workers_reload; + struct uwsgi_string_list *touch_gracefully_stop; struct uwsgi_string_list *touch_logrotate; struct uwsgi_string_list *touch_logreopen; @@ -1873,18 +1887,9 @@ struct uwsgi_server { char *snmp_addr; char *snmp_community; struct uwsgi_lock_item *snmp_lock; + int snmp_fd; - int to_heaven; - int to_hell; - int to_outworld; - - int cleaning; - - int marked_workers; - int ready_to_die; - int ready_to_reload; - - int lazy_respawned; + int udp_fd; uint16_t buffer_size; int signal_bufsize; @@ -2363,7 +2368,8 @@ struct uwsgi_rpc { int sig; uint8_t signum; - time_t stopped_at; + time_t cursed_at; + time_t no_mercy_at; // signals managed by this worker uint64_t signals; @@ -2461,10 +2467,15 @@ void uwsgi_redirect_to_slash(struct wsgi_request *); void manage_snmp(int, uint8_t *, int, struct sockaddr_in *); void snmp_init(void); +void uwsgi_master_manage_snmp(int); + int spool_request(struct uwsgi_spooler *uspool, char *, int, int, char *, int, char *, time_t, char *, size_t); void spooler(struct uwsgi_spooler *); pid_t spooler_start(struct uwsgi_spooler *); +void uwsgi_curse(int, int); +void uwsgi_destroy_processes(void); + void set_harakiri(int); void set_user_harakiri(int); void set_mule_harakiri(int); @@ -3773,6 +3784,28 @@ int uwsgi_upload_progress_update(struct wsgi_request *, int, size_t); void uwsgi_upload_progress_destroy(char *, int); void uwsgi_time_bomb(int, int); +void uwsgi_master_manage_emperor(void); +void uwsgi_master_manage_udp(int); + +void uwsgi_threaded_logger_spawn(void); + +void uwsgi_master_check_idle(void); +void uwsgi_master_check_workers_deadline(void); +void uwsgi_master_check_gateways_deadline(void); +void uwsgi_master_check_mules_deadline(void); +void uwsgi_master_check_spoolers_deadline(void); +int uwsgi_master_check_spoolers_death(int); +int uwsgi_master_check_emperor_death(int); +int uwsgi_master_check_mules_death(int); +int uwsgi_master_check_gateways_death(int); +int uwsgi_master_check_daemons_death(int); + +void uwsgi_master_check_death(void); +int uwsgi_master_check_reload(char **); +void uwsgi_master_commit_status(void); + +void uwsgi_master_fix_request_counters(void); +int uwsgi_master_manage_events(int); #define uwsgi_response_add_connection_close(x) uwsgi_response_add_header(x, "Connection", 10, "close", 5) #define uwsgi_response_add_content_type(x, y, z) uwsgi_response_add_header(x, "Content-Type", 12, y, z) diff --git a/uwsgiconfig.py b/uwsgiconfig.py index b6356aad..189d6867 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -446,10 +446,10 @@ class uConf(object): self.config.read(filename) self.gcc_list = ['core/utils', 'core/protocol', 'core/socket', 'core/logging', 'core/master', 'core/master_utils', 'core/emperor', - 'core/notify', 'core/mule', 'core/subscription', 'core/stats', 'core/sendfile', 'core/async', + 'core/notify', 'core/mule', 'core/subscription', 'core/stats', 'core/sendfile', 'core/async', 'core/master_checks', 'core/offload', 'core/io', 'core/static', 'core/websockets', 'core/spooler', 'core/snmp', 'core/setup_utils', 'core/clock', 'core/init', 'core/buffer', 'core/reader', 'core/writer', - 'core/plugins', 'core/lock', 'core/cache', 'core/daemons', 'core/errors', 'core/hash', + 'core/plugins', 'core/lock', 'core/cache', 'core/daemons', 'core/errors', 'core/hash', 'core/master_events', 'core/queue', 'core/event', 'core/signal', 'core/strings', 'core/progress', 'core/timebomb', 'core/rpc', 'core/gateway', 'core/loop', 'core/rb_timers', 'core/uwsgi'] # add protocols