From 72c31dab7a7b286deda1ae535ea7436ede14cae6 Mon Sep 17 00:00:00 2001 From: Paul Egan Date: Thu, 24 Apr 2014 17:38:27 +0100 Subject: [PATCH] Support for graceful reload of mule processes. - On reload master now signals mule with HUP instead of KILL. - Added optional call to PyOS_AfterFork to enable cpython child processes to override and trap HUP signal. - Copied cursed_at/no_mercy_at pattern used with worker processes. - Added mule_reload_mercy option. --- core/init.c | 1 + core/master.c | 10 ++++++++++ core/master_checks.c | 5 +++++ core/master_utils.c | 18 +++++++++--------- core/mule.c | 2 +- core/uwsgi.c | 22 ++++++++++++++++++++++ plugins/python/python_plugin.c | 7 +++++++ plugins/python/uwsgi_python.h | 2 ++ uwsgi.h | 6 ++++++ 9 files changed, 63 insertions(+), 10 deletions(-) diff --git a/core/init.c b/core/init.c index 8603ca28..a7f23913 100644 --- a/core/init.c +++ b/core/init.c @@ -113,6 +113,7 @@ void uwsgi_init_default() { uwsgi.log_master_bufsize = 8192; uwsgi.worker_reload_mercy = 60; + uwsgi.mule_reload_mercy = 60; uwsgi.max_vars = MAX_VARS; uwsgi.vec_size = 4 + 1 + (4 * MAX_VARS); diff --git a/core/master.c b/core/master.c index 1ccd597a..b5a81dd8 100644 --- a/core/master.c +++ b/core/master.c @@ -135,6 +135,15 @@ void uwsgi_master_check_mercy() { } } } + + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0 && uwsgi.mules[i].cursed_at) { + if (uwsgi_now() > uwsgi.mules[i].no_mercy_at) { + uwsgi_log_verbose("mule %d (pid: %d) is taking too much time to die...NO MERCY !!!\n", i + 1, uwsgi.mules[i].pid); + uwsgi_curse_mule(i, SIGKILL); + } + } + } } @@ -907,6 +916,7 @@ int master_loop(char **argv, char **environ) { for (i = 0; i < uwsgi.mules_cnt; i++) { if (uwsgi.mules[i].pid == diedpid) { uwsgi_log("mule %d (pid: %d) annihilated\n", i + 1, (int) diedpid); + uwsgi.mules[i].pid = 0; goto next; } } diff --git a/core/master_checks.c b/core/master_checks.c index 6fbef58c..85700e82 100644 --- a/core/master_checks.c +++ b/core/master_checks.c @@ -25,6 +25,11 @@ int uwsgi_master_check_reload(char **argv) { return 0; } } + for(i=0;i 0) { + return 0; + } + } uwsgi_reload(argv); // never here (unless in shared library mode) return -1; diff --git a/core/master_utils.c b/core/master_utils.c index 0e70554c..4341c1dd 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -24,6 +24,15 @@ void uwsgi_curse(int wid, int sig) { } } +void uwsgi_curse_mule(int mid, int sig) { + uwsgi.mules[mid].cursed_at = uwsgi_now(); + uwsgi.mules[mid].no_mercy_at = uwsgi.mules[mid].cursed_at + uwsgi.mule_reload_mercy; + + if (sig) { + (void) kill(uwsgi.mules[mid].pid, sig); + } +} + static void uwsgi_signal_spoolers(int signum) { struct uwsgi_spooler *uspool = uwsgi.spoolers; @@ -53,15 +62,6 @@ void uwsgi_destroy_processes() { } } - // TODO mules can be programmed to be gracefully reloaded - for (i = 0; i < uwsgi.mules_cnt; i++) { - if (uwsgi.mules[i].pid > 0) { - kill(uwsgi.mules[i].pid, SIGKILL); - waitpid(uwsgi.mules[i].pid, &waitpid_status, 0); - uwsgi_log("mule %d has been buried (pid: %d)\n", i, (int) uwsgi.mules[i].pid); - } - } - if (uwsgi.emperor_pid > 0) { kill(uwsgi.emperor_pid, SIGINT); time_t timeout = uwsgi_now() + (uwsgi.reload_mercy ? uwsgi.reload_mercy : 3); diff --git a/core/mule.c b/core/mule.c index 8c0ad5cf..a247af6f 100644 --- a/core/mule.c +++ b/core/mule.c @@ -43,7 +43,7 @@ void uwsgi_mule(int id) { #endif signal(SIGALRM, SIG_IGN); - signal(SIGHUP, SIG_IGN); + signal(SIGHUP, end_me); signal(SIGINT, end_me); signal(SIGTERM, end_me); signal(SIGUSR1, SIG_IGN); diff --git a/core/uwsgi.c b/core/uwsgi.c index 76631ab6..1f0c8c46 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -243,6 +243,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"reload-mercy", required_argument, 0, "set the maximum time (in seconds) we wait for workers and other processes to die during reload/shutdown", uwsgi_opt_set_int, &uwsgi.reload_mercy, 0}, {"worker-reload-mercy", required_argument, 0, "set the maximum time (in seconds) a worker can take to reload/shutdown (default is 60)", uwsgi_opt_set_int, &uwsgi.worker_reload_mercy, 0}, + {"mule-reload-mercy", required_argument, 0, "set the maximum time (in seconds) a mule can take to reload/shutdown (default is 60)", uwsgi_opt_set_int, &uwsgi.mule_reload_mercy, 0}, {"exit-on-reload", no_argument, 0, "force exit even if a reload is requested", uwsgi_opt_true, &uwsgi.exit_on_reload, 0}, {"die-on-term", no_argument, 0, "exit instead of brutal reload on SIGTERM", uwsgi_opt_true, &uwsgi.die_on_term, 0}, {"force-gateway", no_argument, 0, "force the spawn of the first registered gateway without a master", uwsgi_opt_true, &uwsgi.force_gateway, 0}, @@ -1247,6 +1248,11 @@ void kill_them_all(int signum) { uwsgi_curse(i, SIGINT); } } + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0) { + uwsgi_curse_mule(i, SIGINT); + } + } uwsgi_destroy_processes(); } @@ -1268,6 +1274,11 @@ void gracefully_kill_them_all(int signum) { uwsgi_curse(i, SIGHUP); } } + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0) { + uwsgi_curse_mule(i, SIGHUP); + } + } uwsgi_destroy_processes(); } @@ -1309,6 +1320,12 @@ void grace_them_all(int signum) { uwsgi_curse(i, SIGHUP); } } + + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0) { + uwsgi_curse_mule(i, SIGHUP); + } + } } void uwsgi_nuclear_blast() { @@ -1355,6 +1372,11 @@ void reap_them_all(int signum) { if (uwsgi.workers[i].pid > 0) uwsgi_curse(i, SIGTERM); } + for (i = 0; i < uwsgi.mules_cnt; i++) { + if (uwsgi.mules[i].pid > 0) { + uwsgi_curse_mule(i, SIGTERM); + } + } } void harakiri() { diff --git a/plugins/python/python_plugin.c b/plugins/python/python_plugin.c index 1f8e4edc..f3e7c073 100644 --- a/plugins/python/python_plugin.c +++ b/plugins/python/python_plugin.c @@ -168,6 +168,8 @@ struct uwsgi_option uwsgi_python_options[] = { {"py-sharedarea", required_argument, 0, "create a sharedarea from a python bytearray object of the specified size", uwsgi_opt_add_string_list, &up.sharedarea, 0}, #endif + {"py-call-osafterfork", no_argument, 0, "enable child processes running cpython to trap OS signals", uwsgi_opt_true, &up.call_osafterfork, 0}, + {0, 0, 0, 0, 0, 0, 0}, }; @@ -377,6 +379,11 @@ void uwsgi_python_post_fork() { UWSGI_GET_GIL } + // reset python signal flags so child processes can trap signals + if (up.call_osafterfork) { + PyOS_AfterFork(); + } + uwsgi_python_reset_random_seed(); // call the post_fork_hook diff --git a/plugins/python/uwsgi_python.h b/plugins/python/uwsgi_python.h index 0c5c1c81..a2cc6e69 100644 --- a/plugins/python/uwsgi_python.h +++ b/plugins/python/uwsgi_python.h @@ -197,6 +197,8 @@ struct uwsgi_python { PyObject *raw_callable; struct uwsgi_string_list *sharedarea; + + int call_osafterfork; }; diff --git a/uwsgi.h b/uwsgi.h index e28e06e6..48d01a2a 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -2706,6 +2706,8 @@ struct uwsgi_server { char *notify_socket; int notify_socket_fd; char *subscription_notify_socket; + + int mule_reload_mercy; }; struct uwsgi_rpc { @@ -2939,6 +2941,9 @@ struct uwsgi_mule { time_t user_harakiri; char name[0xff]; + + time_t cursed_at; + time_t no_mercy_at; }; struct uwsgi_mule_farm { @@ -3006,6 +3011,7 @@ void spooler(struct uwsgi_spooler *); pid_t spooler_start(struct uwsgi_spooler *); void uwsgi_curse(int, int); +void uwsgi_curse_mule(int, int); void uwsgi_destroy_processes(void); void set_harakiri(int);