From e4903ee0bfb640465628f33ea5a433934ef9be27 Mon Sep 17 00:00:00 2001 From: Unbit Date: Thu, 21 Nov 2013 15:29:30 +0100 Subject: [PATCH] first working prototype of embedded plugins builder --- core/plugins_builder.c | 82 +++++++++++++++++++++++++++++++----------- core/uwsgi.c | 39 +++++++++++++++++++- uwsgi.h | 4 ++- uwsgiconfig.py | 28 +++++++++++++-- 4 files changed, 129 insertions(+), 24 deletions(-) diff --git a/core/plugins_builder.c b/core/plugins_builder.c index da708c9d..b4341f6c 100644 --- a/core/plugins_builder.c +++ b/core/plugins_builder.c @@ -8,64 +8,106 @@ mkdir(.uwsgi_plugin_builder) generate .uwsgi_plugin_builder/uwsgi.h - setenv(CFLAGS=uwsgi_cflags) - pipe uwsgiconfig.py to python passing args + generate .uwsgi_plugin_builder/uwsgiconfig.py + setenv(UWSGI_PLUGINS_BUILDER_CFLAGS=uwsgi_cflags) + exec PYTHON .uwsgi_plugin_builder/uwsgiconfig.py --extra-plugin [name] */ -int uwsgi_build_plugin(char *directory) { +void uwsgi_build_plugin(char *directory) { uwsgi_log("building uWSGI plugin %s...\n", directory); if (!uwsgi_file_exists(UWSGI_BUILD_DIR)) { if (mkdir(UWSGI_BUILD_DIR, S_IRWXU) < 0) { uwsgi_error("uwsgi_build_plugin()/mkdir() " UWSGI_BUILD_DIR "/"); - return 1; + _exit(1); } } char *dot_h = uwsgi_get_dot_h(); if (!dot_h) { uwsgi_log("unable to generate uwsgi.h"); - return 1; + _exit(1); } if (strlen(dot_h) == 0) { free(dot_h); uwsgi_log("invalid uwsgi.h"); - return 1; + _exit(1); } int dot_h_fd = open(UWSGI_BUILD_DIR "/uwsgi.h", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); if (dot_h_fd < 0) { uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgi.h"); free(dot_h); - return 1; + _exit(1); } ssize_t dot_h_len = (ssize_t) strlen(dot_h); if (write(dot_h_fd, dot_h, dot_h_len) != dot_h_len) { uwsgi_error("uwsgi_build_plugin()/write()"); - close(dot_h_fd); - free(dot_h); - return 1; + _exit(1); + } + + char *config_py = uwsgi_get_config_py(); + if (!config_py) { + uwsgi_log("unable to generate uwsgiconfig.py"); + _exit(1); + } + + if (strlen(config_py) == 0) { + uwsgi_log("invalid uwsgiconfig.py"); + _exit(1); + } + + int config_py_fd = open(UWSGI_BUILD_DIR "/uwsgiconfig.py", O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); + if (config_py_fd < 0) { + uwsgi_error_open(UWSGI_BUILD_DIR "/uwsgiconfig.py"); + _exit(1); + } + + ssize_t config_py_len = (ssize_t) strlen(config_py); + if (write(config_py_fd, config_py, config_py_len) != config_py_len) { + uwsgi_error("uwsgi_build_plugin()/write()"); + _exit(1); + } + + char *cflags = uwsgi_get_cflags(); + if (!cflags) { + uwsgi_log("unable to find cflags\n"); + _exit(1); + } + if (strlen(cflags) == 0) { + uwsgi_log("invalid cflags\n"); + _exit(1); + } + + if (setenv("UWSGI_PLUGINS_BUILDER_CFLAGS", cflags, 1)) { + uwsgi_error("uwsgi_build_plugin()/setenv()"); + _exit(1); } - free(dot_h); - close(dot_h_fd); - - // now extract the python build script from the process - // and pipe it to python - - char *argv[4]; + // now run the python script + char *argv[6]; argv[0] = getenv("PYTHON"); if (!argv[0]) argv[0] = "python"; - argv[1] = "-"; + argv[1] = UWSGI_BUILD_DIR "/uwsgiconfig.py"; argv[2] = "--extra-plugin"; - argv[3] = directory; + char *space = strchr(directory, ' '); + if (space) { + *space = 0; + argv[3] = directory; + argv[4] = space+1; + argv[5] = NULL; + } + else { + argv[3] = directory; + argv[4] = NULL; + } execvp(argv[0], argv); // never here... - return 1; + _exit(1); } diff --git a/core/uwsgi.c b/core/uwsgi.c index 7a94e2be..a18c0b6b 100644 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -907,6 +907,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"exit", optional_argument, 0, "force exit() of the instance", uwsgi_opt_exit, NULL, UWSGI_OPT_IMMEDIATE}, {"cflags", no_argument, 0, "report uWSGI CFLAGS (useful for building external plugins)", uwsgi_opt_cflags, NULL, UWSGI_OPT_IMMEDIATE}, {"dot-h", no_argument, 0, "dump the uwsgi.h used for building the core (useful for building external plugins)", uwsgi_opt_dot_h, NULL, UWSGI_OPT_IMMEDIATE}, + {"config-py", no_argument, 0, "dump the uwsgiconfig.py used for building the core (useful for building external plugins)", uwsgi_opt_config_py, NULL, UWSGI_OPT_IMMEDIATE}, {"build-plugin", required_argument, 0, "build a uWSGI plugin for the current binary", uwsgi_opt_build_plugin, NULL, UWSGI_OPT_IMMEDIATE}, {"version", no_argument, 0, "print uWSGI version", uwsgi_opt_print, UWSGI_VERSION, 0}, {0, 0, 0, 0, 0, 0, 0} @@ -4507,8 +4508,44 @@ void uwsgi_opt_dot_h(char *opt, char *filename, void *foobar) { exit(0); } +extern char *uwsgi_config_py; +char *uwsgi_get_config_py() { + char *src = uwsgi_config_py; + size_t len = strlen(src); + char *ptr = uwsgi_malloc(len / 2); + char *base = ptr; + size_t i; + unsigned int u; + for (i = 0; i < len; i += 2) { + sscanf(src + i, "%2x", &u); + *ptr++ = (char) u; + } +#ifdef UWSGI_ZLIB + struct uwsgi_buffer *ub = uwsgi_zlib_decompress(base, ptr-base); + if (!ub) { + free(base); + return ""; + } + // add final null byte + uwsgi_buffer_append(ub, "\0", 1); + free(base); + // base is the final blob + base = ub->buf; + ub->buf = NULL; + uwsgi_buffer_destroy(ub); +#endif + return base; +} + +void uwsgi_opt_config_py(char *opt, char *filename, void *foobar) { + fprintf(stdout, "%s\n", uwsgi_get_config_py()); + exit(0); +} + + void uwsgi_opt_build_plugin(char *opt, char *directory, void *foobar) { - _exit(uwsgi_build_plugin(directory)); + uwsgi_build_plugin(directory); + exit(1); } void uwsgi_opt_connect_and_read(char *opt, char *address, void *foobar) { diff --git a/uwsgi.h b/uwsgi.h index daaecc3b..7fd480a7 100644 --- a/uwsgi.h +++ b/uwsgi.h @@ -3763,10 +3763,12 @@ void uwsgi_opt_add_custom_option(char *, char *, void *); void uwsgi_opt_cflags(char *, char *, void *); void uwsgi_opt_build_plugin(char *, char *, void *); void uwsgi_opt_dot_h(char *, char *, void *); +void uwsgi_opt_config_py(char *, char *, void *); void uwsgi_opt_connect_and_read(char *, char *, void *); void uwsgi_opt_extract(char *, char *, void *); char *uwsgi_get_dot_h(); +char *uwsgi_get_config_py(); char *uwsgi_get_cflags(); struct uwsgi_string_list *uwsgi_string_list_has_item(struct uwsgi_string_list *, char *, size_t); @@ -4529,7 +4531,7 @@ struct uwsgi_protocol *uwsgi_register_protocol(char *, void (*)(struct uwsgi_soc void uwsgi_protocols_register(void); -int uwsgi_build_plugin(char *dir); +void uwsgi_build_plugin(char *dir); #ifdef __cplusplus } diff --git a/uwsgiconfig.py b/uwsgiconfig.py index fd9e7af7..c5375098 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -305,6 +305,24 @@ def build_uwsgi(uc, print_only=False, gcll=None): uwsgi_dot_h = uwsgi_dot_h_content.encode('hex') open('core/dot_h.c', 'w').write('char *uwsgi_dot_h = "%s";\n' % uwsgi_dot_h); gcc_list.append('core/dot_h') + + # embed uwsgiconfig.py in the server binary. It increases the binary size, but will be very useful + # if possibile, the blob is compressed + if sys.version_info[0] >= 3: + uwsgi_config_py_content = open('uwsgiconfig.py', 'rb').read() + else: + uwsgi_config_py_content = open('uwsgiconfig.py').read() + if report['zlib']: + import zlib + # maximum level of compression + uwsgi_config_py_content = zlib.compress(uwsgi_config_py_content, 9) + if sys.version_info[0] >= 3: + import binascii + uwsgi_config_py = binascii.b2a_hex(uwsgi_config_py_content).decode('ascii') + else: + uwsgi_config_py = uwsgi_config_py_content.encode('hex') + open('core/config_py.c', 'w').write('char *uwsgi_config_py = "%s";\n' % uwsgi_config_py); + gcc_list.append('core/config_py') cflags.append('-DUWSGI_CFLAGS=\\"%s\\"' % uwsgi_cflags) cflags.append('-DUWSGI_BUILD_DATE="\\"%s\\""' % time.strftime("%d %B %Y %H:%M:%S")) @@ -1363,7 +1381,7 @@ if __name__ == "__main__": parser.add_option("-f", "--cflags", action="callback", callback=vararg_callback, dest="cflags", help="same as --build but less verbose", metavar="PROFILE") parser.add_option("-u", "--unbit", action="store_true", dest="unbit", help="build unbit profile") parser.add_option("-p", "--plugin", action="callback", callback=vararg_callback, dest="plugin", help="build a plugin as shared library, optionally takes a build profile name", metavar="PLUGIN [PROFILE]") - parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback, dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [INCLUDE_DIR]") + parser.add_option("-x", "--extra-plugin", action="callback", callback=vararg_callback, dest="extra_plugin", help="build an external plugin as shared library, takes an optional include dir", metavar="PLUGIN [NAME]") parser.add_option("-c", "--clean", action="store_true", dest="clean", help="clean the build") parser.add_option("-e", "--check", action="store_true", dest="check", help="run cppcheck") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="more verbose build") @@ -1434,7 +1452,12 @@ if __name__ == "__main__": print("*** uWSGI building and linking plugin %s ***" % options.extra_plugin[0]) cflags = os.environ['UWSGI_PLUGINS_BUILDER_CFLAGS'].split() cflags.append('-I.uwsgi_plugins_builder/') - build_plugin(options.extra_plugin[0], None, cflags, [], [], None) + name = None + try: + name = options.extra_plugin[1] + except: + pass + build_plugin(options.extra_plugin[0], None, cflags, [], [], name) elif options.clean: os.system("rm -f core/*.o") os.system("rm -f proto/*.o") @@ -1442,6 +1465,7 @@ if __name__ == "__main__": os.system("rm -f plugins/*/*.o") os.system("rm -f build/*.o") os.system("rm -f core/dot_h.c") + os.system("rm -f core/config_py.c") elif options.check: os.system("cppcheck --max-configs=1000 --enable=all -q core/ plugins/ proto/ lib/ apache2/") else: