From d4b3e0ad628b3fe49e4a0071b1482bb2d9c01f2b Mon Sep 17 00:00:00 2001 From: Gao feng Date: Tue, 15 Sep 2015 20:11:16 +0800 Subject: [PATCH 01/23] introduce WRITE/READFILE command write data to files or read data from files in contianer. Signed-off-by: Gao feng --- src/hyper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hyper.h b/src/hyper.h index 1889ab0..49277f8 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -25,6 +25,8 @@ enum { PING, FINISH, NEXT, + WRITEFILE, + READFILE, }; enum { From ed9a35151847d336aa09df774193c8e93b3dfabb Mon Sep 17 00:00:00 2001 From: Gao feng Date: Tue, 15 Sep 2015 23:22:38 +0800 Subject: [PATCH 02/23] introduce hyper_parse_write_file Signed-off-by: Gao feng --- src/hyper.h | 6 ++++++ src/init.c | 17 +++++++++++++++ src/parse.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/parse.h | 1 + 4 files changed, 83 insertions(+) diff --git a/src/hyper.h b/src/hyper.h index 49277f8..b430beb 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -66,6 +66,12 @@ struct hyper_win_size { uint64_t seq; }; +struct hyper_writter { + char *id; + char *file; + char *data; +}; + struct hyper_ctl { int efd; struct hyper_event sig; diff --git a/src/init.c b/src/init.c index e45b0bd..fc3e56e 100644 --- a/src/init.c +++ b/src/init.c @@ -762,6 +762,16 @@ static int hyper_start_pod(char *json, int length) return 0; } +static int hyper_cmd_write_file(char *json, int length) +{ + return 0; +} + +static int hyper_cmd_read_file(char *json, int length) +{ + return 0; +} + static void hyper_cleanup_shared(struct hyper_pod *pod) { if (pod->tag == NULL) { @@ -941,6 +951,13 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) case EXECCMD: ret = hyper_exec_cmd((char *)buf->data + 8, len - 8); break; + case WRITEFILE: + ret = hyper_cmd_write_file((char *)buf->data + 8, len - 8); + break; + case READFILE: + hyper_cmd_read_file((char *)buf->data + 8, len - 8); + /* hyperstart will send ACK message with file context */ + return 0; case PING: case GETPOD: break; diff --git a/src/parse.c b/src/parse.c index f29e888..cf7e38d 100644 --- a/src/parse.c +++ b/src/parse.c @@ -632,3 +632,62 @@ fail: exec = NULL; goto out; } + +int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length) +{ + int i, n, ret = 0; + + jsmn_parser p; + int toks_num = 10; + jsmntok_t *toks = NULL; + + toks = calloc(toks_num, sizeof(jsmntok_t)); + + jsmn_init(&p); + n = jsmn_parse(&p, json, length, toks, toks_num); + if (n < 0) { + fprintf(stdout, "jsmn parse failed, n is %d\n", n); + ret = -1; + goto out; + } + + for (i = 0; i < n; i++) { + jsmntok_t *t = &toks[i]; + + if (t->type != JSMN_STRING) + continue; + + if (json_token_streq(json, t, "container")) { + if (i++ == n) + goto fail; + + writter->id = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "writefile get container %s\n", writter->id); + } else if (json_token_streq(json, t, "file")) { + if (i++ == n) + goto fail; + + writter->file = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "writefile get file %s\n", writter->file); + } else if (json_token_streq(json, t, "data")) { + if (i++ == n) + goto fail; + + writter->data = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "writefile get data %s\n", writter->data); + } else { + fprintf(stdout, "in writefile incorrect %s\n", json_token_str(json, &toks[i])); + } + } + +out: + free(toks); + return ret; +fail: + free(writter->id); + free(writter->file); + free(writter->data); + + ret = -1; + goto out; +} diff --git a/src/parse.h b/src/parse.h index 7b41ba0..4b93b41 100644 --- a/src/parse.h +++ b/src/parse.h @@ -9,5 +9,6 @@ struct hyper_exec *hyper_parse_execcmd(char *json, int length); char *json_token_str(char *js, jsmntok_t *t); int json_token_streq(char *js, jsmntok_t *t, char *s); int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length); +int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length); #endif From 7d6468f70b10ffe9143991b44ca9d3b85313d69e Mon Sep 17 00:00:00 2001 From: Gao feng Date: Tue, 15 Sep 2015 23:25:25 +0800 Subject: [PATCH 03/23] introduce hyper_parse_read_file Signed-off-by: Gao feng --- src/hyper.h | 5 +++++ src/parse.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/hyper.h b/src/hyper.h index b430beb..689fda9 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -66,6 +66,11 @@ struct hyper_win_size { uint64_t seq; }; +struct hyper_reader { + char *id; + char *file; +}; + struct hyper_writter { char *id; char *file; diff --git a/src/parse.c b/src/parse.c index cf7e38d..0f75a2e 100644 --- a/src/parse.c +++ b/src/parse.c @@ -691,3 +691,55 @@ fail: ret = -1; goto out; } + +int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length) +{ + int i, n, ret = 0; + + jsmn_parser p; + int toks_num = 10; + jsmntok_t *toks = NULL; + + toks = calloc(toks_num, sizeof(jsmntok_t)); + + jsmn_init(&p); + n = jsmn_parse(&p, json, length, toks, toks_num); + if (n < 0) { + fprintf(stdout, "jsmn parse failed, n is %d\n", n); + ret = -1; + goto out; + } + + for (i = 0; i < n; i++) { + jsmntok_t *t = &toks[i]; + + if (t->type != JSMN_STRING) + continue; + + if (json_token_streq(json, t, "container")) { + if (i++ == n) + goto fail; + + reader->id = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "readfile get container %s\n", reader->id); + } else if (json_token_streq(json, t, "file")) { + if (i++ == n) + goto fail; + + reader->file = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "readfile get file %s\n", reader->file); + } else { + fprintf(stdout, "in readfile incorrect %s\n", json_token_str(json, &toks[i])); + } + } + +out: + free(toks); + return ret; +fail: + free(reader->id); + free(reader->file); + + ret = -1; + goto out; +} From 6a15977238bc34b3c3c98e52be6128893efc1936 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 16 Sep 2015 16:13:47 +0800 Subject: [PATCH 04/23] create container process in hyper init Move most of functions from pod init to hyper init. this makes logic more clearer Signed-off-by: Gao feng --- src/container.c | 93 ++++++++++++------- src/container.h | 5 +- src/exec.c | 237 +++++++++++++++++++----------------------------- src/exec.h | 4 +- src/hyper.h | 5 +- src/init.c | 176 +++++++++++++++++------------------ 6 files changed, 251 insertions(+), 269 deletions(-) diff --git a/src/container.c b/src/container.c index 113585f..8721033 100644 --- a/src/container.c +++ b/src/container.c @@ -276,6 +276,9 @@ static int hyper_rescan_scsi(void) struct hyper_container_arg { struct hyper_container *c; + int pidns; + int ipcns; + int utsns; int pipe[2]; }; @@ -291,6 +294,21 @@ static int hyper_container_init(void *data) goto fail; } + if (setns(arg->pidns, CLONE_NEWPID) < 0) { + perror("setns to pidns of pod init faild"); + goto fail; + } + + if (setns(arg->ipcns, CLONE_NEWIPC) < 0) { + perror("setns to ipcns of pod init faild"); + goto fail; + } + + if (setns(arg->utsns, CLONE_NEWUTS) < 0) { + perror("setns to ipcns of pod init faild"); + goto fail; + } + if (hyper_rescan_scsi() < 0) { fprintf(stdout, "rescan scsi failed\n"); goto fail; @@ -405,11 +423,40 @@ fail: _exit(-1); } -int hyper_start_container(struct hyper_container *container) +static int hyper_setup_pty(struct hyper_container *c) +{ + char root[512]; + + sprintf(root, "/tmp/hyper/%s/devpts/", c->id); + + if (hyper_mkdir(root) < 0) { + perror("make container pts directroy failed"); + return -1; + } + + if (mount("devpts", root, "devpts", MS_NOSUID, + "newinstance,ptmxmode=0666,mode=0620") < 0) { + perror("mount devpts failed"); + return -1; + } + + if (hyper_setup_exec_tty(&c->exec) < 0) { + fprintf(stderr, "setup container pts failed\n"); + return -1; + } + + return 0; +} + +int hyper_start_container(struct hyper_container *container, + int pidns, int utsns, int ipcns) { int stacksize = getpagesize() * 4; struct hyper_container_arg arg = { .c = container, + .pidns = pidns, + .utsns = utsns, + .ipcns = ipcns, }; int flags = CLONE_NEWNS | SIGCHLD; uint32_t type; @@ -422,6 +469,11 @@ int hyper_start_container(struct hyper_container *container) goto fail; } + if (hyper_setup_pty(container) < 0) { + fprintf(stderr, "setup pty device for container failed\n"); + goto fail; + } + if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { perror("create pipe between pod init execcmd failed"); goto fail; @@ -430,7 +482,7 @@ int hyper_start_container(struct hyper_container *container) stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); - return -1; + goto fail; } pid = clone(hyper_container_init, stack + stacksize, flags, &arg); @@ -439,54 +491,29 @@ int hyper_start_container(struct hyper_container *container) perror("create child process failed"); goto fail; } - container->exec.pid = pid; /* wait for ready message */ if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { - fprintf(stdout, "wait for container started failed\n"); + fprintf(stderr, "wait for container started failed\n"); goto fail; } close(arg.pipe[0]); close(arg.pipe[1]); + if (hyper_watch_exec_pty(&container->exec) < 0) + fprintf(stderr, "faile to watch container pty\n"); + fprintf(stdout, "container %s init pid is %d\n", container->id, pid); return 0; - fail: fprintf(stdout, "container %s init exit code %d\n", container->id, -1); - container->exec.code = -1; return -1; } -int hyper_start_containers(struct hyper_pod *pod) -{ - int i; - - /* mount new proc directory */ - if (umount("/proc") < 0) { - perror("umount proc filesystem failed\n"); - return -1; - } - - if (mount("proc", "/proc", "proc", 0, NULL) < 0) { - perror("mount proc filesystem failed\n"); - return -1; - } - - if (sethostname(pod->hostname, strlen(pod->hostname)) < 0) { - perror("set host name failed"); - return -1; - } - - for (i = 0; i < pod->c_num; i++) - hyper_start_container(&pod->c[i]); - - return 0; -} - +/* int hyper_restart_containers(struct hyper_pod *pod) { int i; @@ -507,7 +534,7 @@ int hyper_restart_containers(struct hyper_pod *pod) return 0; } - +*/ struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id) { int i; diff --git a/src/container.h b/src/container.h index dc95f8c..db969d9 100644 --- a/src/container.h +++ b/src/container.h @@ -39,9 +39,10 @@ struct hyper_container { struct hyper_pod; -int hyper_start_containers(struct hyper_pod *pod); +int hyper_start_container(struct hyper_container *container, + int pidns, int utsns, int ipcns); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); -int hyper_restart_containers(struct hyper_pod *pod); +//int hyper_restart_containers(struct hyper_pod *pod); void hyper_cleanup_container(struct hyper_pod *pod); #endif diff --git a/src/exec.c b/src/exec.c index 122b795..f080663 100644 --- a/src/exec.c +++ b/src/exec.c @@ -176,33 +176,93 @@ int hyper_dup_exec_tty(int to, struct hyper_exec *e) return 0; } -int hyper_exec_in_container(struct hyper_pod *pod, - struct hyper_exec *exec) +int hyper_watch_exec_pty(struct hyper_exec *exec) { - global_exec = exec; - - if (hyper_send_type_block(ctl.ctl.fd, EXECCMD, 1) < 0) { - global_exec = NULL; - fprintf(stderr, "tell pod init EXECCMD failed\n"); - return -1; - } - - list_add_tail(&exec->list, &pod->ce_head); - if (exec->seq == 0) - return 0; - - fprintf(stdout, "init container exec pts event %p, ops %p, fd %d\n", + fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n", &exec->e, &pts_ops, exec->e.fd); - - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || + if (hyper_init_event(&exec->e, &pts_ops, NULL) < 0 || hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { - fprintf(stderr, "add pts master event failed\n"); + fprintf(stderr, "add container pts master event failed\n"); return -1; } return 0; } +int hyper_enter_container(struct hyper_pod *pod, + struct hyper_exec *exec) +{ + int pidns, ipcns, utsns, mntns, ret; + struct hyper_container *c; + char path[512]; + + ret = pidns = ipcns = utsns = mntns = -1; + + c = hyper_find_container(pod, exec->id); + if (c == NULL) { + fprintf(stderr, "can not find container %s\n", exec->id); + return -1; + } + + sprintf(path, "/proc/%d/ns/pid", c->exec.pid); + pidns = open(path, O_RDONLY| O_CLOEXEC); + if (pidns < 0) { + perror("fail to open pidns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/uts", c->exec.pid); + utsns = open(path, O_RDONLY| O_CLOEXEC); + if (utsns < 0) { + perror("fail to open utsns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/ipc", c->exec.pid); + ipcns = open(path, O_RDONLY| O_CLOEXEC); + if (ipcns < 0) { + perror("fail to open ipcns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); + mntns = open(path, O_RDONLY| O_CLOEXEC); + if (mntns < 0) { + perror("fail to open mntns of pod init"); + goto out; + } + + if (setns(pidns, CLONE_NEWPID) < 0 || + setns(utsns, CLONE_NEWUTS) < 0|| + setns(ipcns, CLONE_NEWIPC) <0 || + setns(mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + goto out; + } + + sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + fprintf(stdout, "root directory for container is %s, exec %s\n", + path, exec->argv[0]); + + /* TODO: wait for container finishing setup root */ + if (chroot(path) < 0) { + perror("chroot for exec command failed"); + goto out; + } + + chdir("/"); + + ret = 0; +out: + close(pidns); + close(ipcns); + close(utsns); + close(mntns); + + return ret; +} + +/* int hyper_request_restart_containers(struct hyper_pod *pod) { int i; @@ -241,6 +301,7 @@ int hyper_request_restart_containers(struct hyper_pod *pod) return 0; } +*/ int hyper_exec_cmd(char *json, int length) { @@ -267,15 +328,6 @@ int hyper_exec_cmd(char *json, int length) return -1; } - if (exec->id) { - if (hyper_exec_in_container(pod, exec) < 0) { - fprintf(stderr, "notify container exec failed\n"); - return -1; - } - - return 0; - } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { perror("create pipe between pod init execcmd failed"); return -1; @@ -299,14 +351,11 @@ int hyper_exec_cmd(char *json, int length) exec->pid = pid; fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); - list_add_tail(&exec->list, &pod->pe_head); + list_add_tail(&exec->list, &pod->exec_head); if (exec->seq == 0) return 0; - fprintf(stdout, "init pod exec pts event %p, ops %p, fd %d\n", - &exec->e, &pts_ops, exec->e.fd); - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { + if (hyper_watch_exec_pty(exec) < 0) { fprintf(stderr, "add pts master event failed\n"); return -1; } @@ -314,6 +363,11 @@ int hyper_exec_cmd(char *json, int length) return 0; } + if (exec->id && hyper_enter_container(pod, exec) < 0) { + fprintf(stderr, "enter container ns failed\n"); + return -1; + } + if (hyper_dup_exec_tty(pipe[1], exec) < 0) { fprintf(stderr, "dup pts to exec stdio failed\n"); _exit(-1); @@ -330,106 +384,6 @@ int hyper_exec_cmd(char *json, int length) _exit(0); } -int hyper_container_execcmd(struct hyper_pod *pod) -{ - struct hyper_exec *exec = global_exec; - struct hyper_container *container = NULL; - int fd, pid, sock = pod->ctl.fd; - char root[512]; int pipe[2]; - - global_exec = NULL; - - container = hyper_find_container(pod, exec->id); - if (container == NULL) { - fprintf(stderr, "can not find container %s\n", exec->id); - return -1; - } - - if (sprintf(root, "/tmp/hyper/%s/devpts/", exec->id) < 0) { - fprintf(stderr, "get container %s pts path failed\n", exec->id); - return -1; - } - - if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { - perror("create pipe between pod init execcmd failed"); - return -1; - } - - pid = fork(); - if (pid < 0) { - fprintf(stderr, "container init fork failed\n"); - return -1; - } else if (pid > 0) { - uint32_t type; - - if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { - fprintf(stderr, "pod init get execcmd ready message failed\n"); - hyper_send_type_block(sock, ERROR, 0); - return 0; - } - - close(pipe[0]); - close(pipe[1]); - fprintf(stdout, "pod init get ready message\n"); - - exec->pid = pid; - fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); - - if (hyper_send_type_block(sock, ACK, 0) < 0) - return -1; - - return 0; - } - - close(pipe[0]); - sprintf(root, "/proc/%d/ns/mnt", container->exec.pid); - - fprintf(stdout, "container %s, init pid %d\n", - exec->id, container->exec.pid); - - fd = open(root, O_RDONLY); - if (fd < 0) { - perror("fail to open container mnt ns\n"); - goto fail; - } - - if (syscall(SYS_setns, fd, CLONE_NEWNS) < 0) { - perror("enter mnt ns failed"); - goto fail; - } - - close(fd); - - sprintf(root, "/tmp/hyper/%s/root/%s/", - container->id, container->rootfs); - - fprintf(stdout, "root directory for container is %s, exec %s\n", - root, exec->argv[0]); - - /* TODO: wait for container finishing setup root */ - if (chroot(root) < 0) { - perror("chroot for exec command failed"); - goto fail; - } - - chdir("/"); - - if (hyper_dup_exec_tty(pipe[1], exec)) { - fprintf(stderr, "dup pts to stdio failed\n"); - goto fail; - } - - close(pipe[1]); - if (execvp(exec->argv[0], exec->argv) < 0) - perror("exec failed"); - - _exit(-1); -fail: - hyper_send_type_block(pipe[1], ERROR, 0); - _exit(-1); -} - - int hyper_release_exec(struct hyper_exec *exec, struct hyper_pod *pod) { @@ -463,7 +417,7 @@ int hyper_release_exec(struct hyper_exec *exec, return 0; } - if (hyper_request_restart_containers(pod) < 0) { + if (hyper_start_containers(pod) < 0) { fprintf(stderr, "restart container failed\n"); return -1; } @@ -503,15 +457,15 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq) { struct hyper_exec *exec; - list_for_each_entry(exec, &pod->ce_head, list) { - fprintf(stdout, "container exec seq %" PRIu64 ", seq %" PRIu64 "\n", + list_for_each_entry(exec, &pod->exec_head, list) { + fprintf(stdout, "exec seq %" PRIu64 ", seq %" PRIu64 "\n", exec->seq, seq); if (exec->seq != seq) continue; return exec; } - +/* list_for_each_entry(exec, &pod->pe_head, list) { fprintf(stdout, "pod exec seq %" PRIu64 ", seq %" PRIu64 "\n", exec->seq, seq); @@ -520,18 +474,17 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq) return exec; } - +*/ return NULL; } int hyper_send_exec_eof(int to, struct hyper_pod *pod, - struct list_head *head, int pid, - uint8_t code) + int pid, uint8_t code) { struct hyper_exec *exec; uint8_t seq[12]; - exec = hyper_find_exec_by_pid(head, pid); + exec = hyper_find_exec_by_pid(&pod->exec_head, pid); if (exec == NULL) { fprintf(stdout, "can not find exec whose pid is %d\n", pid); @@ -562,8 +515,8 @@ void hyper_cleanup_exec(struct hyper_pod *pod) { struct hyper_exec *exec, *next; - list_for_each_entry_safe(exec, next, &pod->ce_head, list) { - fprintf(stdout, "cleanup container exec seq %" PRIu64 "\n", exec->seq); + list_for_each_entry_safe(exec, next, &pod->exec_head, list) { + fprintf(stdout, "cleanup exec seq %" PRIu64 "\n", exec->seq); hyper_release_exec(exec, pod); } } diff --git a/src/exec.h b/src/exec.h index ed9d60a..e26aab2 100644 --- a/src/exec.h +++ b/src/exec.h @@ -28,8 +28,8 @@ int hyper_dup_exec_tty(int fd, struct hyper_exec *e); struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid); struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq); int hyper_send_exec_eof(int to, struct hyper_pod *pod, - struct list_head *head, int pid, - uint8_t code); + int pid, uint8_t code); +int hyper_watch_exec_pty(struct hyper_exec *exec); void hyper_cleanup_exec(struct hyper_pod *pod); extern struct hyper_event_ops pts_ops; diff --git a/src/hyper.h b/src/hyper.h index 689fda9..2a09207 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -40,8 +40,8 @@ struct hyper_pod { struct hyper_interface *iface; struct hyper_route *rt; char **dns; - struct list_head pe_head; - struct list_head ce_head; + struct list_head exec_head; + //struct list_head ce_head; char *hostname; char *tag; int init_pid; @@ -88,6 +88,7 @@ struct hyper_ctl { int hyper_mkdir(char *hyper_path); int hyper_open_serial(char *tty); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); +int hyper_start_containers(struct hyper_pod *pod); extern struct hyper_pod global_pod; extern struct hyper_ctl ctl; diff --git a/src/init.c b/src/init.c index fc3e56e..cb787ac 100644 --- a/src/init.c +++ b/src/init.c @@ -29,8 +29,7 @@ #include "container.h" struct hyper_pod global_pod = { - .ce_head = LIST_HEAD_INIT(global_pod.ce_head), - .pe_head = LIST_HEAD_INIT(global_pod.pe_head), + .exec_head = LIST_HEAD_INIT(global_pod.exec_head), }; struct hyper_exec *global_exec; @@ -39,8 +38,7 @@ struct hyper_exec *global_exec; struct hyper_ctl ctl; static void hyper_cleanup_pod(struct hyper_pod *pod); -static int hyper_handle_exit(struct hyper_pod *pod, int to, - int container, int option); +static int hyper_handle_exit(struct hyper_pod *pod); static int hyper_set_win_size(char *json, int length) { @@ -112,15 +110,18 @@ static int pod_ctl_pipe_handle(struct hyper_event *de, uint32_t len) case STOPPOD: fprintf(stdout, "get type STOPPOD, exit\n"); hyper_cleanup_pod(pod); + /* case RESTARTCONTAINER: fprintf(stdout, "%s get type RESTARTCONTAINER\n", __func__); if (hyper_restart_containers(pod) < 0) return -1; break; + case EXECCMD: if (hyper_container_execcmd(pod) < 0) return -1; break; + */ default: break; } @@ -128,14 +129,13 @@ static int pod_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return 0; } -static int hyper_handle_exit(struct hyper_pod *pod, int to, - int container, int option) +static int hyper_handle_exit(struct hyper_pod *pod) { int pid, status; /* pid + exit code */ uint8_t data[5]; - while ((pid = waitpid(-1, &status, option)) > 0) { + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { data[4] = 0; if (WIFEXITED(status)) { @@ -147,7 +147,7 @@ static int hyper_handle_exit(struct hyper_pod *pod, int to, fprintf(stdout, "pid %d exit by signal, status %d\n", pid, WTERMSIG(status)); } - +/* if (container) { hyper_set_be32(data, pid); if (hyper_send_msg(to, FINISHCMD, 5, data) < 0) { @@ -157,35 +157,30 @@ static int hyper_handle_exit(struct hyper_pod *pod, int to, continue; } - - if (hyper_send_exec_eof(to, pod, &pod->pe_head, pid, data[4]) < 0) +*/ + if (hyper_send_exec_eof(ctl.tty.fd, pod, pid, data[4]) < 0) fprintf(stderr, "signal_loop send eof failed\n"); } - if (option == WNOHANG) - return 0; + return 0; - /* send ack message to hyper init. */ + /* send ack message to hyper init. if (hyper_send_type(to, ACK) < 0) { fprintf(stderr, "send ACK of STOPPOD to hyper init failed\n"); return -1; } return 0; + */ } -static int signal_loop(struct hyper_event *de, int container) +static int hyper_signal_loop(struct hyper_event *de) { - int size, to; + int size; struct signalfd_siginfo sinfo; struct hyper_pod *pod = de->ptr; - if (container) - to = pod->ctl.fd; - else - to = ctl.tty.fd; - - fprintf(stdout, "%s write to %d\n", __func__, to); + fprintf(stdout, "%s write to ctl tty fd\n", __func__); while (1) { size = read(de->fd, &sinfo, sizeof(struct signalfd_siginfo)); @@ -207,12 +202,13 @@ static int signal_loop(struct hyper_event *de, int container) return 0; } - hyper_handle_exit(pod, to, container, WNOHANG); + hyper_handle_exit(pod); } return 0; } +/* static int pod_signal_loop(struct hyper_event *de) { return signal_loop(de, 1); @@ -222,6 +218,7 @@ static int hyper_signal_loop(struct hyper_event *de) { return signal_loop(de, 0); } +*/ static struct hyper_event_ops pod_ctl_pipe_ops = { .read = hyper_event_read, @@ -231,10 +228,12 @@ static struct hyper_event_ops pod_ctl_pipe_ops = { .len_offset = 4, }; +/* static struct hyper_event_ops pod_signal_ops = { .read = pod_signal_loop, .hup = hyper_event_hup, }; +*/ static int pod_init_loop(struct hyper_pod *pod) { @@ -254,7 +253,7 @@ static int pod_init_loop(struct hyper_pod *pod) fprintf(stderr, "hyper add pod ctl pipe event failed\n"); return -1; } - +/* fprintf(stdout, "hyper_init_event pod signal event %p, ops %p, fd %d\n", &pod->sig, &pod_signal_ops, pod->sig.fd); if (hyper_init_event(&pod->sig, &pod_signal_ops, pod) < 0 || @@ -262,7 +261,7 @@ static int pod_init_loop(struct hyper_pod *pod) fprintf(stderr, "hyper add pod tty pipe event failed\n"); return -1; } - +*/ events = calloc(MAXEVENTS, sizeof(*events)); while (1) { @@ -295,7 +294,6 @@ static int hyper_pod_init(void *data) { struct hyper_pod_arg *arg = data; struct hyper_pod *pod = arg->pod; - sigset_t mask; close(arg->ctl_pipe[0]); close(ctl.sig.fd); @@ -308,7 +306,7 @@ static int hyper_pod_init(void *data) perror("set pod init ctl pipe fd FD_CLOEXEC failed"); goto fail; } - +/* sigemptyset(&mask); sigaddset(&mask, SIGCHLD); @@ -322,30 +320,43 @@ static int hyper_pod_init(void *data) perror("create signalfd failed"); goto fail; } - - if (hyper_start_containers(pod) < 0) +*/ + /* mount new proc directory */ + if (umount("/proc") < 0) { + perror("umount proc filesystem failed\n"); goto fail; + } + + if (mount("proc", "/proc", "proc", 0, NULL) < 0) { + perror("mount proc filesystem failed\n"); + goto fail; + } + + if (sethostname(pod->hostname, strlen(pod->hostname)) < 0) { + perror("set host name failed"); + goto fail; + } fprintf(stdout, "pod ctl_pipe %d\n", arg->ctl_pipe[1]); if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) { fprintf(stderr, "container init send ready message failed\n"); goto fail; } -loop: + pod_init_loop(pod); + fprintf(stdout, "pod init exit\n"); +out: _exit(-1); fail: hyper_send_type(arg->ctl_pipe[1], ERROR); - goto loop; + goto out; } static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; - struct hyper_pod *pod = de->ptr; - uint32_t type, pid = 0; - uint8_t code; + uint32_t type; /* container exec finish message */ fprintf(stdout, "%s\n", __func__); @@ -358,6 +369,7 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) * fd is block, and ACK is the last message, exit loop. */ fprintf(stdout, "hyper_ctl_pipe_loop get ack\n"); return 1; + /* case FINISHCMD: pid = hyper_get_be32(buf->data + 8); code = buf->data[12]; @@ -366,6 +378,7 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return -1; } break; + */ default: fprintf(stdout, "get unknown type %" PRIu32"\n", type); break; @@ -374,57 +387,49 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return 0; } -static int hyper_setup_pty(struct hyper_pod *pod) +int hyper_start_containers(struct hyper_pod *pod) { - int i; - char root[512]; + int i, pidns, ipcns, utsns, ret; struct hyper_container *c; + char path[64]; + + ret = pidns = ipcns = utsns = -1; + + fprintf(stdout, "%s\n", __func__); + sprintf(path, "/proc/%d/ns/pid", pod->init_pid); + pidns = open(path, O_RDONLY| O_CLOEXEC); + if (pidns < 0) { + perror("fail to open pidns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/uts", pod->init_pid); + utsns = open(path, O_RDONLY| O_CLOEXEC); + if (utsns < 0) { + perror("fail to open utsns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/ipc", pod->init_pid); + ipcns = open(path, O_RDONLY| O_CLOEXEC); + if (ipcns < 0) { + perror("fail to open ipcns of pod init"); + goto out; + } for (i = 0; i < pod->c_num; i++) { c = &pod->c[i]; - - sprintf(root, "/tmp/hyper/%s/devpts/", c->id); - - if (hyper_mkdir(root) < 0) { - perror("make container pts directroy failed"); - return -1; - } - - if (mount("devpts", root, "devpts", MS_NOSUID, - "newinstance,ptmxmode=0666,mode=0620") < 0) { - perror("mount devpts failed"); - return -1; - } - - list_add_tail(&c->exec.list, &pod->ce_head); - - if (hyper_setup_exec_tty(&c->exec) < 0) { - fprintf(stderr, "setup container pts failed\n"); - return -1; - } + list_add_tail(&c->exec.list, &pod->exec_head); + hyper_start_container(c, pidns, utsns, ipcns); } - return 0; -} + ret = 0; +out: + close(pidns); + close(utsns); + close(ipcns); -static int hyper_watch_pty(struct hyper_pod *pod) -{ - int i; - struct hyper_container *c; - - for (i = 0; i < pod->c_num; i++) { - c = &pod->c[i]; - - fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n", - &c->exec.e, &pts_ops, c->exec.e.fd); - if (hyper_init_event(&c->exec.e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &c->exec.e, EPOLLIN) < 0) { - fprintf(stderr, "add container pts master event failed\n"); - return -1; - } - } - - return 0; + return ret; } static struct hyper_event_ops hyper_ctl_pipe_ops = { @@ -449,11 +454,6 @@ static int hyper_setup_container(struct hyper_pod *pod) uint32_t type; void *stack; - if (hyper_setup_pty(pod) < 0) { - fprintf(stderr, "setup pty failed\n"); - return -1; - } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { perror("create pipe between hyper init and pod init failed"); return -1; @@ -473,10 +473,10 @@ static int hyper_setup_container(struct hyper_pod *pod) perror("create container init process failed"); return -1; } + fprintf(stdout, "pod init pid %d\n", pod->init_pid); close(arg.ctl_pipe[1]); ctl.ctl.fd = arg.ctl_pipe[0]; - fprintf(stdout, "pod init pid %d\n", pod->init_pid); /* Wait for container start */ if (hyper_get_type_block(ctl.ctl.fd, &type) < 0) { @@ -489,8 +489,8 @@ static int hyper_setup_container(struct hyper_pod *pod) return -1; } - if (hyper_watch_pty(pod) < 0) { - fprintf(stderr, "watch pty failed\n"); + if (hyper_start_containers(pod) < 0) { + fprintf(stderr, "start containers failed\n"); return -1; } @@ -724,13 +724,13 @@ static void hyper_term_all(struct hyper_pod *pod) static void hyper_cleanup_pod(struct hyper_pod *pod) { - close(pod->sig.fd); - hyper_reset_event(&pod->sig); + //close(pod->sig.fd); + //hyper_reset_event(&pod->sig); hyper_term_all(pod); - hyper_handle_exit(pod, pod->ctl.fd, 1, 0); + //hyper_handle_exit(pod, pod->ctl.fd, 1, 0); - pod->init_pid = 0; + //pod->init_pid = 0; close(pod->ctl.fd); hyper_reset_event(&pod->ctl); From c862b89079810a6f7f10ed4ed60fe5914ad70e0e Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 16 Sep 2015 18:54:56 +0800 Subject: [PATCH 05/23] running contianers in the pid ns of pod init manpage of setns says: >CLONE_NEWPID behaves somewhat differently from the other nstype values: reassociating the calling thread with a PID namespace changes only the PID namespace that child processes of the caller will >be created in; it does not change the PID namespace of the caller itself. Reassociating with a PID namespace is allowed only if the PID namespace specified by fd is a descendant (child, grandchild, >etc.) of the PID namespace of the caller. For further details on PID namespaces, see pid_namespaces(7). so clone a temporary process to enter pid ns, and then create containers. and join them to the other nss of pod init. Signed-off-by: Gao feng --- src/container.c | 9 +---- src/container.h | 3 +- src/init.c | 93 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/container.c b/src/container.c index 8721033..1c26c5f 100644 --- a/src/container.c +++ b/src/container.c @@ -276,7 +276,6 @@ static int hyper_rescan_scsi(void) struct hyper_container_arg { struct hyper_container *c; - int pidns; int ipcns; int utsns; int pipe[2]; @@ -294,11 +293,6 @@ static int hyper_container_init(void *data) goto fail; } - if (setns(arg->pidns, CLONE_NEWPID) < 0) { - perror("setns to pidns of pod init faild"); - goto fail; - } - if (setns(arg->ipcns, CLONE_NEWIPC) < 0) { perror("setns to ipcns of pod init faild"); goto fail; @@ -449,12 +443,11 @@ static int hyper_setup_pty(struct hyper_container *c) } int hyper_start_container(struct hyper_container *container, - int pidns, int utsns, int ipcns) + int utsns, int ipcns) { int stacksize = getpagesize() * 4; struct hyper_container_arg arg = { .c = container, - .pidns = pidns, .utsns = utsns, .ipcns = ipcns, }; diff --git a/src/container.h b/src/container.h index db969d9..8ab04fd 100644 --- a/src/container.h +++ b/src/container.h @@ -40,9 +40,8 @@ struct hyper_container { struct hyper_pod; int hyper_start_container(struct hyper_container *container, - int pidns, int utsns, int ipcns); + int utsns, int ipcns); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); -//int hyper_restart_containers(struct hyper_pod *pod); void hyper_cleanup_container(struct hyper_pod *pod); #endif diff --git a/src/init.c b/src/init.c index cb787ac..b2a6462 100644 --- a/src/init.c +++ b/src/init.c @@ -110,18 +110,6 @@ static int pod_ctl_pipe_handle(struct hyper_event *de, uint32_t len) case STOPPOD: fprintf(stdout, "get type STOPPOD, exit\n"); hyper_cleanup_pod(pod); - /* - case RESTARTCONTAINER: - fprintf(stdout, "%s get type RESTARTCONTAINER\n", __func__); - if (hyper_restart_containers(pod) < 0) - return -1; - break; - - case EXECCMD: - if (hyper_container_execcmd(pod) < 0) - return -1; - break; - */ default: break; } @@ -147,17 +135,7 @@ static int hyper_handle_exit(struct hyper_pod *pod) fprintf(stdout, "pid %d exit by signal, status %d\n", pid, WTERMSIG(status)); } -/* - if (container) { - hyper_set_be32(data, pid); - if (hyper_send_msg(to, FINISHCMD, 5, data) < 0) { - fprintf(stderr, "pod signal_loop send finishcmd failed\n"); - return -1; - } - continue; - } -*/ if (hyper_send_exec_eof(ctl.tty.fd, pod, pid, data[4]) < 0) fprintf(stderr, "signal_loop send eof failed\n"); } @@ -387,15 +365,19 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return 0; } -int hyper_start_containers(struct hyper_pod *pod) +static int hyper_enter_container_pidns(void *data) { int i, pidns, ipcns, utsns, ret; struct hyper_container *c; + struct hyper_pod_arg *arg; + struct hyper_pod *pod; char path[64]; + arg = data; + pod = arg->pod; + ret = pidns = ipcns = utsns = -1; - fprintf(stdout, "%s\n", __func__); sprintf(path, "/proc/%d/ns/pid", pod->init_pid); pidns = open(path, O_RDONLY| O_CLOEXEC); if (pidns < 0) { @@ -403,6 +385,13 @@ int hyper_start_containers(struct hyper_pod *pod) goto out; } + /* enter pidns of pod init, so the children of this process will run in + * pidns of pod init, see man 2 setns */ + if (setns(pidns, CLONE_NEWPID) < 0) { + perror("enter pidns of pod init failed"); + goto out; + } + sprintf(path, "/proc/%d/ns/uts", pod->init_pid); utsns = open(path, O_RDONLY| O_CLOEXEC); if (utsns < 0) { @@ -420,7 +409,12 @@ int hyper_start_containers(struct hyper_pod *pod) for (i = 0; i < pod->c_num; i++) { c = &pod->c[i]; list_add_tail(&c->exec.list, &pod->exec_head); - hyper_start_container(c, pidns, utsns, ipcns); + hyper_start_container(c, utsns, ipcns); + } + + if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) { + fprintf(stderr, "container init send ready message failed\n"); + goto out; } ret = 0; @@ -428,7 +422,56 @@ out: close(pidns); close(utsns); close(ipcns); + close(arg->ctl_pipe[0]); + close(arg->ctl_pipe[1]); + _exit(ret); +} + +int hyper_start_containers(struct hyper_pod *pod) +{ + int stacksize = getpagesize() * 4; + void *stack = malloc(stacksize); + struct hyper_pod_arg arg = { + .pod = pod, + .ctl_pipe = {-1, -1}, + }; + int ret = -1, pid; + uint32_t type; + + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { + perror("create pipe between hyper init and pod init failed"); + goto out; + } + + pid = clone(hyper_enter_container_pidns, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); + free(stack); + if (pid < 0) { + perror("enter container pid ns failed"); + goto out; + } + + /* Wait for container start */ + if (hyper_get_type_block(arg.ctl_pipe[0], &type) < 0) { + perror("get enter_container_pidns ready message failed"); + goto out; + } + + if (type != READY) { + fprintf(stderr, "get incorrect enter_container_pidns message type %d, expect READY\n", + type); + goto out; + } + + ret = 0; +out: + close(arg.ctl_pipe[0]); + close(arg.ctl_pipe[1]); return ret; } From 69768651764f64d7486a976f673942a3be85e6fa Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 16 Sep 2015 20:14:48 +0800 Subject: [PATCH 06/23] cleanup Signed-off-by: Gao feng --- src/container.c | 22 ---- src/exec.c | 50 --------- src/init.c | 270 +++++++++++++++++------------------------------- 3 files changed, 95 insertions(+), 247 deletions(-) diff --git a/src/container.c b/src/container.c index 1c26c5f..5e8068e 100644 --- a/src/container.c +++ b/src/container.c @@ -506,28 +506,6 @@ fail: return -1; } -/* -int hyper_restart_containers(struct hyper_pod *pod) -{ - int i; - struct hyper_container *c; - - for (i = 0; i < pod->c_num; i++) { - c = &pod->c[i]; - - if (hyper_start_container(c) < 0) { - fprintf(stderr, "restart container %s failed\n", c->id); - hyper_send_type(pod->ctl.fd, ERROR); - return -1; - } - } - - if (hyper_send_type(pod->ctl.fd, ACK) < 0) - return -1; - - return 0; -} -*/ struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id) { int i; diff --git a/src/exec.c b/src/exec.c index f080663..cd4fd66 100644 --- a/src/exec.c +++ b/src/exec.c @@ -262,47 +262,6 @@ out: return ret; } -/* -int hyper_request_restart_containers(struct hyper_pod *pod) -{ - int i; - struct hyper_exec *exec; - - pod->code = 0; - pod->remains = pod->c_num; - - for (i = 0; i < pod->c_num; i++) { - exec = &pod->c[i].exec; - - if (hyper_setup_exec_tty(exec) < 0) { - fprintf(stdout, "restart setup container tty failed\n"); - return -1; - } - } - - if (hyper_send_type_block(ctl.ctl.fd, RESTARTCONTAINER, 1) < 0) { - fprintf(stderr, "tell container init RESTARTCONTAINER failed\n"); - return -1; - } - - for (i = 0; i < pod->c_num; i++) { - exec = &pod->c[i].exec; - - list_add_tail(&exec->list, &pod->ce_head); - if (exec->seq == 0) - continue; - - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { - fprintf(stderr, "add pts master event failed\n"); - return -1; - } - } - - return 0; -} -*/ - int hyper_exec_cmd(char *json, int length) { struct hyper_exec *exec; @@ -465,16 +424,7 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq) return exec; } -/* - list_for_each_entry(exec, &pod->pe_head, list) { - fprintf(stdout, "pod exec seq %" PRIu64 ", seq %" PRIu64 "\n", - exec->seq, seq); - if (exec->seq != seq) - continue; - return exec; - } -*/ return NULL; } diff --git a/src/init.c b/src/init.c index b2a6462..ede12c0 100644 --- a/src/init.c +++ b/src/init.c @@ -37,7 +37,6 @@ struct hyper_exec *global_exec; struct hyper_ctl ctl; -static void hyper_cleanup_pod(struct hyper_pod *pod); static int hyper_handle_exit(struct hyper_pod *pod); static int hyper_set_win_size(char *json, int length) @@ -96,22 +95,106 @@ out: return ret; } +static void hyper_kill_process(int pid) +{ + char path[64]; + char *line = NULL, *ignore = "SigIgn:"; + size_t len = 0; + ssize_t read; + FILE *file; + char *sub; + + sprintf(path, "/proc/%u/status", pid); + + fprintf(stdout, "fopen %s\n", path); + file = fopen(path, "r"); + if (file == NULL) { + perror("can not open process proc status file"); + return; + } + + while ((read = getline(&line, &len, file)) != -1) { + long mask; + + if (strstr(line, ignore) == NULL) + continue; + + sub = line + strlen(ignore); + fprintf(stdout, "find sigign %s", sub); + + mask = atol(sub); + fprintf(stdout, "mask is %ld\n", mask); + + if ((mask >> (SIGTERM - 1)) & 0x1) { + fprintf(stdout, "signal term is ignored, kill it\n"); + kill(pid, SIGKILL); + } + + break; + } + + fclose(file); + free(line); +} + +static void hyper_term_all(struct hyper_pod *pod) +{ + int npids = 0; + int index = 0; + int pid; + DIR *dp; + struct dirent *de; + pid_t *pids = NULL; + + dp = opendir("/proc"); + if (dp == NULL) + return; + + while ((de = readdir(dp)) && de != NULL) { + if (!isdigit(de->d_name[0])) + continue; + pid = atoi(de->d_name); + if (pid == 1) + continue; + if (index <= npids) { + pids = realloc(pids, npids + 16384); + if (pids == NULL) + return; + npids += 16384; + } + + pids[index++] = pid; + } + + fprintf(stdout, "Sending SIGTERM\n"); + + for (--index; index >= 0; --index) { + fprintf(stdout, "kill process %d\n", pids[index]); + kill(pids[index], SIGTERM); + } + + free(pids); + closedir(dp); + + for (index = 0; index < pod->c_num; index++) { + hyper_kill_process(pod->c[index].exec.pid); + } +} + static int pod_ctl_pipe_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; - struct hyper_pod *pod = de->ptr; - uint32_t type; fprintf(stdout, "%s\n", __func__); - type = hyper_get_be32(buf->data); + if (hyper_get_be32(buf->data) == STOPPOD) { + struct hyper_pod *pod = de->ptr; - switch (type) { - case STOPPOD: - fprintf(stdout, "get type STOPPOD, exit\n"); - hyper_cleanup_pod(pod); - default: - break; + fprintf(stdout, "pod init get type STOPPOD, exit\n"); + hyper_term_all(pod); + hyper_reset_event(&pod->ctl); + hyper_unmount_all(); + _exit(0); } return 0; @@ -141,15 +224,6 @@ static int hyper_handle_exit(struct hyper_pod *pod) } return 0; - - /* send ack message to hyper init. - if (hyper_send_type(to, ACK) < 0) { - fprintf(stderr, "send ACK of STOPPOD to hyper init failed\n"); - return -1; - } - - return 0; - */ } static int hyper_signal_loop(struct hyper_event *de) @@ -186,18 +260,6 @@ static int hyper_signal_loop(struct hyper_event *de) return 0; } -/* -static int pod_signal_loop(struct hyper_event *de) -{ - return signal_loop(de, 1); -} - -static int hyper_signal_loop(struct hyper_event *de) -{ - return signal_loop(de, 0); -} -*/ - static struct hyper_event_ops pod_ctl_pipe_ops = { .read = hyper_event_read, .handle = pod_ctl_pipe_handle, @@ -206,13 +268,6 @@ static struct hyper_event_ops pod_ctl_pipe_ops = { .len_offset = 4, }; -/* -static struct hyper_event_ops pod_signal_ops = { - .read = pod_signal_loop, - .hup = hyper_event_hup, -}; -*/ - static int pod_init_loop(struct hyper_pod *pod) { int i, n; @@ -231,15 +286,7 @@ static int pod_init_loop(struct hyper_pod *pod) fprintf(stderr, "hyper add pod ctl pipe event failed\n"); return -1; } -/* - fprintf(stdout, "hyper_init_event pod signal event %p, ops %p, fd %d\n", - &pod->sig, &pod_signal_ops, pod->sig.fd); - if (hyper_init_event(&pod->sig, &pod_signal_ops, pod) < 0 || - hyper_add_event(pod->efd, &pod->sig, EPOLLIN) < 0) { - fprintf(stderr, "hyper add pod tty pipe event failed\n"); - return -1; - } -*/ + events = calloc(MAXEVENTS, sizeof(*events)); while (1) { @@ -284,21 +331,7 @@ static int hyper_pod_init(void *data) perror("set pod init ctl pipe fd FD_CLOEXEC failed"); goto fail; } -/* - sigemptyset(&mask); - sigaddset(&mask, SIGCHLD); - if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) { - perror("sigprocmask SIGCHLD failed"); - goto fail; - } - - pod->sig.fd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC); - if (pod->sig.fd < 0) { - perror("create signalfd failed"); - goto fail; - } -*/ /* mount new proc directory */ if (umount("/proc") < 0) { perror("umount proc filesystem failed\n"); @@ -347,16 +380,6 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) * fd is block, and ACK is the last message, exit loop. */ fprintf(stdout, "hyper_ctl_pipe_loop get ack\n"); return 1; - /* - case FINISHCMD: - pid = hyper_get_be32(buf->data + 8); - code = buf->data[12]; - if (hyper_send_exec_eof(ctl.tty.fd, pod, &pod->ce_head, pid, code) < 0) { - fprintf(stderr, "hyper_ctl_pipe_loop send eof failed\n"); - return -1; - } - break; - */ default: fprintf(stdout, "get unknown type %" PRIu32"\n", type); break; @@ -679,110 +702,6 @@ static void hyper_print_uptime(void) close(fd); } -static void hyper_kill_process(int pid) -{ - char path[64]; - char *line = NULL, *ignore = "SigIgn:"; - size_t len = 0; - ssize_t read; - FILE *file; - char *sub; - - sprintf(path, "/proc/%u/status", pid); - - fprintf(stdout, "fopen %s\n", path); - file = fopen(path, "r"); - if (file == NULL) { - perror("can not open process proc status file"); - return; - } - - while ((read = getline(&line, &len, file)) != -1) { - long mask; - - if (strstr(line, ignore) == NULL) - continue; - - sub = line + strlen(ignore); - fprintf(stdout, "find sigign %s", sub); - - mask = atol(sub); - fprintf(stdout, "mask is %ld\n", mask); - - if ((mask >> (SIGTERM - 1)) & 0x1) { - fprintf(stdout, "signal term is ignored, kill it\n"); - kill(pid, SIGKILL); - } - - break; - } - - fclose(file); - free(line); -} - -static void hyper_term_all(struct hyper_pod *pod) -{ - int npids = 0; - int index = 0; - int pid; - DIR *dp; - struct dirent *de; - pid_t *pids = NULL; - - dp = opendir("/proc"); - if (dp == NULL) - return; - - while ((de = readdir(dp)) && de != NULL) { - if (!isdigit(de->d_name[0])) - continue; - pid = atoi(de->d_name); - if (pid == 1) - continue; - if (index <= npids) { - pids = realloc(pids, npids + 16384); - if (pids == NULL) - return; - npids += 16384; - } - - pids[index++] = pid; - } - - fprintf(stdout, "Sending SIGTERM\n"); - - for (--index; index >= 0; --index) { - fprintf(stdout, "kill process %d\n", pids[index]); - kill(pids[index], SIGTERM); - } - - free(pids); - closedir(dp); - - for (index = 0; index < pod->c_num; index++) { - hyper_kill_process(pod->c[index].exec.pid); - } -} - -static void hyper_cleanup_pod(struct hyper_pod *pod) -{ - //close(pod->sig.fd); - //hyper_reset_event(&pod->sig); - - hyper_term_all(pod); - //hyper_handle_exit(pod, pod->ctl.fd, 1, 0); - - //pod->init_pid = 0; - - close(pod->ctl.fd); - hyper_reset_event(&pod->ctl); - - hyper_unmount_all(); - - _exit(0); -} - static int hyper_start_pod(char *json, int length) { struct hyper_pod *pod = &global_pod; @@ -858,6 +777,7 @@ static int hyper_stop_pod(struct hyper_pod *pod) return 0; } + pod->init_pid = 0; /* Make hyper ctl_pipe blocked */ hyper_send_stoppod(ctl.ctl.fd); From bf0f36093504035b60476d54d88d8afc54991b1e Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 16 Sep 2015 23:19:56 +0800 Subject: [PATCH 07/23] run exec in right pid namespace Signed-off-by: Gao feng --- src/container.c | 16 +---- src/exec.c | 153 ++++++++++++++++++++++++++++++++++-------------- src/util.c | 17 ++++++ src/util.h | 2 + 4 files changed, 130 insertions(+), 58 deletions(-) diff --git a/src/container.c b/src/container.c index 5e8068e..bf3dbc3 100644 --- a/src/container.c +++ b/src/container.c @@ -17,20 +17,6 @@ #include "util.h" #include "hyper.h" -static int container_setup_env(struct hyper_container *container) -{ - int i; - struct env *env; - - for (i = 0; i < container->envs_num; i++) { - env = &container->envs[i]; - - setenv(env->env, env->value, 1); - } - - return 0; -} - static int container_setup_volume(struct hyper_container *container) { int i; @@ -308,7 +294,7 @@ static int hyper_container_init(void *data) goto fail; } - if (container_setup_env(container) < 0) { + if (hyper_setup_env(container->envs, container->envs_num) < 0) { fprintf(stdout, "setup env failed\n"); goto fail; } diff --git a/src/exec.c b/src/exec.c index cd4fd66..424fd8b 100644 --- a/src/exec.c +++ b/src/exec.c @@ -192,11 +192,11 @@ int hyper_watch_exec_pty(struct hyper_exec *exec) int hyper_enter_container(struct hyper_pod *pod, struct hyper_exec *exec) { - int pidns, ipcns, utsns, mntns, ret; + int ipcns, utsns, mntns, ret; struct hyper_container *c; char path[512]; - ret = pidns = ipcns = utsns = mntns = -1; + ret = ipcns = utsns = mntns = -1; c = hyper_find_container(pod, exec->id); if (c == NULL) { @@ -204,13 +204,6 @@ int hyper_enter_container(struct hyper_pod *pod, return -1; } - sprintf(path, "/proc/%d/ns/pid", c->exec.pid); - pidns = open(path, O_RDONLY| O_CLOEXEC); - if (pidns < 0) { - perror("fail to open pidns of pod init"); - goto out; - } - sprintf(path, "/proc/%d/ns/uts", c->exec.pid); utsns = open(path, O_RDONLY| O_CLOEXEC); if (utsns < 0) { @@ -232,8 +225,7 @@ int hyper_enter_container(struct hyper_pod *pod, goto out; } - if (setns(pidns, CLONE_NEWPID) < 0 || - setns(utsns, CLONE_NEWUTS) < 0|| + if (setns(utsns, CLONE_NEWUTS) < 0 || setns(ipcns, CLONE_NEWIPC) <0 || setns(mntns, CLONE_NEWNS) < 0) { perror("fail to enter container ns"); @@ -252,9 +244,8 @@ int hyper_enter_container(struct hyper_pod *pod, chdir("/"); - ret = 0; + ret = hyper_setup_env(c->envs, c->envs_num); out: - close(pidns); close(ipcns); close(utsns); close(mntns); @@ -262,69 +253,79 @@ out: return ret; } -int hyper_exec_cmd(char *json, int length) +struct hyper_exec_arg { + struct hyper_pod *pod; + struct hyper_exec *exec; + int pipe[2]; +}; + +static int hyper_do_exec_cmd(void *data) { - struct hyper_exec *exec; - struct hyper_pod *pod = &global_pod; - int pid, pipe[2]; + struct hyper_exec_arg *arg = data; + struct hyper_exec *exec = arg->exec; + struct hyper_pod *pod = arg->pod; + int pipe[2], pid; - fprintf(stdout, "call hyper_exec_cmd, json %s, len %d\n", json, length); + if (exec->id) { + char path[512]; + int pidns; - exec = hyper_parse_execcmd(json, length); - if (exec == NULL) { - fprintf(stderr, "parse exec cmd failed\n"); - return -1; - } + sprintf(path, "/proc/%d/ns/pid", pod->init_pid); + pidns = open(path, O_RDONLY| O_CLOEXEC); + if (pidns < 0) { + perror("fail to open pidns of pod init"); + _exit(-1); + } - if (exec->argv == NULL) { - fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n", - exec->argv, exec->seq, exec->id); - return -1; - } - - if (hyper_setup_exec_tty(exec) < 0) { - fprintf(stderr, "setup exec tty failed\n"); - return -1; + /* enter pidns of pod init, so the children of this process will run in + * pidns of pod init, see man 2 setns */ + if (setns(pidns, CLONE_NEWPID) < 0) { + perror("enter pidns of pod init failed"); + _exit(-1); + } } if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { - perror("create pipe between pod init execcmd failed"); - return -1; + perror("create pipe in exec command failed"); + _exit(-1); } pid = fork(); if (pid < 0) { - fprintf(stderr, "fork failed\n"); - return -1; + perror("fail to fork"); + _exit(-1); } else if (pid > 0) { uint32_t type; if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); - return -1; + _exit(-1); + } + + if (hyper_send_type_block(arg->pipe[1], READY, 0) < 0) { + fprintf(stderr, "send ready message to hyper init failed\n"); + _exit(-1); } - close(pipe[0]); - close(pipe[1]); fprintf(stdout, "hyper init get ready message\n"); exec->pid = pid; fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); list_add_tail(&exec->list, &pod->exec_head); if (exec->seq == 0) - return 0; + _exit(0); if (hyper_watch_exec_pty(exec) < 0) { fprintf(stderr, "add pts master event failed\n"); - return -1; + _exit(-1); } - return 0; + _exit(0); } if (exec->id && hyper_enter_container(pod, exec) < 0) { fprintf(stderr, "enter container ns failed\n"); - return -1; + _exit(-1); } if (hyper_dup_exec_tty(pipe[1], exec) < 0) { @@ -334,6 +335,8 @@ int hyper_exec_cmd(char *json, int length) close(pipe[0]); close(pipe[1]); + close(arg->pipe[0]); + close(arg->pipe[1]); if (execvp(exec->argv[0], exec->argv) < 0) { perror("exec failed"); @@ -343,6 +346,70 @@ int hyper_exec_cmd(char *json, int length) _exit(0); } +int hyper_exec_cmd(char *json, int length) +{ + struct hyper_exec *exec; + struct hyper_pod *pod = &global_pod; + int stacksize = getpagesize() * 4; + void *stack = malloc(stacksize); + struct hyper_exec_arg arg = { + .pod = pod, + .exec = NULL, + .pipe = {-1, -1}, + }; + int pid, ret = -1; + uint32_t type; + + fprintf(stdout, "call hyper_exec_cmd, json %s, len %d\n", json, length); + + exec = hyper_parse_execcmd(json, length); + if (exec == NULL) { + fprintf(stderr, "parse exec cmd failed\n"); + goto out; + } + + if (exec->argv == NULL) { + fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n", + exec->argv, exec->seq, exec->id); + goto out; + } + + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + if (hyper_setup_exec_tty(exec) < 0) { + fprintf(stderr, "setup exec tty failed\n"); + goto out; + } + + if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + perror("create pipe between pod init execcmd failed"); + goto out; + } + + arg.exec = exec; + pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); + free(stack); + if (pid < 0) { + perror("clone hyper_do_exec_cmd failed"); + goto out; + } + + if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); + return -1; + } + + ret = 0; +out: + close(arg.pipe[0]); + close(arg.pipe[1]); + + return ret; +} + int hyper_release_exec(struct hyper_exec *exec, struct hyper_pod *pod) { diff --git a/src/util.c b/src/util.c index d1212b6..1d541fd 100644 --- a/src/util.c +++ b/src/util.c @@ -16,6 +16,7 @@ #include "util.h" #include "hyper.h" +#include "container.h" #include "../config.h" char *read_cmdline(void) @@ -23,6 +24,22 @@ char *read_cmdline(void) return NULL; } +int hyper_setup_env(struct env *envs, int num) +{ + int i, ret = 0; + struct env *env; + + for (i = 0; i < num; i++) { + env = &envs[i]; + if (setenv(env->env, env->value, 1) < 0) { + perror("fail to setup env"); + ret = -1; + } + } + + return ret; +} + int hyper_list_dir(char *path) { struct dirent **list; diff --git a/src/util.h b/src/util.h index 807a6eb..9471f5c 100644 --- a/src/util.h +++ b/src/util.h @@ -5,6 +5,7 @@ #include "../config.h" struct hyper_pod; +struct env; #ifdef WITH_DEBUG #define dprintf(fmt, ...) \ @@ -14,6 +15,7 @@ struct hyper_pod; #endif char *read_cmdline(void); +int hyper_setup_env(struct env *envs, int num); int hyper_list_dir(char *path); int hyper_mkdir(char *path); int hyper_open_channel(char *channel, int mode); From 2fb02c57b624187fe9c22b6f80cb962a902794a4 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 16 Sep 2015 23:39:01 +0800 Subject: [PATCH 08/23] set cloexec flag on socket pair Signed-off-by: Gao feng --- src/container.c | 5 +---- src/exec.c | 9 ++------- src/init.c | 8 ++++---- src/util.c | 16 ++++++++++++++++ src/util.h | 1 + 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/container.c b/src/container.c index bf3dbc3..93cf3f7 100644 --- a/src/container.c +++ b/src/container.c @@ -389,9 +389,6 @@ static int hyper_container_init(void *data) goto fail; } - close(arg->pipe[0]); - close(arg->pipe[1]); - execvp(container->exec.argv[0], container->exec.argv); perror("exec container command failed"); @@ -453,7 +450,7 @@ int hyper_start_container(struct hyper_container *container, goto fail; } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { perror("create pipe between pod init execcmd failed"); goto fail; } diff --git a/src/exec.c b/src/exec.c index 424fd8b..00c9272 100644 --- a/src/exec.c +++ b/src/exec.c @@ -285,7 +285,7 @@ static int hyper_do_exec_cmd(void *data) } } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { perror("create pipe in exec command failed"); _exit(-1); } @@ -333,11 +333,6 @@ static int hyper_do_exec_cmd(void *data) _exit(-1); } - close(pipe[0]); - close(pipe[1]); - close(arg->pipe[0]); - close(arg->pipe[1]); - if (execvp(exec->argv[0], exec->argv) < 0) { perror("exec failed"); _exit(-1); @@ -384,7 +379,7 @@ int hyper_exec_cmd(char *json, int length) goto out; } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { perror("create pipe between pod init execcmd failed"); goto out; } diff --git a/src/init.c b/src/init.c index ede12c0..2bb7b66 100644 --- a/src/init.c +++ b/src/init.c @@ -388,7 +388,7 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return 0; } -static int hyper_enter_container_pidns(void *data) +static int hyper_do_start_containers(void *data) { int i, pidns, ipcns, utsns, ret; struct hyper_container *c; @@ -467,12 +467,12 @@ int hyper_start_containers(struct hyper_pod *pod) goto out; } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { perror("create pipe between hyper init and pod init failed"); goto out; } - pid = clone(hyper_enter_container_pidns, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); + pid = clone(hyper_do_start_containers, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); free(stack); if (pid < 0) { perror("enter container pid ns failed"); @@ -520,7 +520,7 @@ static int hyper_setup_container(struct hyper_pod *pod) uint32_t type; void *stack; - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { perror("create pipe between hyper init and pod init failed"); return -1; } diff --git a/src/util.c b/src/util.c index 1d541fd..adfdd29 100644 --- a/src/util.c +++ b/src/util.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -323,6 +324,21 @@ int hyper_setfd_nonblock(int fd) return 0; } +int hyper_socketpair(int domain, int type, int protocol, int sv[2]) +{ + if (socketpair(domain, type, protocol, sv) < 0) { + perror("socketpair failed"); + return -1; + } + + if (hyper_setfd_cloexec(sv[0]) < 0 || + hyper_setfd_cloexec(sv[1]) < 0) { + return -1; + } + + return 0; +} + void hyper_unmount_all(void) { FILE *mtab; diff --git a/src/util.h b/src/util.h index 9471f5c..c285a9d 100644 --- a/src/util.h +++ b/src/util.h @@ -23,6 +23,7 @@ int hyper_open_serial_dev(char *tty); int hyper_setfd_cloexec(int fd); int hyper_setfd_block(int fd); int hyper_setfd_nonblock(int fd); +int hyper_socketpair(int domain, int type, int protocol, int sv[2]); void hyper_shutdown(struct hyper_pod *pod); int hyper_send_finish(struct hyper_pod *pod); void hyper_unmount_all(void); From bc0e688f961da6c0ab5e09c57804eed7a605f2ed Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 17 Sep 2015 00:44:33 +0800 Subject: [PATCH 09/23] implemet hyper_cmd_write_file Signed-off-by: Gao feng --- src/hyper.h | 1 + src/init.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/parse.c | 11 +++++-- 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/src/hyper.h b/src/hyper.h index 2a09207..918d4f9 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -75,6 +75,7 @@ struct hyper_writter { char *id; char *file; char *data; + int len; }; struct hyper_ctl { diff --git a/src/init.c b/src/init.c index 2bb7b66..f4b3d5c 100644 --- a/src/init.c +++ b/src/init.c @@ -726,6 +726,98 @@ static int hyper_start_pod(char *json, int length) static int hyper_cmd_write_file(char *json, int length) { + struct hyper_writter writter; + struct hyper_container *c; + struct hyper_pod *pod = &global_pod; + int pipe[2] = {-1, -1}; + int pid, mntns = -1, fd; + char path[512]; + int len = 0, size; + + fprintf(stdout, "%s\n", __func__); + memset(&writter, 0, sizeof(writter)); + + if (hyper_parse_write_file(&writter, json, length) < 0) { + goto out; + } + + c = hyper_find_container(pod, writter.id); + if (c == NULL) { + fprintf(stderr, "can not find container whose id is %s\n", writter.id); + goto out; + } + + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { + perror("create writter pipe failed"); + goto out; + } + + sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); + mntns = open(path, O_RDONLY); + if (mntns < 0) { + perror("fail to open mnt ns"); + goto out; + } + + pid = fork(); + if (pid < 0) { + perror("fail to fork writter process"); + goto out; + } else if (pid > 0) { + uint32_t type; + + if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "get incorrect message type %d, expect READY\n", type); + goto out; + } + + goto out; + } + + if (setns(mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + _exit(-1); + } + + sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + fprintf(stdout, "root directory for container is %s\n",path); + + /* TODO: wait for container finishing setup root */ + if (chroot(path) < 0) { + perror("chroot for exec command failed"); + _exit(-1); + } + + fd = open(writter.file, O_CREAT| O_WRONLY, 0644); + if (fd < 0) { + perror("fail to open target file"); + _exit(-1); + } + + while(len < writter.len) { + size = write(fd, writter.data + len, writter.len - len); + + if (size < 0) { + if (errno == EINTR) + continue; + + perror("fail to write data to file"); + _exit(-1); + } + + len += size; + } + + hyper_send_type(pipe[1], READY); + _exit(0); +out: + close(pipe[0]); + close(pipe[1]); + close(mntns); + free(writter.id); + free(writter.file); + free(writter.data); + return 0; } diff --git a/src/parse.c b/src/parse.c index 0f75a2e..55fcf02 100644 --- a/src/parse.c +++ b/src/parse.c @@ -673,8 +673,15 @@ int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length if (i++ == n) goto fail; - writter->data = strdup(json_token_str(json, &toks[i])); - fprintf(stdout, "writefile get data %s\n", writter->data); + writter->len = toks[i].end - toks[i].start + 1; + writter->data = malloc(writter->len); + + if (writter->data == NULL) + goto fail; + + memcpy(writter->data, json + toks[i].start, writter->len); + writter->data[writter->len - 1] = '\0'; + fprintf(stdout, "writefile get data len %d %s\n", writter->len, writter->data); } else { fprintf(stdout, "in writefile incorrect %s\n", json_token_str(json, &toks[i])); } From 2262bc18fe5872dd4454eb5acfe83b306e7c4761 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 17 Sep 2015 14:39:10 +0800 Subject: [PATCH 10/23] accept read raw data from write message It's difficult to encode the json with special characters such as '"', '>'... encoding and unicoding these characters will change the length of json message, it's a pain to get bytes contains these characters from json. This patch allows to read the bytes contains special character outside the json. for example: {"container":"c5f67934326d2ecef59fee62148bc9237e2481cd803cc0760a6406da15f4ee60","file":"/tmp/aaa"}sssssss Signed-off-by: Gao feng --- src/init.c | 2 +- src/jsmn.c | 15 +++++++++++---- src/parse.c | 26 +++++++++++--------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/init.c b/src/init.c index f4b3d5c..82c72b7 100644 --- a/src/init.c +++ b/src/init.c @@ -780,7 +780,7 @@ static int hyper_cmd_write_file(char *json, int length) } sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); - fprintf(stdout, "root directory for container is %s\n",path); + fprintf(stdout, "write file %s, data len %d\n", writter.file, writter.len); /* TODO: wait for container finishing setup root */ if (chroot(path) < 0) { diff --git a/src/jsmn.c b/src/jsmn.c index b1c3ebb..027b8aa 100644 --- a/src/jsmn.c +++ b/src/jsmn.c @@ -164,14 +164,18 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, int i; jsmntok_t *token; int count = 0; + int num = 0; + int out = 0; - for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + for (; parser->pos < len && js[parser->pos] != '\0' && !out; parser->pos++) { char c; jsmntype_t type; c = js[parser->pos]; switch (c) { - case '{': case '[': + case '{': + num++; + case '[': count++; if (tokens == NULL) break; @@ -189,7 +193,10 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, token->start = parser->pos; parser->toksuper = parser->toknext - 1; break; - case '}': case ']': + case '}': + if (--num <= 0) + out = 1; + case ']': if (tokens == NULL) break; type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); @@ -221,7 +228,7 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, parser->toksuper = -1; token->end = parser->pos + 1; - break; + break; } } /* Error if unmatched closing bracket */ diff --git a/src/parse.c b/src/parse.c index 55fcf02..3e046bf 100644 --- a/src/parse.c +++ b/src/parse.c @@ -645,12 +645,22 @@ int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length jsmn_init(&p); n = jsmn_parse(&p, json, length, toks, toks_num); - if (n < 0) { + /* Must be json first */ + if (n <= 0) { fprintf(stdout, "jsmn parse failed, n is %d\n", n); ret = -1; goto out; } + writter->len = length - toks[0].end; + writter->data = malloc(writter->len); + + if (writter->data == NULL) + goto fail; + + memcpy(writter->data, json + toks[0].end, writter->len); + fprintf(stdout, "writefile get data len %d %s\n", writter->len, writter->data); + for (i = 0; i < n; i++) { jsmntok_t *t = &toks[i]; @@ -669,24 +679,10 @@ int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length writter->file = strdup(json_token_str(json, &toks[i])); fprintf(stdout, "writefile get file %s\n", writter->file); - } else if (json_token_streq(json, t, "data")) { - if (i++ == n) - goto fail; - - writter->len = toks[i].end - toks[i].start + 1; - writter->data = malloc(writter->len); - - if (writter->data == NULL) - goto fail; - - memcpy(writter->data, json + toks[i].start, writter->len); - writter->data[writter->len - 1] = '\0'; - fprintf(stdout, "writefile get data len %d %s\n", writter->len, writter->data); } else { fprintf(stdout, "in writefile incorrect %s\n", json_token_str(json, &toks[i])); } } - out: free(toks); return ret; From 27f4b56e71bae2f779b299c7b13ee643ade3ccbb Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 17 Sep 2015 20:28:31 +0800 Subject: [PATCH 11/23] implement hyper_cmd_read_file Signed-off-by: Gao feng --- src/hyper.h | 2 +- src/init.c | 149 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/parse.c | 5 ++ src/parse.h | 1 + 4 files changed, 150 insertions(+), 7 deletions(-) diff --git a/src/hyper.h b/src/hyper.h index 918d4f9..f7a17fd 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -74,7 +74,7 @@ struct hyper_reader { struct hyper_writter { char *id; char *file; - char *data; + uint8_t *data; int len; }; diff --git a/src/init.c b/src/init.c index 82c72b7..dcde8ba 100644 --- a/src/init.c +++ b/src/init.c @@ -821,9 +821,145 @@ out: return 0; } -static int hyper_cmd_read_file(char *json, int length) +struct hyper_file_arg { + int mntns; + int pipe[2]; + char *file; + char root[128]; + uint32_t *datalen; + uint8_t **data; +}; + +static int hyper_do_cmd_read_file(void *data) { + struct stat st; + int len = 0, size, fd; + struct hyper_file_arg *arg = data; + + if (setns(arg->mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + goto err; + } + + fprintf(stdout, "read file %s\n", arg->file); + + /* TODO: wait for container finishing setup root */ + if (chroot(arg->root) < 0) { + perror("chroot for exec command failed"); + goto err; + } + + if (stat(arg->file, &st) < 0) { + perror("fail to state file"); + goto err; + } + + *arg->datalen = st.st_size; + *arg->data = malloc(*arg->datalen); + if (*arg->data == NULL) { + fprintf(stderr, "allocate memory for reading file failed\n"); + goto err; + } + + fd = open(arg->file, O_RDONLY); + if (fd < 0) { + perror("fail to open target file"); + goto err; + } + + fprintf(stdout, "file length %d\n", *arg->datalen); + while(len < *arg->datalen) { + size = read(fd, *arg->data + len, *arg->datalen - len); + + if (size < 0) { + if (errno == EINTR) + continue; + + perror("fail to read data from file"); + goto err; + } + + len += size; + } + + fprintf(stdout, "read data %s\n", *arg->data); + hyper_send_type(arg->pipe[1], READY); return 0; +err: + hyper_send_type(arg->pipe[1], ERROR); + return -1; +} + +static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_t **data) +{ + struct hyper_reader reader; + struct hyper_container *c; + struct hyper_pod *pod = &global_pod; + struct hyper_file_arg arg = { + .pipe = {-1, -1}, + }; + int stacksize = getpagesize() * 4; + void *stack = malloc(stacksize); + int pid, ret = -1; + char path[128]; + uint32_t type; + + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + fprintf(stdout, "%s\n", __func__); + memset(&reader, 0, sizeof(reader)); + + if (hyper_parse_read_file(&reader, json, length) < 0) { + goto out; + } + + c = hyper_find_container(pod, reader.id); + if (c == NULL) { + fprintf(stderr, "can not find container whose id is %s\n", reader.id); + goto out; + } + + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + perror("create reader pipe failed"); + goto out; + } + + sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); + arg.mntns = open(path, O_RDONLY); + if (arg.mntns < 0) { + perror("fail to open mnt ns"); + goto out; + } + + arg.file = reader.file; + arg.datalen = datalen; + arg.data = data; + sprintf(arg.root, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + + pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM, &arg); + free(stack); + if (pid < 0) { + perror("fail to fork writter process"); + goto out; + } + + if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "%s to incorrect type %" PRIu32 "\n", __func__, type); + goto out; + } + + ret = 0; +out: + close(arg.pipe[0]); + close(arg.pipe[1]); + close(arg.mntns); + free(reader.id); + free(reader.file); + + return ret; } static void hyper_cleanup_shared(struct hyper_pod *pod) @@ -979,7 +1115,8 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; struct hyper_pod *pod = de->ptr; - uint32_t type = 0; + uint32_t type = 0, datalen = 0; + uint8_t *data = NULL; int i, ret = 0; for (i = 0; i < buf->get; i++) @@ -1010,9 +1147,8 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) ret = hyper_cmd_write_file((char *)buf->data + 8, len - 8); break; case READFILE: - hyper_cmd_read_file((char *)buf->data + 8, len - 8); - /* hyperstart will send ACK message with file context */ - return 0; + ret = hyper_cmd_read_file((char *)buf->data + 8, len - 8, &datalen, &data); + break; case PING: case GETPOD: break; @@ -1030,8 +1166,9 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) if (ret < 0) hyper_send_type(de->fd, ERROR); else - hyper_send_type(de->fd, ACK); + hyper_send_msg(de->fd, ACK, datalen, data); + free(data); return 0; } diff --git a/src/parse.c b/src/parse.c index 3e046bf..cf843e6 100644 --- a/src/parse.c +++ b/src/parse.c @@ -704,6 +704,11 @@ int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length) jsmntok_t *toks = NULL; toks = calloc(toks_num, sizeof(jsmntok_t)); + if (toks == NULL) { + fprintf(stderr, "fail to allocate tokens for read file cmd\n"); + ret = -1; + goto out; + } jsmn_init(&p); n = jsmn_parse(&p, json, length, toks, toks_num); diff --git a/src/parse.h b/src/parse.h index 4b93b41..867025b 100644 --- a/src/parse.h +++ b/src/parse.h @@ -10,5 +10,6 @@ char *json_token_str(char *js, jsmntok_t *t); int json_token_streq(char *js, jsmntok_t *t, char *s); int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length); int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length); +int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length); #endif From 330c46cdb21c41a15f9ae9ffecc46a34a7df0b66 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Mon, 21 Sep 2015 10:47:31 +0800 Subject: [PATCH 12/23] enter container namespace with opened ns fd After process exit,the path of pid ns will disappear, open it first to prohibit ns being released. Signed-off-by: Gao feng --- src/container.c | 19 ++++++++++++++++--- src/container.h | 1 + src/exec.c | 8 +++----- src/init.c | 9 ++------- src/parse.c | 1 + 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/container.c b/src/container.c index 93cf3f7..619820d 100644 --- a/src/container.c +++ b/src/container.c @@ -430,11 +430,13 @@ int hyper_start_container(struct hyper_container *container, { int stacksize = getpagesize() * 4; struct hyper_container_arg arg = { - .c = container, - .utsns = utsns, - .ipcns = ipcns, + .c = container, + .utsns = utsns, + .ipcns = ipcns, + .pipe = {-1, -1}, }; int flags = CLONE_NEWNS | SIGCHLD; + char path[128]; uint32_t type; void *stack; int pid; @@ -467,7 +469,13 @@ int hyper_start_container(struct hyper_container *container, perror("create child process failed"); goto fail; } + sprintf(path, "/proc/%d/ns/mnt", pid); container->exec.pid = pid; + container->ns = open(path, O_RDONLY | O_CLOEXEC); + if (container->ns < 0) { + perror("open container mount ns failed"); + goto fail; + } /* wait for ready message */ if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { @@ -484,6 +492,10 @@ int hyper_start_container(struct hyper_container *container, fprintf(stdout, "container %s init pid is %d\n", container->id, pid); return 0; fail: + close(arg.pipe[0]); + close(arg.pipe[1]); + close(container->ns); + container->ns = -1; fprintf(stdout, "container %s init exit code %d\n", container->id, -1); container->exec.code = -1; return -1; @@ -552,6 +564,7 @@ void hyper_cleanup_container(struct hyper_pod *pod) free(map->path); } free(c->maps); + close(c->ns); } free(pod->c); diff --git a/src/container.h b/src/container.h index 8ab04fd..6b2065b 100644 --- a/src/container.h +++ b/src/container.h @@ -33,6 +33,7 @@ struct hyper_container { int vols_num; int envs_num; int maps_num; + int ns; uint32_t code; struct hyper_exec exec; }; diff --git a/src/exec.c b/src/exec.c index 00c9272..10a275b 100644 --- a/src/exec.c +++ b/src/exec.c @@ -204,22 +204,21 @@ int hyper_enter_container(struct hyper_pod *pod, return -1; } - sprintf(path, "/proc/%d/ns/uts", c->exec.pid); + sprintf(path, "/proc/%d/ns/uts", pod->init_pid); utsns = open(path, O_RDONLY| O_CLOEXEC); if (utsns < 0) { perror("fail to open utsns of pod init"); goto out; } - sprintf(path, "/proc/%d/ns/ipc", c->exec.pid); + sprintf(path, "/proc/%d/ns/ipc", pod->init_pid); ipcns = open(path, O_RDONLY| O_CLOEXEC); if (ipcns < 0) { perror("fail to open ipcns of pod init"); goto out; } - sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); - mntns = open(path, O_RDONLY| O_CLOEXEC); + mntns = c->ns; if (mntns < 0) { perror("fail to open mntns of pod init"); goto out; @@ -248,7 +247,6 @@ int hyper_enter_container(struct hyper_pod *pod, out: close(ipcns); close(utsns); - close(mntns); return ret; } diff --git a/src/init.c b/src/init.c index dcde8ba..1aa063e 100644 --- a/src/init.c +++ b/src/init.c @@ -752,8 +752,7 @@ static int hyper_cmd_write_file(char *json, int length) goto out; } - sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); - mntns = open(path, O_RDONLY); + mntns = c->ns; if (mntns < 0) { perror("fail to open mnt ns"); goto out; @@ -813,7 +812,6 @@ static int hyper_cmd_write_file(char *json, int length) out: close(pipe[0]); close(pipe[1]); - close(mntns); free(writter.id); free(writter.file); free(writter.data); @@ -901,7 +899,6 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ int stacksize = getpagesize() * 4; void *stack = malloc(stacksize); int pid, ret = -1; - char path[128]; uint32_t type; if (stack == NULL) { @@ -927,8 +924,7 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ goto out; } - sprintf(path, "/proc/%d/ns/mnt", c->exec.pid); - arg.mntns = open(path, O_RDONLY); + arg.mntns = c->ns; if (arg.mntns < 0) { perror("fail to open mnt ns"); goto out; @@ -955,7 +951,6 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ out: close(arg.pipe[0]); close(arg.pipe[1]); - close(arg.mntns); free(reader.id); free(reader.file); diff --git a/src/parse.c b/src/parse.c index cf843e6..bd61d70 100644 --- a/src/parse.c +++ b/src/parse.c @@ -199,6 +199,7 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container * c->exec.init = 1; c->exec.code = -1; + c->ns = -1; next_container = toks[i].size; fprintf(stdout, "next container %d\n", next_container); From 87fa294a42bed871e89123dc896530004f66ac9c Mon Sep 17 00:00:00 2001 From: Gao feng Date: Mon, 21 Sep 2015 18:54:57 +0800 Subject: [PATCH 13/23] handle pts epoll hup event hup means peer closes the pipe, and the exec will exit, send out all of the outputs of exec. Signed-off-by: Gao feng --- src/exec.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/exec.c b/src/exec.c index 10a275b..cdac0a3 100644 --- a/src/exec.c +++ b/src/exec.c @@ -17,6 +17,57 @@ #include "util.h" #include "parse.h" +static void pts_hup(struct hyper_event *de, int efd) +{ + + struct hyper_buf *buf = &ctl.tty.wbuf; + struct hyper_exec *exec = container_of(de, struct hyper_exec, e); + int i, len, size; + + hyper_setfd_block(de->fd); + hyper_setfd_block(ctl.tty.fd); + dprintf("%s\n", __func__); + +again: + while (buf->get + 12 < buf->size) { + size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12); + fprintf(stdout, "%s: read %d data\n", __func__, size); + if (size <= 0) { + if (errno == EINTR) + continue; + perror("read from pts failed"); + goto out; + } + + hyper_set_be64(buf->data + buf->get, exec->seq); + hyper_set_be32(buf->data + buf->get + 8, size + 12); + buf->get += size + 12; + + fprintf(stdout, "%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size); + for (i = 0; i < size; i++) + dprintf("%0x ", buf->data[i]); + } + + len = 0; + while (len < buf->get) { + size = write(ctl.tty.fd, buf->data + len, buf->get - len); + if (size <= 0) { + if (errno == EINTR) + continue; + perror("write to tty failed"); + goto out; + } + len += size; + } + + buf->get -= len; + memmove(buf->data, buf->data + len, buf->get); + goto again; +out: + hyper_setfd_nonblock(ctl.tty.fd); + return hyper_event_hup(de, efd); +} + static int pts_loop(struct hyper_event *de) { int size = 0, i; @@ -57,7 +108,7 @@ static int pts_loop(struct hyper_event *de) struct hyper_event_ops pts_ops = { .read = pts_loop, .write = hyper_event_write, - .hup = hyper_event_hup, + .hup = pts_hup, .wbuf_size = 512, /* don't need read buff, the pts data will store in tty buffer */ }; From a8362c1a70b969e0a7eb92b8269be480f6e01dc6 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 23 Sep 2015 03:13:24 +0800 Subject: [PATCH 14/23] hold the fd of container pts device in hyper init 1, Hold the the file description of pts device and ptmx in hyper-init process 2, Dup fd to the stdio of Exec cmd 3, Exec cmd exited, no EPOLLHUP event since the file is still open 4, Try to read ptmx fd before close pts fd 4, signal handler get signal SIGCHLD, close the pts fd to trigger EPOLLHUP event 5, EPOLLHUP event is received, send eof message to ttyfd. release exec. Signed-off-by: Gao feng --- src/container.c | 4 +- src/container.h | 2 +- src/event.c | 38 +++++++------ src/exec.c | 141 ++++++++++++++++++++---------------------------- src/exec.h | 4 +- src/init.c | 25 ++++----- src/parse.c | 1 + 7 files changed, 96 insertions(+), 119 deletions(-) diff --git a/src/container.c b/src/container.c index 619820d..37b447e 100644 --- a/src/container.c +++ b/src/container.c @@ -426,7 +426,7 @@ static int hyper_setup_pty(struct hyper_container *c) } int hyper_start_container(struct hyper_container *container, - int utsns, int ipcns) + int utsns, int ipcns, struct hyper_pod *pod) { int stacksize = getpagesize() * 4; struct hyper_container_arg arg = { @@ -486,7 +486,7 @@ int hyper_start_container(struct hyper_container *container, close(arg.pipe[0]); close(arg.pipe[1]); - if (hyper_watch_exec_pty(&container->exec) < 0) + if (hyper_watch_exec_pty(&container->exec, pod) < 0) fprintf(stderr, "faile to watch container pty\n"); fprintf(stdout, "container %s init pid is %d\n", container->id, pid); diff --git a/src/container.h b/src/container.h index 6b2065b..07e54c3 100644 --- a/src/container.h +++ b/src/container.h @@ -41,7 +41,7 @@ struct hyper_container { struct hyper_pod; int hyper_start_container(struct hyper_container *container, - int utsns, int ipcns); + int utsns, int ipcns, struct hyper_pod *pod); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); void hyper_cleanup_container(struct hyper_pod *pod); diff --git a/src/event.c b/src/event.c index 9edbd92..9c4585f 100644 --- a/src/event.c +++ b/src/event.c @@ -226,6 +226,27 @@ void hyper_event_hup(struct hyper_event *de, int efd) int hyper_handle_event(int efd, struct epoll_event *event) { struct hyper_event *de = event->data.ptr; + fprintf(stdout, "%s get event %d, de %p, fd %d. ops %p\n", + __func__, event->events, de, de->fd, de->ops); + + if (event->events & EPOLLIN) { + fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n", + __func__, de, de->fd, de->ops); + if (de->ops->read(de) < 0) + return -1; + } + + if (event->events & EPOLLOUT) { + fprintf(stdout, "%s event EPOLLOUT, de %p, fd %d, %p\n", + __func__, de, de->fd, de->ops); + if (de->ops->write && de->ops->write(de) < 0) + return -1; + } + + if (event->events & EPOLLERR) { + fprintf(stderr, "get epoll err of not epool in event\n"); + return -1; + } if (event->events & EPOLLHUP) { fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n", @@ -233,22 +254,7 @@ int hyper_handle_event(int efd, struct epoll_event *event) if (de->ops->hup) de->ops->hup(de, efd); return 0; - } else if (event->events & EPOLLIN) { - fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n", - __func__, de, de->fd, de->ops); - return de->ops->read(de); - } else if (event->events & EPOLLOUT) { - fprintf(stdout, "%s event EPOLLOUT, de %p, fd %d, %p\n", - __func__, de, de->fd, de->ops); - if (de->ops->write) - return de->ops->write(de); - fprintf(stderr, "warning: %p received unexpected write event\n", de); - return 0; - } else if (event->events & EPOLLERR) { - fprintf(stderr, "get epoll err of not epool in event\n"); - return -1; } - fprintf(stdout, "%s get unknown event %d\n", __func__, event->events); - return -1; + return 0; } diff --git a/src/exec.c b/src/exec.c index cdac0a3..7d7d8e1 100644 --- a/src/exec.c +++ b/src/exec.c @@ -19,53 +19,25 @@ static void pts_hup(struct hyper_event *de, int efd) { - struct hyper_buf *buf = &ctl.tty.wbuf; struct hyper_exec *exec = container_of(de, struct hyper_exec, e); - int i, len, size; + struct hyper_pod *pod = de->ptr; - hyper_setfd_block(de->fd); - hyper_setfd_block(ctl.tty.fd); dprintf("%s\n", __func__); -again: - while (buf->get + 12 < buf->size) { - size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12); - fprintf(stdout, "%s: read %d data\n", __func__, size); - if (size <= 0) { - if (errno == EINTR) - continue; - perror("read from pts failed"); - goto out; - } - + if (buf->get + 12 < buf->size) { hyper_set_be64(buf->data + buf->get, exec->seq); - hyper_set_be32(buf->data + buf->get + 8, size + 12); - buf->get += size + 12; + hyper_set_be32(buf->data + buf->get + 8, 12); + buf->get += 12; - fprintf(stdout, "%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size); - for (i = 0; i < size; i++) - dprintf("%0x ", buf->data[i]); + fprintf(stdout, "%s: seq %" PRIu64"\n", __func__, exec->seq); } - len = 0; - while (len < buf->get) { - size = write(ctl.tty.fd, buf->data + len, buf->get - len); - if (size <= 0) { - if (errno == EINTR) - continue; - perror("write to tty failed"); - goto out; - } - len += size; + if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) { + fprintf(stderr, "modify ctl tty event to in & out failed\n"); } - buf->get -= len; - memmove(buf->data, buf->data + len, buf->get); - goto again; -out: - hyper_setfd_nonblock(ctl.tty.fd); - return hyper_event_hup(de, efd); + hyper_release_exec(exec, pod); } static int pts_loop(struct hyper_event *de) @@ -74,20 +46,23 @@ static int pts_loop(struct hyper_event *de) struct hyper_buf *buf = &ctl.tty.wbuf; struct hyper_exec *exec = container_of(de, struct hyper_exec, e); - dprintf("%s\n", __func__); + fprintf(stdout, "%s\n", __func__); while (buf->get + 12 < buf->size) { size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12); - dprintf("%s: read %d data\n", __func__, size); + fprintf(stdout, "%s: read %d data\n", __func__, size); if (size <= 0) { if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EIO) + + if (errno == EAGAIN || errno == EIO) { break; + } - perror("fail to read tty fd"); - return -1; + if (size != 0) { + perror("fail to read tty fd"); + return -1; + } } - hyper_set_be64(buf->data + buf->get, exec->seq); hyper_set_be32(buf->data + buf->get + 8, size + 12); buf->get += size + 12; @@ -95,6 +70,8 @@ static int pts_loop(struct hyper_event *de) dprintf("%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size); for (i = 0; i < size; i++) dprintf("%0x ", buf->data[i]); + if (size == 0) + break; } if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) { @@ -159,84 +136,84 @@ int hyper_setup_exec_tty(struct hyper_exec *e) return -1; } - e->pty = strdup(ptmx); - fprintf(stdout, "get pty device for exec %s\n", e->pty); + e->ptyfd = open(ptmx, O_RDWR | O_NOCTTY); + fprintf(stdout, "get pty device for exec %s\n", ptmx); return 0; } int hyper_dup_exec_tty(int to, struct hyper_exec *e) { - int fd; + int fd = -1, ret = -1; char pty[128]; + fprintf(stdout, "%s\n", __func__); setsid(); if (e->seq) { - if (sprintf(pty, "/dev/pts/%d", e->ptyno) < 0) { - perror("get pts device name failed"); - return -1; - } + fd = e->ptyfd; } else { if (sprintf(pty, "/dev/null") < 0) { perror("get pts device name failed"); - return -1; + goto out; } + fd = open(pty, O_RDWR | O_NOCTTY); } - fprintf(stdout, "setup pty device %s for exec\n", pty); - - fd = open(pty, O_RDWR | O_NOCTTY); if (fd < 0) { perror("open pty device for execcmd failed"); - return -1; + goto out; } if (e->seq && (ioctl(fd, TIOCSCTTY, NULL) < 0)) { perror("ioctl pty device for execcmd failed"); - return -1; + goto out; } if (hyper_send_type_block(to, READY, 0) < 0) { - fprintf(stderr, "send ready message to hyper init failed\n"); - return -1; + fprintf(stderr, "%s send ready message failed\n", __func__); + goto out; } fflush(stdout); if (dup2(fd, STDIN_FILENO) < 0) { perror("dup tty device to stdin failed"); - close(fd); - return -1; + goto out; } if (dup2(fd, STDOUT_FILENO) < 0) { perror("dup tty device to stdout failed"); - close(fd); - return -1; + goto out; } if (dup2(fd, STDERR_FILENO) < 0) { perror("dup tty device to stderr failed"); - close(fd); - return -1; + goto out; } + ret = 0; +out: close(fd); - return 0; + return ret; } -int hyper_watch_exec_pty(struct hyper_exec *exec) +int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod) { fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n", &exec->e, &pts_ops, exec->e.fd); - if (hyper_init_event(&exec->e, &pts_ops, NULL) < 0 || + + if (exec->seq == 0) + return 0; + + if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { fprintf(stderr, "add container pts master event failed\n"); return -1; } + hyper_list_dir("/proc/1/fd/"); return 0; } @@ -332,6 +309,7 @@ static int hyper_do_exec_cmd(void *data) perror("enter pidns of pod init failed"); _exit(-1); } + close(pidns); } if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { @@ -348,12 +326,13 @@ static int hyper_do_exec_cmd(void *data) if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); - _exit(-1); + hyper_send_type_block(arg->pipe[1], ERROR, 0); + goto out; } if (hyper_send_type_block(arg->pipe[1], READY, 0) < 0) { - fprintf(stderr, "send ready message to hyper init failed\n"); - _exit(-1); + fprintf(stderr, "%s send ready message failed\n", __func__); + goto out; } fprintf(stdout, "hyper init get ready message\n"); @@ -361,14 +340,14 @@ static int hyper_do_exec_cmd(void *data) fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); list_add_tail(&exec->list, &pod->exec_head); - if (exec->seq == 0) - _exit(0); - if (hyper_watch_exec_pty(exec) < 0) { + if (hyper_watch_exec_pty(exec, pod) < 0) { fprintf(stderr, "add pts master event failed\n"); - _exit(-1); + goto out; } - +out: + close(pipe[0]); + close(pipe[1]); _exit(0); } @@ -460,7 +439,8 @@ int hyper_release_exec(struct hyper_exec *exec, int i; close(exec->e.fd); - free(exec->pty); + hyper_reset_event(&exec->e); + //close(exec->ptyfd); list_del_init(&exec->list); @@ -498,7 +478,7 @@ int hyper_release_exec(struct hyper_exec *exec, free(exec->id); for (i = 0; i < exec->argc; i++) { - fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); + //fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); free(exec->argv[i]); } @@ -543,7 +523,6 @@ int hyper_send_exec_eof(int to, struct hyper_pod *pod, int pid, uint8_t code) { struct hyper_exec *exec; - uint8_t seq[12]; exec = hyper_find_exec_by_pid(&pod->exec_head, pid); if (exec == NULL) { @@ -560,14 +539,8 @@ int hyper_send_exec_eof(int to, struct hyper_pod *pod, if (exec->seq == 0) goto out; - hyper_set_be64(seq, exec->seq); - hyper_set_be32(seq + 8, 12); - if (hyper_send_data(to, seq, 12) < 0) { - fprintf(stderr, "pod signal_loop send finishcmd failed\n"); - return -1; - } out: - hyper_release_exec(exec, pod); + close(exec->ptyfd); return 0; } diff --git a/src/exec.h b/src/exec.h index e26aab2..3de500b 100644 --- a/src/exec.h +++ b/src/exec.h @@ -8,13 +8,13 @@ struct hyper_exec { struct list_head list; struct hyper_event e; char *id; - char *pty; char **argv; int argc; uint64_t seq; int pid; int ptyno; int init; + int ptyfd; uint8_t code; }; @@ -29,7 +29,7 @@ struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid); struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq); int hyper_send_exec_eof(int to, struct hyper_pod *pod, int pid, uint8_t code); -int hyper_watch_exec_pty(struct hyper_exec *exec); +int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod); void hyper_cleanup_exec(struct hyper_pod *pod); extern struct hyper_event_ops pts_ops; diff --git a/src/init.c b/src/init.c index 1aa063e..7f60111 100644 --- a/src/init.c +++ b/src/init.c @@ -63,27 +63,23 @@ static int hyper_set_win_size(char *json, int length) return 0; } - fprintf(stdout, "find exec %s, pts %s, pid is %d, seq is %" PRIu64"\n", - exec->id ? exec->id : "pod", exec->pty, exec->pid, ws.seq); - name = exec->pty; + fprintf(stdout, "find exec %s, pid is %d, seq is %" PRIu64"\n", + exec->id ? exec->id : "pod", exec->pid, ws.seq); + fd = exec->ptyfd; } else { if (sprintf(path, "/dev/%s", ws.tty) < 0) { fprintf(stderr, "get tty device failed\n"); return -1; } - name = path; - } - - fprintf(stdout, "try to open %s\n", name); - ret = hyper_open_serial_dev(name); - if (ret < 0) { - fprintf(stderr, "cannot open %s to set term size\n", name); - goto out; + fd = hyper_open_serial_dev(name); + if (fd < 0) { + fprintf(stderr, "cannot open %s to set term size\n", name); + goto out; + } } size.ws_row = ws.row; size.ws_col = ws.column; - fd = ret; ret = ioctl(fd, TIOCSWINSZ, &size); if (ret < 0) @@ -226,7 +222,7 @@ static int hyper_handle_exit(struct hyper_pod *pod) return 0; } -static int hyper_signal_loop(struct hyper_event *de) +static int hyper_signal_loop(struct hyper_event *de, int closed) { int size; struct signalfd_siginfo sinfo; @@ -432,7 +428,7 @@ static int hyper_do_start_containers(void *data) for (i = 0; i < pod->c_num; i++) { c = &pod->c[i]; list_add_tail(&c->exec.list, &pod->exec_head); - hyper_start_container(c, utsns, ipcns); + hyper_start_container(c, utsns, ipcns, pod); } if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) { @@ -1239,6 +1235,7 @@ static int hyper_loop(void) if (hyper_handle_event(ctl.efd, &events[i]) < 0) return -1; } + hyper_modify_event(ctl.efd, &ctl.sig, EPOLLIN); } free(events); diff --git a/src/parse.c b/src/parse.c index bd61d70..9e1ac0b 100644 --- a/src/parse.c +++ b/src/parse.c @@ -576,6 +576,7 @@ realloc: if (exec == NULL) goto out; + exec->ptyfd = -1; INIT_LIST_HEAD(&exec->list); for (i = 0, j = 0; i < n; i++) { From d260dda29e53f961b6f737c0a00ea9079280ec93 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 23 Sep 2015 10:20:29 +0800 Subject: [PATCH 15/23] remove debug message Signed-off-by: Gao feng --- src/exec.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/exec.c b/src/exec.c index 7d7d8e1..f36a0b8 100644 --- a/src/exec.c +++ b/src/exec.c @@ -213,7 +213,6 @@ int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod) return -1; } - hyper_list_dir("/proc/1/fd/"); return 0; } From 88bb908c76011981940ba89733ae1811bfa65a6e Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 23 Sep 2015 17:01:20 +0800 Subject: [PATCH 16/23] do not handle hup event if get in event Should handle in event to get reamin data before close fd in hup. Signed-off-by: Gao feng --- src/event.c | 15 +++++++------- src/exec.c | 60 +++++++++++++++++++++-------------------------------- src/exec.h | 1 + src/init.c | 2 +- 4 files changed, 33 insertions(+), 45 deletions(-) diff --git a/src/event.c b/src/event.c index 9c4585f..c075801 100644 --- a/src/event.c +++ b/src/event.c @@ -229,11 +229,18 @@ int hyper_handle_event(int efd, struct epoll_event *event) fprintf(stdout, "%s get event %d, de %p, fd %d. ops %p\n", __func__, event->events, de, de->fd, de->ops); + /* do not handle hup event if have in event */ if (event->events & EPOLLIN) { fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n", __func__, de, de->fd, de->ops); if (de->ops->read(de) < 0) return -1; + } else if (event->events & EPOLLHUP) { + fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n", + __func__, de, de->fd, de->ops); + if (de->ops->hup) + de->ops->hup(de, efd); + return 0; } if (event->events & EPOLLOUT) { @@ -248,13 +255,5 @@ int hyper_handle_event(int efd, struct epoll_event *event) return -1; } - if (event->events & EPOLLHUP) { - fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n", - __func__, de, de->fd, de->ops); - if (de->ops->hup) - de->ops->hup(de, efd); - return 0; - } - return 0; } diff --git a/src/exec.c b/src/exec.c index f36a0b8..bf9fd51 100644 --- a/src/exec.c +++ b/src/exec.c @@ -19,59 +19,45 @@ static void pts_hup(struct hyper_event *de, int efd) { - struct hyper_buf *buf = &ctl.tty.wbuf; - struct hyper_exec *exec = container_of(de, struct hyper_exec, e); struct hyper_pod *pod = de->ptr; + struct hyper_exec *exec = container_of(de, struct hyper_exec, e); - dprintf("%s\n", __func__); - - if (buf->get + 12 < buf->size) { - hyper_set_be64(buf->data + buf->get, exec->seq); - hyper_set_be32(buf->data + buf->get + 8, 12); - buf->get += 12; - - fprintf(stdout, "%s: seq %" PRIu64"\n", __func__, exec->seq); - } - - if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) { - fprintf(stderr, "modify ctl tty event to in & out failed\n"); - } - + fprintf(stdout, "%s\n", __func__); hyper_release_exec(exec, pod); } static int pts_loop(struct hyper_event *de) { - int size = 0, i; + int size = -1; struct hyper_buf *buf = &ctl.tty.wbuf; struct hyper_exec *exec = container_of(de, struct hyper_exec, e); fprintf(stdout, "%s\n", __func__); - while (buf->get + 12 < buf->size) { + while ((buf->get + 12 < buf->size) && size) { size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12); fprintf(stdout, "%s: read %d data\n", __func__, size); if (size <= 0) { if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EIO) { - break; - } - - if (size != 0) { + if (errno != EAGAIN && errno != EIO) { perror("fail to read tty fd"); return -1; } + + if (!exec->exit && size != 0) + break; + + /* container task exited, No more data from pts of container, release exec */ + size = 0; + fprintf(stdout, "%s: get eof from pts of contaienr\n", __func__); } + hyper_set_be64(buf->data + buf->get, exec->seq); hyper_set_be32(buf->data + buf->get + 8, size + 12); buf->get += size + 12; dprintf("%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size); - for (i = 0; i < size; i++) - dprintf("%0x ", buf->data[i]); - if (size == 0) - break; } if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) { @@ -84,8 +70,8 @@ static int pts_loop(struct hyper_event *de) struct hyper_event_ops pts_ops = { .read = pts_loop, - .write = hyper_event_write, .hup = pts_hup, + .write = hyper_event_write, .wbuf_size = 512, /* don't need read buff, the pts data will store in tty buffer */ }; @@ -437,9 +423,16 @@ int hyper_release_exec(struct hyper_exec *exec, { int i; + if (!exec->exit) { + fprintf(stdout, "first user of exec exit\n"); + exec->exit = 1; + return 0; + } + + fprintf(stdout, "second user of exec exit, release\n"); close(exec->e.fd); + close(exec->ptyfd); hyper_reset_event(&exec->e); - //close(exec->ptyfd); list_del_init(&exec->list); @@ -530,16 +523,11 @@ int hyper_send_exec_eof(int to, struct hyper_pod *pod, return 0; } - fprintf(stdout, "%s exec pid %d, seq %" PRIu64 ", container %s\n", + fprintf(stdout, "%s exec exit pid %d, seq %" PRIu64 ", container %s\n", __func__, exec->pid, exec->seq, exec->id ? exec->id : "pod"); exec->code = code; - - if (exec->seq == 0) - goto out; - -out: - close(exec->ptyfd); + hyper_release_exec(exec, pod); return 0; } diff --git a/src/exec.h b/src/exec.h index 3de500b..ba2dba0 100644 --- a/src/exec.h +++ b/src/exec.h @@ -16,6 +16,7 @@ struct hyper_exec { int init; int ptyfd; uint8_t code; + uint8_t exit; }; struct hyper_pod; diff --git a/src/init.c b/src/init.c index 7f60111..068dffd 100644 --- a/src/init.c +++ b/src/init.c @@ -222,7 +222,7 @@ static int hyper_handle_exit(struct hyper_pod *pod) return 0; } -static int hyper_signal_loop(struct hyper_event *de, int closed) +static int hyper_signal_loop(struct hyper_event *de) { int size; struct signalfd_siginfo sinfo; From 749387bbfac88f884dff2b6fac708f5164809f2b Mon Sep 17 00:00:00 2001 From: Gao feng Date: Wed, 23 Sep 2015 23:07:22 +0800 Subject: [PATCH 17/23] fix possible missing eof data It happend that pts_loop reads all of the data before the, SIGCHLD signal being handled, so only hup handler will be triggered, move the sending eof message to hup handler. Signed-off-by: Gao feng --- src/exec.c | 21 +++++++++++++++------ src/init.c | 1 - 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/exec.c b/src/exec.c index bf9fd51..ce56fbe 100644 --- a/src/exec.c +++ b/src/exec.c @@ -20,9 +20,23 @@ static void pts_hup(struct hyper_event *de, int efd) { struct hyper_pod *pod = de->ptr; + struct hyper_buf *buf = &ctl.tty.wbuf; struct hyper_exec *exec = container_of(de, struct hyper_exec, e); fprintf(stdout, "%s\n", __func__); + + if (buf->get + 12 > buf->size) { + fprintf(stdout, "%s: tty buf full\n", __func__); + return; + } + + /* no in event, no more data, send eof */ + hyper_set_be64(buf->data + buf->get, exec->seq); + hyper_set_be32(buf->data + buf->get + 8, 12); + buf->get += 12; + + hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT); + hyper_release_exec(exec, pod); } @@ -45,12 +59,7 @@ static int pts_loop(struct hyper_event *de) return -1; } - if (!exec->exit && size != 0) - break; - - /* container task exited, No more data from pts of container, release exec */ - size = 0; - fprintf(stdout, "%s: get eof from pts of contaienr\n", __func__); + break; } hyper_set_be64(buf->data + buf->get, exec->seq); diff --git a/src/init.c b/src/init.c index 068dffd..7cdbbc0 100644 --- a/src/init.c +++ b/src/init.c @@ -1235,7 +1235,6 @@ static int hyper_loop(void) if (hyper_handle_event(ctl.efd, &events[i]) < 0) return -1; } - hyper_modify_event(ctl.efd, &ctl.sig, EPOLLIN); } free(events); From 08764d5bbc8c651832e431d3deacc708b64c2e00 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 24 Sep 2015 10:34:12 +0800 Subject: [PATCH 18/23] fix incorrect logic of hyper_send_type_block It means set fd block first and send out the type. And fix the access to exec consurrently, hyper init should be blocked until do_exec_cmd processes the exec struct. Signed-off-by: Gao feng --- src/exec.c | 13 +++++++------ src/net.c | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/exec.c b/src/exec.c index ce56fbe..a8170dc 100644 --- a/src/exec.c +++ b/src/exec.c @@ -284,7 +284,7 @@ static int hyper_do_exec_cmd(void *data) struct hyper_exec_arg *arg = data; struct hyper_exec *exec = arg->exec; struct hyper_pod *pod = arg->pod; - int pipe[2], pid; + int pipe[2] = {-1, -1}, pid; if (exec->id) { char path[512]; @@ -324,11 +324,6 @@ static int hyper_do_exec_cmd(void *data) goto out; } - if (hyper_send_type_block(arg->pipe[1], READY, 0) < 0) { - fprintf(stderr, "%s send ready message failed\n", __func__); - goto out; - } - fprintf(stdout, "hyper init get ready message\n"); exec->pid = pid; fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); @@ -339,6 +334,11 @@ static int hyper_do_exec_cmd(void *data) fprintf(stderr, "add pts master event failed\n"); goto out; } + + if (hyper_send_type_block(arg->pipe[1], READY, 0) < 0) { + fprintf(stderr, "%s send ready message failed\n", __func__); + goto out; + } out: close(pipe[0]); close(pipe[1]); @@ -419,6 +419,7 @@ int hyper_exec_cmd(char *json, int length) return -1; } + fprintf(stdout, "%s get ready message %"PRIu32 "\n", __func__, type); ret = 0; out: close(arg.pipe[0]); diff --git a/src/net.c b/src/net.c index ff1f3c6..4ece161 100644 --- a/src/net.c +++ b/src/net.c @@ -90,7 +90,7 @@ int hyper_send_type(int fd, uint32_t type) return hyper_send_msg(fd, type, 0, NULL); } -int hyper_get_type_block(int fd, uint32_t *type) +int hyper_get_type(int fd, uint32_t *type) { int len = 0, size; uint8_t buf[8]; @@ -111,6 +111,34 @@ int hyper_get_type_block(int fd, uint32_t *type) return 0; } +int hyper_get_type_block(int fd, uint32_t *type) +{ + int ret = 0, flags; + + flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) { + fprintf(stderr, "%s get fd flag failed\n", __func__); + return -1; + } + + if (fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0) { + perror("set fd BLOCK failed"); + return -1; + } + + ret = hyper_get_type(fd, type); + if (ret < 0) { + fprintf(stderr, "%s can not get type\n", __func__); + } + + if (fcntl(fd, F_SETFL, flags) < 0) { + perror("restore fd flag failed"); + return -1; + } + + return ret; +} + int hyper_send_type_block(int fd, uint32_t type, int need_ack) { int ret = 0, flags; @@ -134,7 +162,7 @@ int hyper_send_type_block(int fd, uint32_t type, int need_ack) if (need_ack == 0) goto out; - ret = hyper_get_type_block(fd, &t); + ret = hyper_get_type(fd, &t); if (ret < 0) { fprintf(stderr, "can not get type\n"); goto out; @@ -145,8 +173,8 @@ int hyper_send_type_block(int fd, uint32_t type, int need_ack) if (t != ACK) ret = -1; out: - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - perror("set fd BLOCK failed"); + if (fcntl(fd, F_SETFL, flags) < 0) { + perror("restore fd flag failed"); return -1; } From 600e8dd3c860839339a0123a868f2940ebba1f15 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 24 Sep 2015 10:55:57 +0800 Subject: [PATCH 19/23] release exec immediately if exec has no tty Only signal handler operates the exex struct, we can release exec immediately Signed-off-by: Gao feng --- src/exec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/exec.c b/src/exec.c index a8170dc..990efbc 100644 --- a/src/exec.c +++ b/src/exec.c @@ -433,13 +433,14 @@ int hyper_release_exec(struct hyper_exec *exec, { int i; - if (!exec->exit) { + if (!exec->exit && exec->seq) { fprintf(stdout, "first user of exec exit\n"); exec->exit = 1; return 0; } - fprintf(stdout, "second user of exec exit, release\n"); + /* exec has no pty or the pty user already exited */ + fprintf(stdout, "last user of exec exit, release\n"); close(exec->e.fd); close(exec->ptyfd); hyper_reset_event(&exec->e); From 5c3bd1b19f9096a56881e0f6a12473391b9eac3a Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 24 Sep 2015 11:02:45 +0800 Subject: [PATCH 20/23] initialize ptyfd and eventfd for exec initialize them to -1 Signed-off-by: Gao feng --- src/parse.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/parse.c b/src/parse.c index 9e1ac0b..9d5bffa 100644 --- a/src/parse.c +++ b/src/parse.c @@ -199,6 +199,8 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container * c->exec.init = 1; c->exec.code = -1; + c->exec.e.fd = -1; + c->exec.ptyfd = -1; c->ns = -1; next_container = toks[i].size; @@ -577,6 +579,7 @@ realloc: goto out; exec->ptyfd = -1; + exec->e.fd = -1; INIT_LIST_HEAD(&exec->list); for (i = 0, j = 0; i < n; i++) { From d45be919cd10a5d7c7ccc5a3ea8fd2d1b8fee493 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 24 Sep 2015 14:38:46 +0800 Subject: [PATCH 21/23] change socketpair to pipe If one pair of pipe created by socketpair closed, the other side will receice error, so other side cannot know the exec result of peer. Use pipe to replace socketpair. Signed-off-by: Gao feng --- src/container.c | 6 +++--- src/exec.c | 54 +++++++++++++++++++++++++------------------------ src/init.c | 43 ++++++++++++++++++++------------------- src/net.h | 1 + 4 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/container.c b/src/container.c index 37b447e..dec8703 100644 --- a/src/container.c +++ b/src/container.c @@ -396,7 +396,7 @@ static int hyper_container_init(void *data) fail: container->exec.code = -1; - hyper_send_type_block(arg->pipe[1], ERROR, 0); + hyper_send_type(arg->pipe[1], ERROR); _exit(-1); } @@ -452,7 +452,7 @@ int hyper_start_container(struct hyper_container *container, goto fail; } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); goto fail; } @@ -478,7 +478,7 @@ int hyper_start_container(struct hyper_container *container, } /* wait for ready message */ - if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "wait for container started failed\n"); goto fail; } diff --git a/src/exec.c b/src/exec.c index 990efbc..6fea52e 100644 --- a/src/exec.c +++ b/src/exec.c @@ -165,12 +165,8 @@ int hyper_dup_exec_tty(int to, struct hyper_exec *e) goto out; } - if (hyper_send_type_block(to, READY, 0) < 0) { - fprintf(stderr, "%s send ready message failed\n", __func__); - goto out; - } - fflush(stdout); + hyper_send_type(to, READY); if (dup2(fd, STDIN_FILENO) < 0) { perror("dup tty device to stdin failed"); @@ -285,6 +281,7 @@ static int hyper_do_exec_cmd(void *data) struct hyper_exec *exec = arg->exec; struct hyper_pod *pod = arg->pod; int pipe[2] = {-1, -1}, pid; + int ret = -1; if (exec->id) { char path[512]; @@ -294,33 +291,33 @@ static int hyper_do_exec_cmd(void *data) pidns = open(path, O_RDONLY| O_CLOEXEC); if (pidns < 0) { perror("fail to open pidns of pod init"); - _exit(-1); + goto out; } /* enter pidns of pod init, so the children of this process will run in * pidns of pod init, see man 2 setns */ if (setns(pidns, CLONE_NEWPID) < 0) { perror("enter pidns of pod init failed"); - _exit(-1); + goto out; } close(pidns); } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { + if (pipe2(pipe, O_CLOEXEC) < 0) { perror("create pipe in exec command failed"); - _exit(-1); + goto out; } pid = fork(); if (pid < 0) { perror("fail to fork"); - _exit(-1); + goto out; } else if (pid > 0) { uint32_t type; - if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { + if (hyper_get_type(pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); - hyper_send_type_block(arg->pipe[1], ERROR, 0); + hyper_send_type(arg->pipe[1], ERROR); goto out; } @@ -335,32 +332,35 @@ static int hyper_do_exec_cmd(void *data) goto out; } - if (hyper_send_type_block(arg->pipe[1], READY, 0) < 0) { - fprintf(stderr, "%s send ready message failed\n", __func__); - goto out; - } -out: - close(pipe[0]); - close(pipe[1]); - _exit(0); + ret = 0; + goto out; } if (exec->id && hyper_enter_container(pod, exec) < 0) { fprintf(stderr, "enter container ns failed\n"); - _exit(-1); + goto exit; } if (hyper_dup_exec_tty(pipe[1], exec) < 0) { fprintf(stderr, "dup pts to exec stdio failed\n"); - _exit(-1); + goto exit; } if (execvp(exec->argv[0], exec->argv) < 0) { perror("exec failed"); - _exit(-1); + goto exit; } - _exit(0); + ret = 0; +exit: + hyper_send_type(pipe[1], ERROR); + _exit(ret); + +out: + hyper_send_type(arg->pipe[1], ret ? ERROR : READY); + close(pipe[0]); + close(pipe[1]); + _exit(ret); } int hyper_exec_cmd(char *json, int length) @@ -401,7 +401,7 @@ int hyper_exec_cmd(char *json, int length) goto out; } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); goto out; } @@ -414,7 +414,7 @@ int hyper_exec_cmd(char *json, int length) goto out; } - if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); return -1; } @@ -436,6 +436,8 @@ int hyper_release_exec(struct hyper_exec *exec, if (!exec->exit && exec->seq) { fprintf(stdout, "first user of exec exit\n"); exec->exit = 1; + close(exec->ptyfd); + exec->ptyfd = -1; return 0; } diff --git a/src/init.c b/src/init.c index 7cdbbc0..dfd6a96 100644 --- a/src/init.c +++ b/src/init.c @@ -431,13 +431,13 @@ static int hyper_do_start_containers(void *data) hyper_start_container(c, utsns, ipcns, pod); } - if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) { + ret = 0; +out: + if (hyper_send_type(arg->ctl_pipe[1], ret ? ERROR : READY) < 0) { fprintf(stderr, "container init send ready message failed\n"); goto out; } - ret = 0; -out: close(pidns); close(utsns); close(ipcns); @@ -463,7 +463,7 @@ int hyper_start_containers(struct hyper_pod *pod) goto out; } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { + if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) { perror("create pipe between hyper init and pod init failed"); goto out; } @@ -476,7 +476,7 @@ int hyper_start_containers(struct hyper_pod *pod) } /* Wait for container start */ - if (hyper_get_type_block(arg.ctl_pipe[0], &type) < 0) { + if (hyper_get_type(arg.ctl_pipe[0], &type) < 0) { perror("get enter_container_pidns ready message failed"); goto out; } @@ -728,7 +728,7 @@ static int hyper_cmd_write_file(char *json, int length) int pipe[2] = {-1, -1}; int pid, mntns = -1, fd; char path[512]; - int len = 0, size; + int len = 0, size, ret = -1; fprintf(stdout, "%s\n", __func__); memset(&writter, 0, sizeof(writter)); @@ -743,7 +743,7 @@ static int hyper_cmd_write_file(char *json, int length) goto out; } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { + if (pipe2(pipe, O_CLOEXEC) < 0) { perror("create writter pipe failed"); goto out; } @@ -761,17 +761,18 @@ static int hyper_cmd_write_file(char *json, int length) } else if (pid > 0) { uint32_t type; - if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { + if (hyper_get_type(pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "get incorrect message type %d, expect READY\n", type); goto out; } + ret = 0; goto out; } if (setns(mntns, CLONE_NEWNS) < 0) { perror("fail to enter container ns"); - _exit(-1); + goto exit; } sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); @@ -780,13 +781,13 @@ static int hyper_cmd_write_file(char *json, int length) /* TODO: wait for container finishing setup root */ if (chroot(path) < 0) { perror("chroot for exec command failed"); - _exit(-1); + goto exit; } fd = open(writter.file, O_CREAT| O_WRONLY, 0644); if (fd < 0) { perror("fail to open target file"); - _exit(-1); + goto exit; } while(len < writter.len) { @@ -797,13 +798,14 @@ static int hyper_cmd_write_file(char *json, int length) continue; perror("fail to write data to file"); - _exit(-1); + goto exit; } len += size; } - - hyper_send_type(pipe[1], READY); + ret = 0; +exit: + hyper_send_type(pipe[1], ret ? ERROR : READY); _exit(0); out: close(pipe[0]); @@ -827,7 +829,7 @@ struct hyper_file_arg { static int hyper_do_cmd_read_file(void *data) { struct stat st; - int len = 0, size, fd; + int len = 0, size, fd, ret = -1; struct hyper_file_arg *arg = data; if (setns(arg->mntns, CLONE_NEWNS) < 0) { @@ -877,11 +879,10 @@ static int hyper_do_cmd_read_file(void *data) } fprintf(stdout, "read data %s\n", *arg->data); - hyper_send_type(arg->pipe[1], READY); - return 0; + ret = 0; err: - hyper_send_type(arg->pipe[1], ERROR); - return -1; + hyper_send_type(arg->pipe[1], ret ? ERROR : READY); + return ret; } static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_t **data) @@ -915,7 +916,7 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ goto out; } - if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create reader pipe failed"); goto out; } @@ -938,7 +939,7 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ goto out; } - if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "%s to incorrect type %" PRIu32 "\n", __func__, type); goto out; } diff --git a/src/net.h b/src/net.h index af8450f..e096f21 100644 --- a/src/net.h +++ b/src/net.h @@ -50,6 +50,7 @@ int hyper_setup_network(struct hyper_pod *pod); void hyper_cleanup_network(struct hyper_pod *pod); int hyper_setup_dns(struct hyper_pod *pod); void hyper_cleanup_dns(struct hyper_pod *pod); +int hyper_get_type(int fd, uint32_t *type); int hyper_get_type_block(int fd, uint32_t *type); int hyper_send_type(int fd, uint32_t type); int hyper_send_type_block(int fd, uint32_t type, int need_ack); From 684e85cc5ebf12cbacc5a72dd24c16aa2a1ca4c1 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Thu, 24 Sep 2015 16:58:42 +0800 Subject: [PATCH 22/23] recyle resources of process which created by clone Signed-off-by: Gao feng --- src/exec.c | 4 +++- src/init.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/exec.c b/src/exec.c index 6fea52e..e0a04f3 100644 --- a/src/exec.c +++ b/src/exec.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include "syscall.h" @@ -407,7 +408,8 @@ int hyper_exec_cmd(char *json, int length) } arg.exec = exec; - pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); + pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); + fprintf(stdout, "do_exec_cmd pid %d\n", pid); free(stack); if (pid < 0) { perror("clone hyper_do_exec_cmd failed"); diff --git a/src/init.c b/src/init.c index dfd6a96..7893c08 100644 --- a/src/init.c +++ b/src/init.c @@ -468,7 +468,7 @@ int hyper_start_containers(struct hyper_pod *pod) goto out; } - pid = clone(hyper_do_start_containers, stack + stacksize, CLONE_VM| CLONE_FILES, &arg); + pid = clone(hyper_do_start_containers, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); free(stack); if (pid < 0) { perror("enter container pid ns failed"); @@ -932,7 +932,7 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ arg.data = data; sprintf(arg.root, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); - pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM, &arg); + pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGCHLD, &arg); free(stack); if (pid < 0) { perror("fail to fork writter process"); From f0d9fc7bcec1fe43426101fdb72c1d8bd49e5483 Mon Sep 17 00:00:00 2001 From: Gao feng Date: Fri, 25 Sep 2015 09:17:45 +0800 Subject: [PATCH 23/23] fix possible memory leak Signed-off-by: Gao feng --- src/exec.c | 15 +++++++------- src/init.c | 58 +++++++++++++++++++++++++++++++----------------------- src/util.c | 10 ++++++---- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/exec.c b/src/exec.c index e0a04f3..ae88c6e 100644 --- a/src/exec.c +++ b/src/exec.c @@ -369,8 +369,7 @@ int hyper_exec_cmd(char *json, int length) struct hyper_exec *exec; struct hyper_pod *pod = &global_pod; int stacksize = getpagesize() * 4; - void *stack = malloc(stacksize); - struct hyper_exec_arg arg = { + void *stack = NULL; struct hyper_exec_arg arg = { .pod = pod, .exec = NULL, .pipe = {-1, -1}, @@ -392,11 +391,6 @@ int hyper_exec_cmd(char *json, int length) goto out; } - if (stack == NULL) { - perror("fail to allocate stack for container init"); - goto out; - } - if (hyper_setup_exec_tty(exec) < 0) { fprintf(stderr, "setup exec tty failed\n"); goto out; @@ -408,6 +402,13 @@ int hyper_exec_cmd(char *json, int length) } arg.exec = exec; + + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); fprintf(stdout, "do_exec_cmd pid %d\n", pid); free(stack); diff --git a/src/init.c b/src/init.c index 7893c08..cdb793c 100644 --- a/src/init.c +++ b/src/init.c @@ -450,7 +450,7 @@ out: int hyper_start_containers(struct hyper_pod *pod) { int stacksize = getpagesize() * 4; - void *stack = malloc(stacksize); + void *stack = NULL; struct hyper_pod_arg arg = { .pod = pod, .ctl_pipe = {-1, -1}, @@ -458,16 +458,18 @@ int hyper_start_containers(struct hyper_pod *pod) int ret = -1, pid; uint32_t type; - if (stack == NULL) { - perror("fail to allocate stack for container init"); - goto out; - } if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) { perror("create pipe between hyper init and pod init failed"); goto out; } + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + pid = clone(hyper_do_start_containers, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); free(stack); if (pid < 0) { @@ -515,16 +517,17 @@ static int hyper_setup_container(struct hyper_pod *pod) uint32_t type; void *stack; + int ret = -1; if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { perror("create pipe between hyper init and pod init failed"); - return -1; + goto out; } stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); - return -1; + goto out; } arg.pod = pod; @@ -533,41 +536,45 @@ static int hyper_setup_container(struct hyper_pod *pod) free(stack); if (pod->init_pid < 0) { perror("create container init process failed"); - return -1; + goto out; } fprintf(stdout, "pod init pid %d\n", pod->init_pid); - close(arg.ctl_pipe[1]); - ctl.ctl.fd = arg.ctl_pipe[0]; - /* Wait for container start */ - if (hyper_get_type_block(ctl.ctl.fd, &type) < 0) { + if (hyper_get_type_block(arg.ctl_pipe[0], &type) < 0) { perror("get container init ready message failed"); - return -1; + goto out; } if (type != READY) { fprintf(stderr, "get incorrect message type %d, expect READY\n", type); - return -1; + goto out; } if (hyper_start_containers(pod) < 0) { fprintf(stderr, "start containers failed\n"); - return -1; + goto out; } - if (hyper_setfd_cloexec(ctl.ctl.fd) < 0) { + if (hyper_setfd_cloexec(arg.ctl_pipe[0]) < 0) { perror("set ctl pipe fd FD_CLOEXEC failed"); - return -1; + goto out; } + ctl.ctl.fd = arg.ctl_pipe[0]; fprintf(stdout, "hyper_init_event hyper ctl pipe fd %d\n", ctl.ctl.fd); if (hyper_init_event(&ctl.ctl, &hyper_ctl_pipe_ops, pod) < 0 || hyper_add_event(ctl.efd, &ctl.ctl, EPOLLIN) < 0) { - return -1; + goto out; } - return 0; + ret = 0; +out: + close(arg.ctl_pipe[1]); + if (ret < 0) { + close(arg.ctl_pipe[0]); + } + return ret; } #ifdef WITH_VBOX @@ -894,15 +901,10 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ .pipe = {-1, -1}, }; int stacksize = getpagesize() * 4; - void *stack = malloc(stacksize); + void *stack = NULL; int pid, ret = -1; uint32_t type; - if (stack == NULL) { - perror("fail to allocate stack for container init"); - goto out; - } - fprintf(stdout, "%s\n", __func__); memset(&reader, 0, sizeof(reader)); @@ -932,6 +934,12 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_ arg.data = data; sprintf(arg.root, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGCHLD, &arg); free(stack); if (pid < 0) { diff --git a/src/util.c b/src/util.c index adfdd29..57c5ca1 100644 --- a/src/util.c +++ b/src/util.c @@ -57,6 +57,7 @@ int hyper_list_dir(char *path) for (i = 0; i < num; i++) { dir = list[i]; fprintf(stdout, "%s get %s\n", path, dir->d_name); + free(dir); } free(list); @@ -192,8 +193,7 @@ int hyper_insmod(char *module) ret = 0; out: close(fd); - if (buf) - free(buf); + free(buf); return ret; err: @@ -426,13 +426,15 @@ void hyper_kill_all(void) int hyper_send_finish(struct hyper_pod *pod) { - int i; + int i, ret; uint8_t *data = calloc(pod->c_num, 4); for (i = 0; i < pod->c_num; i++) hyper_set_be32(data + (i * 4), pod->c[i].exec.code); - return hyper_send_msg(ctl.chan.fd, FINISH, pod->c_num * 4, data); + ret = hyper_send_msg(ctl.chan.fd, FINISH, pod->c_num * 4, data); + free(data); + return ret; } void hyper_shutdown(struct hyper_pod *pod)