diff --git a/src/container.c b/src/container.c index 113585f..dec8703 100644 --- a/src/container.c +++ b/src/container.c @@ -17,20 +17,6 @@ #include "util.h" #include "hyper.h" -static int container_setup_env(struct hyper_container *container) -{ - int i; - struct env *env; - - for (i = 0; i < container->envs_num; i++) { - env = &container->envs[i]; - - setenv(env->env, env->value, 1); - } - - return 0; -} - static int container_setup_volume(struct hyper_container *container) { int i; @@ -276,6 +262,8 @@ static int hyper_rescan_scsi(void) struct hyper_container_arg { struct hyper_container *c; + int ipcns; + int utsns; int pipe[2]; }; @@ -291,12 +279,22 @@ static int hyper_container_init(void *data) goto fail; } + if (setns(arg->ipcns, CLONE_NEWIPC) < 0) { + perror("setns to ipcns of pod init faild"); + goto fail; + } + + if (setns(arg->utsns, CLONE_NEWUTS) < 0) { + perror("setns to ipcns of pod init faild"); + goto fail; + } + if (hyper_rescan_scsi() < 0) { fprintf(stdout, "rescan scsi failed\n"); goto fail; } - if (container_setup_env(container) < 0) { + if (hyper_setup_env(container->envs, container->envs_num) < 0) { fprintf(stdout, "setup env failed\n"); goto fail; } @@ -391,9 +389,6 @@ static int hyper_container_init(void *data) goto fail; } - close(arg->pipe[0]); - close(arg->pipe[1]); - execvp(container->exec.argv[0], container->exec.argv); perror("exec container command failed"); @@ -401,17 +396,47 @@ static int hyper_container_init(void *data) fail: container->exec.code = -1; - hyper_send_type_block(arg->pipe[1], ERROR, 0); + hyper_send_type(arg->pipe[1], ERROR); _exit(-1); } -int hyper_start_container(struct hyper_container *container) +static int hyper_setup_pty(struct hyper_container *c) +{ + char root[512]; + + sprintf(root, "/tmp/hyper/%s/devpts/", c->id); + + if (hyper_mkdir(root) < 0) { + perror("make container pts directroy failed"); + return -1; + } + + if (mount("devpts", root, "devpts", MS_NOSUID, + "newinstance,ptmxmode=0666,mode=0620") < 0) { + perror("mount devpts failed"); + return -1; + } + + if (hyper_setup_exec_tty(&c->exec) < 0) { + fprintf(stderr, "setup container pts failed\n"); + return -1; + } + + return 0; +} + +int hyper_start_container(struct hyper_container *container, + int utsns, int ipcns, struct hyper_pod *pod) { int stacksize = getpagesize() * 4; struct hyper_container_arg arg = { - .c = container, + .c = container, + .utsns = utsns, + .ipcns = ipcns, + .pipe = {-1, -1}, }; int flags = CLONE_NEWNS | SIGCHLD; + char path[128]; uint32_t type; void *stack; int pid; @@ -422,7 +447,12 @@ int hyper_start_container(struct hyper_container *container) goto fail; } - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) { + if (hyper_setup_pty(container) < 0) { + fprintf(stderr, "setup pty device for container failed\n"); + goto fail; + } + + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); goto fail; } @@ -430,7 +460,7 @@ int hyper_start_container(struct hyper_container *container) stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); - return -1; + goto fail; } pid = clone(hyper_container_init, stack + stacksize, flags, &arg); @@ -439,75 +469,38 @@ int hyper_start_container(struct hyper_container *container) perror("create child process failed"); goto fail; } - + sprintf(path, "/proc/%d/ns/mnt", pid); container->exec.pid = pid; + container->ns = open(path, O_RDONLY | O_CLOEXEC); + if (container->ns < 0) { + perror("open container mount ns failed"); + goto fail; + } /* wait for ready message */ - if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) { - fprintf(stdout, "wait for container started failed\n"); + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "wait for container started failed\n"); goto fail; } close(arg.pipe[0]); close(arg.pipe[1]); + if (hyper_watch_exec_pty(&container->exec, pod) < 0) + fprintf(stderr, "faile to watch container pty\n"); + fprintf(stdout, "container %s init pid is %d\n", container->id, pid); return 0; - fail: + close(arg.pipe[0]); + close(arg.pipe[1]); + close(container->ns); + container->ns = -1; fprintf(stdout, "container %s init exit code %d\n", container->id, -1); - container->exec.code = -1; return -1; } -int hyper_start_containers(struct hyper_pod *pod) -{ - int i; - - /* mount new proc directory */ - if (umount("/proc") < 0) { - perror("umount proc filesystem failed\n"); - return -1; - } - - if (mount("proc", "/proc", "proc", 0, NULL) < 0) { - perror("mount proc filesystem failed\n"); - return -1; - } - - if (sethostname(pod->hostname, strlen(pod->hostname)) < 0) { - perror("set host name failed"); - return -1; - } - - for (i = 0; i < pod->c_num; i++) - hyper_start_container(&pod->c[i]); - - return 0; -} - -int hyper_restart_containers(struct hyper_pod *pod) -{ - int i; - struct hyper_container *c; - - for (i = 0; i < pod->c_num; i++) { - c = &pod->c[i]; - - if (hyper_start_container(c) < 0) { - fprintf(stderr, "restart container %s failed\n", c->id); - hyper_send_type(pod->ctl.fd, ERROR); - return -1; - } - } - - if (hyper_send_type(pod->ctl.fd, ACK) < 0) - return -1; - - return 0; -} - struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id) { int i; @@ -571,6 +564,7 @@ void hyper_cleanup_container(struct hyper_pod *pod) free(map->path); } free(c->maps); + close(c->ns); } free(pod->c); diff --git a/src/container.h b/src/container.h index dc95f8c..07e54c3 100644 --- a/src/container.h +++ b/src/container.h @@ -33,15 +33,16 @@ struct hyper_container { int vols_num; int envs_num; int maps_num; + int ns; uint32_t code; struct hyper_exec exec; }; struct hyper_pod; -int hyper_start_containers(struct hyper_pod *pod); +int hyper_start_container(struct hyper_container *container, + int utsns, int ipcns, struct hyper_pod *pod); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); -int hyper_restart_containers(struct hyper_pod *pod); void hyper_cleanup_container(struct hyper_pod *pod); #endif diff --git a/src/event.c b/src/event.c index 9edbd92..c075801 100644 --- a/src/event.c +++ b/src/event.c @@ -226,29 +226,34 @@ void hyper_event_hup(struct hyper_event *de, int efd) int hyper_handle_event(int efd, struct epoll_event *event) { struct hyper_event *de = event->data.ptr; + fprintf(stdout, "%s get event %d, de %p, fd %d. ops %p\n", + __func__, event->events, de, de->fd, de->ops); - if (event->events & EPOLLHUP) { + /* do not handle hup event if have in event */ + if (event->events & EPOLLIN) { + fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n", + __func__, de, de->fd, de->ops); + if (de->ops->read(de) < 0) + return -1; + } else if (event->events & EPOLLHUP) { fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n", __func__, de, de->fd, de->ops); if (de->ops->hup) de->ops->hup(de, efd); return 0; - } else if (event->events & EPOLLIN) { - fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n", - __func__, de, de->fd, de->ops); - return de->ops->read(de); - } else if (event->events & EPOLLOUT) { + } + + if (event->events & EPOLLOUT) { fprintf(stdout, "%s event EPOLLOUT, de %p, fd %d, %p\n", __func__, de, de->fd, de->ops); - if (de->ops->write) - return de->ops->write(de); - fprintf(stderr, "warning: %p received unexpected write event\n", de); - return 0; - } else if (event->events & EPOLLERR) { + if (de->ops->write && de->ops->write(de) < 0) + return -1; + } + + if (event->events & EPOLLERR) { fprintf(stderr, "get epoll err of not epool in event\n"); return -1; } - fprintf(stdout, "%s get unknown event %d\n", __func__, event->events); - return -1; + return 0; } diff --git a/src/exec.c b/src/exec.c index 122b795..ae88c6e 100644 --- a/src/exec.c +++ b/src/exec.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include "syscall.h" @@ -17,24 +18,49 @@ #include "util.h" #include "parse.h" -static int pts_loop(struct hyper_event *de) +static void pts_hup(struct hyper_event *de, int efd) { - int size = 0, i; + struct hyper_pod *pod = de->ptr; struct hyper_buf *buf = &ctl.tty.wbuf; struct hyper_exec *exec = container_of(de, struct hyper_exec, e); - dprintf("%s\n", __func__); - while (buf->get + 12 < buf->size) { + fprintf(stdout, "%s\n", __func__); + + if (buf->get + 12 > buf->size) { + fprintf(stdout, "%s: tty buf full\n", __func__); + return; + } + + /* no in event, no more data, send eof */ + hyper_set_be64(buf->data + buf->get, exec->seq); + hyper_set_be32(buf->data + buf->get + 8, 12); + buf->get += 12; + + hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT); + + hyper_release_exec(exec, pod); +} + +static int pts_loop(struct hyper_event *de) +{ + int size = -1; + struct hyper_buf *buf = &ctl.tty.wbuf; + struct hyper_exec *exec = container_of(de, struct hyper_exec, e); + + fprintf(stdout, "%s\n", __func__); + while ((buf->get + 12 < buf->size) && size) { size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12); - dprintf("%s: read %d data\n", __func__, size); + fprintf(stdout, "%s: read %d data\n", __func__, size); if (size <= 0) { if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EIO) - break; - perror("fail to read tty fd"); - return -1; + if (errno != EAGAIN && errno != EIO) { + perror("fail to read tty fd"); + return -1; + } + + break; } hyper_set_be64(buf->data + buf->get, exec->seq); @@ -42,8 +68,6 @@ static int pts_loop(struct hyper_event *de) buf->get += size + 12; dprintf("%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size); - for (i = 0; i < size; i++) - dprintf("%0x ", buf->data[i]); } if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) { @@ -56,8 +80,8 @@ static int pts_loop(struct hyper_event *de) struct hyper_event_ops pts_ops = { .read = pts_loop, + .hup = pts_hup, .write = hyper_event_write, - .hup = hyper_event_hup, .wbuf_size = 512, /* don't need read buff, the pts data will store in tty buffer */ }; @@ -108,335 +132,323 @@ int hyper_setup_exec_tty(struct hyper_exec *e) return -1; } - e->pty = strdup(ptmx); - fprintf(stdout, "get pty device for exec %s\n", e->pty); + e->ptyfd = open(ptmx, O_RDWR | O_NOCTTY); + fprintf(stdout, "get pty device for exec %s\n", ptmx); return 0; } int hyper_dup_exec_tty(int to, struct hyper_exec *e) { - int fd; + int fd = -1, ret = -1; char pty[128]; + fprintf(stdout, "%s\n", __func__); setsid(); if (e->seq) { - if (sprintf(pty, "/dev/pts/%d", e->ptyno) < 0) { - perror("get pts device name failed"); - return -1; - } + fd = e->ptyfd; } else { if (sprintf(pty, "/dev/null") < 0) { perror("get pts device name failed"); - return -1; + goto out; } + fd = open(pty, O_RDWR | O_NOCTTY); } - fprintf(stdout, "setup pty device %s for exec\n", pty); - - fd = open(pty, O_RDWR | O_NOCTTY); if (fd < 0) { perror("open pty device for execcmd failed"); - return -1; + goto out; } if (e->seq && (ioctl(fd, TIOCSCTTY, NULL) < 0)) { perror("ioctl pty device for execcmd failed"); - return -1; - } - - if (hyper_send_type_block(to, READY, 0) < 0) { - fprintf(stderr, "send ready message to hyper init failed\n"); - return -1; + goto out; } fflush(stdout); + hyper_send_type(to, READY); if (dup2(fd, STDIN_FILENO) < 0) { perror("dup tty device to stdin failed"); - close(fd); - return -1; + goto out; } if (dup2(fd, STDOUT_FILENO) < 0) { perror("dup tty device to stdout failed"); - close(fd); - return -1; + goto out; } if (dup2(fd, STDERR_FILENO) < 0) { perror("dup tty device to stderr failed"); - close(fd); - return -1; + goto out; } + ret = 0; +out: close(fd); - return 0; + return ret; } -int hyper_exec_in_container(struct hyper_pod *pod, - struct hyper_exec *exec) +int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod) { - global_exec = exec; + fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n", + &exec->e, &pts_ops, exec->e.fd); - if (hyper_send_type_block(ctl.ctl.fd, EXECCMD, 1) < 0) { - global_exec = NULL; - fprintf(stderr, "tell pod init EXECCMD failed\n"); - return -1; - } - - list_add_tail(&exec->list, &pod->ce_head); if (exec->seq == 0) return 0; - fprintf(stdout, "init container exec pts event %p, ops %p, fd %d\n", - &exec->e, &pts_ops, exec->e.fd); - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { - fprintf(stderr, "add pts master event failed\n"); + fprintf(stderr, "add container pts master event failed\n"); return -1; } return 0; } -int hyper_request_restart_containers(struct hyper_pod *pod) +int hyper_enter_container(struct hyper_pod *pod, + struct hyper_exec *exec) { - int i; - struct hyper_exec *exec; + int ipcns, utsns, mntns, ret; + struct hyper_container *c; + char path[512]; - pod->code = 0; - pod->remains = pod->c_num; + ret = ipcns = utsns = mntns = -1; - for (i = 0; i < pod->c_num; i++) { - exec = &pod->c[i].exec; - - if (hyper_setup_exec_tty(exec) < 0) { - fprintf(stdout, "restart setup container tty failed\n"); - return -1; - } - } - - if (hyper_send_type_block(ctl.ctl.fd, RESTARTCONTAINER, 1) < 0) { - fprintf(stderr, "tell container init RESTARTCONTAINER failed\n"); + c = hyper_find_container(pod, exec->id); + if (c == NULL) { + fprintf(stderr, "can not find container %s\n", exec->id); return -1; } - for (i = 0; i < pod->c_num; i++) { - exec = &pod->c[i].exec; - - list_add_tail(&exec->list, &pod->ce_head); - if (exec->seq == 0) - continue; - - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { - fprintf(stderr, "add pts master event failed\n"); - return -1; - } + sprintf(path, "/proc/%d/ns/uts", pod->init_pid); + utsns = open(path, O_RDONLY| O_CLOEXEC); + if (utsns < 0) { + perror("fail to open utsns of pod init"); + goto out; } - return 0; + sprintf(path, "/proc/%d/ns/ipc", pod->init_pid); + ipcns = open(path, O_RDONLY| O_CLOEXEC); + if (ipcns < 0) { + perror("fail to open ipcns of pod init"); + goto out; + } + + mntns = c->ns; + if (mntns < 0) { + perror("fail to open mntns of pod init"); + goto out; + } + + if (setns(utsns, CLONE_NEWUTS) < 0 || + setns(ipcns, CLONE_NEWIPC) <0 || + setns(mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + goto out; + } + + sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + fprintf(stdout, "root directory for container is %s, exec %s\n", + path, exec->argv[0]); + + /* TODO: wait for container finishing setup root */ + if (chroot(path) < 0) { + perror("chroot for exec command failed"); + goto out; + } + + chdir("/"); + + ret = hyper_setup_env(c->envs, c->envs_num); +out: + close(ipcns); + close(utsns); + + return ret; +} + +struct hyper_exec_arg { + struct hyper_pod *pod; + struct hyper_exec *exec; + int pipe[2]; +}; + +static int hyper_do_exec_cmd(void *data) +{ + struct hyper_exec_arg *arg = data; + struct hyper_exec *exec = arg->exec; + struct hyper_pod *pod = arg->pod; + int pipe[2] = {-1, -1}, pid; + int ret = -1; + + if (exec->id) { + char path[512]; + int pidns; + + sprintf(path, "/proc/%d/ns/pid", pod->init_pid); + pidns = open(path, O_RDONLY| O_CLOEXEC); + if (pidns < 0) { + perror("fail to open pidns of pod init"); + goto out; + } + + /* enter pidns of pod init, so the children of this process will run in + * pidns of pod init, see man 2 setns */ + if (setns(pidns, CLONE_NEWPID) < 0) { + perror("enter pidns of pod init failed"); + goto out; + } + close(pidns); + } + + if (pipe2(pipe, O_CLOEXEC) < 0) { + perror("create pipe in exec command failed"); + goto out; + } + + pid = fork(); + if (pid < 0) { + perror("fail to fork"); + goto out; + } else if (pid > 0) { + uint32_t type; + + if (hyper_get_type(pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); + hyper_send_type(arg->pipe[1], ERROR); + goto out; + } + + fprintf(stdout, "hyper init get ready message\n"); + exec->pid = pid; + fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); + + list_add_tail(&exec->list, &pod->exec_head); + + if (hyper_watch_exec_pty(exec, pod) < 0) { + fprintf(stderr, "add pts master event failed\n"); + goto out; + } + + ret = 0; + goto out; + } + + if (exec->id && hyper_enter_container(pod, exec) < 0) { + fprintf(stderr, "enter container ns failed\n"); + goto exit; + } + + if (hyper_dup_exec_tty(pipe[1], exec) < 0) { + fprintf(stderr, "dup pts to exec stdio failed\n"); + goto exit; + } + + if (execvp(exec->argv[0], exec->argv) < 0) { + perror("exec failed"); + goto exit; + } + + ret = 0; +exit: + hyper_send_type(pipe[1], ERROR); + _exit(ret); + +out: + hyper_send_type(arg->pipe[1], ret ? ERROR : READY); + close(pipe[0]); + close(pipe[1]); + _exit(ret); } int hyper_exec_cmd(char *json, int length) { struct hyper_exec *exec; struct hyper_pod *pod = &global_pod; - int pid, pipe[2]; + int stacksize = getpagesize() * 4; + void *stack = NULL; struct hyper_exec_arg arg = { + .pod = pod, + .exec = NULL, + .pipe = {-1, -1}, + }; + int pid, ret = -1; + uint32_t type; fprintf(stdout, "call hyper_exec_cmd, json %s, len %d\n", json, length); exec = hyper_parse_execcmd(json, length); if (exec == NULL) { fprintf(stderr, "parse exec cmd failed\n"); - return -1; + goto out; } if (exec->argv == NULL) { fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n", exec->argv, exec->seq, exec->id); - return -1; + goto out; } if (hyper_setup_exec_tty(exec) < 0) { fprintf(stderr, "setup exec tty failed\n"); - return -1; + goto out; } - if (exec->id) { - if (hyper_exec_in_container(pod, exec) < 0) { - fprintf(stderr, "notify container exec failed\n"); - return -1; - } - - return 0; - } - - if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); - return -1; + goto out; } - pid = fork(); + arg.exec = exec; + + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); + fprintf(stdout, "do_exec_cmd pid %d\n", pid); + free(stack); if (pid < 0) { - fprintf(stderr, "fork failed\n"); + perror("clone hyper_do_exec_cmd failed"); + goto out; + } + + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); return -1; - } else if (pid > 0) { - uint32_t type; - - if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { - fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); - return -1; - } - - close(pipe[0]); - close(pipe[1]); - fprintf(stdout, "hyper init get ready message\n"); - exec->pid = pid; - fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); - - list_add_tail(&exec->list, &pod->pe_head); - if (exec->seq == 0) - return 0; - - fprintf(stdout, "init pod exec pts event %p, ops %p, fd %d\n", - &exec->e, &pts_ops, exec->e.fd); - if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) { - fprintf(stderr, "add pts master event failed\n"); - return -1; - } - - return 0; } - if (hyper_dup_exec_tty(pipe[1], exec) < 0) { - fprintf(stderr, "dup pts to exec stdio failed\n"); - _exit(-1); - } + fprintf(stdout, "%s get ready message %"PRIu32 "\n", __func__, type); + ret = 0; +out: + close(arg.pipe[0]); + close(arg.pipe[1]); - close(pipe[0]); - close(pipe[1]); - - if (execvp(exec->argv[0], exec->argv) < 0) { - perror("exec failed"); - _exit(-1); - } - - _exit(0); + return ret; } -int hyper_container_execcmd(struct hyper_pod *pod) -{ - struct hyper_exec *exec = global_exec; - struct hyper_container *container = NULL; - int fd, pid, sock = pod->ctl.fd; - char root[512]; int pipe[2]; - - global_exec = NULL; - - container = hyper_find_container(pod, exec->id); - if (container == NULL) { - fprintf(stderr, "can not find container %s\n", exec->id); - return -1; - } - - if (sprintf(root, "/tmp/hyper/%s/devpts/", exec->id) < 0) { - fprintf(stderr, "get container %s pts path failed\n", exec->id); - return -1; - } - - if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) { - perror("create pipe between pod init execcmd failed"); - return -1; - } - - pid = fork(); - if (pid < 0) { - fprintf(stderr, "container init fork failed\n"); - return -1; - } else if (pid > 0) { - uint32_t type; - - if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) { - fprintf(stderr, "pod init get execcmd ready message failed\n"); - hyper_send_type_block(sock, ERROR, 0); - return 0; - } - - close(pipe[0]); - close(pipe[1]); - fprintf(stdout, "pod init get ready message\n"); - - exec->pid = pid; - fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); - - if (hyper_send_type_block(sock, ACK, 0) < 0) - return -1; - - return 0; - } - - close(pipe[0]); - sprintf(root, "/proc/%d/ns/mnt", container->exec.pid); - - fprintf(stdout, "container %s, init pid %d\n", - exec->id, container->exec.pid); - - fd = open(root, O_RDONLY); - if (fd < 0) { - perror("fail to open container mnt ns\n"); - goto fail; - } - - if (syscall(SYS_setns, fd, CLONE_NEWNS) < 0) { - perror("enter mnt ns failed"); - goto fail; - } - - close(fd); - - sprintf(root, "/tmp/hyper/%s/root/%s/", - container->id, container->rootfs); - - fprintf(stdout, "root directory for container is %s, exec %s\n", - root, exec->argv[0]); - - /* TODO: wait for container finishing setup root */ - if (chroot(root) < 0) { - perror("chroot for exec command failed"); - goto fail; - } - - chdir("/"); - - if (hyper_dup_exec_tty(pipe[1], exec)) { - fprintf(stderr, "dup pts to stdio failed\n"); - goto fail; - } - - close(pipe[1]); - if (execvp(exec->argv[0], exec->argv) < 0) - perror("exec failed"); - - _exit(-1); -fail: - hyper_send_type_block(pipe[1], ERROR, 0); - _exit(-1); -} - - int hyper_release_exec(struct hyper_exec *exec, struct hyper_pod *pod) { int i; + if (!exec->exit && exec->seq) { + fprintf(stdout, "first user of exec exit\n"); + exec->exit = 1; + close(exec->ptyfd); + exec->ptyfd = -1; + return 0; + } + + /* exec has no pty or the pty user already exited */ + fprintf(stdout, "last user of exec exit, release\n"); close(exec->e.fd); - free(exec->pty); + close(exec->ptyfd); + hyper_reset_event(&exec->e); list_del_init(&exec->list); @@ -463,7 +475,7 @@ int hyper_release_exec(struct hyper_exec *exec, return 0; } - if (hyper_request_restart_containers(pod) < 0) { + if (hyper_start_containers(pod) < 0) { fprintf(stderr, "restart container failed\n"); return -1; } @@ -474,7 +486,7 @@ int hyper_release_exec(struct hyper_exec *exec, free(exec->id); for (i = 0; i < exec->argc; i++) { - fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); + //fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); free(exec->argv[i]); } @@ -503,17 +515,8 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq) { struct hyper_exec *exec; - list_for_each_entry(exec, &pod->ce_head, list) { - fprintf(stdout, "container exec seq %" PRIu64 ", seq %" PRIu64 "\n", - exec->seq, seq); - if (exec->seq != seq) - continue; - - return exec; - } - - list_for_each_entry(exec, &pod->pe_head, list) { - fprintf(stdout, "pod exec seq %" PRIu64 ", seq %" PRIu64 "\n", + list_for_each_entry(exec, &pod->exec_head, list) { + fprintf(stdout, "exec seq %" PRIu64 ", seq %" PRIu64 "\n", exec->seq, seq); if (exec->seq != seq) continue; @@ -525,34 +528,21 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq) } int hyper_send_exec_eof(int to, struct hyper_pod *pod, - struct list_head *head, int pid, - uint8_t code) + int pid, uint8_t code) { struct hyper_exec *exec; - uint8_t seq[12]; - exec = hyper_find_exec_by_pid(head, pid); + exec = hyper_find_exec_by_pid(&pod->exec_head, pid); if (exec == NULL) { fprintf(stdout, "can not find exec whose pid is %d\n", pid); return 0; } - fprintf(stdout, "%s exec pid %d, seq %" PRIu64 ", container %s\n", + fprintf(stdout, "%s exec exit pid %d, seq %" PRIu64 ", container %s\n", __func__, exec->pid, exec->seq, exec->id ? exec->id : "pod"); exec->code = code; - - if (exec->seq == 0) - goto out; - - hyper_set_be64(seq, exec->seq); - hyper_set_be32(seq + 8, 12); - if (hyper_send_data(to, seq, 12) < 0) { - fprintf(stderr, "pod signal_loop send finishcmd failed\n"); - return -1; - } -out: hyper_release_exec(exec, pod); return 0; @@ -562,8 +552,8 @@ void hyper_cleanup_exec(struct hyper_pod *pod) { struct hyper_exec *exec, *next; - list_for_each_entry_safe(exec, next, &pod->ce_head, list) { - fprintf(stdout, "cleanup container exec seq %" PRIu64 "\n", exec->seq); + list_for_each_entry_safe(exec, next, &pod->exec_head, list) { + fprintf(stdout, "cleanup exec seq %" PRIu64 "\n", exec->seq); hyper_release_exec(exec, pod); } } diff --git a/src/exec.h b/src/exec.h index ed9d60a..ba2dba0 100644 --- a/src/exec.h +++ b/src/exec.h @@ -8,14 +8,15 @@ struct hyper_exec { struct list_head list; struct hyper_event e; char *id; - char *pty; char **argv; int argc; uint64_t seq; int pid; int ptyno; int init; + int ptyfd; uint8_t code; + uint8_t exit; }; struct hyper_pod; @@ -28,8 +29,8 @@ int hyper_dup_exec_tty(int fd, struct hyper_exec *e); struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid); struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq); int hyper_send_exec_eof(int to, struct hyper_pod *pod, - struct list_head *head, int pid, - uint8_t code); + int pid, uint8_t code); +int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod); void hyper_cleanup_exec(struct hyper_pod *pod); extern struct hyper_event_ops pts_ops; diff --git a/src/hyper.h b/src/hyper.h index 1889ab0..f7a17fd 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -25,6 +25,8 @@ enum { PING, FINISH, NEXT, + WRITEFILE, + READFILE, }; enum { @@ -38,8 +40,8 @@ struct hyper_pod { struct hyper_interface *iface; struct hyper_route *rt; char **dns; - struct list_head pe_head; - struct list_head ce_head; + struct list_head exec_head; + //struct list_head ce_head; char *hostname; char *tag; int init_pid; @@ -64,6 +66,18 @@ struct hyper_win_size { uint64_t seq; }; +struct hyper_reader { + char *id; + char *file; +}; + +struct hyper_writter { + char *id; + char *file; + uint8_t *data; + int len; +}; + struct hyper_ctl { int efd; struct hyper_event sig; @@ -75,6 +89,7 @@ struct hyper_ctl { int hyper_mkdir(char *hyper_path); int hyper_open_serial(char *tty); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); +int hyper_start_containers(struct hyper_pod *pod); extern struct hyper_pod global_pod; extern struct hyper_ctl ctl; diff --git a/src/init.c b/src/init.c index e45b0bd..cdb793c 100644 --- a/src/init.c +++ b/src/init.c @@ -29,8 +29,7 @@ #include "container.h" struct hyper_pod global_pod = { - .ce_head = LIST_HEAD_INIT(global_pod.ce_head), - .pe_head = LIST_HEAD_INIT(global_pod.pe_head), + .exec_head = LIST_HEAD_INIT(global_pod.exec_head), }; struct hyper_exec *global_exec; @@ -38,9 +37,7 @@ struct hyper_exec *global_exec; struct hyper_ctl ctl; -static void hyper_cleanup_pod(struct hyper_pod *pod); -static int hyper_handle_exit(struct hyper_pod *pod, int to, - int container, int option); +static int hyper_handle_exit(struct hyper_pod *pod); static int hyper_set_win_size(char *json, int length) { @@ -66,27 +63,23 @@ static int hyper_set_win_size(char *json, int length) return 0; } - fprintf(stdout, "find exec %s, pts %s, pid is %d, seq is %" PRIu64"\n", - exec->id ? exec->id : "pod", exec->pty, exec->pid, ws.seq); - name = exec->pty; + fprintf(stdout, "find exec %s, pid is %d, seq is %" PRIu64"\n", + exec->id ? exec->id : "pod", exec->pid, ws.seq); + fd = exec->ptyfd; } else { if (sprintf(path, "/dev/%s", ws.tty) < 0) { fprintf(stderr, "get tty device failed\n"); return -1; } - name = path; - } - - fprintf(stdout, "try to open %s\n", name); - ret = hyper_open_serial_dev(name); - if (ret < 0) { - fprintf(stderr, "cannot open %s to set term size\n", name); - goto out; + fd = hyper_open_serial_dev(name); + if (fd < 0) { + fprintf(stderr, "cannot open %s to set term size\n", name); + goto out; + } } size.ws_row = ws.row; size.ws_col = ws.column; - fd = ret; ret = ioctl(fd, TIOCSWINSZ, &size); if (ret < 0) @@ -98,44 +91,118 @@ out: return ret; } +static void hyper_kill_process(int pid) +{ + char path[64]; + char *line = NULL, *ignore = "SigIgn:"; + size_t len = 0; + ssize_t read; + FILE *file; + char *sub; + + sprintf(path, "/proc/%u/status", pid); + + fprintf(stdout, "fopen %s\n", path); + file = fopen(path, "r"); + if (file == NULL) { + perror("can not open process proc status file"); + return; + } + + while ((read = getline(&line, &len, file)) != -1) { + long mask; + + if (strstr(line, ignore) == NULL) + continue; + + sub = line + strlen(ignore); + fprintf(stdout, "find sigign %s", sub); + + mask = atol(sub); + fprintf(stdout, "mask is %ld\n", mask); + + if ((mask >> (SIGTERM - 1)) & 0x1) { + fprintf(stdout, "signal term is ignored, kill it\n"); + kill(pid, SIGKILL); + } + + break; + } + + fclose(file); + free(line); +} + +static void hyper_term_all(struct hyper_pod *pod) +{ + int npids = 0; + int index = 0; + int pid; + DIR *dp; + struct dirent *de; + pid_t *pids = NULL; + + dp = opendir("/proc"); + if (dp == NULL) + return; + + while ((de = readdir(dp)) && de != NULL) { + if (!isdigit(de->d_name[0])) + continue; + pid = atoi(de->d_name); + if (pid == 1) + continue; + if (index <= npids) { + pids = realloc(pids, npids + 16384); + if (pids == NULL) + return; + npids += 16384; + } + + pids[index++] = pid; + } + + fprintf(stdout, "Sending SIGTERM\n"); + + for (--index; index >= 0; --index) { + fprintf(stdout, "kill process %d\n", pids[index]); + kill(pids[index], SIGTERM); + } + + free(pids); + closedir(dp); + + for (index = 0; index < pod->c_num; index++) { + hyper_kill_process(pod->c[index].exec.pid); + } +} + static int pod_ctl_pipe_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; - struct hyper_pod *pod = de->ptr; - uint32_t type; fprintf(stdout, "%s\n", __func__); - type = hyper_get_be32(buf->data); + if (hyper_get_be32(buf->data) == STOPPOD) { + struct hyper_pod *pod = de->ptr; - switch (type) { - case STOPPOD: - fprintf(stdout, "get type STOPPOD, exit\n"); - hyper_cleanup_pod(pod); - case RESTARTCONTAINER: - fprintf(stdout, "%s get type RESTARTCONTAINER\n", __func__); - if (hyper_restart_containers(pod) < 0) - return -1; - break; - case EXECCMD: - if (hyper_container_execcmd(pod) < 0) - return -1; - break; - default: - break; + fprintf(stdout, "pod init get type STOPPOD, exit\n"); + hyper_term_all(pod); + hyper_reset_event(&pod->ctl); + hyper_unmount_all(); + _exit(0); } return 0; } -static int hyper_handle_exit(struct hyper_pod *pod, int to, - int container, int option) +static int hyper_handle_exit(struct hyper_pod *pod) { int pid, status; /* pid + exit code */ uint8_t data[5]; - while ((pid = waitpid(-1, &status, option)) > 0) { + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { data[4] = 0; if (WIFEXITED(status)) { @@ -148,44 +215,20 @@ static int hyper_handle_exit(struct hyper_pod *pod, int to, pid, WTERMSIG(status)); } - if (container) { - hyper_set_be32(data, pid); - if (hyper_send_msg(to, FINISHCMD, 5, data) < 0) { - fprintf(stderr, "pod signal_loop send finishcmd failed\n"); - return -1; - } - - continue; - } - - if (hyper_send_exec_eof(to, pod, &pod->pe_head, pid, data[4]) < 0) + if (hyper_send_exec_eof(ctl.tty.fd, pod, pid, data[4]) < 0) fprintf(stderr, "signal_loop send eof failed\n"); } - if (option == WNOHANG) - return 0; - - /* send ack message to hyper init. */ - if (hyper_send_type(to, ACK) < 0) { - fprintf(stderr, "send ACK of STOPPOD to hyper init failed\n"); - return -1; - } - return 0; } -static int signal_loop(struct hyper_event *de, int container) +static int hyper_signal_loop(struct hyper_event *de) { - int size, to; + int size; struct signalfd_siginfo sinfo; struct hyper_pod *pod = de->ptr; - if (container) - to = pod->ctl.fd; - else - to = ctl.tty.fd; - - fprintf(stdout, "%s write to %d\n", __func__, to); + fprintf(stdout, "%s write to ctl tty fd\n", __func__); while (1) { size = read(de->fd, &sinfo, sizeof(struct signalfd_siginfo)); @@ -207,22 +250,12 @@ static int signal_loop(struct hyper_event *de, int container) return 0; } - hyper_handle_exit(pod, to, container, WNOHANG); + hyper_handle_exit(pod); } return 0; } -static int pod_signal_loop(struct hyper_event *de) -{ - return signal_loop(de, 1); -} - -static int hyper_signal_loop(struct hyper_event *de) -{ - return signal_loop(de, 0); -} - static struct hyper_event_ops pod_ctl_pipe_ops = { .read = hyper_event_read, .handle = pod_ctl_pipe_handle, @@ -231,11 +264,6 @@ static struct hyper_event_ops pod_ctl_pipe_ops = { .len_offset = 4, }; -static struct hyper_event_ops pod_signal_ops = { - .read = pod_signal_loop, - .hup = hyper_event_hup, -}; - static int pod_init_loop(struct hyper_pod *pod) { int i, n; @@ -255,14 +283,6 @@ static int pod_init_loop(struct hyper_pod *pod) return -1; } - fprintf(stdout, "hyper_init_event pod signal event %p, ops %p, fd %d\n", - &pod->sig, &pod_signal_ops, pod->sig.fd); - if (hyper_init_event(&pod->sig, &pod_signal_ops, pod) < 0 || - hyper_add_event(pod->efd, &pod->sig, EPOLLIN) < 0) { - fprintf(stderr, "hyper add pod tty pipe event failed\n"); - return -1; - } - events = calloc(MAXEVENTS, sizeof(*events)); while (1) { @@ -295,7 +315,6 @@ static int hyper_pod_init(void *data) { struct hyper_pod_arg *arg = data; struct hyper_pod *pod = arg->pod; - sigset_t mask; close(arg->ctl_pipe[0]); close(ctl.sig.fd); @@ -309,43 +328,42 @@ static int hyper_pod_init(void *data) goto fail; } - sigemptyset(&mask); - sigaddset(&mask, SIGCHLD); - - if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) { - perror("sigprocmask SIGCHLD failed"); + /* mount new proc directory */ + if (umount("/proc") < 0) { + perror("umount proc filesystem failed\n"); goto fail; } - pod->sig.fd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC); - if (pod->sig.fd < 0) { - perror("create signalfd failed"); + if (mount("proc", "/proc", "proc", 0, NULL) < 0) { + perror("mount proc filesystem failed\n"); goto fail; } - if (hyper_start_containers(pod) < 0) + if (sethostname(pod->hostname, strlen(pod->hostname)) < 0) { + perror("set host name failed"); goto fail; + } fprintf(stdout, "pod ctl_pipe %d\n", arg->ctl_pipe[1]); if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) { fprintf(stderr, "container init send ready message failed\n"); goto fail; } -loop: + pod_init_loop(pod); + fprintf(stdout, "pod init exit\n"); +out: _exit(-1); fail: hyper_send_type(arg->ctl_pipe[1], ERROR); - goto loop; + goto out; } static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; - struct hyper_pod *pod = de->ptr; - uint32_t type, pid = 0; - uint8_t code; + uint32_t type; /* container exec finish message */ fprintf(stdout, "%s\n", __func__); @@ -358,14 +376,6 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) * fd is block, and ACK is the last message, exit loop. */ fprintf(stdout, "hyper_ctl_pipe_loop get ack\n"); return 1; - case FINISHCMD: - pid = hyper_get_be32(buf->data + 8); - code = buf->data[12]; - if (hyper_send_exec_eof(ctl.tty.fd, pod, &pod->ce_head, pid, code) < 0) { - fprintf(stderr, "hyper_ctl_pipe_loop send eof failed\n"); - return -1; - } - break; default: fprintf(stdout, "get unknown type %" PRIu32"\n", type); break; @@ -374,57 +384,116 @@ static int hyper_ctl_pipe_handle(struct hyper_event *de, uint32_t len) return 0; } -static int hyper_setup_pty(struct hyper_pod *pod) +static int hyper_do_start_containers(void *data) { - int i; - char root[512]; + int i, pidns, ipcns, utsns, ret; struct hyper_container *c; + struct hyper_pod_arg *arg; + struct hyper_pod *pod; + char path[64]; + + arg = data; + pod = arg->pod; + + ret = pidns = ipcns = utsns = -1; + + sprintf(path, "/proc/%d/ns/pid", pod->init_pid); + pidns = open(path, O_RDONLY| O_CLOEXEC); + if (pidns < 0) { + perror("fail to open pidns of pod init"); + goto out; + } + + /* enter pidns of pod init, so the children of this process will run in + * pidns of pod init, see man 2 setns */ + if (setns(pidns, CLONE_NEWPID) < 0) { + perror("enter pidns of pod init failed"); + goto out; + } + + sprintf(path, "/proc/%d/ns/uts", pod->init_pid); + utsns = open(path, O_RDONLY| O_CLOEXEC); + if (utsns < 0) { + perror("fail to open utsns of pod init"); + goto out; + } + + sprintf(path, "/proc/%d/ns/ipc", pod->init_pid); + ipcns = open(path, O_RDONLY| O_CLOEXEC); + if (ipcns < 0) { + perror("fail to open ipcns of pod init"); + goto out; + } for (i = 0; i < pod->c_num; i++) { c = &pod->c[i]; - - sprintf(root, "/tmp/hyper/%s/devpts/", c->id); - - if (hyper_mkdir(root) < 0) { - perror("make container pts directroy failed"); - return -1; - } - - if (mount("devpts", root, "devpts", MS_NOSUID, - "newinstance,ptmxmode=0666,mode=0620") < 0) { - perror("mount devpts failed"); - return -1; - } - - list_add_tail(&c->exec.list, &pod->ce_head); - - if (hyper_setup_exec_tty(&c->exec) < 0) { - fprintf(stderr, "setup container pts failed\n"); - return -1; - } + list_add_tail(&c->exec.list, &pod->exec_head); + hyper_start_container(c, utsns, ipcns, pod); } - return 0; + ret = 0; +out: + if (hyper_send_type(arg->ctl_pipe[1], ret ? ERROR : READY) < 0) { + fprintf(stderr, "container init send ready message failed\n"); + goto out; + } + + close(pidns); + close(utsns); + close(ipcns); + close(arg->ctl_pipe[0]); + close(arg->ctl_pipe[1]); + + _exit(ret); } -static int hyper_watch_pty(struct hyper_pod *pod) +int hyper_start_containers(struct hyper_pod *pod) { - int i; - struct hyper_container *c; + int stacksize = getpagesize() * 4; + void *stack = NULL; + struct hyper_pod_arg arg = { + .pod = pod, + .ctl_pipe = {-1, -1}, + }; + int ret = -1, pid; + uint32_t type; - for (i = 0; i < pod->c_num; i++) { - c = &pod->c[i]; - fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n", - &c->exec.e, &pts_ops, c->exec.e.fd); - if (hyper_init_event(&c->exec.e, &pts_ops, pod) < 0 || - hyper_add_event(ctl.efd, &c->exec.e, EPOLLIN) < 0) { - fprintf(stderr, "add container pts master event failed\n"); - return -1; - } + if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) { + perror("create pipe between hyper init and pod init failed"); + goto out; } - return 0; + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + pid = clone(hyper_do_start_containers, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); + free(stack); + if (pid < 0) { + perror("enter container pid ns failed"); + goto out; + } + + /* Wait for container start */ + if (hyper_get_type(arg.ctl_pipe[0], &type) < 0) { + perror("get enter_container_pidns ready message failed"); + goto out; + } + + if (type != READY) { + fprintf(stderr, "get incorrect enter_container_pidns message type %d, expect READY\n", + type); + goto out; + } + + ret = 0; +out: + close(arg.ctl_pipe[0]); + close(arg.ctl_pipe[1]); + return ret; } static struct hyper_event_ops hyper_ctl_pipe_ops = { @@ -448,21 +517,17 @@ static int hyper_setup_container(struct hyper_pod *pod) uint32_t type; void *stack; + int ret = -1; - if (hyper_setup_pty(pod) < 0) { - fprintf(stderr, "setup pty failed\n"); - return -1; - } - - if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { + if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) { perror("create pipe between hyper init and pod init failed"); - return -1; + goto out; } stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); - return -1; + goto out; } arg.pod = pod; @@ -471,41 +536,45 @@ static int hyper_setup_container(struct hyper_pod *pod) free(stack); if (pod->init_pid < 0) { perror("create container init process failed"); - return -1; + goto out; } - - close(arg.ctl_pipe[1]); - ctl.ctl.fd = arg.ctl_pipe[0]; fprintf(stdout, "pod init pid %d\n", pod->init_pid); /* Wait for container start */ - if (hyper_get_type_block(ctl.ctl.fd, &type) < 0) { + if (hyper_get_type_block(arg.ctl_pipe[0], &type) < 0) { perror("get container init ready message failed"); - return -1; + goto out; } if (type != READY) { fprintf(stderr, "get incorrect message type %d, expect READY\n", type); - return -1; + goto out; } - if (hyper_watch_pty(pod) < 0) { - fprintf(stderr, "watch pty failed\n"); - return -1; + if (hyper_start_containers(pod) < 0) { + fprintf(stderr, "start containers failed\n"); + goto out; } - if (hyper_setfd_cloexec(ctl.ctl.fd) < 0) { + if (hyper_setfd_cloexec(arg.ctl_pipe[0]) < 0) { perror("set ctl pipe fd FD_CLOEXEC failed"); - return -1; + goto out; } + ctl.ctl.fd = arg.ctl_pipe[0]; fprintf(stdout, "hyper_init_event hyper ctl pipe fd %d\n", ctl.ctl.fd); if (hyper_init_event(&ctl.ctl, &hyper_ctl_pipe_ops, pod) < 0 || hyper_add_event(ctl.efd, &ctl.ctl, EPOLLIN) < 0) { - return -1; + goto out; } - return 0; + ret = 0; +out: + close(arg.ctl_pipe[1]); + if (ret < 0) { + close(arg.ctl_pipe[0]); + } + return ret; } #ifdef WITH_VBOX @@ -636,110 +705,6 @@ static void hyper_print_uptime(void) close(fd); } -static void hyper_kill_process(int pid) -{ - char path[64]; - char *line = NULL, *ignore = "SigIgn:"; - size_t len = 0; - ssize_t read; - FILE *file; - char *sub; - - sprintf(path, "/proc/%u/status", pid); - - fprintf(stdout, "fopen %s\n", path); - file = fopen(path, "r"); - if (file == NULL) { - perror("can not open process proc status file"); - return; - } - - while ((read = getline(&line, &len, file)) != -1) { - long mask; - - if (strstr(line, ignore) == NULL) - continue; - - sub = line + strlen(ignore); - fprintf(stdout, "find sigign %s", sub); - - mask = atol(sub); - fprintf(stdout, "mask is %ld\n", mask); - - if ((mask >> (SIGTERM - 1)) & 0x1) { - fprintf(stdout, "signal term is ignored, kill it\n"); - kill(pid, SIGKILL); - } - - break; - } - - fclose(file); - free(line); -} - -static void hyper_term_all(struct hyper_pod *pod) -{ - int npids = 0; - int index = 0; - int pid; - DIR *dp; - struct dirent *de; - pid_t *pids = NULL; - - dp = opendir("/proc"); - if (dp == NULL) - return; - - while ((de = readdir(dp)) && de != NULL) { - if (!isdigit(de->d_name[0])) - continue; - pid = atoi(de->d_name); - if (pid == 1) - continue; - if (index <= npids) { - pids = realloc(pids, npids + 16384); - if (pids == NULL) - return; - npids += 16384; - } - - pids[index++] = pid; - } - - fprintf(stdout, "Sending SIGTERM\n"); - - for (--index; index >= 0; --index) { - fprintf(stdout, "kill process %d\n", pids[index]); - kill(pids[index], SIGTERM); - } - - free(pids); - closedir(dp); - - for (index = 0; index < pod->c_num; index++) { - hyper_kill_process(pod->c[index].exec.pid); - } -} - -static void hyper_cleanup_pod(struct hyper_pod *pod) -{ - close(pod->sig.fd); - hyper_reset_event(&pod->sig); - - hyper_term_all(pod); - hyper_handle_exit(pod, pod->ctl.fd, 1, 0); - - pod->init_pid = 0; - - close(pod->ctl.fd); - hyper_reset_event(&pod->ctl); - - hyper_unmount_all(); - - _exit(0); -} - static int hyper_start_pod(char *json, int length) { struct hyper_pod *pod = &global_pod; @@ -762,6 +727,241 @@ static int hyper_start_pod(char *json, int length) return 0; } +static int hyper_cmd_write_file(char *json, int length) +{ + struct hyper_writter writter; + struct hyper_container *c; + struct hyper_pod *pod = &global_pod; + int pipe[2] = {-1, -1}; + int pid, mntns = -1, fd; + char path[512]; + int len = 0, size, ret = -1; + + fprintf(stdout, "%s\n", __func__); + memset(&writter, 0, sizeof(writter)); + + if (hyper_parse_write_file(&writter, json, length) < 0) { + goto out; + } + + c = hyper_find_container(pod, writter.id); + if (c == NULL) { + fprintf(stderr, "can not find container whose id is %s\n", writter.id); + goto out; + } + + if (pipe2(pipe, O_CLOEXEC) < 0) { + perror("create writter pipe failed"); + goto out; + } + + mntns = c->ns; + if (mntns < 0) { + perror("fail to open mnt ns"); + goto out; + } + + pid = fork(); + if (pid < 0) { + perror("fail to fork writter process"); + goto out; + } else if (pid > 0) { + uint32_t type; + + if (hyper_get_type(pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "get incorrect message type %d, expect READY\n", type); + goto out; + } + + ret = 0; + goto out; + } + + if (setns(mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + goto exit; + } + + sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + fprintf(stdout, "write file %s, data len %d\n", writter.file, writter.len); + + /* TODO: wait for container finishing setup root */ + if (chroot(path) < 0) { + perror("chroot for exec command failed"); + goto exit; + } + + fd = open(writter.file, O_CREAT| O_WRONLY, 0644); + if (fd < 0) { + perror("fail to open target file"); + goto exit; + } + + while(len < writter.len) { + size = write(fd, writter.data + len, writter.len - len); + + if (size < 0) { + if (errno == EINTR) + continue; + + perror("fail to write data to file"); + goto exit; + } + + len += size; + } + ret = 0; +exit: + hyper_send_type(pipe[1], ret ? ERROR : READY); + _exit(0); +out: + close(pipe[0]); + close(pipe[1]); + free(writter.id); + free(writter.file); + free(writter.data); + + return 0; +} + +struct hyper_file_arg { + int mntns; + int pipe[2]; + char *file; + char root[128]; + uint32_t *datalen; + uint8_t **data; +}; + +static int hyper_do_cmd_read_file(void *data) +{ + struct stat st; + int len = 0, size, fd, ret = -1; + struct hyper_file_arg *arg = data; + + if (setns(arg->mntns, CLONE_NEWNS) < 0) { + perror("fail to enter container ns"); + goto err; + } + + fprintf(stdout, "read file %s\n", arg->file); + + /* TODO: wait for container finishing setup root */ + if (chroot(arg->root) < 0) { + perror("chroot for exec command failed"); + goto err; + } + + if (stat(arg->file, &st) < 0) { + perror("fail to state file"); + goto err; + } + + *arg->datalen = st.st_size; + *arg->data = malloc(*arg->datalen); + if (*arg->data == NULL) { + fprintf(stderr, "allocate memory for reading file failed\n"); + goto err; + } + + fd = open(arg->file, O_RDONLY); + if (fd < 0) { + perror("fail to open target file"); + goto err; + } + + fprintf(stdout, "file length %d\n", *arg->datalen); + while(len < *arg->datalen) { + size = read(fd, *arg->data + len, *arg->datalen - len); + + if (size < 0) { + if (errno == EINTR) + continue; + + perror("fail to read data from file"); + goto err; + } + + len += size; + } + + fprintf(stdout, "read data %s\n", *arg->data); + ret = 0; +err: + hyper_send_type(arg->pipe[1], ret ? ERROR : READY); + return ret; +} + +static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_t **data) +{ + struct hyper_reader reader; + struct hyper_container *c; + struct hyper_pod *pod = &global_pod; + struct hyper_file_arg arg = { + .pipe = {-1, -1}, + }; + int stacksize = getpagesize() * 4; + void *stack = NULL; + int pid, ret = -1; + uint32_t type; + + fprintf(stdout, "%s\n", __func__); + memset(&reader, 0, sizeof(reader)); + + if (hyper_parse_read_file(&reader, json, length) < 0) { + goto out; + } + + c = hyper_find_container(pod, reader.id); + if (c == NULL) { + fprintf(stderr, "can not find container whose id is %s\n", reader.id); + goto out; + } + + if (pipe2(arg.pipe, O_CLOEXEC) < 0) { + perror("create reader pipe failed"); + goto out; + } + + arg.mntns = c->ns; + if (arg.mntns < 0) { + perror("fail to open mnt ns"); + goto out; + } + + arg.file = reader.file; + arg.datalen = datalen; + arg.data = data; + sprintf(arg.root, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs); + + stack = malloc(stacksize); + if (stack == NULL) { + perror("fail to allocate stack for container init"); + goto out; + } + + pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGCHLD, &arg); + free(stack); + if (pid < 0) { + perror("fail to fork writter process"); + goto out; + } + + if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { + fprintf(stderr, "%s to incorrect type %" PRIu32 "\n", __func__, type); + goto out; + } + + ret = 0; +out: + close(arg.pipe[0]); + close(arg.pipe[1]); + free(reader.id); + free(reader.file); + + return ret; +} + static void hyper_cleanup_shared(struct hyper_pod *pod) { if (pod->tag == NULL) { @@ -805,6 +1005,7 @@ static int hyper_stop_pod(struct hyper_pod *pod) return 0; } + pod->init_pid = 0; /* Make hyper ctl_pipe blocked */ hyper_send_stoppod(ctl.ctl.fd); @@ -914,7 +1115,8 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) { struct hyper_buf *buf = &de->rbuf; struct hyper_pod *pod = de->ptr; - uint32_t type = 0; + uint32_t type = 0, datalen = 0; + uint8_t *data = NULL; int i, ret = 0; for (i = 0; i < buf->get; i++) @@ -941,6 +1143,12 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) case EXECCMD: ret = hyper_exec_cmd((char *)buf->data + 8, len - 8); break; + case WRITEFILE: + ret = hyper_cmd_write_file((char *)buf->data + 8, len - 8); + break; + case READFILE: + ret = hyper_cmd_read_file((char *)buf->data + 8, len - 8, &datalen, &data); + break; case PING: case GETPOD: break; @@ -958,8 +1166,9 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len) if (ret < 0) hyper_send_type(de->fd, ERROR); else - hyper_send_type(de->fd, ACK); + hyper_send_msg(de->fd, ACK, datalen, data); + free(data); return 0; } diff --git a/src/jsmn.c b/src/jsmn.c index b1c3ebb..027b8aa 100644 --- a/src/jsmn.c +++ b/src/jsmn.c @@ -164,14 +164,18 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, int i; jsmntok_t *token; int count = 0; + int num = 0; + int out = 0; - for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + for (; parser->pos < len && js[parser->pos] != '\0' && !out; parser->pos++) { char c; jsmntype_t type; c = js[parser->pos]; switch (c) { - case '{': case '[': + case '{': + num++; + case '[': count++; if (tokens == NULL) break; @@ -189,7 +193,10 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, token->start = parser->pos; parser->toksuper = parser->toknext - 1; break; - case '}': case ']': + case '}': + if (--num <= 0) + out = 1; + case ']': if (tokens == NULL) break; type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); @@ -221,7 +228,7 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, parser->toksuper = -1; token->end = parser->pos + 1; - break; + break; } } /* Error if unmatched closing bracket */ diff --git a/src/net.c b/src/net.c index ff1f3c6..4ece161 100644 --- a/src/net.c +++ b/src/net.c @@ -90,7 +90,7 @@ int hyper_send_type(int fd, uint32_t type) return hyper_send_msg(fd, type, 0, NULL); } -int hyper_get_type_block(int fd, uint32_t *type) +int hyper_get_type(int fd, uint32_t *type) { int len = 0, size; uint8_t buf[8]; @@ -111,6 +111,34 @@ int hyper_get_type_block(int fd, uint32_t *type) return 0; } +int hyper_get_type_block(int fd, uint32_t *type) +{ + int ret = 0, flags; + + flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) { + fprintf(stderr, "%s get fd flag failed\n", __func__); + return -1; + } + + if (fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0) { + perror("set fd BLOCK failed"); + return -1; + } + + ret = hyper_get_type(fd, type); + if (ret < 0) { + fprintf(stderr, "%s can not get type\n", __func__); + } + + if (fcntl(fd, F_SETFL, flags) < 0) { + perror("restore fd flag failed"); + return -1; + } + + return ret; +} + int hyper_send_type_block(int fd, uint32_t type, int need_ack) { int ret = 0, flags; @@ -134,7 +162,7 @@ int hyper_send_type_block(int fd, uint32_t type, int need_ack) if (need_ack == 0) goto out; - ret = hyper_get_type_block(fd, &t); + ret = hyper_get_type(fd, &t); if (ret < 0) { fprintf(stderr, "can not get type\n"); goto out; @@ -145,8 +173,8 @@ int hyper_send_type_block(int fd, uint32_t type, int need_ack) if (t != ACK) ret = -1; out: - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - perror("set fd BLOCK failed"); + if (fcntl(fd, F_SETFL, flags) < 0) { + perror("restore fd flag failed"); return -1; } diff --git a/src/net.h b/src/net.h index af8450f..e096f21 100644 --- a/src/net.h +++ b/src/net.h @@ -50,6 +50,7 @@ int hyper_setup_network(struct hyper_pod *pod); void hyper_cleanup_network(struct hyper_pod *pod); int hyper_setup_dns(struct hyper_pod *pod); void hyper_cleanup_dns(struct hyper_pod *pod); +int hyper_get_type(int fd, uint32_t *type); int hyper_get_type_block(int fd, uint32_t *type); int hyper_send_type(int fd, uint32_t type); int hyper_send_type_block(int fd, uint32_t type, int need_ack); diff --git a/src/parse.c b/src/parse.c index f29e888..9d5bffa 100644 --- a/src/parse.c +++ b/src/parse.c @@ -199,6 +199,9 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container * c->exec.init = 1; c->exec.code = -1; + c->exec.e.fd = -1; + c->exec.ptyfd = -1; + c->ns = -1; next_container = toks[i].size; fprintf(stdout, "next container %d\n", next_container); @@ -575,6 +578,8 @@ realloc: if (exec == NULL) goto out; + exec->ptyfd = -1; + exec->e.fd = -1; INIT_LIST_HEAD(&exec->list); for (i = 0, j = 0; i < n; i++) { @@ -632,3 +637,122 @@ fail: exec = NULL; goto out; } + +int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length) +{ + int i, n, ret = 0; + + jsmn_parser p; + int toks_num = 10; + jsmntok_t *toks = NULL; + + toks = calloc(toks_num, sizeof(jsmntok_t)); + + jsmn_init(&p); + n = jsmn_parse(&p, json, length, toks, toks_num); + /* Must be json first */ + if (n <= 0) { + fprintf(stdout, "jsmn parse failed, n is %d\n", n); + ret = -1; + goto out; + } + + writter->len = length - toks[0].end; + writter->data = malloc(writter->len); + + if (writter->data == NULL) + goto fail; + + memcpy(writter->data, json + toks[0].end, writter->len); + fprintf(stdout, "writefile get data len %d %s\n", writter->len, writter->data); + + for (i = 0; i < n; i++) { + jsmntok_t *t = &toks[i]; + + if (t->type != JSMN_STRING) + continue; + + if (json_token_streq(json, t, "container")) { + if (i++ == n) + goto fail; + + writter->id = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "writefile get container %s\n", writter->id); + } else if (json_token_streq(json, t, "file")) { + if (i++ == n) + goto fail; + + writter->file = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "writefile get file %s\n", writter->file); + } else { + fprintf(stdout, "in writefile incorrect %s\n", json_token_str(json, &toks[i])); + } + } +out: + free(toks); + return ret; +fail: + free(writter->id); + free(writter->file); + free(writter->data); + + ret = -1; + goto out; +} + +int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length) +{ + int i, n, ret = 0; + + jsmn_parser p; + int toks_num = 10; + jsmntok_t *toks = NULL; + + toks = calloc(toks_num, sizeof(jsmntok_t)); + if (toks == NULL) { + fprintf(stderr, "fail to allocate tokens for read file cmd\n"); + ret = -1; + goto out; + } + + jsmn_init(&p); + n = jsmn_parse(&p, json, length, toks, toks_num); + if (n < 0) { + fprintf(stdout, "jsmn parse failed, n is %d\n", n); + ret = -1; + goto out; + } + + for (i = 0; i < n; i++) { + jsmntok_t *t = &toks[i]; + + if (t->type != JSMN_STRING) + continue; + + if (json_token_streq(json, t, "container")) { + if (i++ == n) + goto fail; + + reader->id = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "readfile get container %s\n", reader->id); + } else if (json_token_streq(json, t, "file")) { + if (i++ == n) + goto fail; + + reader->file = strdup(json_token_str(json, &toks[i])); + fprintf(stdout, "readfile get file %s\n", reader->file); + } else { + fprintf(stdout, "in readfile incorrect %s\n", json_token_str(json, &toks[i])); + } + } + +out: + free(toks); + return ret; +fail: + free(reader->id); + free(reader->file); + + ret = -1; + goto out; +} diff --git a/src/parse.h b/src/parse.h index 7b41ba0..867025b 100644 --- a/src/parse.h +++ b/src/parse.h @@ -9,5 +9,7 @@ struct hyper_exec *hyper_parse_execcmd(char *json, int length); char *json_token_str(char *js, jsmntok_t *t); int json_token_streq(char *js, jsmntok_t *t, char *s); int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length); +int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length); +int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length); #endif diff --git a/src/util.c b/src/util.c index d1212b6..57c5ca1 100644 --- a/src/util.c +++ b/src/util.c @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include "util.h" #include "hyper.h" +#include "container.h" #include "../config.h" char *read_cmdline(void) @@ -23,6 +25,22 @@ char *read_cmdline(void) return NULL; } +int hyper_setup_env(struct env *envs, int num) +{ + int i, ret = 0; + struct env *env; + + for (i = 0; i < num; i++) { + env = &envs[i]; + if (setenv(env->env, env->value, 1) < 0) { + perror("fail to setup env"); + ret = -1; + } + } + + return ret; +} + int hyper_list_dir(char *path) { struct dirent **list; @@ -39,6 +57,7 @@ int hyper_list_dir(char *path) for (i = 0; i < num; i++) { dir = list[i]; fprintf(stdout, "%s get %s\n", path, dir->d_name); + free(dir); } free(list); @@ -174,8 +193,7 @@ int hyper_insmod(char *module) ret = 0; out: close(fd); - if (buf) - free(buf); + free(buf); return ret; err: @@ -306,6 +324,21 @@ int hyper_setfd_nonblock(int fd) return 0; } +int hyper_socketpair(int domain, int type, int protocol, int sv[2]) +{ + if (socketpair(domain, type, protocol, sv) < 0) { + perror("socketpair failed"); + return -1; + } + + if (hyper_setfd_cloexec(sv[0]) < 0 || + hyper_setfd_cloexec(sv[1]) < 0) { + return -1; + } + + return 0; +} + void hyper_unmount_all(void) { FILE *mtab; @@ -393,13 +426,15 @@ void hyper_kill_all(void) int hyper_send_finish(struct hyper_pod *pod) { - int i; + int i, ret; uint8_t *data = calloc(pod->c_num, 4); for (i = 0; i < pod->c_num; i++) hyper_set_be32(data + (i * 4), pod->c[i].exec.code); - return hyper_send_msg(ctl.chan.fd, FINISH, pod->c_num * 4, data); + ret = hyper_send_msg(ctl.chan.fd, FINISH, pod->c_num * 4, data); + free(data); + return ret; } void hyper_shutdown(struct hyper_pod *pod) diff --git a/src/util.h b/src/util.h index 807a6eb..c285a9d 100644 --- a/src/util.h +++ b/src/util.h @@ -5,6 +5,7 @@ #include "../config.h" struct hyper_pod; +struct env; #ifdef WITH_DEBUG #define dprintf(fmt, ...) \ @@ -14,6 +15,7 @@ struct hyper_pod; #endif char *read_cmdline(void); +int hyper_setup_env(struct env *envs, int num); int hyper_list_dir(char *path); int hyper_mkdir(char *path); int hyper_open_channel(char *channel, int mode); @@ -21,6 +23,7 @@ int hyper_open_serial_dev(char *tty); int hyper_setfd_cloexec(int fd); int hyper_setfd_block(int fd); int hyper_setfd_nonblock(int fd); +int hyper_socketpair(int domain, int type, int protocol, int sv[2]); void hyper_shutdown(struct hyper_pod *pod); int hyper_send_finish(struct hyper_pod *pod); void hyper_unmount_all(void);