diff --git a/src/container.c b/src/container.c index 169eb42..63a2d11 100644 --- a/src/container.c +++ b/src/container.c @@ -505,6 +505,11 @@ int hyper_start_container(struct hyper_container *container, goto fail; } + if (hyper_watch_exec_pty(&container->exec, pod) < 0) { + fprintf(stderr, "faile to watch container pty\n"); + goto fail; + } + stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); @@ -518,7 +523,7 @@ int hyper_start_container(struct hyper_container *container, goto fail; } sprintf(path, "/proc/%d/ns/mnt", pid); - container->exec.pid = pid; + container->ns = open(path, O_RDONLY | O_CLOEXEC); if (container->ns < 0) { perror("open container mount ns failed"); @@ -531,24 +536,26 @@ int hyper_start_container(struct hyper_container *container, goto fail; } + container->exec.pid = pid; + list_add_tail(&container->exec.list, &pod->exec_head); + container->exec.ref++; + close(arg.pipe[0]); close(arg.pipe[1]); - if (hyper_watch_exec_pty(&container->exec, pod) < 0) { - fprintf(stderr, "faile to watch container pty\n"); - goto fail; - } - - fprintf(stdout, "container %s init pid is %d\n", container->id, pid); + fprintf(stdout, "container %s,init pid %d,ref %d\n", container->id, pid, container->exec.ref); return 0; fail: close(arg.pipe[0]); close(arg.pipe[1]); close(container->ns); + hyper_reset_event(&container->exec.e); + hyper_reset_event(&container->exec.errev); container->ns = -1; fprintf(stdout, "container %s init exit code %d\n", container->id, -1); container->exec.code = -1; container->exec.seq = 0; + container->exec.ref = 0; return -1; } @@ -624,6 +631,13 @@ void hyper_cleanup_container(struct hyper_pod *pod) } free(c->maps); close(c->ns); + + free(c->exec.id); + for (i = 0; i < c->exec.argc; i++) { + //fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); + free(c->exec.argv[i]); + } + free(c->exec.argv); } free(pod->c); diff --git a/src/event.c b/src/event.c index a6ec6a7..81b6375 100644 --- a/src/event.c +++ b/src/event.c @@ -14,8 +14,9 @@ void hyper_reset_event(struct hyper_event *de) { free(de->rbuf.data); free(de->wbuf.data); - + close(de->fd); memset(de, 0, sizeof(*de)); + de->fd = -1; } int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops, void *arg) @@ -219,8 +220,6 @@ void hyper_event_hup(struct hyper_event *de, int efd) { if (epoll_ctl(efd, EPOLL_CTL_DEL, de->fd, NULL) < 0) perror("epoll_ctl del epoll event failed"); - close(de->fd); - de->fd = -1; hyper_reset_event(de); } diff --git a/src/exec.c b/src/exec.c index 64e42a7..95826f2 100644 --- a/src/exec.c +++ b/src/exec.c @@ -18,13 +18,24 @@ #include "util.h" #include "parse.h" -static void pts_hup(struct hyper_event *de, int efd) +static void pts_hup(struct hyper_event *de, int efd, int out) { + struct hyper_exec *exec; struct hyper_pod *pod = de->ptr; struct hyper_buf *buf = &ctl.tty.wbuf; - struct hyper_exec *exec = container_of(de, struct hyper_exec, e); + uint64_t seq; - fprintf(stdout, "%s\n", __func__); + if (out) { + exec = container_of(de, struct hyper_exec, e); + seq = exec->seq; + } else { + exec = container_of(de, struct hyper_exec, errev); + seq = exec->errseq; + } + + fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, seq); + + hyper_event_hup(de, efd); if (buf->get + 12 > buf->size) { fprintf(stdout, "%s: tty buf full\n", __func__); @@ -32,24 +43,27 @@ static void pts_hup(struct hyper_event *de, int efd) } /* no in event, no more data, send eof */ - hyper_set_be64(buf->data + buf->get, exec->seq); + hyper_set_be64(buf->data + buf->get, seq); hyper_set_be32(buf->data + buf->get + 8, 12); buf->get += 12; - if (buf->get + 12 > buf->size) { - fprintf(stdout, "%s: tty buf full (for stderr)\n", __func__); - } else { - /* no in event, no more data, send eof in stderr */ - hyper_set_be64(buf->data + buf->get, exec->errseq); - hyper_set_be32(buf->data + buf->get + 8, 12); - buf->get += 12; - } - hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT); hyper_release_exec(exec, pod); } +static void stdout_hup(struct hyper_event *de, int efd) +{ + fprintf(stdout, "%s\n", __func__); + return pts_hup(de, efd, 1); +} + +static void stderr_hup(struct hyper_event *de, int efd) +{ + fprintf(stdout, "%s\n", __func__); + return pts_hup(de, efd, 0); +} + static int pts_loop(struct hyper_event *de, uint64_t seq) { int size = -1; @@ -93,7 +107,7 @@ static int stdout_loop(struct hyper_event *de) struct hyper_event_ops pts_ops = { .read = stdout_loop, - .hup = pts_hup, + .hup = stdout_hup, .write = hyper_event_write, .wbuf_size = 512, /* don't need read buff, the pts data will store in tty buffer */ @@ -108,8 +122,8 @@ static int stderr_loop(struct hyper_event *de) } struct hyper_event_ops err_ops = { - /* don't need to deal with hup, the hup will be dealed by pts*/ .read = stderr_loop, + .hup = stderr_hup, /* don't need read buff, the stderr data will store in tty buffer */ /* don't need write buff, the stderr data is one way */ }; @@ -173,6 +187,8 @@ int hyper_setup_exec_tty(struct hyper_exec *e) e->ptyfd = open(ptmx, O_RDWR | O_NOCTTY | O_CLOEXEC); fprintf(stdout, "get pty device for exec %s\n", ptmx); + fprintf(stdout, "%s pts event %p, fd %d %d\n", + __func__, &e->e, e->e.fd, e->ptyfd); return 0; } @@ -249,16 +265,17 @@ int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod) fprintf(stderr, "add container pts master event failed\n"); return -1; } + exec->ref++; if (exec->errseq == 0) return 0; - if (hyper_init_event(&exec->errev, &err_ops, NULL) < 0 || + if (hyper_init_event(&exec->errev, &err_ops, pod) < 0 || hyper_add_event(ctl.efd, &exec->errev, EPOLLIN) < 0) { fprintf(stderr, "add container stderr event failed\n"); return -1; } - + exec->ref++; return 0; } @@ -363,6 +380,11 @@ static int hyper_do_exec_cmd(void *data) goto out; } + if (hyper_watch_exec_pty(exec, pod) < 0) { + fprintf(stderr, "add pts master event failed\n"); + goto out; + } + pid = fork(); if (pid < 0) { perror("fail to fork"); @@ -378,15 +400,10 @@ static int hyper_do_exec_cmd(void *data) fprintf(stdout, "hyper init get ready message\n"); exec->pid = pid; - fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid); - + //TODO combin ref++ and add to list. list_add_tail(&exec->list, &pod->exec_head); - - if (hyper_watch_exec_pty(exec, pod) < 0) { - fprintf(stderr, "add pts master event failed\n"); - goto out; - } - + exec->ref++; + fprintf(stdout, "create exec cmd %s pid %d,ref %d\n", exec->argv[0], pid, exec->ref); ret = 0; goto out; } @@ -418,12 +435,28 @@ out: _exit(ret); } +static void hyper_free_exec(struct hyper_exec *exec) +{ + int i; + + free(exec->id); + + for (i = 0; i < exec->argc; i++) { + //fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); + free(exec->argv[i]); + } + + free(exec->argv); + free(exec); +} + int hyper_exec_cmd(char *json, int length) { struct hyper_exec *exec; struct hyper_pod *pod = &global_pod; int stacksize = getpagesize() * 4; - void *stack = NULL; struct hyper_exec_arg arg = { + void *stack = NULL; + struct hyper_exec_arg arg = { .pod = pod, .exec = NULL, .pipe = {-1, -1}, @@ -442,17 +475,17 @@ int hyper_exec_cmd(char *json, int length) if (exec->argv == NULL) { fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n", exec->argv, exec->seq, exec->id); - goto out; + goto free_exec; } if (hyper_setup_exec_tty(exec) < 0) { fprintf(stderr, "setup exec tty failed\n"); - goto out; + goto free_exec; } if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); - goto out; + goto close_tty; } arg.exec = exec; @@ -460,7 +493,7 @@ int hyper_exec_cmd(char *json, int length) stack = malloc(stacksize); if (stack == NULL) { perror("fail to allocate stack for container init"); - goto out; + goto close_tty; } pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg); @@ -468,12 +501,12 @@ int hyper_exec_cmd(char *json, int length) free(stack); if (pid < 0) { perror("clone hyper_do_exec_cmd failed"); - goto out; + goto close_tty; } if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) { fprintf(stderr, "hyper init doesn't get execcmd ready message\n"); - return -1; + goto close_tty; } fprintf(stdout, "%s get ready message %"PRIu32 "\n", __func__, type); @@ -481,29 +514,26 @@ int hyper_exec_cmd(char *json, int length) out: close(arg.pipe[0]); close(arg.pipe[1]); - return ret; +close_tty: + close(exec->ptyfd); + close(exec->errfd); + close(exec->e.fd); +free_exec: + hyper_free_exec(exec); + goto out; } int hyper_release_exec(struct hyper_exec *exec, struct hyper_pod *pod) { - int i; - - if (!exec->exit && exec->seq) { - fprintf(stdout, "first user of exec exit\n"); - exec->exit = 1; - close(exec->ptyfd); - exec->ptyfd = -1; + if (--exec->ref != 0) { + fprintf(stdout, "still have %d user of exec\n", exec->ref); return 0; } /* exec has no pty or the pty user already exited */ fprintf(stdout, "last user of exec exit, release\n"); - close(exec->e.fd); - close(exec->errev.fd); - close(exec->ptyfd); - close(exec->errfd); hyper_reset_event(&exec->e); hyper_reset_event(&exec->errev); @@ -544,15 +574,7 @@ int hyper_release_exec(struct hyper_exec *exec, return 0; } - free(exec->id); - - for (i = 0; i < exec->argc; i++) { - //fprintf(stdout, "argv %d %s\n", i, exec->argv[i]); - free(exec->argv[i]); - } - - free(exec->argv); - free(exec); + hyper_free_exec(exec); return 0; } @@ -604,11 +626,18 @@ int hyper_send_exec_eof(int to, struct hyper_pod *pod, __func__, exec->pid, exec->seq, exec->id ? exec->id : "pod"); exec->code = code; + exec->exit = 1; + + close(exec->ptyfd); + exec->ptyfd = -1; + close(exec->errfd); + exec->errfd = -1; + hyper_release_exec(exec, pod); return 0; } - +/* void hyper_cleanup_exec(struct hyper_pod *pod) { struct hyper_exec *exec, *next; @@ -618,3 +647,4 @@ void hyper_cleanup_exec(struct hyper_pod *pod) hyper_release_exec(exec, pod); } } +*/ diff --git a/src/exec.h b/src/exec.h index 2726e08..2da0dd8 100644 --- a/src/exec.h +++ b/src/exec.h @@ -20,6 +20,7 @@ struct hyper_exec { int errfd; uint8_t code; uint8_t exit; + uint8_t ref; }; struct hyper_pod; diff --git a/src/init.c b/src/init.c index 8af778b..58c2ecb 100644 --- a/src/init.c +++ b/src/init.c @@ -48,7 +48,7 @@ static int hyper_set_win_size(char *json, int length) }; struct winsize size; struct hyper_exec *exec; - char *name, path[128]; + char path[128]; int fd, ret; fprintf(stdout, "call hyper_win_size, json %s, len %d\n", json, length); @@ -57,7 +57,6 @@ static int hyper_set_win_size(char *json, int length) return -1; } - name = ws.tty; if (!ws.tty) { exec = hyper_find_exec_by_seq(&global_pod, ws.seq); if (exec == NULL) { @@ -67,17 +66,18 @@ static int hyper_set_win_size(char *json, int length) fprintf(stdout, "find exec %s, pid is %d, seq is %" PRIu64"\n", exec->id ? exec->id : "pod", exec->pid, ws.seq); - fd = exec->ptyfd; + fd = dup(exec->ptyfd); } else { if (sprintf(path, "/dev/%s", ws.tty) < 0) { fprintf(stderr, "get tty device failed\n"); return -1; } - fd = hyper_open_serial_dev(name); - if (fd < 0) { - fprintf(stderr, "cannot open %s to set term size\n", name); - goto out; - } + fd = hyper_open_serial_dev(path); + } + + if (fd < 0) { + perror("cannot open pty device to set term size"); + goto out; } size.ws_row = ws.row; @@ -85,7 +85,7 @@ static int hyper_set_win_size(char *json, int length) ret = ioctl(fd, TIOCSWINSZ, &size); if (ret < 0) - fprintf(stderr, "cannot ioctl to set %s term size\n", name); + perror("cannot ioctl to set pty device term size"); close(fd); out: @@ -392,7 +392,6 @@ static int hyper_do_start_containers(void *data) for (i = 0; i < pod->c_num; i++) { c = &pod->c[i]; - list_add_tail(&c->exec.list, &pod->exec_head); if (hyper_start_container(c, utsns, ipcns, pod) < 0) { fprintf(stderr, "fail to start container\n"); goto out; @@ -403,7 +402,6 @@ static int hyper_do_start_containers(void *data) out: if (hyper_send_type(arg->ctl_pipe[1], ret ? ERROR : READY) < 0) { fprintf(stderr, "container init send ready message failed\n"); - goto out; } close(pidns); @@ -940,7 +938,6 @@ static void hyper_cleanup_shared(struct hyper_pod *pod) void hyper_cleanup_pod(struct hyper_pod *pod) { - hyper_cleanup_exec(pod); hyper_cleanup_container(pod); hyper_cleanup_network(pod); hyper_cleanup_shared(pod); @@ -1025,6 +1022,12 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len) dprintf(stdout, "find exec %s pid %d, seq is %" PRIu64 "\n", exec->id ? exec->id : "pod", exec->pid, exec->seq); + // if exec is exited, the event fd of exec is invalid. don't accept any input. + if (exec->exit) { + fprintf(stdout, "exec seq %" PRIu64 " exited, don't accept any input\n", exec->seq); + return 0; + } + wbuf = &exec->e.wbuf; size = wbuf->size - wbuf->get; diff --git a/src/net.c b/src/net.c index bd4a58a..822d18b 100644 --- a/src/net.c +++ b/src/net.c @@ -707,6 +707,11 @@ static int hyper_cleanup_interface(struct rtnl_handle *rth, return -1; } + /* Don't down&remove lo device */ + if (strcmp(iface->device, "lo") == 0) { + return 0; + } + if (hyper_down_nic(rth, iface->ifindex) < 0) { fprintf(stderr, "up device %d failed\n", iface->ifindex); return -1; diff --git a/src/parse.c b/src/parse.c index 98c25eb..dfb11e3 100644 --- a/src/parse.c +++ b/src/parse.c @@ -226,7 +226,9 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container * c->exec.init = 1; c->exec.code = -1; c->exec.e.fd = -1; + c->exec.errev.fd = -1; c->exec.ptyfd = -1; + c->exec.errfd = -1; c->ns = -1; next_container = toks[i].size; @@ -614,7 +616,9 @@ realloc: goto out; exec->ptyfd = -1; + exec->errfd = -1; exec->e.fd = -1; + exec->errev.fd = -1; INIT_LIST_HEAD(&exec->list); for (i = 0, j = 0; i < n; i++) {