2 * A Process Isolator based in Linux Containers
4 * (c) 2012 Martin Mares <mj@ucw.cz>
27 #include <sys/ptrace.h>
28 #include <sys/signal.h>
29 #include <sys/sysinfo.h>
30 #include <sys/resource.h>
31 #include <sys/mount.h>
34 #define NONRET __attribute__((noreturn))
35 #define UNUSED __attribute__((unused))
36 #define ARRAY_SIZE(a) (int)(sizeof(a)/sizeof(a[0]))
38 // FIXME: Make configurable, probably in compile time
39 #define BOX_DIR "/tmp/box"
43 static int timeout; /* milliseconds */
44 static int wall_timeout;
45 static int extra_timeout;
46 static int pass_environ;
48 static int memory_limit;
49 static int stack_limit;
50 static int max_processes = 1;
51 static char *redir_stdin, *redir_stdout, *redir_stderr;
54 static int cg_memory_limit;
56 static char *cg_root = "/sys/fs/cgroup";
58 static uid_t orig_uid;
59 static gid_t orig_gid;
62 static volatile sig_atomic_t timer_tick;
63 static struct timeval start_time;
64 static int ticks_per_sec;
65 static int partial_line;
66 static char cleanup_cmd[256];
68 static int total_ms, wall_ms;
70 static void die(char *msg, ...) NONRET;
71 static void cg_stats(void);
72 static int get_wall_time_ms(void);
73 static int get_run_time_ms(void);
77 static FILE *metafile;
80 meta_open(const char *name)
82 if (!strcmp(name, "-"))
87 metafile = fopen(name, "w");
89 die("Failed to open metafile '%s'",name);
95 if (metafile && metafile != stdout)
99 static void __attribute__((format(printf,1,2)))
100 meta_printf(const char *fmt, ...)
107 vfprintf(metafile, fmt, args);
112 final_stats(struct rusage *rus)
114 total_ms = get_run_time_ms();
115 wall_ms = get_wall_time_ms();
117 meta_printf("time:%d.%03d\n", total_ms/1000, total_ms%1000);
118 meta_printf("time-wall:%d.%03d\n", wall_ms/1000, wall_ms%1000);
119 meta_printf("max-rss:%ld\n", rus->ru_maxrss);
120 meta_printf("csw-voluntary:%ld\n", rus->ru_nvcsw);
121 meta_printf("csw-forced:%ld\n", rus->ru_nivcsw);
126 /*** Messages and exits ***/
129 xsystem(const char *cmd)
131 int ret = system(cmd);
133 die("system(\"%s\"): %m", cmd);
134 if (!WIFEXITED(ret) || WEXITSTATUS(ret))
135 die("system(\"%s\"): Exited with status %d", cmd, ret);
143 kill(-box_pid, SIGKILL);
144 kill(box_pid, SIGKILL);
145 meta_printf("killed:1\n");
150 p = wait4(box_pid, &stat, 0, &rus);
151 while (p < 0 && errno == EINTR);
153 fprintf(stderr, "UGH: Lost track of the process (%m)\n");
158 if (rc < 2 && cleanup_cmd[0])
159 xsystem(cleanup_cmd);
173 /* Report an error of the sandbox itself */
174 static void NONRET __attribute__((format(printf,1,2)))
181 vsnprintf(buf, sizeof(buf), msg, args);
182 meta_printf("status:XX\nmessage:%s\n", buf);
188 /* Report an error of the program inside the sandbox */
189 static void NONRET __attribute__((format(printf,1,2)))
195 if (msg[0] && msg[1] && msg[2] == ':' && msg[3] == ' ')
197 meta_printf("status:%c%c\n", msg[0], msg[1]);
201 vsnprintf(buf, sizeof(buf), msg, args);
202 meta_printf("message:%s\n", buf);
208 /* Write a message, but only if in verbose mode */
209 static void __attribute__((format(printf,1,2)))
216 int len = strlen(msg);
218 partial_line = (msg[len-1] != '\n');
219 vfprintf(stderr, msg, args);
228 void *p = malloc(size);
230 die("Out of memory");
234 /*** Environment rules ***/
237 char *var; // Variable to match
238 char *val; // ""=clear, NULL=inherit
240 struct env_rule *next;
243 static struct env_rule *first_env_rule;
244 static struct env_rule **last_env_rule = &first_env_rule;
246 static struct env_rule default_env_rules[] = {
247 { "LIBC_FATAL_STDERR_", "1" }
251 set_env_action(char *a0)
253 struct env_rule *r = xmalloc(sizeof(*r) + strlen(a0) + 1);
254 char *a = (char *)(r+1);
257 char *sep = strchr(a, '=');
269 last_env_rule = &r->next;
275 match_env_var(char *env_entry, struct env_rule *r)
277 if (strncmp(env_entry, r->var, r->var_len))
279 return (env_entry[r->var_len] == '=');
283 apply_env_rule(char **env, int *env_sizep, struct env_rule *r)
285 // First remove the variable if already set
287 while (pos < *env_sizep && !match_env_var(env[pos], r))
289 if (pos < *env_sizep)
292 env[pos] = env[*env_sizep];
293 env[*env_sizep] = NULL;
296 // What is the new value?
302 new = xmalloc(r->var_len + 1 + strlen(r->val) + 1);
303 sprintf(new, "%s=%s", r->var, r->val);
308 while (environ[pos] && !match_env_var(environ[pos], r))
310 if (!(new = environ[pos]))
314 // Add it at the end of the array
315 env[(*env_sizep)++] = new;
316 env[*env_sizep] = NULL;
320 setup_environment(void)
322 // Link built-in rules with user rules
323 for (int i=ARRAY_SIZE(default_env_rules)-1; i >= 0; i--)
325 default_env_rules[i].next = first_env_rule;
326 first_env_rule = &default_env_rules[i];
329 // Scan the original environment
330 char **orig_env = environ;
332 while (orig_env[orig_size])
335 // For each rule, reserve one more slot and calculate length
337 for (struct env_rule *r = first_env_rule; r; r=r->next)
340 r->var_len = strlen(r->var);
343 // Create a new environment
344 char **env = xmalloc((orig_size + num_rules + 1) * sizeof(char *));
348 memcpy(env, environ, orig_size * sizeof(char *));
355 // Apply the rules one by one
356 for (struct env_rule *r = first_env_rule; r; r=r->next)
357 apply_env_rule(env, &size, r);
359 // Return the new env and pass some gossip
362 fprintf(stderr, "Passing environment:\n");
363 for (int i=0; env[i]; i++)
364 fprintf(stderr, "\t%s\n", env[i]);
369 /*** Control groups ***/
371 static char cg_path[256];
373 #define CG_BUFSIZE 1024
376 cg_read(char *attr, char *buf)
386 snprintf(path, sizeof(path), "%s/%s", cg_path, attr);
388 int fd = open(path, O_RDONLY);
393 die("Cannot read %s: %m", path);
396 int n = read(fd, buf, CG_BUFSIZE);
398 die("Cannot read %s: %m", path);
399 if (n >= CG_BUFSIZE - 1)
400 die("Attribute %s too long", path);
401 if (n > 0 && buf[n-1] == '\n')
406 msg("CG: Read %s = %s\n", attr, buf);
412 static void __attribute__((format(printf,2,3)))
413 cg_write(char *attr, char *fmt, ...)
418 char buf[CG_BUFSIZE];
419 int n = vsnprintf(buf, sizeof(buf), fmt, args);
421 die("cg_writef: Value for attribute %s is too long", attr);
424 msg("CG: Write %s = %s", attr, buf);
427 snprintf(path, sizeof(path), "%s/%s", cg_path, attr);
429 int fd = open(path, O_WRONLY | O_TRUNC);
431 die("Cannot write %s: %m", path);
433 int written = write(fd, buf, n);
435 die("Cannot set %s to %s: %m", path, buf);
437 die("Short write to %s (%d out of %d bytes)", path, written, n);
450 if (stat(cg_root, &st) < 0 || !S_ISDIR(st.st_mode))
451 die("Control group filesystem at %s not mounted", cg_root);
453 snprintf(cg_path, sizeof(cg_path), "%s/box-%d", cg_root, BOX_UID);
454 msg("Using control group %s\n", cg_path);
464 char buf[CG_BUFSIZE];
466 if (stat(cg_path, &st) >= 0 || errno != ENOENT)
468 msg("Control group %s already exists, trying to empty it.\n", cg_path);
469 if (rmdir(cg_path) < 0)
470 die("Failed to reset control group %s: %m", cg_path);
473 if (mkdir(cg_path, 0777) < 0)
474 die("Failed to create control group %s: %m", cg_path);
476 // If cpuset module is enabled, copy allowed cpus and memory nodes from parent group
477 if (cg_read("?../cpuset.cpus", buf))
478 cg_write("cpuset.cpus", "%s", buf);
479 if (cg_read("?../cpuset.mems", buf))
480 cg_write("cpuset.mems", "%s", buf);
489 msg("Entering control group %s\n", cg_path);
492 if (stat(cg_path, &st) < 0)
493 die("Control group %s does not exist: %m", cg_path);
497 cg_write("memory.limit_in_bytes", "%lld\n", (long long) cg_memory_limit << 10);
498 cg_write("memory.memsw.limit_in_bytes", "%lld\n", (long long) cg_memory_limit << 10);
502 cg_write("cpuacct.usage", "0\n");
504 cg_write("tasks", "%d\n", (int) getpid());
508 cg_get_run_time_ms(void)
513 char buf[CG_BUFSIZE];
514 cg_read("cpuacct.usage", buf);
515 unsigned long long ns = atoll(buf);
525 char buf[CG_BUFSIZE];
527 // Memory usage statistics
528 unsigned long long mem=0, memsw=0;
529 if (cg_read("?memory.max_usage_in_bytes", buf))
531 if (cg_read("?memory.memsw.max_usage_in_bytes", buf))
538 meta_printf("cg-mem:%lld\n", mem >> 10);
544 char buf[CG_BUFSIZE];
549 cg_read("tasks", buf);
551 die("Some tasks left in control group %s, failed to remove it", cg_path);
553 // FIXME: Is this needed?
554 // cg_write("memory.force_empty", "0\n");
556 if (rmdir(cg_path) < 0)
557 die("Cannot remove control group %s: %m", cg_path);
560 /*** The keeper process ***/
563 signal_alarm(int unused UNUSED)
565 /* Time limit checks are synchronous, so we only schedule them there. */
571 signal_int(int unused UNUSED)
573 /* Interrupts are fatal, so no synchronization requirements. */
574 meta_printf("exitsig:%d\n", SIGINT);
575 err("SG: Interrupted");
578 #define PROC_BUF_SIZE 4096
580 read_proc_file(char *buf, char *name, int *fdp)
586 sprintf(buf, "/proc/%d/%s", (int) box_pid, name);
587 *fdp = open(buf, O_RDONLY);
589 die("open(%s): %m", buf);
591 lseek(*fdp, 0, SEEK_SET);
592 if ((c = read(*fdp, buf, PROC_BUF_SIZE-1)) < 0)
593 die("read on /proc/$pid/%s: %m", name);
594 if (c >= PROC_BUF_SIZE-1)
595 die("/proc/$pid/%s too long", name);
600 get_wall_time_ms(void)
602 struct timeval now, wall;
603 gettimeofday(&now, NULL);
604 timersub(&now, &start_time, &wall);
605 return wall.tv_sec*1000 + wall.tv_usec/1000;
609 get_run_time_ms(void)
612 return cg_get_run_time_ms();
614 char buf[PROC_BUF_SIZE], *x;
616 static int proc_stat_fd;
618 read_proc_file(buf, "stat", &proc_stat_fd);
620 while (*x && *x != ' ')
625 die("proc stat syntax error 1");
626 while (*x && (*x != ')' || x[1] != ' '))
628 while (*x == ')' || *x == ' ')
630 if (sscanf(x, "%*c %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d %d %d", &utime, &stime) != 2)
631 die("proc stat syntax error 2");
633 return (utime + stime) * 1000 / ticks_per_sec;
641 int wall_ms = get_wall_time_ms();
642 if (wall_ms > wall_timeout)
643 err("TO: Time limit exceeded (wall clock)");
645 fprintf(stderr, "[wall time check: %d msec]\n", wall_ms);
649 int ms = get_run_time_ms();
651 fprintf(stderr, "[time check: %d msec]\n", ms);
652 if (ms > timeout && ms > extra_timeout)
653 err("TO: Time limit exceeded");
662 bzero(&sa, sizeof(sa));
663 sa.sa_handler = signal_int;
664 sigaction(SIGINT, &sa, NULL);
666 gettimeofday(&start_time, NULL);
667 ticks_per_sec = sysconf(_SC_CLK_TCK);
668 if (ticks_per_sec <= 0)
669 die("Invalid ticks_per_sec!");
671 if (timeout || wall_timeout)
673 sa.sa_handler = signal_alarm;
674 sigaction(SIGALRM, &sa, NULL);
688 p = wait4(box_pid, &stat, 0, &rus);
696 die("wait4: unknown pid %d exited!", p);
701 if (WEXITSTATUS(stat))
703 // FIXME: Recognize internal errors during setup
704 meta_printf("exitcode:%d\n", WEXITSTATUS(stat));
705 err("RE: Exited with error status %d", WEXITSTATUS(stat));
707 if (timeout && total_ms > timeout)
708 err("TO: Time limit exceeded");
709 if (wall_timeout && wall_ms > wall_timeout)
710 err("TO: Time limit exceeded (wall clock)");
712 fprintf(stderr, "OK (%d.%03d sec real, %d.%03d sec wall)\n",
713 total_ms/1000, total_ms%1000,
714 wall_ms/1000, wall_ms%1000);
717 if (WIFSIGNALED(stat))
720 meta_printf("exitsig:%d\n", WTERMSIG(stat));
722 err("SG: Caught fatal signal %d", WTERMSIG(stat));
724 if (WIFSTOPPED(stat))
727 meta_printf("exitsig:%d\n", WSTOPSIG(stat));
729 err("SG: Stopped by signal %d", WSTOPSIG(stat));
732 die("wait4: unknown status %x, giving up!", stat);
736 /*** The process running inside the box ***/
741 if (mkdir("root", 0750) < 0 && errno != EEXIST)
742 die("mkdir('root'): %m");
744 if (mount("none", "root", "tmpfs", 0, "mode=755") < 0)
745 die("Cannot mount root ramdisk: %m");
747 // FIXME: Make the list of bind-mounts configurable
748 // FIXME: Virtual /dev?
749 // FIXME: Read-only mounts?
751 static const char * const dirs[] = { "box", "/bin", "/lib", "/usr", "/dev" };
752 for (int i=0; i < ARRAY_SIZE(dirs); i++)
754 const char *d = dirs[i];
755 char buf[1024]; // FIXME
756 sprintf(buf, "root/%s", (d[0] == '/' ? d+1 : d));
757 msg("Binding %s on %s\n", d, buf);
758 if (mkdir(buf, 0755) < 0)
759 die("mkdir(%s): %m", buf);
760 if (mount(d, buf, "none", MS_BIND | MS_NOSUID | MS_NODEV, "") < 0)
761 die("Cannot bind %s on %s: %m", d, buf);
764 if (mkdir("root/proc", 0755) < 0)
765 die("Cannot create proc: %m");
766 if (mount("none", "root/proc", "proc", 0, "") < 0)
767 die("Cannot mount proc: %m");
769 if (chroot("root") < 0)
770 die("Chroot failed: %m");
772 if (chdir("root/box") < 0)
773 die("Cannot change current directory: %m");
777 box_inside(void *arg)
787 memcpy(args, argv, argc * sizeof(char *));
793 if (setresgid(BOX_GID, BOX_GID, BOX_GID) < 0)
794 die("setresgid: %m");
795 if (setgroups(0, NULL) < 0)
796 die("setgroups: %m");
797 if (setresuid(BOX_UID, BOX_UID, BOX_UID) < 0)
798 die("setresuid: %m");
803 if (open(redir_stdin, O_RDONLY) != 0)
804 die("open(\"%s\"): %m", redir_stdin);
809 if (open(redir_stdout, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 1)
810 die("open(\"%s\"): %m", redir_stdout);
815 if (open(redir_stderr, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 2)
816 die("open(\"%s\"): %m", redir_stderr);
824 rl.rlim_cur = rl.rlim_max = memory_limit * 1024;
825 if (setrlimit(RLIMIT_AS, &rl) < 0)
826 die("setrlimit(RLIMIT_AS): %m");
829 rl.rlim_cur = rl.rlim_max = (stack_limit ? (rlim_t)stack_limit * 1024 : RLIM_INFINITY);
830 if (setrlimit(RLIMIT_STACK, &rl) < 0)
831 die("setrlimit(RLIMIT_STACK): %m");
833 rl.rlim_cur = rl.rlim_max = 64;
834 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
835 die("setrlimit(RLIMIT_NOFILE): %m");
839 rl.rlim_cur = rl.rlim_max = max_processes;
840 if (setrlimit(RLIMIT_NPROC, &rl) < 0)
841 die("setrlimit(RLIMIT_NPROC): %m");
844 rl.rlim_cur = rl.rlim_max = 0;
845 if (setrlimit(RLIMIT_MEMLOCK, &rl) < 0)
846 die("setrlimit(RLIMIT_MEMLOCK): %m");
848 char **env = setup_environment();
849 execve(args[0], args, env);
850 die("execve(\"%s\"): %m", args[0]);
856 msg("Preparing sandbox directory\n");
857 xsystem("rm -rf box");
858 if (mkdir("box", 0700) < 0)
859 die("Cannot create box: %m");
860 if (chown("box", orig_uid, orig_gid) < 0)
861 die("Cannot chown box: %m");
870 if (stat("box", &st) < 0 || !S_ISDIR(st.st_mode))
871 die("Box directory not found, there isn't anything to clean up");
873 msg("Deleting sandbox directory\n");
874 xsystem("rm -rf box");
882 if (stat("box", &st) < 0 || !S_ISDIR(st.st_mode))
883 die("Box directory not found, did you run `isolate --init'?");
886 snprintf(cmd, sizeof(cmd), "chown -R %d.%d box", BOX_UID, BOX_GID);
888 snprintf(cleanup_cmd, sizeof(cleanup_cmd), "chown -R %d.%d box", orig_uid, orig_gid);
891 box_inside, // Function to execute as the body of the new process
892 argv, // Pass our stack
893 SIGCHLD | CLONE_NEWIPC | CLONE_NEWNET | CLONE_NEWNS | CLONE_NEWPID,
894 argv); // Pass the arguments
898 die("clone returned 0");
906 printf("Process isolator 0.0\n");
907 printf("(c) 2012 Martin Mares <mj@ucw.cz>\n\n");
908 printf("Sandbox directory: %s\n", BOX_DIR);
909 printf("Sandbox credentials: uid=%u gid=%u\n", BOX_UID, BOX_GID);
915 fprintf(stderr, "Invalid arguments!\n");
917 Usage: isolate [<options>] <command>\n\
920 -c, --cg[=<parent>]\tPut process in a control group (optionally a sub-group of <parent>)\n\
921 --cg-mem=<size>\tLimit memory usage of the control group to <size> KB\n\
922 --cg-timing\t\tTime limits affects total run time of the control group\n\
923 -E, --env=<var>\tInherit the environment variable <var> from the parent process\n\
924 -E, --env=<var>=<val>\tSet the environment variable <var> to <val>; unset it if <var> is empty\n\
925 -x, --extra-time=<time>\tSet extra timeout, before which a timing-out program is not yet killed,\n\
926 \t\t\tso that its real execution time is reported (seconds, fractions allowed)\n\
927 -e, --full-env\t\tInherit full environment of the parent process\n\
928 -m, --mem=<size>\tLimit address space to <size> KB\n\
929 -M, --meta=<file>\tOutput process information to <file> (name:value)\n\
930 -k, --stack=<size>\tLimit stack size to <size> KB (default: 0=unlimited)\n\
931 -r, --stderr=<file>\tRedirect stderr to <file>\n\
932 -i, --stdin=<file>\tRedirect stdin from <file>\n\
933 -o, --stdout=<file>\tRedirect stdout to <file>\n\
934 -p, --processes[=<max>]\tEnable multiple processes (at most <max> of them); needs --cg\n\
935 -t, --time=<time>\tSet run time limit (seconds, fractions allowed)\n\
936 -v, --verbose\t\tBe verbose (use multiple times for even more verbosity)\n\
937 -w, --wall-time=<time>\tSet wall clock time limit (seconds, fractions allowed)\n\
940 --init\t\tInitialize sandbox (and its control group when --cg is used)\n\
941 --run -- <cmd> ...\tRun given command within sandbox\n\
942 --cleanup\t\tClean up sandbox\n\
943 --version\t\tDisplay program version and configuration\n\
957 static const char short_opts[] = "c::eE:i:k:m:M:o:p::r:t:vw:x:";
959 static const struct option long_opts[] = {
960 { "cg", 2, NULL, 'c' },
961 { "cg-mem", 1, NULL, OPT_CG_MEM },
962 { "cg-timing", 0, NULL, OPT_CG_TIMING },
963 { "cleanup", 0, NULL, OPT_CLEANUP },
964 { "env", 1, NULL, 'E' },
965 { "extra-time", 1, NULL, 'x' },
966 { "full-env", 0, NULL, 'e' },
967 { "init", 0, NULL, OPT_INIT },
968 { "mem", 1, NULL, 'm' },
969 { "meta", 1, NULL, 'M' },
970 { "processes", 2, NULL, 'p' },
971 { "run", 0, NULL, OPT_RUN },
972 { "stack", 1, NULL, 'k' },
973 { "stderr", 1, NULL, 'r' },
974 { "stdin", 1, NULL, 'i' },
975 { "stdout", 1, NULL, 'o' },
976 { "time", 1, NULL, 't' },
977 { "verbose", 0, NULL, 'v' },
978 { "version", 0, NULL, OPT_VERSION },
979 { "wall-time", 1, NULL, 'w' },
984 main(int argc, char **argv)
987 enum opt_code mode = 0;
989 while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) >= 0)
1001 if (!set_env_action(optarg))
1005 stack_limit = atoi(optarg);
1008 redir_stdin = optarg;
1011 memory_limit = atoi(optarg);
1017 redir_stdout = optarg;
1021 max_processes = atoi(optarg);
1026 redir_stderr = optarg;
1029 timeout = 1000*atof(optarg);
1035 wall_timeout = 1000*atof(optarg);
1038 extra_timeout = 1000*atof(optarg);
1047 cg_memory_limit = atoi(optarg);
1058 if (mode == OPT_VERSION)
1065 die("Must be started as root");
1066 orig_uid = getuid();
1067 orig_gid = getgid();
1070 if (chdir(BOX_DIR) < 0)
1071 die("chdir(%s): %m", BOX_DIR);
1092 die("Internal error: mode mismatch");