diff --git a/LibOS/.gitignore b/LibOS/.gitignore index 9caf427d..9e801aa5 100644 --- a/LibOS/.gitignore +++ b/LibOS/.gitignore @@ -2,3 +2,8 @@ /glibc-*.*/ /glibc-build/ /build.log + +/gcc-*.tar.gz +/gcc-*.*.*/ +/gcc-build/ +/gcc-build.log diff --git a/LibOS/Makefile b/LibOS/Makefile index 5777140d..f6e7be0f 100644 --- a/LibOS/Makefile +++ b/LibOS/Makefile @@ -1,12 +1,27 @@ include ../Scripts/Makefile.configs include ../Scripts/Makefile.rules +RUNTIME_DIR = $(CURDIR)/../Runtime + +define LN_SF_TO_RUNTIME_DIR_template = +$(RUNTIME_DIR)/$(notdir $(1)): $(1) + $$(call cmd,ln_sfr) +endef + +GNU_MIRRORS ?= https://ftp.gnu.org/gnu/ \ + https://mirrors.kernel.org/gnu/ \ + https://mirrors.ocf.berkeley.edu/gnu/ + +# ------------------------------------------------------------------------------------------------ +# Glibc build: patch libc.so and other C standard libraries to replace raw SYSCALL instructions +# with function calls into Graphene (plus some smaller patches for Graphene). The resulting +# libraries are symlinked under $RUNTIME_DIR. +# ------------------------------------------------------------------------------------------------ GLIBC_VERSION ?= 2.31 GLIBC_SRC = glibc-$(GLIBC_VERSION) -GLIBC_CHECKSUM = $(firstword $(shell grep $(GLIBC_SRC).tar.gz glibc-checksums)) +GLIBC_HASH = $(firstword $(shell grep $(GLIBC_SRC).tar.gz glibc-checksums)) SHIM_DIR = shim BUILD_DIR = glibc-build -RUNTIME_DIR = $(CURDIR)/../Runtime GLIBC_LIBS = \ csu/crt1.o \ csu/crti.o \ @@ -38,10 +53,6 @@ export GLIBC_CFLAGS all: $(GLIBC_TARGET) $(GLIBC_RUNTIME) $(MAKE) -C $(SHIM_DIR) all -.PHONY: format -format: - $(MAKE) -C $(SHIM_DIR) format - ifeq ($(findstring x86_64,$(SYS))$(findstring linux,$(SYS)),x86_64linux) .SECONDARY: $(BUILD_DIR)/Build.success @@ -52,27 +63,18 @@ $(BUILD_DIR)/Build.success: $(BUILD_DIR)/Makefile $(GLIBC_TARGET): $(BUILD_DIR)/Build.success -$(BUILD_DIR)/Makefile: $(GLIBC_SRC)/configure +$(BUILD_DIR)/Makefile: $(GLIBC_SRC)/.configured mkdir -p $(BUILD_DIR) (cd $(BUILD_DIR) || exit 1; \ - CFLAGS=$$GLIBC_CFLAGS ../$< --prefix=$(RUNTIME_DIR) \ + CFLAGS=$$GLIBC_CFLAGS ../$(GLIBC_SRC)/configure --prefix=$(RUNTIME_DIR) \ --with-tls \ --without-selinux \ --disable-test \ --disable-nscd \ ) -define LN_SF_TO_RUNTIME_DIR_template = -$(RUNTIME_DIR)/$(notdir $(1)): $(1) - $$(call cmd,ln_sfr) -endef - $(foreach lib,$(GLIBC_TARGET),$(eval $(call LN_SF_TO_RUNTIME_DIR_template,$(lib)))) -GLIBC_MIRRORS ?= https://ftp.gnu.org/gnu/ \ - https://mirrors.kernel.org/gnu/ \ - https://mirrors.ocf.berkeley.edu/gnu/ - GLIBC_PATCHES = \ glibc-patches/$(GLIBC_SRC).patch \ glibc-patches/syscalldb-api.patch @@ -93,28 +95,86 @@ GLIBC_PATCHES_2.31 = \ GLIBC_PATCHES += $(GLIBC_PATCHES_$(GLIBC_VERSION)) -$(GLIBC_SRC)/configure: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz +.SECONDARY: $(GLIBC_SRC)/.configured +$(GLIBC_SRC)/.configured: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz $(RM) -r $(GLIBC_SRC) - tar -xzf $(GLIBC_SRC).tar.gz + tar -mxzf $(GLIBC_SRC).tar.gz cd $(GLIBC_SRC) && \ for p in $(GLIBC_PATCHES); do \ echo applying $$p; \ - patch -p1 < ../$$p || exit 255; \ + patch -p1 -l < ../$$p || exit 255; \ done touch $@ $(GLIBC_SRC).tar.gz: - ../Scripts/download --output $@ --sha256 $(GLIBC_CHECKSUM) $(foreach mirror,$(GLIBC_MIRRORS),--url $(mirror)glibc/$(GLIBC_SRC).tar.gz) + ../Scripts/download --output $@ --sha256 $(GLIBC_HASH) $(foreach mirror,$(GNU_MIRRORS),--url $(mirror)glibc/$(GLIBC_SRC).tar.gz) -$(GLIBC_SRC)/elf/Versions: $(GLIBC_SRC)/configure +$(GLIBC_SRC)/elf/Versions: $(GLIBC_SRC)/.configured -$(GLIBC_SRC)/nptl/Versions: $(GLIBC_SRC)/configure +$(GLIBC_SRC)/nptl/Versions: $(GLIBC_SRC)/.configured -$(GLIBC_SRC)/dlfcn/Versions: $(GLIBC_SRC)/configure +$(GLIBC_SRC)/dlfcn/Versions: $(GLIBC_SRC)/.configured +# ------------------------------------------------------------------------------------------------ +# GCC build: patch libgomp.so.1 (OpenMP runtime library) to replace raw SYSCALL instruction with +# function call into Graphene. GCC is not built by default with Graphene; use `make -C LibOS gcc` +# to build it. The resulting libgomp.so.1 is symlinked under $RUNTIME_DIR. This patched version +# makes sense only on x86_64 platforms. NOTE: We'd prefer to build libgomp.so.1 alone but it is +# impossible (the only way to build it is as part of the complete GCC build). +# ------------------------------------------------------------------------------------------------ +GCC_VERSION ?= 10.2.0 +GCC_SRC = gcc-$(GCC_VERSION) +GCC_HASH = 27e879dccc639cd7b0cc08ed575c1669492579529b53c9ff27b0b96265fa867d +GCC_BUILD_DIR = gcc-build +GCC_LIBS = x86_64-pc-linux-gnu/libgomp/.libs/libgomp.so.1 +GCC_TARGET = $(addprefix $(GCC_BUILD_DIR)/, $(GCC_LIBS)) +GCC_RUNTIME = $(addprefix $(RUNTIME_DIR)/, $(notdir $(GCC_TARGET))) + +.SECONDARY: $(GCC_BUILD_DIR)/Build.success + +$(GCC_BUILD_DIR)/Build.success: $(GCC_BUILD_DIR)/Makefile + @echo "Building gcc, may take a while to finish. Warning messages may show up. If this process terminates with failures, see \"$(GCC_BUILD_DIR)/gcc-build.log\" for more information." + ($(MAKE) -C $(GCC_BUILD_DIR) 2>&1 > gcc-build.log) && touch $@ + +$(GCC_TARGET): $(GCC_BUILD_DIR)/Build.success + +$(GCC_BUILD_DIR)/Makefile: $(GCC_SRC)/.configured + mkdir -p $(GCC_BUILD_DIR) + (cd $(GCC_BUILD_DIR) || exit 1; \ + ../$(GCC_SRC)/configure --prefix=$(RUNTIME_DIR) \ + --enable-languages=c \ + --disable-multilib \ + ) + +$(foreach lib,$(GCC_TARGET),$(eval $(call LN_SF_TO_RUNTIME_DIR_template,$(lib)))) + +GCC_PATCHES = \ + gcc-patches/libgomp-replace-futex-instruction.patch + +.SECONDARY: $(GCC_SRC)/.configured +$(GCC_SRC)/.configured: $(GCC_PATCHES) $(GCC_SRC).tar.gz + $(RM) -r $(GCC_SRC) + tar -mxzf $(GCC_SRC).tar.gz + cd $(GCC_SRC) && \ + for p in $(GCC_PATCHES); do \ + echo applying $$p; \ + patch -p1 -l < ../$$p || exit 255; \ + done + cd $(GCC_SRC) && ./contrib/download_prerequisites + touch $@ + +$(GCC_SRC).tar.gz: + ../Scripts/download --output $@ --sha256 $(GCC_HASH) $(foreach mirror,$(GNU_MIRRORS),--url $(mirror)gcc/$(GCC_SRC)/$(GCC_SRC).tar.gz) + +.PHONY: gcc +gcc: $(GCC_TARGET) $(GCC_RUNTIME) + +# ------------------------------------------------------------------------------------------------ +# Common targets +# ------------------------------------------------------------------------------------------------ .PHONY: clean_ clean_: - $(RM) -r $(BUILD_DIR) $(GLIBC_SRC) build.log + $(RM) -r $(BUILD_DIR) $(GLIBC_SRC) $(GCC_BUILD_DIR) $(GCC_SRC) build.log gcc-build.log .PHONY: clean clean: clean_ @@ -123,20 +183,27 @@ clean: clean_ .PHONY: distclean distclean: clean_ $(MAKE) -C $(SHIM_DIR) distclean - $(RM) $(GLIBC_SRC).tar.gz + $(RM) $(GLIBC_SRC).tar.gz $(GCC_SRC).tar.gz else .IGNORE: $(GLIBC_TARGET) $(GLIBC_TARGET): +.IGNORE: $(GCC_TARGET) +$(GCC_TARGET): + .PHONY: clean clean: - $(RM) -r $(BUILD_DIR) + $(RM) -r $(BUILD_DIR) $(GCC_BUILD_DIR) .PHONY: distclean distclean: clean endif +.PHONY: format +format: + $(MAKE) -C $(SHIM_DIR) format + .PHONY: test test: $(MAKE) -C $(SHIM_DIR) test diff --git a/LibOS/gcc-patches/libgomp-replace-futex-instruction.patch b/LibOS/gcc-patches/libgomp-replace-futex-instruction.patch new file mode 100644 index 00000000..c964ab90 --- /dev/null +++ b/LibOS/gcc-patches/libgomp-replace-futex-instruction.patch @@ -0,0 +1,49 @@ +diff --git a/libgomp/config/linux/x86/futex.h b/libgomp/config/linux/x86/futex.h +index ead74d1496736a49694ef6b9b2b4da50f9852664..3c82859ad8b82a09ec95f720727937ee5a2863c1 100644 +--- a/libgomp/config/linux/x86/futex.h ++++ b/libgomp/config/linux/x86/futex.h +@@ -30,13 +30,16 @@ + # define SYS_futex 202 + # endif + ++asm (".weak syscalldb\r\n" ++ ".type syscalldb, @function\r\n"); ++ + static inline void + futex_wait (int *addr, int val) + { + long res; + + register long r10 __asm__("%r10") = 0; +- __asm volatile ("syscall" ++ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) +@@ -45,7 +48,7 @@ futex_wait (int *addr, int val) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; +- __asm volatile ("syscall" ++ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) +@@ -58,7 +61,7 @@ futex_wake (int *addr, int count) + { + long res; + +- __asm volatile ("syscall" ++ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) +@@ -67,7 +70,7 @@ futex_wake (int *addr, int count) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; +- __asm volatile ("syscall" ++ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) diff --git a/LibOS/shim/test/regression/openmp.manifest.template b/LibOS/shim/test/regression/openmp.manifest.template index f3dae162..dbc85cef 100644 --- a/LibOS/shim/test/regression/openmp.manifest.template +++ b/LibOS/shim/test/regression/openmp.manifest.template @@ -22,6 +22,14 @@ sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 sgx.trusted_files.libdl = file:../../../../Runtime/libdl.so.2 -sgx.trusted_files.libgomp = file:/usr$(ARCH_LIBDIR)/libgomp.so.1 + +sgx.trusted_files.libgomp_native = file:/usr$(ARCH_LIBDIR)/libgomp.so.1 + +# Graphene optionally provides patched OpenMP runtime library that runs faster +# inside SGX enclaves (execute `make -C LibOS gcc` to generate it). Uncomment +# the line below to use the patched library. This library will replace the +# native one because Graphene's Runtime path has priority in LD_LIBRARY_PATH. + +#sgx.trusted_files.libgomp_graphene = file:../../../../Runtime/libgomp.so.1 sgx.static_address = 1