mirror of
https://github.com/clearlinux/graphene.git
synced 2026-10-03 23:48:19 +00:00
[LibOS] Build GCC with patched libgomp.so (OpenMP runtime library)
Many multi-threaded applications rely on OpenMP for efficient parallelism. Such applications link against the libgomp.so lib. Unfortunately, the native libgomp.so uses a raw SYSCALL instruction to issues futex() syscalls instead of syscall() Glibc wrapper (for performance reasons). SYSCALL instructions are executed on hot paths of OpenMP, but they are forbidden inside SGX enclaves, so Graphene traps-and-emulates them. This is very expensive, so this commit introduces an optional `make -C LibOS gcc` to build libgomp.so where SYSCALL is replaced by direct call into Graphene, similar to how we patch Glibc. See `openmp` LibOS regression manifest for example. This patched libgomp.so improves SGX performance of e.g. PyTorch: from 25% perf overhead over native to 8% on some workloads. Note that libgomp.so is built as part of GCC, and the build takes 30 min to 2 hours. That's why we make it optional and do not build in Jenkins.
This commit is contained in:
Dmitrii Kuvaiskii
@@ -2,3 +2,8 @@
|
||||
/glibc-*.*/
|
||||
/glibc-build/
|
||||
/build.log
|
||||
|
||||
/gcc-*.tar.gz
|
||||
/gcc-*.*.*/
|
||||
/gcc-build/
|
||||
/gcc-build.log
|
||||
|
||||
+94
-27
@@ -1,12 +1,27 @@
|
||||
include ../Scripts/Makefile.configs
|
||||
include ../Scripts/Makefile.rules
|
||||
|
||||
RUNTIME_DIR = $(CURDIR)/../Runtime
|
||||
|
||||
define LN_SF_TO_RUNTIME_DIR_template =
|
||||
$(RUNTIME_DIR)/$(notdir $(1)): $(1)
|
||||
$$(call cmd,ln_sfr)
|
||||
endef
|
||||
|
||||
GNU_MIRRORS ?= https://ftp.gnu.org/gnu/ \
|
||||
https://mirrors.kernel.org/gnu/ \
|
||||
https://mirrors.ocf.berkeley.edu/gnu/
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# Glibc build: patch libc.so and other C standard libraries to replace raw SYSCALL instructions
|
||||
# with function calls into Graphene (plus some smaller patches for Graphene). The resulting
|
||||
# libraries are symlinked under $RUNTIME_DIR.
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
GLIBC_VERSION ?= 2.31
|
||||
GLIBC_SRC = glibc-$(GLIBC_VERSION)
|
||||
GLIBC_CHECKSUM = $(firstword $(shell grep $(GLIBC_SRC).tar.gz glibc-checksums))
|
||||
GLIBC_HASH = $(firstword $(shell grep $(GLIBC_SRC).tar.gz glibc-checksums))
|
||||
SHIM_DIR = shim
|
||||
BUILD_DIR = glibc-build
|
||||
RUNTIME_DIR = $(CURDIR)/../Runtime
|
||||
GLIBC_LIBS = \
|
||||
csu/crt1.o \
|
||||
csu/crti.o \
|
||||
@@ -38,10 +53,6 @@ export GLIBC_CFLAGS
|
||||
all: $(GLIBC_TARGET) $(GLIBC_RUNTIME)
|
||||
$(MAKE) -C $(SHIM_DIR) all
|
||||
|
||||
.PHONY: format
|
||||
format:
|
||||
$(MAKE) -C $(SHIM_DIR) format
|
||||
|
||||
ifeq ($(findstring x86_64,$(SYS))$(findstring linux,$(SYS)),x86_64linux)
|
||||
|
||||
.SECONDARY: $(BUILD_DIR)/Build.success
|
||||
@@ -52,27 +63,18 @@ $(BUILD_DIR)/Build.success: $(BUILD_DIR)/Makefile
|
||||
|
||||
$(GLIBC_TARGET): $(BUILD_DIR)/Build.success
|
||||
|
||||
$(BUILD_DIR)/Makefile: $(GLIBC_SRC)/configure
|
||||
$(BUILD_DIR)/Makefile: $(GLIBC_SRC)/.configured
|
||||
mkdir -p $(BUILD_DIR)
|
||||
(cd $(BUILD_DIR) || exit 1; \
|
||||
CFLAGS=$$GLIBC_CFLAGS ../$< --prefix=$(RUNTIME_DIR) \
|
||||
CFLAGS=$$GLIBC_CFLAGS ../$(GLIBC_SRC)/configure --prefix=$(RUNTIME_DIR) \
|
||||
--with-tls \
|
||||
--without-selinux \
|
||||
--disable-test \
|
||||
--disable-nscd \
|
||||
)
|
||||
|
||||
define LN_SF_TO_RUNTIME_DIR_template =
|
||||
$(RUNTIME_DIR)/$(notdir $(1)): $(1)
|
||||
$$(call cmd,ln_sfr)
|
||||
endef
|
||||
|
||||
$(foreach lib,$(GLIBC_TARGET),$(eval $(call LN_SF_TO_RUNTIME_DIR_template,$(lib))))
|
||||
|
||||
GLIBC_MIRRORS ?= https://ftp.gnu.org/gnu/ \
|
||||
https://mirrors.kernel.org/gnu/ \
|
||||
https://mirrors.ocf.berkeley.edu/gnu/
|
||||
|
||||
GLIBC_PATCHES = \
|
||||
glibc-patches/$(GLIBC_SRC).patch \
|
||||
glibc-patches/syscalldb-api.patch
|
||||
@@ -93,28 +95,86 @@ GLIBC_PATCHES_2.31 = \
|
||||
|
||||
GLIBC_PATCHES += $(GLIBC_PATCHES_$(GLIBC_VERSION))
|
||||
|
||||
$(GLIBC_SRC)/configure: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz
|
||||
.SECONDARY: $(GLIBC_SRC)/.configured
|
||||
$(GLIBC_SRC)/.configured: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz
|
||||
$(RM) -r $(GLIBC_SRC)
|
||||
tar -xzf $(GLIBC_SRC).tar.gz
|
||||
tar -mxzf $(GLIBC_SRC).tar.gz
|
||||
cd $(GLIBC_SRC) && \
|
||||
for p in $(GLIBC_PATCHES); do \
|
||||
echo applying $$p; \
|
||||
patch -p1 < ../$$p || exit 255; \
|
||||
patch -p1 -l < ../$$p || exit 255; \
|
||||
done
|
||||
touch $@
|
||||
|
||||
$(GLIBC_SRC).tar.gz:
|
||||
../Scripts/download --output $@ --sha256 $(GLIBC_CHECKSUM) $(foreach mirror,$(GLIBC_MIRRORS),--url $(mirror)glibc/$(GLIBC_SRC).tar.gz)
|
||||
../Scripts/download --output $@ --sha256 $(GLIBC_HASH) $(foreach mirror,$(GNU_MIRRORS),--url $(mirror)glibc/$(GLIBC_SRC).tar.gz)
|
||||
|
||||
$(GLIBC_SRC)/elf/Versions: $(GLIBC_SRC)/configure
|
||||
$(GLIBC_SRC)/elf/Versions: $(GLIBC_SRC)/.configured
|
||||
|
||||
$(GLIBC_SRC)/nptl/Versions: $(GLIBC_SRC)/configure
|
||||
$(GLIBC_SRC)/nptl/Versions: $(GLIBC_SRC)/.configured
|
||||
|
||||
$(GLIBC_SRC)/dlfcn/Versions: $(GLIBC_SRC)/configure
|
||||
$(GLIBC_SRC)/dlfcn/Versions: $(GLIBC_SRC)/.configured
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# GCC build: patch libgomp.so.1 (OpenMP runtime library) to replace raw SYSCALL instruction with
|
||||
# function call into Graphene. GCC is not built by default with Graphene; use `make -C LibOS gcc`
|
||||
# to build it. The resulting libgomp.so.1 is symlinked under $RUNTIME_DIR. This patched version
|
||||
# makes sense only on x86_64 platforms. NOTE: We'd prefer to build libgomp.so.1 alone but it is
|
||||
# impossible (the only way to build it is as part of the complete GCC build).
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
GCC_VERSION ?= 10.2.0
|
||||
GCC_SRC = gcc-$(GCC_VERSION)
|
||||
GCC_HASH = 27e879dccc639cd7b0cc08ed575c1669492579529b53c9ff27b0b96265fa867d
|
||||
GCC_BUILD_DIR = gcc-build
|
||||
GCC_LIBS = x86_64-pc-linux-gnu/libgomp/.libs/libgomp.so.1
|
||||
GCC_TARGET = $(addprefix $(GCC_BUILD_DIR)/, $(GCC_LIBS))
|
||||
GCC_RUNTIME = $(addprefix $(RUNTIME_DIR)/, $(notdir $(GCC_TARGET)))
|
||||
|
||||
.SECONDARY: $(GCC_BUILD_DIR)/Build.success
|
||||
|
||||
$(GCC_BUILD_DIR)/Build.success: $(GCC_BUILD_DIR)/Makefile
|
||||
@echo "Building gcc, may take a while to finish. Warning messages may show up. If this process terminates with failures, see \"$(GCC_BUILD_DIR)/gcc-build.log\" for more information."
|
||||
($(MAKE) -C $(GCC_BUILD_DIR) 2>&1 > gcc-build.log) && touch $@
|
||||
|
||||
$(GCC_TARGET): $(GCC_BUILD_DIR)/Build.success
|
||||
|
||||
$(GCC_BUILD_DIR)/Makefile: $(GCC_SRC)/.configured
|
||||
mkdir -p $(GCC_BUILD_DIR)
|
||||
(cd $(GCC_BUILD_DIR) || exit 1; \
|
||||
../$(GCC_SRC)/configure --prefix=$(RUNTIME_DIR) \
|
||||
--enable-languages=c \
|
||||
--disable-multilib \
|
||||
)
|
||||
|
||||
$(foreach lib,$(GCC_TARGET),$(eval $(call LN_SF_TO_RUNTIME_DIR_template,$(lib))))
|
||||
|
||||
GCC_PATCHES = \
|
||||
gcc-patches/libgomp-replace-futex-instruction.patch
|
||||
|
||||
.SECONDARY: $(GCC_SRC)/.configured
|
||||
$(GCC_SRC)/.configured: $(GCC_PATCHES) $(GCC_SRC).tar.gz
|
||||
$(RM) -r $(GCC_SRC)
|
||||
tar -mxzf $(GCC_SRC).tar.gz
|
||||
cd $(GCC_SRC) && \
|
||||
for p in $(GCC_PATCHES); do \
|
||||
echo applying $$p; \
|
||||
patch -p1 -l < ../$$p || exit 255; \
|
||||
done
|
||||
cd $(GCC_SRC) && ./contrib/download_prerequisites
|
||||
touch $@
|
||||
|
||||
$(GCC_SRC).tar.gz:
|
||||
../Scripts/download --output $@ --sha256 $(GCC_HASH) $(foreach mirror,$(GNU_MIRRORS),--url $(mirror)gcc/$(GCC_SRC)/$(GCC_SRC).tar.gz)
|
||||
|
||||
.PHONY: gcc
|
||||
gcc: $(GCC_TARGET) $(GCC_RUNTIME)
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# Common targets
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
.PHONY: clean_
|
||||
clean_:
|
||||
$(RM) -r $(BUILD_DIR) $(GLIBC_SRC) build.log
|
||||
$(RM) -r $(BUILD_DIR) $(GLIBC_SRC) $(GCC_BUILD_DIR) $(GCC_SRC) build.log gcc-build.log
|
||||
|
||||
.PHONY: clean
|
||||
clean: clean_
|
||||
@@ -123,20 +183,27 @@ clean: clean_
|
||||
.PHONY: distclean
|
||||
distclean: clean_
|
||||
$(MAKE) -C $(SHIM_DIR) distclean
|
||||
$(RM) $(GLIBC_SRC).tar.gz
|
||||
$(RM) $(GLIBC_SRC).tar.gz $(GCC_SRC).tar.gz
|
||||
|
||||
else
|
||||
.IGNORE: $(GLIBC_TARGET)
|
||||
$(GLIBC_TARGET):
|
||||
|
||||
.IGNORE: $(GCC_TARGET)
|
||||
$(GCC_TARGET):
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) -r $(BUILD_DIR)
|
||||
$(RM) -r $(BUILD_DIR) $(GCC_BUILD_DIR)
|
||||
|
||||
.PHONY: distclean
|
||||
distclean: clean
|
||||
endif
|
||||
|
||||
.PHONY: format
|
||||
format:
|
||||
$(MAKE) -C $(SHIM_DIR) format
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
$(MAKE) -C $(SHIM_DIR) test
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
diff --git a/libgomp/config/linux/x86/futex.h b/libgomp/config/linux/x86/futex.h
|
||||
index ead74d1496736a49694ef6b9b2b4da50f9852664..3c82859ad8b82a09ec95f720727937ee5a2863c1 100644
|
||||
--- a/libgomp/config/linux/x86/futex.h
|
||||
+++ b/libgomp/config/linux/x86/futex.h
|
||||
@@ -30,13 +30,16 @@
|
||||
# define SYS_futex 202
|
||||
# endif
|
||||
|
||||
+asm (".weak syscalldb\r\n"
|
||||
+ ".type syscalldb, @function\r\n");
|
||||
+
|
||||
static inline void
|
||||
futex_wait (int *addr, int val)
|
||||
{
|
||||
long res;
|
||||
|
||||
register long r10 __asm__("%r10") = 0;
|
||||
- __asm volatile ("syscall"
|
||||
+ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;"
|
||||
: "=a" (res)
|
||||
: "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait),
|
||||
"d" (val), "r" (r10)
|
||||
@@ -45,7 +48,7 @@ futex_wait (int *addr, int val)
|
||||
{
|
||||
gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
|
||||
gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
|
||||
- __asm volatile ("syscall"
|
||||
+ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;"
|
||||
: "=a" (res)
|
||||
: "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait),
|
||||
"d" (val), "r" (r10)
|
||||
@@ -58,7 +61,7 @@ futex_wake (int *addr, int count)
|
||||
{
|
||||
long res;
|
||||
|
||||
- __asm volatile ("syscall"
|
||||
+ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;"
|
||||
: "=a" (res)
|
||||
: "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake),
|
||||
"d" (count)
|
||||
@@ -67,7 +70,7 @@ futex_wake (int *addr, int count)
|
||||
{
|
||||
gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
|
||||
gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
|
||||
- __asm volatile ("syscall"
|
||||
+ __asm volatile ("subq $128, %%rsp; callq *syscalldb@GOTPCREL(%%rip); addq $128, %%rsp;"
|
||||
: "=a" (res)
|
||||
: "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake),
|
||||
"d" (count)
|
||||
@@ -22,6 +22,14 @@ sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2
|
||||
sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6
|
||||
sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0
|
||||
sgx.trusted_files.libdl = file:../../../../Runtime/libdl.so.2
|
||||
sgx.trusted_files.libgomp = file:/usr$(ARCH_LIBDIR)/libgomp.so.1
|
||||
|
||||
sgx.trusted_files.libgomp_native = file:/usr$(ARCH_LIBDIR)/libgomp.so.1
|
||||
|
||||
# Graphene optionally provides patched OpenMP runtime library that runs faster
|
||||
# inside SGX enclaves (execute `make -C LibOS gcc` to generate it). Uncomment
|
||||
# the line below to use the patched library. This library will replace the
|
||||
# native one because Graphene's Runtime path has priority in LD_LIBRARY_PATH.
|
||||
|
||||
#sgx.trusted_files.libgomp_graphene = file:../../../../Runtime/libgomp.so.1
|
||||
|
||||
sgx.static_address = 1
|
||||
|
||||
Reference in New Issue
Block a user