8368247: RISC-V: enable vectorapi test for expand operation

make/Hsdis.gmk

@@ -114,6 +114,7 @@ ifeq ($(HSDIS_BACKEND), binutils)
     TOOLCHAIN_TYPE := gcc
     OPENJDK_TARGET_OS := linux
     OPENJDK_TARGET_OS_TYPE := unix
+    CC_OUT_OPTION := -o$(SPACE)
     GENDEPS_FLAGS := -MMD -MF
     CFLAGS_DEBUG_SYMBOLS := -g
     DISABLED_WARNINGS :=

make/autoconf/boot-jdk.m4

@@ -597,9 +597,10 @@ AC_DEFUN([BOOTJDK_SETUP_BUILD_JDK],
   AC_ARG_WITH(build-jdk, [AS_HELP_STRING([--with-build-jdk],
       [path to JDK of same version as is being built@<:@the newly built JDK@:>@])])
 
-  EXTERNAL_BUILDJDK_PATH=""
+  CREATE_BUILDJDK=false
+  EXTERNAL_BUILDJDK=false
+  BUILD_JDK_FOUND="no"
   if test "x$with_build_jdk" != "x"; then
-    BUILD_JDK_FOUND=no
     BOOTJDK_CHECK_BUILD_JDK([
       if test "x$with_build_jdk" != x; then
         BUILD_JDK=$with_build_jdk
@@ -607,15 +608,40 @@ AC_DEFUN([BOOTJDK_SETUP_BUILD_JDK],
         AC_MSG_NOTICE([Found potential Build JDK using configure arguments])
       fi
     ])
-    if test "x$BUILD_JDK_FOUND" != "xyes"; then
+    EXTERNAL_BUILDJDK=true
+  else
+    if test "x$COMPILE_TYPE" = "xcross"; then
+      BUILD_JDK="\$(BUILDJDK_OUTPUTDIR)/jdk"
+      BUILD_JDK_FOUND=yes
+      CREATE_BUILDJDK=true
       AC_MSG_CHECKING([for Build JDK])
-      AC_MSG_RESULT([no])
-      AC_MSG_ERROR([Could not find a suitable Build JDK])
+      AC_MSG_RESULT([yes, will build it for the host platform])
+    else
+      BUILD_JDK="\$(JDK_OUTPUTDIR)"
+      BUILD_JDK_FOUND=yes
+      AC_MSG_CHECKING([for Build JDK])
+      AC_MSG_RESULT([yes, will use output dir])
     fi
-    EXTERNAL_BUILDJDK_PATH="$BUILD_JDK"
   fi
 
-  AC_SUBST(EXTERNAL_BUILDJDK_PATH)
+  # Since these tools do not yet exist, we cannot use UTIL_FIXUP_EXECUTABLE to
+  # detect the need of fixpath
+  JMOD="$BUILD_JDK/bin/jmod"
+  UTIL_ADD_FIXPATH(JMOD)
+  JLINK="$BUILD_JDK/bin/jlink"
+  UTIL_ADD_FIXPATH(JLINK)
+  AC_SUBST(JMOD)
+  AC_SUBST(JLINK)
+
+  if test "x$BUILD_JDK_FOUND" != "xyes"; then
+    AC_MSG_CHECKING([for Build JDK])
+    AC_MSG_RESULT([no])
+    AC_MSG_ERROR([Could not find a suitable Build JDK])
+  fi
+
+  AC_SUBST(CREATE_BUILDJDK)
+  AC_SUBST(BUILD_JDK)
+  AC_SUBST(EXTERNAL_BUILDJDK)
 ])
 
 # The docs-reference JDK is used to run javadoc for the docs-reference targets.

make/autoconf/flags-cflags.m4

@@ -37,25 +37,56 @@ AC_DEFUN([FLAGS_SETUP_SHARED_LIBS],
   if test "x$TOOLCHAIN_TYPE" = xgcc; then
     # Default works for linux, might work on other platforms as well.
     SHARED_LIBRARY_FLAGS='-shared'
+    # --disable-new-dtags forces use of RPATH instead of RUNPATH for rpaths.
+    # This protects internal library dependencies within the JDK from being
+    # overridden using LD_LIBRARY_PATH. See JDK-8326891 for more information.
+    SET_EXECUTABLE_ORIGIN='-Wl,-rpath,\$$ORIGIN[$]1 -Wl,--disable-new-dtags'
+    SET_SHARED_LIBRARY_ORIGIN="-Wl,-z,origin $SET_EXECUTABLE_ORIGIN"
+    SET_SHARED_LIBRARY_NAME='-Wl,-soname=[$]1'
 
   elif test "x$TOOLCHAIN_TYPE" = xclang; then
     if test "x$OPENJDK_TARGET_OS" = xmacosx; then
       # Linking is different on MacOSX
       SHARED_LIBRARY_FLAGS="-dynamiclib -compatibility_version 1.0.0 -current_version 1.0.0"
+      SET_EXECUTABLE_ORIGIN='-Wl,-rpath,@loader_path$(or [$]1,/.)'
+      SET_SHARED_LIBRARY_ORIGIN="$SET_EXECUTABLE_ORIGIN"
+      SET_SHARED_LIBRARY_NAME='-Wl,-install_name,@rpath/[$]1'
 
     elif test "x$OPENJDK_TARGET_OS" = xaix; then
       # Linking is different on aix
       SHARED_LIBRARY_FLAGS="-shared -Wl,-bM:SRE -Wl,-bnoentry"
+      SET_EXECUTABLE_ORIGIN=""
+      SET_SHARED_LIBRARY_ORIGIN=''
+      SET_SHARED_LIBRARY_NAME=''
 
     else
       # Default works for linux, might work on other platforms as well.
       SHARED_LIBRARY_FLAGS='-shared'
+      SET_EXECUTABLE_ORIGIN='-Wl,-rpath,\$$ORIGIN[$]1'
+      if test "x$OPENJDK_TARGET_OS" = xlinux; then
+        SET_EXECUTABLE_ORIGIN="$SET_EXECUTABLE_ORIGIN -Wl,--disable-new-dtags"
+      fi
+      SET_SHARED_LIBRARY_NAME='-Wl,-soname=[$]1'
+
+      # arm specific settings
+      if test "x$OPENJDK_TARGET_CPU" = "xarm"; then
+        # '-Wl,-z,origin' isn't used on arm.
+        SET_SHARED_LIBRARY_ORIGIN='-Wl,-rpath,\$$$$ORIGIN[$]1'
+      else
+        SET_SHARED_LIBRARY_ORIGIN="-Wl,-z,origin $SET_EXECUTABLE_ORIGIN"
+      fi
     fi
 
   elif test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
     SHARED_LIBRARY_FLAGS="-dll"
+    SET_EXECUTABLE_ORIGIN=''
+    SET_SHARED_LIBRARY_ORIGIN=''
+    SET_SHARED_LIBRARY_NAME=''
   fi
 
+  AC_SUBST(SET_EXECUTABLE_ORIGIN)
+  AC_SUBST(SET_SHARED_LIBRARY_ORIGIN)
+  AC_SUBST(SET_SHARED_LIBRARY_NAME)
   AC_SUBST(SHARED_LIBRARY_FLAGS)
 ])
 

make/autoconf/flags-ldflags.m4

@@ -98,7 +98,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
 
   # Setup OS-dependent LDFLAGS
   if test "x$OPENJDK_TARGET_OS" = xmacosx && test "x$TOOLCHAIN_TYPE" = xclang; then
-    # FIXME: We should really generalize SetSharedLibraryOrigin instead.
+    # FIXME: We should really generalize SET_SHARED_LIBRARY_ORIGIN instead.
     OS_LDFLAGS_JVM_ONLY="-Wl,-rpath,@loader_path/. -Wl,-rpath,@loader_path/.."
     OS_LDFLAGS="-mmacosx-version-min=$MACOSX_VERSION_MIN -Wl,-reproducible"
   fi

make/autoconf/flags.m4

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -319,10 +319,16 @@ AC_DEFUN_ONCE([FLAGS_PRE_TOOLCHAIN],
 AC_DEFUN([FLAGS_SETUP_TOOLCHAIN_CONTROL],
 [
   if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
+    CC_OUT_OPTION=-Fo
     if test "x$OPENJDK_TARGET_CPU" != xaarch64; then
       AS_NON_ASM_EXTENSION_OPTION=-Ta
     fi
+  else
+    # The option used to specify the target .o,.a or .so file.
+    # When compiling, how to specify the to be created object file.
+    CC_OUT_OPTION='-o$(SPACE)'
   fi
+  AC_SUBST(CC_OUT_OPTION)
   AC_SUBST(AS_NON_ASM_EXTENSION_OPTION)
 
   # Generate make dependency files

make/autoconf/spec.gmk.template

@@ -386,22 +386,9 @@ CAPSTONE_ARCH_AARCH64_NAME := @CAPSTONE_ARCH_AARCH64_NAME@
 # it in sync.
 BOOT_JDK := @BOOT_JDK@
 
-EXTERNAL_BUILDJDK_PATH := @EXTERNAL_BUILDJDK_PATH@
-
-ifneq ($(EXTERNAL_BUILDJDK_PATH), )
-  EXTERNAL_BUILDJDK := true
-  CREATE_BUILDJDK := false
-  BUILD_JDK := $(EXTERNAL_BUILDJDK_PATH)
-else
-  EXTERNAL_BUILDJDK := false
-  ifeq ($(COMPILE_TYPE), cross)
-    CREATE_BUILDJDK := true
-    BUILD_JDK := $(BUILDJDK_OUTPUTDIR)/jdk
-  else
-    CREATE_BUILDJDK := false
-    BUILD_JDK := $(JDK_OUTPUTDIR)
-  endif
-endif
+BUILD_JDK := @BUILD_JDK@
+CREATE_BUILDJDK := @CREATE_BUILDJDK@
+EXTERNAL_BUILDJDK := @EXTERNAL_BUILDJDK@
 
 # Whether the boot jdk jar supports --date=TIMESTAMP
 BOOT_JDK_JAR_SUPPORTS_DATE := @BOOT_JDK_JAR_SUPPORTS_DATE@
@@ -504,6 +491,7 @@ CXX_VERSION_NUMBER := @CXX_VERSION_NUMBER@
 # Legacy support
 HOTSPOT_TOOLCHAIN_TYPE := @HOTSPOT_TOOLCHAIN_TYPE@
 
+CC_OUT_OPTION := @CC_OUT_OPTION@
 AS_NON_ASM_EXTENSION_OPTION := @AS_NON_ASM_EXTENSION_OPTION@
 
 # Flags used for overriding the default opt setting for a C/C++ source file.
@@ -636,8 +624,17 @@ ASFLAGS_DEBUG_SYMBOLS := @ASFLAGS_DEBUG_SYMBOLS@
 # Compress (or not) jars
 COMPRESS_JARS := @COMPRESS_JARS@
 
+# Options to linker to specify the library name.
+# (Note absence of := assignment, because we do not want to evaluate the macro body here)
+SET_SHARED_LIBRARY_NAME = @SET_SHARED_LIBRARY_NAME@
+
 SHARED_LIBRARY_FLAGS := @SHARED_LIBRARY_FLAGS@
 
+# Set origin using the linker, ie use the relative path to the dependent library to find the dependencies.
+# (Note absence of := assignment, because we do not want to evaluate the macro body here)
+SET_SHARED_LIBRARY_ORIGIN = @SET_SHARED_LIBRARY_ORIGIN@
+SET_EXECUTABLE_ORIGIN = @SET_EXECUTABLE_ORIGIN@
+
 LIBRARY_PREFIX := @LIBRARY_PREFIX@
 SHARED_LIBRARY_SUFFIX := @SHARED_LIBRARY_SUFFIX@
 STATIC_LIBRARY_SUFFIX := @STATIC_LIBRARY_SUFFIX@
@@ -660,8 +657,8 @@ JAVA_CMD := @JAVA@
 JAVAC_CMD := @JAVAC@
 JAVADOC_CMD := @JAVADOC@
 JAR_CMD := @JAR@
-JLINK_CMD := @FIXPATH@ $(BUILD_JDK)/bin/jlink
-JMOD_CMD := @FIXPATH@ $(BUILD_JDK)/bin/jmod
+JLINK_CMD := @JLINK@
+JMOD_CMD := @JMOD@
 # These variables are meant to be used. They are defined with = instead of := to make
 # it possible to override only the *_CMD variables.
 JAVA = $(JAVA_CMD) $(JAVA_FLAGS_BIG) $(JAVA_FLAGS)

make/common/Execute.gmk

@@ -149,7 +149,7 @@ define SetupExecuteBody
   endif
 
   $1_VARDEPS := $$($1_COMMAND) $$($1_PRE_COMMAND) $$($1_POST_COMMAND)
-  $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_BASE)_exec.vardeps)
+  $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS)
 
   ifneq ($$($1_PRE_COMMAND), )
 

make/common/JdkNativeCompilation.gmk

@@ -30,47 +30,6 @@ ifeq ($(INCLUDE), true)
 
 include NativeCompilation.gmk
 
-ifeq ($(call isCompiler, gcc), true)
-  # --disable-new-dtags forces use of RPATH instead of RUNPATH for rpaths.
-  # This protects internal library dependencies within the JDK from being
-  # overridden using LD_LIBRARY_PATH. See JDK-8326891 for more information.
-  SetExecutableOrigin = \
-      -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
-  SetSharedLibraryOrigin = \
-      -Wl,-z,origin -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
-else ifeq ($(call isCompiler, clang), true)
-  ifeq ($(call isTargetOs, macosx), true)
-    SetExecutableOrigin = \
-        -Wl,-rpath,@loader_path$(or $1,/.)
-    SetSharedLibraryOrigin = \
-        -Wl,-rpath,@loader_path$(or $1,/.)
-  else ifeq ($(call isTargetOs, aix), true)
-    SetExecutableOrigin =
-    SetSharedLibraryOrigin =
-  else
-    ifeq ($(call isTargetOs, linux), true)
-      SetExecutableOrigin = \
-          -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
-    else
-      SetExecutableOrigin = \
-          -Wl,-rpath,\$(DOLLAR)ORIGIN$1
-    endif
-
-    ifeq ($(call isTargetOs, arm), true)
-      SetSharedLibraryOrigin = \
-          -Wl,-rpath,\$(DOLLAR)ORIGIN$1
-    else
-      SetSharedLibraryOrigin = \
-          -Wl,-z,origin -Wl,-rpath,\$(DOLLAR)ORIGIN$1
-    endif
-  endif
-else ifeq ($(call isCompiler, microsoft), true)
-  SetExecutableOrigin =
-  SetSharedLibraryOrigin =
-else
-  $(error Unknown toolchain)
-endif
-
 FindSrcDirsForComponent += \
   $(call uniq, $(wildcard \
       $(TOPDIR)/src/$(strip $1)/$(OPENJDK_TARGET_OS)/native/$(strip $2) \
@@ -485,9 +444,9 @@ define SetupJdkNativeCompilationBody
 
   ifneq ($$($1_LD_SET_ORIGIN), false)
     ifeq ($$($1_TYPE), EXECUTABLE)
-      $1_LDFLAGS += $$(call SetExecutableOrigin)
+      $1_LDFLAGS += $$(call SET_EXECUTABLE_ORIGIN)
     else
-      $1_LDFLAGS += $$(call SetSharedLibraryOrigin)
+      $1_LDFLAGS += $$(call SET_SHARED_LIBRARY_ORIGIN)
     endif
   endif
   # APPEND_LDFLAGS, if it exists, must be set after the origin flags

make/common/modules/LauncherCommon.gmk

@@ -156,8 +156,8 @@ define SetupBuildLauncherBody
       DISABLED_WARNINGS_gcc := unused-function unused-variable, \
       DISABLED_WARNINGS_clang := unused-function, \
       LDFLAGS := $$($1_LDFLAGS), \
-      LDFLAGS_linux := $$(call SetExecutableOrigin,/../lib), \
-      LDFLAGS_macosx := $$(call SetExecutableOrigin,/../lib), \
+      LDFLAGS_linux := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
+      LDFLAGS_macosx := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
       LDFLAGS_FILTER_OUT := $$($1_LDFLAGS_FILTER_OUT), \
       JDK_LIBS := $$($1_JDK_LIBS), \
       JDK_LIBS_windows := $$($1_JDK_LIBS_windows), \

make/common/native/CompileFile.gmk

@@ -93,14 +93,6 @@ DEPENDENCY_TARGET_SED_PATTERN := \
     -e 's/$$$$/ :/' \
     #
 
-################################################################################
-# Setup compiler-specific argument to specify output file
-ifeq ($(call isCompiler, microsoft), true)
-  CC_OUT_OPTION := -Fo
-else
-  CC_OUT_OPTION := -o$(SPACE)
-endif
-
 ################################################################################
 # Create the recipe needed to compile a single native source file.
 #
@@ -342,7 +334,7 @@ define CreateWindowsResourceFile
 		$$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
 		$$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
 		$$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
-		    $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) -Fo$$@ \
+		    $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
 		    $$($1_VERSIONINFO_RESOURCE) 2>&1 ))
                 # Windows RC compiler does not support -showIncludes, so we mis-use CL
                 # for this. Filter out RC specific arguments that are unknown to CL.
@@ -352,7 +344,7 @@ define CreateWindowsResourceFile
 		$$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
 		    $$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
 		        $$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
-		        -Fo$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
+		        $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
 		        $$($1_VERSIONINFO_RESOURCE)) 2>&1 \
 		    | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
 		        -e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \

make/common/native/Link.gmk

@@ -50,26 +50,6 @@ GetEntitlementsFile = \
       $(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
     )
 
-ifeq ($(call isCompiler, gcc), true)
-  SetSharedLibraryName = \
-      -Wl,-soname=$1
-else ifeq ($(call isCompiler, clang), true)
-  ifeq ($(call isTargetOs, macosx), true)
-    SetSharedLibraryName = \
-        -Wl,-install_name,@rpath/$1
-  else ifeq ($(call isTargetOs, aix), true)
-    SetSharedLibraryName =
-  else
-    # Default works for linux, might work on other platforms as well.
-    SetSharedLibraryName = \
-        -Wl,-soname=$1
-  endif
-else ifeq ($(call isCompiler, microsoft), true)
-  SetSharedLibraryName =
-else
-  $(error Unknown toolchain)
-endif
-
 ################################################################################
 define SetupLinking
   # Unless specifically set, stripping should only happen if symbols are also
@@ -151,7 +131,7 @@ define CreateDynamicLibraryOrExecutable
   # A shared dynamic library or an executable binary has been specified
   ifeq ($$($1_TYPE), LIBRARY)
     # Generating a dynamic library.
-    $1_EXTRA_LDFLAGS += $$(call SetSharedLibraryName,$$($1_BASENAME))
+    $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
   endif
 
   ifeq ($(MACOSX_CODESIGN_MODE), hardened)

make/hotspot/lib/CompileGtest.gmk

@@ -152,7 +152,7 @@ $(eval $(call SetupJdkExecutable, BUILD_GTEST_LAUNCHER, \
         -I$(GTEST_FRAMEWORK_SRC)/googlemock \
         -I$(GTEST_FRAMEWORK_SRC)/googlemock/include, \
     LD_SET_ORIGIN := false, \
-    LDFLAGS_unix := $(call SetSharedLibraryOrigin), \
+    LDFLAGS_unix := $(call SET_SHARED_LIBRARY_ORIGIN), \
     JDK_LIBS := gtest:libjvm, \
     COPY_DEBUG_SYMBOLS := $(GTEST_COPY_DEBUG_SYMBOLS), \
     ZIP_EXTERNAL_DEBUG_SYMBOLS := false, \

make/hotspot/lib/JvmFeatures.gmk

@@ -57,7 +57,7 @@ ifeq ($(call check-jvm-feature, zero), true)
       -DZERO_LIBARCH='"$(OPENJDK_TARGET_CPU_LEGACY_LIB)"' $(LIBFFI_CFLAGS)
   JVM_LIBS_FEATURES += $(LIBFFI_LIBS)
   ifeq ($(ENABLE_LIBFFI_BUNDLING), true)
-    JVM_LDFLAGS_FEATURES += $(call SetExecutableOrigin,/..)
+    JVM_LDFLAGS_FEATURES += $(call SET_EXECUTABLE_ORIGIN,/..)
   endif
 else
   JVM_EXCLUDE_PATTERNS += /zero/

make/modules/java.base/Lib.gmk

@@ -154,8 +154,6 @@ endif
 
 ################################################################################
 ## Build libsyslookup
-## The LIBDL dependency on Linux is needed to dynamically access libdl symbols,
-## which may be needed as part of resolving some standard symbols
 ################################################################################
 
 $(eval $(call SetupJdkLibrary, BUILD_LIBSYSLOOKUP, \
@@ -198,7 +196,7 @@ ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2)
       OPTIMIZATION := HIGH, \
       CXXFLAGS := -std=c++17, \
       DISABLED_WARNINGS_gcc := unused-variable, \
-      LIBS_linux := $(LIBM), \
+      LIBS_linux := $(LIBDL) $(LIBM), \
   ))
 
   TARGETS += $(BUILD_LIBSIMD_SORT)

make/modules/java.desktop/lib/ClientLibraries.gmk

@@ -51,7 +51,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBMLIB_IMAGE, \
         $(LIBMLIB_IMAGE_CFLAGS), \
     DISABLED_WARNINGS_gcc := unused-function, \
     DISABLED_WARNINGS_clang_mlib_ImageCreate.c := unused-function, \
-    LIBS_unix := $(LIBM), \
+    LIBS_unix := $(LIBDL) $(LIBM), \
 ))
 
 TARGETS += $(BUILD_LIBMLIB_IMAGE)
@@ -264,7 +264,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
       JDK_LIBS_macosx := libosxapp, \
       LIBS := $(GIFLIB_LIBS) $(LIBJPEG_LIBS) $(LIBZ_LIBS) $(PNG_LIBS) $(ICONV_LIBS), \
       LIBS_unix := $(LIBM) $(LIBPTHREAD), \
-      LIBS_linux := $(X_LIBS) -lX11 -lXext, \
+      LIBS_linux := $(LIBDL) $(X_LIBS) -lX11 -lXext, \
       LIBS_macosx := \
           -framework ApplicationServices \
           -framework Cocoa \

make/modules/java.instrument/Lib.gmk

@@ -43,6 +43,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBINSTRUMENT, \
     JDK_LIBS := java.base:libjava java.base:libjli java.base:libjvm, \
     LIBS := $(ICONV_LIBS), \
     LIBS_unix := $(LIBZ_LIBS), \
+    LIBS_linux := $(LIBDL), \
+    LIBS_aix := $(LIBDL), \
     LIBS_macosx := \
         -framework ApplicationServices \
         -framework Cocoa \

make/modules/jdk.sctp/Lib.gmk

@@ -41,7 +41,7 @@ ifeq ($(call isTargetOs, linux), true)
           java.base:libnio \
           java.base:libnio/ch, \
       JDK_LIBS := java.base:libjava java.base:libnet, \
-      LIBS_linux := $(LIBDL), \
+      LIBS_linux := $(LIBDL) $(LIBPTHREAD), \
   ))
 
   TARGETS += $(BUILD_LIBSCTP)

src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp

@@ -216,10 +216,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
-
-  // lock_reg was destroyed by fast unlocking attempt => recompute it
-  ce->monitor_address(_monitor_ix, _lock_reg);
-
+  if (_compute_lock) {
+    // lock_reg was destroyed by fast unlocking attempt => recompute it
+    ce->monitor_address(_monitor_ix, _lock_reg);
+  }
   ce->store_parameter(_lock_reg->as_register(), 0);
   // note: non-blocking leaf routine => no call info needed
   StubId exit_id;

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -409,7 +409,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::r0_opr);
-    stub = new MonitorExitStub(FrameMap::r0_opr, 0);
+    stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
     __ unlock_object(r5, r4, r0, r6, *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -2481,6 +2481,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register lock = op->lock_opr()->as_register();
   Register temp = op->scratch_opr()->as_register();
   if (op->code() == lir_lock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // add debug info for NullPointerException only if one is possible
     int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
     if (op->info() != nullptr) {
@@ -2488,6 +2489,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     }
     // done
   } else if (op->code() == lir_unlock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
   } else {
     Unimplemented();

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -59,28 +59,28 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
   }
 }
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
-  assert_different_registers(hdr, obj, basic_lock, temp, rscratch2);
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
+  assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
   int null_check_offset = -1;
 
   verify_oop(obj);
 
   // save object being locked into the BasicObjectLock
-  str(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 
   null_check_offset = offset();
 
-  lightweight_lock(basic_lock, obj, hdr, temp, rscratch2, slow_case);
+  lightweight_lock(disp_hdr, obj, hdr, temp, rscratch2, slow_case);
 
   return null_check_offset;
 }
 
 
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
-  assert_different_registers(hdr, obj, basic_lock, temp, rscratch2);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
+  assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
 
   // load object
-  ldr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
   verify_oop(obj);
 
   lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -55,19 +55,19 @@ using MacroAssembler::null_check;
                  Register result);
 
   // locking
-  // hdr       : must be r0, contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must point to the basic lock, contents preserved
-  // temp      : temporary register, must not be rscratch1 or rscratch2
+  // hdr     : must be r0, contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must point to the displaced header location, contents preserved
+  // temp    : temporary register, must not be rscratch1 or rscratch2
   // returns code offset at which to add null check debug information
-  int lock_object  (Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
+  int lock_object  (Register swap, Register obj, Register disp_hdr, Register temp, Label& slow_case);
 
   // unlocking
-  // hdr       : contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must be r0 & must point to the basic lock, contents destroyed
-  // temp      : temporary register, must not be rscratch1 or rscratch2
-  void unlock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
+  // hdr     : contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must be r0 & must point to the displaced header location, contents destroyed
+  // temp    : temporary register, must not be rscratch1 or rscratch2
+  void unlock_object(Register swap, Register obj, Register lock, Register temp, Label& slow_case);
 
   void initialize_object(
     Register obj,                      // result: pointer to object after successful allocation

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp

@@ -86,48 +86,15 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
   }
 }
 
-void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm,
-                                                             DecoratorSet decorators,
-                                                             Register start,
-                                                             Register count,
-                                                             Register scratch,
-                                                             RegSet saved_regs) {
-
-  Label done;
-  Label loop;
-  Label next;
-
-  __ cbz(count, done);
-
-  // Calculate the number of card marks to set. Since the object might start and
-  // end within a card, we need to calculate this via the card table indexes of
-  // the actual start and last addresses covered by the object.
-  // Temporarily use the count register for the last element address.
-  __ lea(count, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
-  __ sub(count, count, BytesPerHeapOop);                                  // Use last element address for end.
-
-  __ lsr(start, start, CardTable::card_shift());
-  __ lsr(count, count, CardTable::card_shift());
-  __ sub(count, count, start);                                            // Number of bytes to mark - 1.
-
-  // Add card table base offset to start.
-  __ ldr(scratch, Address(rthread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
-  __ add(start, start, scratch);
-
-  __ bind(loop);
-  if (UseCondCardMark) {
-    __ ldrb(scratch, Address(start, count));
-    // Instead of loading clean_card_val and comparing, we exploit the fact that
-    // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
-    __ tbz(scratch, 0, next);
-  }
-  static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
-  __ strb(zr, Address(start, count));
-  __ bind(next);
-  __ subs(count, count, 1);
-  __ br(Assembler::GE, loop);
-
-  __ bind(done);
+void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                             Register start, Register count, Register scratch, RegSet saved_regs) {
+  __ push(saved_regs, sp);
+  assert_different_registers(start, count, scratch);
+  assert_different_registers(c_rarg0, count);
+  __ mov(c_rarg0, start);
+  __ mov(c_rarg1, count);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
+  __ pop(saved_regs, sp);
 }
 
 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
@@ -232,17 +199,13 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
 
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register thread,
-                                  const Register tmp1,
-                                  const Register tmp2,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-  assert(thread == rthread, "must be");
-  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
-
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+                                            const Register store_addr,
+                                            const Register new_val,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            Label& done,
+                                            bool new_val_may_be_null) {
   // Does store cross heap regions?
   __ eor(tmp1, store_addr, new_val);                     // tmp1 := store address ^ new value
   __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);   // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
@@ -251,19 +214,33 @@ static void generate_post_barrier(MacroAssembler* masm,
   if (new_val_may_be_null) {
     __ cbz(new_val, done);
   }
-  // Storing region crossing non-null.
+  // Storing region crossing non-null, is card young?
   __ lsr(tmp1, store_addr, CardTable::card_shift());     // tmp1 := card address relative to card table base
+  __ load_byte_map_base(tmp2);                           // tmp2 := card table base address
+  __ add(tmp1, tmp1, tmp2);                              // tmp1 := card address
+  __ ldrb(tmp2, Address(tmp1));                          // tmp2 := card
+  __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());  // tmp2 := card == young_card_val?
+}
 
-  Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
-  __ ldr(tmp2, card_table_addr);                         // tmp2 := card table base address
-  if (UseCondCardMark) {
-    __ ldrb(rscratch1, Address(tmp1, tmp2));             // rscratch1 := card
-    // Instead of loading clean_card_val and comparing, we exploit the fact that
-    // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
-    __ tbz(rscratch1, 0, done);
-  }
-  static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
-  __ strb(zr, Address(tmp1, tmp2));                      // *(card address) := dirty_card_val
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+                                            const Register thread,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            Label& done,
+                                            Label& runtime) {
+  __ membar(Assembler::StoreLoad);  // StoreLoad membar
+  __ ldrb(tmp2, Address(tmp1));     // tmp2 := card
+  __ cbzw(tmp2, done);
+  // Storing a region crossing, non-null oop, card is clean.
+  // Dirty card and log.
+  STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+  __ strb(zr, Address(tmp1));       // *(card address) := dirty_card_val
+  generate_queue_test_and_insertion(masm,
+                                    G1ThreadLocalData::dirty_card_queue_index_offset(),
+                                    G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                                    runtime,
+                                    thread, tmp1, tmp2, rscratch1);
+  __ b(done);
 }
 
 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
@@ -272,8 +249,27 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
                                                   Register thread,
                                                   Register tmp1,
                                                   Register tmp2) {
+  assert(thread == rthread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
+                             rscratch1);
+  assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
+         && tmp2 != noreg, "expecting a register");
+
   Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, false /* new_val_may_be_null */);
+  Label runtime;
+
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+  // If card is young, jump to done
+  __ br(Assembler::EQ, done);
+  generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(store_addr);
+  __ push(saved, sp);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
+  __ pop(saved, sp);
+
   __ bind(done);
 }
 
@@ -333,10 +329,38 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register thread,
                                                      Register tmp1,
                                                      Register tmp2,
-                                                     bool new_val_may_be_null) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
-  __ bind(done);
+                                                     G1PostBarrierStubC2* stub) {
+  assert(thread == rthread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
+                             rscratch1);
+  assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
+         && tmp2 != noreg, "expecting a register");
+
+  stub->initialize_registers(thread, tmp1, tmp2);
+
+  bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+  // If card is not young, jump to stub (slow path)
+  __ br(Assembler::NE, *stub->entry());
+
+  __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+  Register thread = stub->thread();
+  Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+  Register tmp2 = stub->tmp2();
+  assert(stub->tmp3() == noreg, "not needed in this platform");
+
+  __ bind(*stub->entry());
+  generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
+
+  __ bind(runtime);
+  generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+  __ b(*stub->continuation());
 }
 
 #endif // COMPILER2
@@ -432,19 +456,20 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   __ b(*stub->continuation());
 }
 
-#undef __
-
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
-  masm->bind(done);
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+  assert(stub->addr()->is_register(), "Precondition.");
+  assert(stub->new_val()->is_register(), "Precondition.");
+  Register new_val_reg = stub->new_val()->as_register();
+  __ cbz(new_val_reg, *stub->continuation());
+  ce->store_parameter(stub->addr()->as_pointer_register(), 0);
+  __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
+  __ b(*stub->continuation());
 }
 
+#undef __
+
 #define __ sasm->
 
 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -496,6 +521,74 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ epilogue();
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("g1_post_barrier", false);
+
+  // arg0: store_address
+  Address store_addr(rfp, 2*BytesPerWord);
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+  CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
+  CardTable* ct = ctbs->card_table();
+
+  Label done;
+  Label runtime;
+
+  // At this point we know new_value is non-null and the new_value crosses regions.
+  // Must check to see if card is already dirty
+
+  const Register thread = rthread;
+
+  Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
+  Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
+
+  const Register card_offset = rscratch2;
+  // LR is free here, so we can use it to hold the byte_map_base.
+  const Register byte_map_base = lr;
+
+  assert_different_registers(card_offset, byte_map_base, rscratch1);
+
+  __ load_parameter(0, card_offset);
+  __ lsr(card_offset, card_offset, CardTable::card_shift());
+  __ load_byte_map_base(byte_map_base);
+  __ ldrb(rscratch1, Address(byte_map_base, card_offset));
+  __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
+  __ br(Assembler::EQ, done);
+
+  assert((int)CardTable::dirty_card_val() == 0, "must be 0");
+
+  __ membar(Assembler::StoreLoad);
+  __ ldrb(rscratch1, Address(byte_map_base, card_offset));
+  __ cbzw(rscratch1, done);
+
+  // storing region crossing non-null, card is clean.
+  // dirty card and log.
+  __ strb(zr, Address(byte_map_base, card_offset));
+
+  // Convert card offset into an address in card_addr
+  Register card_addr = card_offset;
+  __ add(card_addr, byte_map_base, card_addr);
+
+  __ ldr(rscratch1, queue_index);
+  __ cbz(rscratch1, runtime);
+  __ sub(rscratch1, rscratch1, wordSize);
+  __ str(rscratch1, queue_index);
+
+  // Reuse LR to hold buffer_addr
+  const Register buffer_addr = lr;
+
+  __ ldr(buffer_addr, buffer);
+  __ str(card_addr, Address(buffer_addr, rscratch1));
+  __ b(done);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
+  __ pop_call_clobbered_registers();
+  __ bind(done);
+  __ epilogue();
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,9 @@
 class LIR_Assembler;
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
 protected:
@@ -63,15 +65,10 @@ protected:
 public:
 #ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
-
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
 #ifdef COMPILER2
@@ -90,7 +87,9 @@ public:
                                 Register thread,
                                 Register tmp1,
                                 Register tmp2,
-                                bool new_val_may_be_null);
+                                G1PostBarrierStubC2* c2_stub);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif
 
   void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -62,13 +62,13 @@ static void write_barrier_post(MacroAssembler* masm,
                                Register new_val,
                                Register tmp1,
                                Register tmp2) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, new_val_may_be_null);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
 }
 
 %}

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp

@@ -398,18 +398,14 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
 OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) {
   switch (node->ideal_reg()) {
     case Op_RegF:
-    case Op_RegI: // RA may place scalar values (Op_RegI/N/L/P) in FP registers when UseFPUForSpilling is enabled
-    case Op_RegN:
       // No need to refine. The original encoding is already fine to distinguish.
-      assert(opto_reg % 4 == 0, "32-bit register should only occupy a single slot");
+      assert(opto_reg % 4 == 0, "Float register should only occupy a single slot");
       break;
     // Use different encoding values of the same fp/vector register to help distinguish different sizes.
     // Such as V16. The OptoReg::name and its corresponding slot value are
     // "V16": 64, "V16_H": 65, "V16_J": 66, "V16_K": 67.
     case Op_RegD:
     case Op_VecD:
-    case Op_RegL:
-    case Op_RegP:
       opto_reg &= ~3;
       opto_reg |= 1;
       break;

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp

@@ -275,6 +275,7 @@ public:
   Label* entry();
 };
 
+
 class ZStoreBarrierStubC2Aarch64 : public ZStoreBarrierStubC2 {
 private:
   bool _deferred_emit;

src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad

@@ -239,6 +239,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
   ins_pipe(pipe_slow);
 %}
 
+
 instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -683,12 +683,12 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
-  entry_address[AdapterBlob::I2C] = __ pc();
+                                            AdapterHandlerEntry* handler) {
+  address i2c_entry = __ pc();
 
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
-  entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+  address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
   Register data = rscratch2;
@@ -718,10 +718,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ block_comment("} c2i_unverified_entry");
   }
 
-  entry_address[AdapterBlob::C2I] = __ pc();
+  address c2i_entry = __ pc();
 
   // Class initialization barrier for static methods
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_no_clinit_check_entry = nullptr;
   if (VM_Version::supports_fast_class_init_checks()) {
     Label L_skip_barrier;
 
@@ -736,13 +736,15 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 
     __ bind(L_skip_barrier);
-    entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
+    c2i_no_clinit_check_entry = __ pc();
   }
 
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
   bs->c2i_entry_barrier(masm);
 
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
   return;
 }
 
@@ -1761,6 +1763,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   Label lock_done;
 
   if (method->is_synchronized()) {
+    Label count;
+    const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
+
     // Get the handle (the 2nd argument)
     __ mov(oop_handle_reg, c_rarg1);
 

src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp

@@ -200,10 +200,9 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
-
-  // lock_reg was destroyed by fast unlocking attempt => recompute it
-  ce->monitor_address(_monitor_ix, _lock_reg);
-
+  if (_compute_lock) {
+    ce->monitor_address(_monitor_ix, _lock_reg);
+  }
   const Register lock_reg = _lock_reg->as_pointer_register();
 
   ce->verify_reserved_argument_area_size(1);

src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp

@@ -245,7 +245,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::R0_opr);
-    stub = new MonitorExitStub(FrameMap::R0_opr, 0);
+    stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
     __ unlock_object(R2, R1, R0, *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -2427,6 +2427,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register lock = op->lock_opr()->as_pointer_register();
 
   if (op->code() == lir_lock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
     if (op->info() != nullptr) {
       add_debug_info_for_null_check(null_check_offset, op->info());

src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp

@@ -176,17 +176,17 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len,
   initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB);
 }
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   int null_check_offset = 0;
 
   const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level
-  assert_different_registers(hdr, obj, basic_lock, tmp2);
+  assert_different_registers(hdr, obj, disp_hdr, tmp2);
 
   assert(BasicObjectLock::lock_offset() == 0, "adjust this code");
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
   // save object being locked into the BasicObjectLock
-  str(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 
   null_check_offset = offset();
 
@@ -197,26 +197,26 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lo
     b(slow_case, ne);
   }
 
-  Register t1 = basic_lock; // Needs saving, probably
-  Register t2 = hdr;        // blow
-  Register t3 = Rtemp;      // blow
+  Register t1 = disp_hdr; // Needs saving, probably
+  Register t2 = hdr;      // blow
+  Register t3 = Rtemp;    // blow
 
   lightweight_lock(obj, t1, t2, t3, 1 /* savemask - save t1 */, slow_case);
   // Success: fall through
   return null_check_offset;
 }
 
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
-  assert_different_registers(hdr, obj, basic_lock, Rtemp);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
+  assert_different_registers(hdr, obj, disp_hdr, Rtemp);
 
   assert(BasicObjectLock::lock_offset() == 0, "adjust this code");
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
-  ldr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 
-  Register t1 = basic_lock; // Needs saving, probably
-  Register t2 = hdr;        // blow
-  Register t3 = Rtemp;      // blow
+  Register t1 = disp_hdr; // Needs saving, probably
+  Register t2 = hdr;      // blow
+  Register t3 = Rtemp;    // blow
 
   lightweight_unlock(obj, t1, t2, t3, 1 /* savemask - save t1 */, slow_case);
   // Success: fall through

src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -59,9 +59,9 @@
     max_array_allocation_length = 0x01000000
   };
 
-  int lock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case);
+  int lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
 
-  void unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case);
+  void unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
 
   // This platform only uses signal-based null checks. The Label is not needed.
   void null_check(Register r, Label *Lnull = nullptr) { MacroAssembler::null_check(r); }

src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp

@@ -198,18 +198,15 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
   __ bind(done);
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register thread,
-                                  const Register tmp1,
-                                  const Register tmp2,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-  assert(thread == Rthread, "must be");
-  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
-
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+                                            const Register store_addr,
+                                            const Register new_val,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            Label& done,
+                                            bool new_val_may_be_null) {
   // Does store cross heap regions?
+
   __ eor(tmp1, store_addr, new_val);
   __ movs(tmp1, AsmOperand(tmp1, lsr, G1HeapRegion::LogOfHRGrainBytes));
   __ b(done, eq);
@@ -218,34 +215,76 @@ static void generate_post_barrier(MacroAssembler* masm,
   if (new_val_may_be_null) {
     __ cbz(new_val, done);
   }
+  // storing region crossing non-null, is card already dirty?
+  const Register card_addr = tmp1;
 
-  // storing region crossing non-null, is card already non-clean?
-  Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
-  __ ldr(tmp2, card_table_addr);
-  __ add(tmp1, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift()));
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+  __ mov_address(tmp2, (address)ct->card_table()->byte_map_base());
+  __ add(card_addr, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift()));
 
-  if (UseCondCardMark) {
-    __ ldrb(tmp2, Address(tmp1));
-    // Instead of loading clean_card_val and comparing, we exploit the fact that
-    // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
-    __ tbz(tmp2, 0, done);
-  }
-
-  static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zero_register()");
-  __ zero_register(tmp2);
-  __ strb(tmp2, Address(tmp1));                   // *(card address) := dirty_card_val
+  __ ldrb(tmp2, Address(card_addr));
+  __ cmp(tmp2, (int)G1CardTable::g1_young_card_val());
 }
 
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+                                            const Register thread,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            const Register tmp3,
+                                            Label& done,
+                                            Label& runtime) {
+  __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp2);
+  assert(CardTable::dirty_card_val() == 0, "adjust this code");
+  // card_addr is loaded by generate_post_barrier_fast_path
+  const Register card_addr = tmp1;
+  __ ldrb(tmp2, Address(card_addr));
+  __ cbz(tmp2, done);
+
+  // storing a region crossing, non-null oop, card is clean.
+  // dirty card and log.
+
+  __ strb(__ zero_register(tmp2), Address(card_addr));
+  generate_queue_test_and_insertion(masm,
+                                    G1ThreadLocalData::dirty_card_queue_index_offset(),
+                                    G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                                    runtime,
+                                    thread, card_addr, tmp2, tmp3);
+  __ b(done);
+}
+
+
 // G1 post-barrier.
 // Blows all volatile registers R0-R3,  LR).
 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
-                                                  Register store_addr,
-                                                  Register new_val,
-                                                  Register tmp1,
-                                                  Register tmp2,
-                                                  Register tmp3) {
+                                           Register store_addr,
+                                           Register new_val,
+                                           Register tmp1,
+                                           Register tmp2,
+                                           Register tmp3) {
   Label done;
-  generate_post_barrier(masm, store_addr, new_val, Rthread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+  Label runtime;
+
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+  // If card is young, jump to done
+  // card_addr and card are loaded by generate_post_barrier_fast_path
+  const Register card      = tmp2;
+  const Register card_addr = tmp1;
+   __ b(done, eq);
+  generate_post_barrier_slow_path(masm, Rthread, card_addr, tmp2, tmp3, done, runtime);
+
+  __ bind(runtime);
+
+  RegisterSet set = RegisterSet(store_addr) | RegisterSet(R0, R3) | RegisterSet(R12);
+  __ push(set);
+
+  if (card_addr != R0) {
+    __ mov(R0, card_addr);
+  }
+  __ mov(R1, Rthread);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), R0, R1);
+
+  __ pop(set);
+
   __ bind(done);
 }
 
@@ -305,10 +344,35 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register tmp1,
                                                      Register tmp2,
                                                      Register tmp3,
-                                                     bool new_val_may_be_null) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
-  __ bind(done);
+                                                     G1PostBarrierStubC2* stub) {
+  assert(thread == Rthread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
+
+  stub->initialize_registers(thread, tmp1, tmp2, tmp3);
+
+  bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+  // If card is not young, jump to stub (slow path)
+  __ b(*stub->entry(), ne);
+
+  __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+  Register thread = stub->thread();
+  Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+  Register tmp2 = stub->tmp2();
+  Register tmp3 = stub->tmp3();
+
+  __ bind(*stub->entry());
+  generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, tmp3,  *stub->continuation(), runtime);
+
+  __ bind(runtime);
+  generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp2);
+  __ b(*stub->continuation());
 }
 
 #endif // COMPILER2
@@ -399,19 +463,20 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   __ b(*stub->continuation());
 }
 
-#undef __
-
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
-  masm->bind(done);
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+  assert(stub->addr()->is_register(), "Precondition.");
+  assert(stub->new_val()->is_register(), "Precondition.");
+  Register new_val_reg = stub->new_val()->as_register();
+  __ cbz(new_val_reg, *stub->continuation());
+  ce->verify_reserved_argument_area_size(1);
+  __ str(stub->addr()->as_pointer_register(), Address(SP));
+  __ call(bs->post_barrier_c1_runtime_code_blob()->code_begin(), relocInfo::runtime_call_type);
+  __ b(*stub->continuation());
 }
 
+#undef __
 #define __ sasm->
 
 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -471,6 +536,102 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ b(done);
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  // Input:
+  // - store_addr, pushed on the stack
+
+  __ set_info("g1_post_barrier_slow_id", false);
+
+  Label done;
+  Label recheck;
+  Label runtime;
+
+  Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
+  Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
+
+  AddressLiteral cardtable(ci_card_table_address_as<address>(), relocInfo::none);
+
+  // save at least the registers that need saving if the runtime is called
+  const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);
+  const int nb_saved_regs = 6;
+  assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");
+  __ push(saved_regs);
+
+  const Register r_card_addr_0 = R0; // must be R0 for the slow case
+  const Register r_obj_0 = R0;
+  const Register r_card_base_1 = R1;
+  const Register r_tmp2 = R2;
+  const Register r_index_2 = R2;
+  const Register r_buffer_3 = R3;
+  const Register tmp1 = Rtemp;
+
+  __ ldr(r_obj_0, Address(SP, nb_saved_regs*wordSize));
+  // Note: there is a comment in x86 code about not using
+  // ExternalAddress / lea, due to relocation not working
+  // properly for that address. Should be OK for arm, where we
+  // explicitly specify that 'cardtable' has a relocInfo::none
+  // type.
+  __ lea(r_card_base_1, cardtable);
+  __ add(r_card_addr_0, r_card_base_1, AsmOperand(r_obj_0, lsr, CardTable::card_shift()));
+
+  // first quick check without barrier
+  __ ldrb(r_tmp2, Address(r_card_addr_0));
+
+  __ cmp(r_tmp2, (int)G1CardTable::g1_young_card_val());
+  __ b(recheck, ne);
+
+  __ bind(done);
+
+  __ pop(saved_regs);
+
+  __ ret();
+
+  __ bind(recheck);
+
+  __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp1);
+
+  // reload card state after the barrier that ensures the stored oop was visible
+  __ ldrb(r_tmp2, Address(r_card_addr_0));
+
+  assert(CardTable::dirty_card_val() == 0, "adjust this code");
+  __ cbz(r_tmp2, done);
+
+  // storing region crossing non-null, card is clean.
+  // dirty card and log.
+
+  assert(0 == (int)CardTable::dirty_card_val(), "adjust this code");
+  if ((ci_card_table_address_as<intptr_t>() & 0xff) == 0) {
+    // Card table is aligned so the lowest byte of the table address base is zero.
+    __ strb(r_card_base_1, Address(r_card_addr_0));
+  } else {
+    __ strb(__ zero_register(r_tmp2), Address(r_card_addr_0));
+  }
+
+  __ ldr(r_index_2, queue_index);
+  __ ldr(r_buffer_3, buffer);
+
+  __ subs(r_index_2, r_index_2, wordSize);
+  __ b(runtime, lt); // go to runtime if now negative
+
+  __ str(r_index_2, queue_index);
+
+  __ str(r_card_addr_0, Address(r_buffer_3, r_index_2));
+
+  __ b(done);
+
+  __ bind(runtime);
+
+  __ save_live_registers();
+
+  assert(r_card_addr_0 == c_rarg0, "card_addr should be in R0");
+  __ mov(c_rarg1, Rthread);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), c_rarg0, c_rarg1);
+
+  __ restore_live_registers_without_return();
+
+  __ b(done);
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,9 @@
 class LIR_Assembler;
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
 protected:
@@ -64,15 +66,10 @@ public:
 #ifdef COMPILER1
 public:
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
-
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
 #ifdef COMPILER2
@@ -92,7 +89,9 @@ public:
                                 Register tmp1,
                                 Register tmp2,
                                 Register tmp3,
-                                bool new_val_may_be_null);
+                                G1PostBarrierStubC2* c2_stub);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif
 
 };

src/hotspot/cpu/arm/gc/g1/g1_arm.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -63,13 +63,13 @@ static void write_barrier_post(MacroAssembler* masm,
                                Register tmp1,
                                Register tmp2,
                                Register tmp3) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Rthread, tmp1, tmp2, tmp3, new_val_may_be_null);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Rthread, tmp1, tmp2, tmp3, stub);
 }
 
 %}

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -617,11 +617,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
-  entry_address[AdapterBlob::I2C] = __ pc();
+                                            AdapterHandlerEntry* handler) {
+  address i2c_entry = __ pc();
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
-  entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+  address c2i_unverified_entry = __ pc();
   Label skip_fixup;
   const Register receiver       = R0;
   const Register holder_klass   = Rtemp; // XXX should be OK for C2 but not 100% sure
@@ -634,9 +634,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
   __ b(skip_fixup, eq);
   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 
-  entry_address[AdapterBlob::C2I] = __ pc();
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_entry = __ pc();
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr);
   return;
 }
 
@@ -1128,7 +1129,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   const Register sync_handle = R5;
   const Register sync_obj    = R6;
-  const Register basic_lock  = altFP_7_11;
+  const Register disp_hdr    = altFP_7_11;
   const Register tmp         = R8;
 
   Label slow_lock, lock_done, fast_lock;
@@ -1139,7 +1140,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ mov(sync_handle, R1);
 
     log_trace(fastlock)("SharedRuntime lock fast");
-    __ lightweight_lock(sync_obj /* object */, basic_lock /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
+    __ lightweight_lock(sync_obj /* object */, disp_hdr /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
                         0x7 /* savemask */, slow_lock);
       // Fall through to lock_done
     __ bind(lock_done);
@@ -1254,7 +1255,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
     // last_Java_frame is already set, so do call_VM manually; no exception can occur
     __ mov(R0, sync_obj);
-    __ mov(R1, basic_lock);
+    __ mov(R1, disp_hdr);
     __ mov(R2, Rthread);
     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C));
 
@@ -1269,12 +1270,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
     // Clear pending exception before reentering VM.
     // Can store the oop in register since it is a leaf call.
-    assert_different_registers(Rtmp_save1, sync_obj, basic_lock);
+    assert_different_registers(Rtmp_save1, sync_obj, disp_hdr);
     __ ldr(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
     Register zero = __ zero_register(Rtemp);
     __ str(zero, Address(Rthread, Thread::pending_exception_offset()));
     __ mov(R0, sync_obj);
-    __ mov(R1, basic_lock);
+    __ mov(R1, disp_hdr);
     __ mov(R2, Rthread);
     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
     __ str(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));

src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp

@@ -268,10 +268,9 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
-
-  // lock_reg was destroyed by fast unlocking attempt => recompute it
-  ce->monitor_address(_monitor_ix, _lock_reg);
-
+  if (_compute_lock) {
+    ce->monitor_address(_monitor_ix, _lock_reg);
+  }
   address stub = Runtime1::entry_for(ce->compilation()->has_fpu_code() ? StubId::c1_monitorexit_id : StubId::c1_monitorexit_nofpu_id);
   //__ load_const_optimized(R0, stub);
   __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(stub));

src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp

@@ -227,7 +227,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::R4_opr);
-    stub = new MonitorExitStub(FrameMap::R4_opr, 0);
+    stub = new MonitorExitStub(FrameMap::R4_opr, true, 0);
     __ unlock_object(R5, R6, R4, *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -2614,6 +2614,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   // Obj may not be an oop.
   if (op->code() == lir_lock) {
     MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // Add debug info for NullPointerException only if one is possible.
     if (op->info() != nullptr) {
       if (!os::zero_page_read_protected() || !ImplicitNullChecks) {
@@ -2625,6 +2626,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     __ lock_object(hdr, obj, lock, op->scratch_opr()->as_register(), *op->stub()->entry());
   } else {
     assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
   }
   __ bind(*op->stub()->continuation());

src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp

@@ -28,6 +28,7 @@
 #include "gc/g1/g1BarrierSetAssembler.hpp"
 #include "gc/g1/g1BarrierSetRuntime.hpp"
 #include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1DirtyCardQueue.hpp"
 #include "gc/g1/g1HeapRegion.hpp"
 #include "gc/g1/g1SATBMarkQueueSet.hpp"
 #include "gc/g1/g1ThreadLocalData.hpp"
@@ -229,52 +230,78 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
   __ bind(filtered);
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register thread,
-                                  const Register tmp1,
-                                  const Register tmp2,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-  assert_different_registers(store_addr, new_val, tmp1, R0);
-  assert_different_registers(store_addr, tmp1, tmp2, R0);
+static void generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val) {
+  __ xorr(R0, store_addr, new_val);                  // tmp1 := store address ^ new value
+  __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+}
 
-  __ xorr(R0, store_addr, new_val);                          // R0 := store address ^ new value
-  __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes);         // R0 := ((store address ^ new value) >> LogOfHRGrainBytes)
-  __ beq(CR0, done);
+static Address generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) {
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+  __ load_const_optimized(tmp1, (address)(ct->card_table()->byte_map_base()), tmp2);
+  __ srdi(tmp2, store_addr, CardTable::card_shift());        // tmp1 := card address relative to card table base
+  __ lbzx(R0, tmp1, tmp2);                                   // tmp1 := card address
+  __ cmpwi(CR0, R0, (int)G1CardTable::g1_young_card_val());
+  return Address(tmp1, tmp2); // return card address
+}
 
-  // Crosses regions, storing null?
-  if (!new_val_may_be_null) {
-#ifdef ASSERT
-    __ cmpdi(CR0, new_val, 0);
-    __ asm_assert_ne("null oop not allowed (G1 post)");      // Checked by caller.
-#endif
-  } else {
-    __ cmpdi(CR0, new_val, 0);
-    __ beq(CR0, done);
-  }
-
-  __ ld(tmp1, G1ThreadLocalData::card_table_base_offset(), thread);
-  __ srdi(tmp2, store_addr, CardTable::card_shift());        // tmp2 := card address relative to card table base
-  if (UseCondCardMark) {
-    __ lbzx(R0, tmp1, tmp2);
-    __ cmpwi(CR0, R0, (int)G1CardTable::clean_card_val());
-    __ bne(CR0, done);
-  }
-
-  __ li(R0, G1CardTable::dirty_card_val());
-  __ stbx(R0, tmp1, tmp2);
+static void generate_card_dirty_test(MacroAssembler* masm, Address card_addr) {
+  __ membar(Assembler::StoreLoad);                        // Must reload after StoreLoad membar due to concurrent refinement
+  __ lbzx(R0, card_addr.base(), card_addr.index());       // tmp2 := card
+  __ cmpwi(CR0, R0, (int)G1CardTable::dirty_card_val()); // tmp2 := card == dirty_card_val?
 }
 
 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators,
                                                   Register store_addr, Register new_val,
-                                                  Register tmp1, Register tmp2) {
+                                                  Register tmp1, Register tmp2, Register tmp3,
+                                                  MacroAssembler::PreservationLevel preservation_level) {
   bool not_null = (decorators & IS_NOT_NULL) != 0;
 
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, R16_thread, tmp1, tmp2, done, !not_null);
-  __ bind(done);
+  Label runtime, filtered;
+  assert_different_registers(store_addr, new_val, tmp1, tmp2);
+
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+
+  generate_region_crossing_test(masm, store_addr, new_val);
+  __ beq(CR0, filtered);
+
+  // Crosses regions, storing null?
+  if (not_null) {
+#ifdef ASSERT
+    __ cmpdi(CR0, new_val, 0);
+    __ asm_assert_ne("null oop not allowed (G1 post)"); // Checked by caller.
+#endif
+  } else {
+    __ cmpdi(CR0, new_val, 0);
+    __ beq(CR0, filtered);
+  }
+
+  Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
+  __ beq(CR0, filtered);
+
+  generate_card_dirty_test(masm, card_addr);
+  __ beq(CR0, filtered);
+
+  __ li(R0, (int)G1CardTable::dirty_card_val());
+  __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
+
+  Register Rcard_addr = tmp3;
+  __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
+
+  generate_queue_insertion(masm,
+                           G1ThreadLocalData::dirty_card_queue_index_offset(),
+                           G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                           runtime, Rcard_addr, tmp1);
+  __ b(filtered);
+
+  __ bind(runtime);
+
+  assert(preservation_level == MacroAssembler::PRESERVATION_NONE,
+         "g1_write_barrier_post doesn't support preservation levels higher than PRESERVATION_NONE");
+
+  // Save the live input values.
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, R16_thread);
+
+  __ bind(filtered);
 }
 
 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -306,7 +333,8 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
     }
     g1_write_barrier_post(masm, decorators,
                           base, val,
-                          tmp1, tmp2);
+                          tmp1, tmp2, tmp3,
+                          preservation_level);
   }
 }
 
@@ -429,29 +457,70 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register new_val,
                                                      Register tmp1,
                                                      Register tmp2,
-                                                     bool new_val_may_be_null,
+                                                     G1PostBarrierStubC2* stub,
                                                      bool decode_new_val) {
   assert_different_registers(store_addr, new_val, tmp1, R0);
   assert_different_registers(store_addr, tmp1, tmp2, R0);
 
-  Label done;
+  stub->initialize_registers(R16_thread, tmp1, tmp2);
 
+  bool null_check_required = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
   Register new_val_decoded = new_val;
 
   if (decode_new_val) {
     assert(UseCompressedOops, "or should not be here");
-    if (new_val_may_be_null && CompressedOops::base() != nullptr) {
+    if (null_check_required && CompressedOops::base() != nullptr) {
       // We prefer doing the null check after the region crossing check.
       // Only compressed oop modes with base != null require a null check here.
       __ cmpwi(CR0, new_val, 0);
-      __ beq(CR0, done);
-      new_val_may_be_null = false;
+      __ beq(CR0, *stub->continuation());
+      null_check_required = false;
     }
     new_val_decoded = __ decode_heap_oop_not_null(tmp2, new_val);
   }
 
-  generate_post_barrier(masm, store_addr, new_val_decoded, R16_thread, tmp1, tmp2, done, new_val_may_be_null);
-  __ bind(done);
+  generate_region_crossing_test(masm, store_addr, new_val_decoded);
+  __ beq(CR0, *stub->continuation());
+
+  // crosses regions, storing null?
+  if (null_check_required) {
+    __ cmpdi(CR0, new_val_decoded, 0);
+    __ beq(CR0, *stub->continuation());
+  }
+
+  Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
+  assert(card_addr.base() == tmp1 && card_addr.index() == tmp2, "needed by post barrier stub");
+  __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry());
+
+  __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+  Address card_addr(stub->tmp1(), stub->tmp2()); // See above.
+
+  __ bind(*stub->entry());
+
+  generate_card_dirty_test(masm, card_addr);
+  __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation());
+
+  __ li(R0, (int)G1CardTable::dirty_card_val());
+  __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
+
+  Register Rcard_addr = stub->tmp1();
+  __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
+
+  generate_queue_insertion(masm,
+                           G1ThreadLocalData::dirty_card_queue_index_offset(),
+                           G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                           runtime, Rcard_addr, stub->tmp2());
+  __ b(*stub->continuation());
+
+  __ bind(runtime);
+  generate_c2_barrier_runtime_call(masm, stub, Rcard_addr, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+  __ b(*stub->continuation());
 }
 
 #endif // COMPILER2
@@ -489,19 +558,28 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   __ b(*stub->continuation());
 }
 
-#undef __
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
 
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
-  masm->bind(done);
+  assert(stub->addr()->is_register(), "Precondition.");
+  assert(stub->new_val()->is_register(), "Precondition.");
+  Register addr_reg = stub->addr()->as_pointer_register();
+  Register new_val_reg = stub->new_val()->as_register();
+
+  __ cmpdi(CR0, new_val_reg, 0);
+  __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation());
+
+  address c_code = bs->post_barrier_c1_runtime_code_blob()->code_begin();
+  //__ load_const_optimized(R0, c_code);
+  __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code));
+  __ mtctr(R0);
+  __ mr(R0, addr_reg); // Pass addr in R0.
+  __ bctrl();
+  __ b(*stub->continuation());
 }
 
+#undef __
 #define __ sasm->
 
 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -564,6 +642,86 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ b(restart);
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
+
+  __ set_info("g1_post_barrier_slow_id", false);
+
+  // Using stack slots: spill addr, spill tmp2
+  const int stack_slots = 2;
+  Register tmp = R0;
+  Register addr = R14;
+  Register tmp2 = R15;
+  CardTable::CardValue* byte_map_base = bs->card_table()->byte_map_base();
+
+  Label restart, refill, ret;
+
+  // Spill
+  __ std(addr, -8, R1_SP);
+  __ std(tmp2, -16, R1_SP);
+
+  __ srdi(addr, R0, CardTable::card_shift()); // Addr is passed in R0.
+  __ load_const_optimized(/*cardtable*/ tmp2, byte_map_base, tmp);
+  __ add(addr, tmp2, addr);
+  __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
+
+  // Return if young card.
+  __ cmpwi(CR0, tmp, G1CardTable::g1_young_card_val());
+  __ beq(CR0, ret);
+
+  // Return if sequential consistent value is already dirty.
+  __ membar(Assembler::StoreLoad);
+  __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
+
+  __ cmpwi(CR0, tmp, G1CardTable::dirty_card_val());
+  __ beq(CR0, ret);
+
+  // Not dirty.
+
+  // First, dirty it.
+  __ li(tmp, G1CardTable::dirty_card_val());
+  __ stb(tmp, 0, addr);
+
+  int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
+  int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
+
+  __ bind(restart);
+
+  // Get the index into the update buffer. G1DirtyCardQueue::_index is
+  // a size_t so ld_ptr is appropriate here.
+  __ ld(tmp2, dirty_card_q_index_byte_offset, R16_thread);
+
+  // index == 0?
+  __ cmpdi(CR0, tmp2, 0);
+  __ beq(CR0, refill);
+
+  __ ld(tmp, dirty_card_q_buf_byte_offset, R16_thread);
+  __ addi(tmp2, tmp2, -oopSize);
+
+  __ std(tmp2, dirty_card_q_index_byte_offset, R16_thread);
+  __ add(tmp2, tmp, tmp2);
+  __ std(addr, 0, tmp2); // [_buf + index] := <address_of_card>
+
+  // Restore temp registers and return-from-leaf.
+  __ bind(ret);
+  __ ld(tmp2, -16, R1_SP);
+  __ ld(addr, -8, R1_SP);
+  __ blr();
+
+  __ bind(refill);
+  const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord;
+  __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0
+  __ mflr(R0);
+  __ std(R0, _abi0(lr), R1_SP);
+  __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread), R16_thread);
+  __ pop_frame();
+  __ ld(R0, _abi0(lr), R1_SP);
+  __ mtlr(R0);
+  __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
+  __ b(restart);
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, 2021 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -37,7 +37,9 @@
 class LIR_Assembler;
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
 protected:
@@ -54,7 +56,8 @@ protected:
                             MacroAssembler::PreservationLevel preservation_level);
   void g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators,
                              Register store_addr, Register new_val,
-                             Register tmp1, Register tmp2);
+                             Register tmp1, Register tmp2, Register tmp3,
+                             MacroAssembler::PreservationLevel preservation_level);
 
   virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                             Register base, RegisterOrConstant ind_or_offs, Register val,
@@ -76,21 +79,17 @@ public:
                                 Register new_val,
                                 Register tmp1,
                                 Register tmp2,
-                                bool new_val_may_be_null,
+                                G1PostBarrierStubC2* c2_stub,
                                 bool decode_new_val);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif
 #ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
-
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
-
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
 // Copyright (c) 2025 SAP SE. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
@@ -64,13 +64,13 @@ static void post_write_barrier(MacroAssembler* masm,
                                Register tmp1,
                                Register tmp2,
                                bool decode_new_val = false) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, new_val_may_be_null, decode_new_val);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, stub, decode_new_val);
 }
 
 %}

src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp

@@ -114,6 +114,7 @@ public:
   }
 };
 
+
 void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                    Register base, RegisterOrConstant ind_or_offs, Register dst,
                                    Register tmp1, Register tmp2,
@@ -560,6 +561,7 @@ void ZBarrierSetAssembler::generate_conjoint_oop_copy(MacroAssembler* masm, bool
   copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized);
 }
 
+
 // Verify a colored pointer.
 void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
   if (!VerifyOops) {
@@ -581,6 +583,7 @@ void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const c
   __ bind(done);
 }
 
+
 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
                                                          Register obj, Register tmp, Label& slowpath) {
   __ block_comment("try_resolve_jobject_in_native (zgc) {");

src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp

@@ -1199,11 +1199,16 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
+                                            AdapterHandlerEntry* handler) {
+  address i2c_entry;
+  address c2i_unverified_entry;
+  address c2i_entry;
+
+
   // entry: i2c
 
   __ align(CodeEntryAlignment);
-  entry_address[AdapterBlob::I2C] = __ pc();
+  i2c_entry = __ pc();
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
 
@@ -1211,7 +1216,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 
   __ align(CodeEntryAlignment);
   BLOCK_COMMENT("c2i unverified entry");
-  entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+  c2i_unverified_entry = __ pc();
 
   // inline_cache contains a CompiledICData
   const Register ic             = R19_inline_cache_reg;
@@ -1239,10 +1244,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 
   // entry: c2i
 
-  entry_address[AdapterBlob::C2I] = __ pc();
+  c2i_entry = __ pc();
 
   // Class initialization barrier for static methods
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_no_clinit_check_entry = nullptr;
   if (VM_Version::supports_fast_class_init_checks()) {
     Label L_skip_barrier;
 
@@ -1261,13 +1266,15 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ bctr();
 
     __ bind(L_skip_barrier);
-    entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
+    c2i_no_clinit_check_entry = __ pc();
   }
 
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
   bs->c2i_entry_barrier(masm, /* tmp register*/ ic_klass, /* tmp register*/ receiver_klass, /* tmp register*/ code);
 
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
   return;
 }
 

src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp

@@ -204,10 +204,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
-
-  // lock_reg was destroyed by fast unlocking attempt => recompute it
-  ce->monitor_address(_monitor_ix, _lock_reg);
-
+  if (_compute_lock) {
+    // lock_reg was destroyed by fast unlocking attempt => recompute it
+    ce->monitor_address(_monitor_ix, _lock_reg);
+  }
   ce->store_parameter(_lock_reg->as_register(), 0);
   // note: non-blocking leaf routine => no call info needed
   StubId exit_id;

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

@@ -338,7 +338,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::r10_opr);
-    stub = new MonitorExitStub(FrameMap::r10_opr, 0);
+    stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
     __ unlock_object(x15, x14, x10, x16, *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -1494,12 +1494,14 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register lock = op->lock_opr()->as_register();
   Register temp = op->scratch_opr()->as_register();
   if (op->code() == lir_lock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // add debug info for NullPointerException only if one is possible
     int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
     if (op->info() != nullptr) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }
   } else if (op->code() == lir_unlock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
   } else {
     Unimplemented();

src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp

@@ -48,27 +48,27 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
   }
 }
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
-  assert_different_registers(hdr, obj, basic_lock, temp, t0, t1);
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
+  assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
   int null_check_offset = -1;
 
   verify_oop(obj);
 
   // save object being locked into the BasicObjectLock
-  sd(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  sd(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
 
   null_check_offset = offset();
 
-  lightweight_lock(basic_lock, obj, hdr, temp, t1, slow_case);
+  lightweight_lock(disp_hdr, obj, hdr, temp, t1, slow_case);
 
   return null_check_offset;
 }
 
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
-  assert_different_registers(hdr, obj, basic_lock, temp, t0, t1);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
+  assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
 
   // load object
-  ld(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
   verify_oop(obj);
 
   lightweight_unlock(obj, hdr, temp, t1, slow_case);

src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -56,19 +56,19 @@ using MacroAssembler::null_check;
                  Register result);
 
   // locking
-  // hdr       : must be x10, contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must point to the basic_lock, contents preserved
-  // temp      : temporary register, must not be scratch register t0 or t1
+  // hdr     : must be x10, contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must point to the displaced header location, contents preserved
+  // temp : temporary register, must not be scratch register t0 or t1
   // returns code offset at which to add null check debug information
-  int lock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
+  int lock_object(Register swap, Register obj, Register disp_hdr, Register temp, Label& slow_case);
 
   // unlocking
-  // hdr       : contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must be x10 & must point to the basic lock, contents destroyed
-  // temp      : temporary register, must not be scratch register t0 or t1
-  void unlock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
+  // hdr     : contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must be x10 & must point to the displaced header location, contents destroyed
+  // temp : temporary register, must not be scratch register t0 or t1
+  void unlock_object(Register swap, Register obj, Register lock, Register temp, Label& slow_case);
 
   void initialize_object(
     Register obj,                      // result: pointer to object after successful allocation

src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp

@@ -87,54 +87,15 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
   }
 }
 
-void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm,
-                                                             DecoratorSet decorators,
-                                                             Register start,
-                                                             Register count,
-                                                             Register tmp,
-                                                             RegSet saved_regs) {
+void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                             Register start, Register count, Register tmp, RegSet saved_regs) {
+  __ push_reg(saved_regs, sp);
   assert_different_registers(start, count, tmp);
-
-  Label loop, next, done;
-
-  // Zero count? Nothing to do.
-  __ beqz(count, done);
-
-  // Calculate the number of card marks to set. Since the object might start and
-  // end within a card, we need to calculate this via the card table indexes of
-  // the actual start and last addresses covered by the object.
-  // Temporarily use the count register for the last element address.
-  __ shadd(count, count, start, tmp, LogBytesPerHeapOop); // end = start + count << LogBytesPerHeapOop
-  __ subi(count, count, BytesPerHeapOop);                 // Use last element address for end.
-
-  __ srli(start, start, CardTable::card_shift());
-  __ srli(count, count, CardTable::card_shift());
-  __ sub(count, count, start);                            // Number of bytes to mark - 1.
-
-  // Add card table base offset to start.
-  Address card_table_address(xthread, G1ThreadLocalData::card_table_base_offset());
-  __ ld(tmp, card_table_address);
-  __ add(start, start, tmp);
-
-  __ bind(loop);
-  if (UseCondCardMark) {
-    __ add(tmp, start, count);
-    __ lbu(tmp, Address(tmp, 0));
-    static_assert((uint)G1CardTable::clean_card_val() == 0xff, "must be");
-    __ subi(tmp, tmp, G1CardTable::clean_card_val()); // Convert to clean_card_value() to a comparison
-                                                      // against zero to avoid use of an extra temp.
-    __ bnez(tmp, next);
-  }
-
-  __ add(tmp, start, count);
-  static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
-  __ sb(zr, Address(tmp, 0));
-
-  __ bind(next);
-  __ subi(count, count, 1);
-  __ bgez(count, loop);
-
-  __ bind(done);
+  assert_different_registers(c_rarg0, count);
+  __ mv(c_rarg0, start);
+  __ mv(c_rarg1, count);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
+  __ pop_reg(saved_regs, sp);
 }
 
 static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
@@ -228,40 +189,47 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
 
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register thread,
-                                  const Register tmp1,
-                                  const Register tmp2,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-  assert(thread == xthread, "must be");
-  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+                                            const Register store_addr,
+                                            const Register new_val,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            Label& done,
+                                            bool new_val_may_be_null) {
   // Does store cross heap regions?
   __ xorr(tmp1, store_addr, new_val);                    // tmp1 := store address ^ new value
   __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);  // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
   __ beqz(tmp1, done);
-
   // Crosses regions, storing null?
   if (new_val_may_be_null) {
     __ beqz(new_val, done);
   }
-  // Storing region crossing non-null, is card clean?
+  // Storing region crossing non-null, is card young?
   __ srli(tmp1, store_addr, CardTable::card_shift());    // tmp1 := card address relative to card table base
-
-  Address card_table_address(xthread, G1ThreadLocalData::card_table_base_offset());
-  __ ld(tmp2, card_table_address);                       // tmp2 := card table base address
+  __ load_byte_map_base(tmp2);                           // tmp2 := card table base address
   __ add(tmp1, tmp1, tmp2);                              // tmp1 := card address
-  if (UseCondCardMark) {
-    static_assert((uint)G1CardTable::clean_card_val() == 0xff, "must be");
-    __ lbu(tmp2, Address(tmp1, 0));                      // tmp2 := card
-    __ subi(tmp2, tmp2, G1CardTable::clean_card_val());  // Convert to clean_card_value() to a comparison
-                                                         // against zero to avoid use of an extra temp.
-    __ bnez(tmp2, done);
-  }
-  static_assert((uint)G1CardTable::dirty_card_val() == 0, "must be to use zr");
-  __ sb(zr, Address(tmp1, 0));
+  __ lbu(tmp2, Address(tmp1));                           // tmp2 := card
+}
+
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+                                            const Register thread,
+                                            const Register tmp1,
+                                            const Register tmp2,
+                                            Label& done,
+                                            Label& runtime) {
+  __ membar(MacroAssembler::StoreLoad);  // StoreLoad membar
+  __ lbu(tmp2, Address(tmp1));           // tmp2 := card
+  __ beqz(tmp2, done, true);
+  // Storing a region crossing, non-null oop, card is clean.
+  // Dirty card and log.
+  STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+  __ sb(zr, Address(tmp1));       // *(card address) := dirty_card_val
+  generate_queue_test_and_insertion(masm,
+                                    G1ThreadLocalData::dirty_card_queue_index_offset(),
+                                    G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                                    runtime,
+                                    thread, tmp1, tmp2, t0);
+  __ j(done);
 }
 
 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
@@ -270,8 +238,27 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
                                                   Register thread,
                                                   Register tmp1,
                                                   Register tmp2) {
+  assert(thread == xthread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
+  assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
+         "expecting a register");
+
   Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+  Label runtime;
+
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+  // If card is young, jump to done (tmp2 holds the card value)
+  __ mv(t0, (int)G1CardTable::g1_young_card_val());
+  __ beq(tmp2, t0, done);   // card == young_card_val?
+  generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(store_addr);
+  __ push_reg(saved, sp);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
+  __ pop_reg(saved, sp);
+
   __ bind(done);
 }
 
@@ -331,10 +318,37 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register thread,
                                                      Register tmp1,
                                                      Register tmp2,
-                                                     bool new_val_may_be_null) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
-  __ bind(done);
+                                                     G1PostBarrierStubC2* stub) {
+  assert(thread == xthread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
+  assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
+         "expecting a register");
+
+  stub->initialize_registers(thread, tmp1, tmp2);
+
+  bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+  // If card is not young, jump to stub (slow path) (tmp2 holds the card value)
+  __ mv(t0, (int)G1CardTable::g1_young_card_val());
+  __ bne(tmp2, t0, *stub->entry(), true);
+
+  __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+  Register thread = stub->thread();
+  Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+  Register tmp2 = stub->tmp2();
+
+  __ bind(*stub->entry());
+  generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
+
+  __ bind(runtime);
+  generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+  __ j(*stub->continuation());
 }
 
 #endif // COMPILER2
@@ -429,19 +443,20 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   __ j(*stub->continuation());
 }
 
-#undef __
-
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
-  masm->bind(done);
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+  assert(stub->addr()->is_register(), "Precondition");
+  assert(stub->new_val()->is_register(), "Precondition");
+  Register new_val_reg = stub->new_val()->as_register();
+  __ beqz(new_val_reg, *stub->continuation(), /* is_far */ true);
+  ce->store_parameter(stub->addr()->as_pointer_register(), 0);
+  __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
+  __ j(*stub->continuation());
 }
 
+#undef __
+
 #define __ sasm->
 
 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -492,6 +507,74 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ epilogue();
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("g1_post_barrier", false);
+
+  // arg0 : store_address
+  Address store_addr(fp, 2 * BytesPerWord); // 2 BytesPerWord from fp
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+  CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
+
+  Label done;
+  Label runtime;
+
+  // At this point we know new_value is non-null and the new_value crosses regions.
+  // Must check to see if card is already dirty
+  const Register thread = xthread;
+
+  Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
+  Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
+
+  const Register card_offset = t1;
+  // RA is free here, so we can use it to hold the byte_map_base.
+  const Register byte_map_base = ra;
+
+  assert_different_registers(card_offset, byte_map_base, t0);
+
+  __ load_parameter(0, card_offset);
+  __ srli(card_offset, card_offset, CardTable::card_shift());
+  __ load_byte_map_base(byte_map_base);
+
+  // Convert card offset into an address in card_addr
+  Register card_addr = card_offset;
+  __ add(card_addr, byte_map_base, card_addr);
+
+  __ lbu(t0, Address(card_addr, 0));
+  __ sub(t0, t0, (int)G1CardTable::g1_young_card_val());
+  __ beqz(t0, done);
+
+  assert((int)CardTable::dirty_card_val() == 0, "must be 0");
+
+  __ membar(MacroAssembler::StoreLoad);
+  __ lbu(t0, Address(card_addr, 0));
+  __ beqz(t0, done);
+
+  // storing region crossing non-null, card is clean.
+  // dirty card and log.
+  __ sb(zr, Address(card_addr, 0));
+
+  __ ld(t0, queue_index);
+  __ beqz(t0, runtime);
+  __ subi(t0, t0, wordSize);
+  __ sd(t0, queue_index);
+
+  // Reuse RA to hold buffer_addr
+  const Register buffer_addr = ra;
+
+  __ ld(buffer_addr, buffer);
+  __ add(t0, buffer_addr, t0);
+  __ sd(card_addr, Address(t0, 0));
+  __ j(done);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
+  __ pop_call_clobbered_registers();
+  __ bind(done);
+  __ epilogue();
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -35,7 +35,9 @@ class LIR_Assembler;
 #endif
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
 protected:
@@ -66,16 +68,10 @@ protected:
 public:
 #ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
-
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
-
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
 #ifdef COMPILER2
@@ -94,7 +90,9 @@ public:
                                 Register thread,
                                 Register tmp1,
                                 Register tmp2,
-                                bool new_val_may_be_null);
+                                G1PostBarrierStubC2* c2_stub);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif
 
   void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
 // Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
@@ -63,13 +63,13 @@ static void write_barrier_post(MacroAssembler* masm,
                                Register new_val,
                                Register tmp1,
                                Register tmp2) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, new_val_may_be_null);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, stub);
 }
 
 %}

src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp

@@ -602,11 +602,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
-  entry_address[AdapterBlob::I2C] = __ pc();
+                                            AdapterHandlerEntry* handler) {
+  address i2c_entry = __ pc();
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
-  entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+  address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
   const Register receiver = j_rarg0;
@@ -633,10 +633,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ block_comment("} c2i_unverified_entry");
   }
 
-  entry_address[AdapterBlob::C2I] = __ pc();
+  address c2i_entry = __ pc();
 
   // Class initialization barrier for static methods
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_no_clinit_check_entry = nullptr;
   if (VM_Version::supports_fast_class_init_checks()) {
     Label L_skip_barrier;
 
@@ -651,13 +651,15 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 
     __ bind(L_skip_barrier);
-    entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
+    c2i_no_clinit_check_entry = __ pc();
   }
 
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
   bs->c2i_entry_barrier(masm);
 
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
   return;
 }
 
@@ -1677,6 +1679,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   Label lock_done;
 
   if (method->is_synchronized()) {
+    const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
+
     // Get the handle (the 2nd argument)
     __ mv(oop_handle_reg, c_rarg1);
 

src/hotspot/cpu/riscv/vm_version_riscv.cpp

@@ -35,28 +35,15 @@
 
 uint32_t VM_Version::_initial_vector_length = 0;
 
-#define DEF_RV_EXT_FEATURE(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF) \
-VM_Version::NAME##RVExtFeatureValue VM_Version::NAME;
-RV_EXT_FEATURE_FLAGS(DEF_RV_EXT_FEATURE)
-#undef DEF_RV_EXT_FEATURE
+#define DEF_RV_FEATURE(NAME, PRETTY, BIT, FSTRING, FLAGF)       \
+VM_Version::NAME##RVFeatureValue VM_Version::NAME(PRETTY, BIT, FSTRING);
+RV_FEATURE_FLAGS(DEF_RV_FEATURE)
 
-#define DEF_RV_NON_EXT_FEATURE(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF) \
-VM_Version::NAME##RVNonExtFeatureValue VM_Version::NAME;
-RV_NON_EXT_FEATURE_FLAGS(DEF_RV_NON_EXT_FEATURE)
-#undef DEF_RV_NON_EXT_FEATURE
-
-#define ADD_RV_EXT_FEATURE_IN_LIST(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF) \
-     &VM_Version::NAME,
-#define ADD_RV_NON_EXT_FEATURE_IN_LIST(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF) \
-     &VM_Version::NAME,
- VM_Version::RVFeatureValue* VM_Version::_feature_list[] = {
- RV_EXT_FEATURE_FLAGS(ADD_RV_EXT_FEATURE_IN_LIST)
- RV_NON_EXT_FEATURE_FLAGS(ADD_RV_NON_EXT_FEATURE_IN_LIST)
+#define ADD_RV_FEATURE_IN_LIST(NAME, PRETTY, BIT, FSTRING, FLAGF) \
+    &VM_Version::NAME,
+VM_Version::RVFeatureValue* VM_Version::_feature_list[] = {
+RV_FEATURE_FLAGS(ADD_RV_FEATURE_IN_LIST)
   nullptr};
-#undef ADD_RV_NON_EXT_FEATURE_IN_LIST
-#undef ADD_RV_EXT_FEATURE_IN_LIST
-
-VM_Version::RVExtFeatures* VM_Version::_rv_ext_features = new VM_Version::RVExtFeatures();
 
 void VM_Version::useRVA20U64Profile() {
   RV_USE_RVA20U64;
@@ -160,7 +147,7 @@ void VM_Version::common_initialize() {
 
   if (FLAG_IS_DEFAULT(AvoidUnalignedAccesses)) {
     FLAG_SET_DEFAULT(AvoidUnalignedAccesses,
-      unaligned_scalar.value() != MISALIGNED_SCALAR_FAST);
+      unaligned_access.value() != MISALIGNED_FAST);
   }
 
   if (!AvoidUnalignedAccesses) {
@@ -175,12 +162,7 @@ void VM_Version::common_initialize() {
   // This machine has fast unaligned memory accesses
   if (FLAG_IS_DEFAULT(UseUnalignedAccesses)) {
     FLAG_SET_DEFAULT(UseUnalignedAccesses,
-      (unaligned_scalar.value() == MISALIGNED_SCALAR_FAST));
-  }
-
-  if (FLAG_IS_DEFAULT(AlignVector) && unaligned_vector.enabled()) {
-    FLAG_SET_DEFAULT(AlignVector,
-      unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
+      unaligned_access.value() == MISALIGNED_FAST);
   }
 
 #ifdef __riscv_ztso
@@ -495,6 +477,10 @@ void VM_Version::c2_initialize() {
     warning("AES/CTR intrinsics are not available on this CPU");
     FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
   }
+
+  if (FLAG_IS_DEFAULT(AlignVector)) {
+    FLAG_SET_DEFAULT(AlignVector, AvoidUnalignedAccesses);
+  }
 }
 
 #endif // COMPILER2

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -46,29 +46,30 @@ class VM_Version : public Abstract_VM_Version {
     RIVOS = 0x6cf, // JEDEC: 0x4f, Bank: 14
   };
 
-  class RVExtFeatures;
-
   class RVFeatureValue {
     const char* const _pretty;
     const bool        _feature_string;
-    const uint64_t    _linux_feature_bit;
+    const uint64_t    _feature_bit;
+    bool              _enabled;
     int64_t           _value;
    public:
-    RVFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
-      _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)),
-      _value(-1) {
+    RVFeatureValue(const char* pretty, int bit_num, bool fstring) :
+      _pretty(pretty), _feature_string(fstring), _feature_bit(nth_bit(bit_num)),
+      _enabled(false), _value(-1) {
     }
-    virtual void enable_feature(int64_t value = 0) {
+    void enable_feature(int64_t value = 0) {
+      _enabled = true;
       _value = value;
     }
-    virtual void disable_feature() {
+    void disable_feature() {
+      _enabled = false;
       _value = -1;
     }
     const char* pretty()         { return _pretty; }
-    uint64_t feature_bit()       { return _linux_feature_bit; }
+    uint64_t feature_bit()       { return _feature_bit; }
     bool feature_string()        { return _feature_string; }
+    bool enabled()               { return _enabled; }
     int64_t value()              { return _value; }
-    virtual bool enabled() = 0;
     virtual void update_flag() = 0;
   };
 
@@ -110,45 +111,6 @@ class VM_Version : public Abstract_VM_Version {
   #define NO_UPDATE_DEFAULT                \
   void update_flag() {}                    \
 
-
-  class RVExtFeatureValue : public RVFeatureValue {
-    const uint32_t _cpu_feature_index;
-   public:
-    RVExtFeatureValue(const char* pretty, int linux_bit_num, uint32_t cpu_feature_index, bool fstring) :
-      RVFeatureValue(pretty, linux_bit_num, fstring),
-      _cpu_feature_index(cpu_feature_index) {
-    }
-    bool enabled() {
-      return RVExtFeatures::current()->support_feature(_cpu_feature_index);
-    }
-    void enable_feature(int64_t value = 0) {
-      RVFeatureValue::enable_feature(value);
-      RVExtFeatures::current()->set_feature(_cpu_feature_index);
-    }
-    void disable_feature() {
-      RVFeatureValue::disable_feature();
-      RVExtFeatures::current()->clear_feature(_cpu_feature_index);
-    }
-  };
-
-  class RVNonExtFeatureValue : public RVFeatureValue {
-    bool _enabled;
-   public:
-    RVNonExtFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
-      RVFeatureValue(pretty, linux_bit_num, fstring),
-      _enabled(false) {
-    }
-    bool enabled()               { return _enabled; }
-    void enable_feature(int64_t value = 0) {
-      RVFeatureValue::enable_feature(value);
-      _enabled = true;
-    }
-    void disable_feature() {
-      RVFeatureValue::disable_feature();
-      _enabled = false;
-    }
-  };
-
   // Frozen standard extensions
   // I RV64I
   // M Integer Multiplication and Division
@@ -191,8 +153,7 @@ class VM_Version : public Abstract_VM_Version {
   // mvendorid Manufactory JEDEC id encoded, ISA vol 2 3.1.2..
   // marchid   Id for microarch. Mvendorid plus marchid uniquely identify the microarch.
   // mimpid    A unique encoding of the version of the processor implementation.
-  // unaligned_scalar Performance of misaligned scalar accesses (unknown, emulated, slow, fast, unsupported)
-  // unaligned_vector Performance of misaligned vector accesses (unknown, unspported, slow, fast)
+  // unaligned_access Unaligned memory accesses (unknown, unspported, emulated, slow, firmware, fast)
   // satp mode SATP bits (number of virtual addr bits) mbare, sv39, sv48, sv57, sv64
 
  public:
@@ -200,141 +161,58 @@ class VM_Version : public Abstract_VM_Version {
   #define RV_NO_FLAG_BIT (BitsPerWord+1) // nth_bit will return 0 on values larger than BitsPerWord
 
   // Note: the order matters, depender should be after their dependee. E.g. ext_V before ext_Zvbb.
-  //
-  // Fields description in `decl`:
-  //    declaration name, extension name, bit value from linux, feature string?, mapped flag)
-  #define RV_EXT_FEATURE_FLAGS(decl)                                                                   \
-  decl(ext_I            ,  i           ,     ('I' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_M            ,  m           ,     ('M' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_A            ,  a           ,     ('A' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_F            ,  f           ,     ('F' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_D            ,  d           ,     ('D' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_C            ,  c           ,     ('C' - 'A'),  true ,  UPDATE_DEFAULT(UseRVC))             \
-  decl(ext_Q            ,  q           ,     ('Q' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_H            ,  h           ,     ('H' - 'A'),  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_V            ,  v           ,     ('V' - 'A'),  true ,  UPDATE_DEFAULT(UseRVV))             \
-  decl(ext_Zicbom       ,  Zicbom      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZicbom))          \
-  decl(ext_Zicboz       ,  Zicboz      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZicboz))          \
-  decl(ext_Zicbop       ,  Zicbop      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZicbop))          \
-  decl(ext_Zba          ,  Zba         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZba))             \
-  decl(ext_Zbb          ,  Zbb         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZbb))             \
-  decl(ext_Zbc          ,  Zbc         ,  RV_NO_FLAG_BIT,  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zbs          ,  Zbs         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZbs))             \
-  decl(ext_Zbkb         ,  Zbkb        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZbkb))            \
-  decl(ext_Zcb          ,  Zcb         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZcb))             \
-  decl(ext_Zfa          ,  Zfa         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZfa))             \
-  decl(ext_Zfh          ,  Zfh         ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZfh))             \
-  decl(ext_Zfhmin       ,  Zfhmin      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZfhmin))          \
-  decl(ext_Zicsr        ,  Zicsr       ,  RV_NO_FLAG_BIT,  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zicntr       ,  Zicntr      ,  RV_NO_FLAG_BIT,  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zifencei     ,  Zifencei    ,  RV_NO_FLAG_BIT,  true ,  NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zic64b       ,  Zic64b      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZic64b))          \
-  decl(ext_Ztso         ,  Ztso        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZtso))            \
-  decl(ext_Zihintpause  ,  Zihintpause ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZihintpause))     \
-  decl(ext_Zacas        ,  Zacas       ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZacas))           \
-  decl(ext_Zvbb         ,  Zvbb        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvbb, ext_V)) \
-  decl(ext_Zvbc         ,  Zvbc        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvbc, ext_V)) \
-  decl(ext_Zvfh         ,  Zvfh        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvfh, ext_V)) \
-  decl(ext_Zvkn         ,  Zvkn        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvkn, ext_V)) \
-  decl(ext_Zicond       ,  Zicond      ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT(UseZicond))          \
+  // declaration name  , extension name, bit pos       ,in str, mapped flag)
+  #define RV_FEATURE_FLAGS(decl)                                                                        \
+  decl(ext_I            , "i"              ,    ('I' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_M            , "m"              ,    ('M' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_A            , "a"              ,    ('A' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_F            , "f"              ,    ('F' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_D            , "d"              ,    ('D' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_C            , "c"              ,    ('C' - 'A'), true , UPDATE_DEFAULT(UseRVC))             \
+  decl(ext_Q            , "q"              ,    ('Q' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_H            , "h"              ,    ('H' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_V            , "v"              ,    ('V' - 'A'), true , UPDATE_DEFAULT(UseRVV))             \
+  decl(ext_Zicbom       , "Zicbom"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbom))          \
+  decl(ext_Zicboz       , "Zicboz"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicboz))          \
+  decl(ext_Zicbop       , "Zicbop"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbop))          \
+  decl(ext_Zba          , "Zba"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZba))             \
+  decl(ext_Zbb          , "Zbb"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb))             \
+  decl(ext_Zbc          , "Zbc"            , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zbs          , "Zbs"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs))             \
+  decl(ext_Zbkb         , "Zbkb"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbkb))            \
+  decl(ext_Zcb          , "Zcb"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb))             \
+  decl(ext_Zfa          , "Zfa"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfa))             \
+  decl(ext_Zfh          , "Zfh"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh))             \
+  decl(ext_Zfhmin       , "Zfhmin"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfhmin))          \
+  decl(ext_Zicsr        , "Zicsr"          , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zicntr       , "Zicntr"         , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zifencei     , "Zifencei"       , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zic64b       , "Zic64b"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b))          \
+  decl(ext_Ztso         , "Ztso"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))            \
+  decl(ext_Zihintpause  , "Zihintpause"    , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause))     \
+  decl(ext_Zacas        , "Zacas"          , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas))           \
+  decl(ext_Zvbb         , "Zvbb"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbb, ext_V)) \
+  decl(ext_Zvbc         , "Zvbc"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbc, ext_V)) \
+  decl(ext_Zvfh         , "Zvfh"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvfh, ext_V)) \
+  decl(ext_Zvkn         , "Zvkn"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvkn, ext_V)) \
+  decl(ext_Zicond       , "Zicond"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond))          \
+  decl(mvendorid        , "VendorId"       , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(marchid          , "ArchId"         , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(mimpid           , "ImpId"          , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(unaligned_access , "Unaligned"      , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(satp_mode        , "SATP"           , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(zicboz_block_size, "ZicbozBlockSize", RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
 
-  #define DECLARE_RV_EXT_FEATURE(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF)               \
-  struct NAME##RVExtFeatureValue : public RVExtFeatureValue {                           \
-    NAME##RVExtFeatureValue() :                                                         \
-      RVExtFeatureValue(#PRETTY, LINUX_BIT, RVExtFeatures::CPU_##NAME, FSTRING) {}      \
-    FLAGF;                                                                              \
-  };                                                                                    \
-  static NAME##RVExtFeatureValue NAME;                                                  \
+  #define DECLARE_RV_FEATURE(NAME, PRETTY, BIT, FSTRING, FLAGF)        \
+  struct NAME##RVFeatureValue : public RVFeatureValue {                \
+    NAME##RVFeatureValue(const char* pretty, int bit, bool fstring) :  \
+      RVFeatureValue(pretty, bit, fstring) {}                          \
+    FLAGF;                                                             \
+  };                                                                   \
+  static NAME##RVFeatureValue NAME;                                    \
 
-  RV_EXT_FEATURE_FLAGS(DECLARE_RV_EXT_FEATURE)
-  #undef DECLARE_RV_EXT_FEATURE
-
-  // Non-extension features
-  //
-  #define RV_NON_EXT_FEATURE_FLAGS(decl)                                                       \
-  decl(mvendorid        ,  VendorId        ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(marchid          ,  ArchId          ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(mimpid           ,  ImpId           ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(satp_mode        ,  SATP            ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(unaligned_scalar ,  UnalignedScalar ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(unaligned_vector ,  UnalignedVector ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-  decl(zicboz_block_size,  ZicbozBlockSize ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)      \
-
-  #define DECLARE_RV_NON_EXT_FEATURE(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF)      \
-  struct NAME##RVNonExtFeatureValue : public RVNonExtFeatureValue {                \
-    NAME##RVNonExtFeatureValue() :                                                 \
-      RVNonExtFeatureValue(#PRETTY, LINUX_BIT, FSTRING) {}                         \
-    FLAGF;                                                                         \
-  };                                                                               \
-  static NAME##RVNonExtFeatureValue NAME;                                          \
-
-  RV_NON_EXT_FEATURE_FLAGS(DECLARE_RV_NON_EXT_FEATURE)
-  #undef DECLARE_RV_NON_EXT_FEATURE
-
-private:
-  // Utility for AOT CPU feature store/check.
-  class RVExtFeatures : public CHeapObj<mtCode> {
-   public:
-    enum RVFeatureIndex {
-      #define DECLARE_RV_FEATURE_ENUM(NAME, PRETTY, LINUX_BIT, FSTRING, FLAGF) CPU_##NAME,
-
-      RV_EXT_FEATURE_FLAGS(DECLARE_RV_FEATURE_ENUM)
-      MAX_CPU_FEATURE_INDEX
-      #undef DECLARE_RV_FEATURE_ENUM
-    };
-   private:
-    uint64_t _features_bitmap[(MAX_CPU_FEATURE_INDEX / BitsPerLong) + 1];
-    STATIC_ASSERT(sizeof(_features_bitmap) * BitsPerByte >= MAX_CPU_FEATURE_INDEX);
-
-    // Number of 8-byte elements in _features_bitmap.
-    constexpr static int element_count() {
-      return sizeof(_features_bitmap) / sizeof(uint64_t);
-    }
-
-    static int element_index(RVFeatureIndex feature) {
-      int idx = feature / BitsPerLong;
-      assert(idx < element_count(), "Features array index out of bounds");
-      return idx;
-    }
-
-    static uint64_t feature_bit(RVFeatureIndex feature) {
-      return (1ULL << (feature % BitsPerLong));
-    }
-
-    static RVFeatureIndex convert(uint32_t index) {
-      assert(index < MAX_CPU_FEATURE_INDEX, "must");
-      return (RVFeatureIndex)index;
-    }
-
-   public:
-    static RVExtFeatures* current() {
-      return _rv_ext_features;
-    }
-
-    RVExtFeatures() {
-      for (int i = 0; i < element_count(); i++) {
-        _features_bitmap[i] = 0;
-      }
-    }
-
-    void set_feature(uint32_t feature) {
-      RVFeatureIndex f = convert(feature);
-      int idx = element_index(f);
-      _features_bitmap[idx] |= feature_bit(f);
-    }
-
-    void clear_feature(uint32_t feature) {
-      RVFeatureIndex f = convert(feature);
-      int idx = element_index(f);
-      _features_bitmap[idx] &= ~feature_bit(f);
-    }
-
-    bool support_feature(uint32_t feature) {
-      RVFeatureIndex f = convert(feature);
-      int idx = element_index(f);
-      return (_features_bitmap[idx] & feature_bit(f)) != 0;
-    }
-  };
+  RV_FEATURE_FLAGS(DECLARE_RV_FEATURE)
+  #undef DECLARE_RV_FEATURE
 
   // enable extensions based on profile, current supported profiles:
   //  RVA20U64
@@ -398,24 +276,16 @@ private:
   static VM_MODE parse_satp_mode(const char* vm_mode);
 
   // Values from riscv_hwprobe()
-  enum UNALIGNED_SCALAR_ACCESS : int {
-    MISALIGNED_SCALAR_UNKNOWN     = 0,
-    MISALIGNED_SCALAR_EMULATED    = 1,
-    MISALIGNED_SCALAR_SLOW        = 2,
-    MISALIGNED_SCALAR_FAST        = 3,
-    MISALIGNED_SCALAR_UNSUPPORTED = 4
-  };
-
-  enum UNALIGNED_VECTOR_ACCESS : int {
-    MISALIGNED_VECTOR_UNKNOWN     = 0,
-    MISALIGNED_VECTOR_SLOW        = 2,
-    MISALIGNED_VECTOR_FAST        = 3,
-    MISALIGNED_VECTOR_UNSUPPORTED = 4
+  enum UNALIGNED_ACCESS : int {
+    MISALIGNED_UNKNOWN     = 0,
+    MISALIGNED_EMULATED    = 1,
+    MISALIGNED_SLOW        = 2,
+    MISALIGNED_FAST        = 3,
+    MISALIGNED_UNSUPPORTED = 4
   };
 
   // Null terminated list
   static RVFeatureValue* _feature_list[];
-  static RVExtFeatures* _rv_ext_features;
 
   // Enables features in _feature_list
   static void setup_cpu_available_features();

src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp

@@ -234,10 +234,12 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
   // Move address of the BasicObjectLock into Z_R1_scratch.
-
-  // Lock_reg was destroyed by fast unlocking attempt => recompute it.
-  ce->monitor_address(_monitor_ix, FrameMap::as_opr(Z_R1_scratch));
-
+  if (_compute_lock) {
+    // Lock_reg was destroyed by fast unlocking attempt => recompute it.
+    ce->monitor_address(_monitor_ix, FrameMap::as_opr(Z_R1_scratch));
+  } else {
+    __ lgr_if_needed(Z_R1_scratch, _lock_reg->as_register());
+  }
   // Note: non-blocking leaf routine => no call info needed.
   StubId exit_id;
   if (ce->compilation()->has_fpu_code()) {

src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp

@@ -228,7 +228,7 @@ int LIR_Assembler::emit_unwind_handler() {
     // StubId::c1_monitorexit_id expects lock address in Z_R1_scratch.
     LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
     monitor_address(0, lock);
-    stub = new MonitorExitStub(lock, 0);
+    stub = new MonitorExitStub(lock, true, 0);
     __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -2711,6 +2711,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
   if (op->code() == lir_lock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // Add debug info for NullPointerException only if one is possible.
     if (op->info() != nullptr) {
       add_debug_info_for_null_check_here(op->info());
@@ -2718,6 +2719,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     __ lock_object(hdr, obj, lock, *op->stub()->entry());
     // done
   } else if (op->code() == lir_unlock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
   } else {
     ShouldNotReachHere();

src/hotspot/cpu/s390/c2_globals_s390.hpp

@@ -45,7 +45,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         4);
 define_pd_global(intx, FreqInlineSize,               175);
-define_pd_global(intx, InteriorEntryAlignment,       4);
+// 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
+define_pd_global(intx, InteriorEntryAlignment,       2);
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 define_pd_global(intx, RegisterCostAreaRatio,        12000);
 define_pd_global(intx, LoopUnrollLimit,              60);

src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp

@@ -28,6 +28,7 @@
 #include "gc/g1/g1BarrierSetAssembler.hpp"
 #include "gc/g1/g1BarrierSetRuntime.hpp"
 #include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1DirtyCardQueue.hpp"
 #include "gc/g1/g1HeapRegion.hpp"
 #include "gc/g1/g1SATBMarkQueueSet.hpp"
 #include "gc/g1/g1ThreadLocalData.hpp"
@@ -204,71 +205,104 @@ void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
   BLOCK_COMMENT("} generate_c2_pre_barrier_stub");
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register thread,
-                                  const Register tmp1,
-                                  const Register tmp2,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-
-  __ block_comment("generate_post_barrier {");
-
-  assert(thread == Z_thread, "must be");
-  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
-
-  // Does store cross heap regions?
-  if (VM_Version::has_DistinctOpnds()) {
-    __ z_xgrk(tmp1, store_addr, new_val);    // tmp1 := store address ^ new value
-  } else {
-    __ z_lgr(tmp1, store_addr);
-    __ z_xgr(tmp1, new_val);
-  }
-  __ z_srag(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
-  __ branch_optimized(Assembler::bcondEqual, done);
-
-  // Crosses regions, storing null?
-  if (new_val_may_be_null) {
-    __ z_ltgr(new_val, new_val);
-    __ z_bre(done);
-  } else {
-#ifdef ASSERT
-    __ z_ltgr(new_val, new_val);
-    __ asm_assert(Assembler::bcondNotZero, "null oop not allowed (G1 post)", 0x322); // Checked by caller.
-#endif
-  }
-
-  __ z_srag(tmp1, store_addr, CardTable::card_shift());
-
-  Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
-  __ z_alg(tmp1, card_table_addr);     // tmp1 := card address
-
-  if(UseCondCardMark) {
-    __ z_cli(0, tmp1, G1CardTable::clean_card_val());
-    __ branch_optimized(Assembler::bcondNotEqual, done);
-  }
-
-  static_assert(G1CardTable::dirty_card_val() == 0, "must be to use z_mvi");
-  __ z_mvi(0, tmp1, G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
-
-  __ block_comment("} generate_post_barrier");
-}
-
 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register store_addr,
                                                      Register new_val,
                                                      Register thread,
                                                      Register tmp1,
                                                      Register tmp2,
-                                                     bool new_val_may_be_null) {
+                                                     G1PostBarrierStubC2* stub) {
   BLOCK_COMMENT("g1_write_barrier_post_c2 {");
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
-  __ bind(done);
+
+  assert(thread == Z_thread, "must be");
+  assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, Z_R1_scratch);
+
+  assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
+
+  stub->initialize_registers(thread, tmp1, tmp2);
+
+  BLOCK_COMMENT("generate_region_crossing_test {");
+  if (VM_Version::has_DistinctOpnds()) {
+    __ z_xgrk(tmp1, store_addr, new_val);
+  } else {
+    __ z_lgr(tmp1, store_addr);
+    __ z_xgr(tmp1, new_val);
+  }
+  __ z_srag(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
+  __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+  BLOCK_COMMENT("} generate_region_crossing_test");
+
+  // crosses regions, storing null?
+  if ((stub->barrier_data() & G1C2BarrierPostNotNull) == 0) {
+    __ z_ltgr(new_val, new_val);
+    __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+  }
+
+  BLOCK_COMMENT("generate_card_young_test {");
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+  // calculate address of card
+  __ load_const_optimized(tmp2, (address)ct->card_table()->byte_map_base());      // Card table base.
+  __ z_srlg(tmp1, store_addr, CardTable::card_shift());         // Index into card table.
+  __ z_algr(tmp1, tmp2);                                      // Explicit calculation needed for cli.
+
+  // Filter young.
+  __ z_cli(0, tmp1, G1CardTable::g1_young_card_val());
+
+  BLOCK_COMMENT("} generate_card_young_test");
+
+  // From here on, tmp1 holds the card address.
+  __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
+
+  __ bind(*stub->continuation());
+
   BLOCK_COMMENT("} g1_write_barrier_post_c2");
 }
 
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+
+  BLOCK_COMMENT("generate_c2_post_barrier_stub {");
+
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+
+  Register thread     = stub->thread();
+  Register tmp1       = stub->tmp1(); // tmp1 holds the card address.
+  Register tmp2       = stub->tmp2();
+  Register Rcard_addr = tmp1;
+
+  __ bind(*stub->entry());
+
+  BLOCK_COMMENT("generate_card_clean_test {");
+  __ z_sync(); // Required to support concurrent cleaning.
+  __ z_cli(0, Rcard_addr, 0); // Reload after membar.
+  __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+  BLOCK_COMMENT("} generate_card_clean_test");
+
+  BLOCK_COMMENT("generate_dirty_card {");
+  // Storing a region crossing, non-null oop, card is clean.
+  // Dirty card and log.
+  STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+  __ z_mvi(0, Rcard_addr, CardTable::dirty_card_val());
+  BLOCK_COMMENT("} generate_dirty_card");
+
+  generate_queue_test_and_insertion(masm,
+                                    G1ThreadLocalData::dirty_card_queue_index_offset(),
+                                    G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                                    runtime,
+                                    Z_thread, tmp1, tmp2);
+
+  __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+  __ bind(runtime);
+
+  generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+
+  __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+  BLOCK_COMMENT("} generate_c2_post_barrier_stub");
+}
+
 #endif //COMPILER2
 
 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -417,9 +451,99 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Decorato
                                                   Register Rtmp1, Register Rtmp2, Register Rtmp3) {
   bool not_null = (decorators & IS_NOT_NULL) != 0;
 
-  Label done;
-  generate_post_barrier(masm, Rstore_addr, Rnew_val, Z_thread, Rtmp1, Rtmp2, done, !not_null);
-  __ bind(done);
+  assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); // Most probably, Rnew_val == Rtmp3.
+
+  Label callRuntime, filtered;
+
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+
+  BLOCK_COMMENT("g1_write_barrier_post {");
+
+  // Does store cross heap regions?
+  // It does if the two addresses specify different grain addresses.
+  if (VM_Version::has_DistinctOpnds()) {
+    __ z_xgrk(Rtmp1, Rstore_addr, Rnew_val);
+  } else {
+    __ z_lgr(Rtmp1, Rstore_addr);
+    __ z_xgr(Rtmp1, Rnew_val);
+  }
+  __ z_srag(Rtmp1, Rtmp1, G1HeapRegion::LogOfHRGrainBytes);
+  __ z_bre(filtered);
+
+  // Crosses regions, storing null?
+  if (not_null) {
+#ifdef ASSERT
+    __ z_ltgr(Rnew_val, Rnew_val);
+    __ asm_assert(Assembler::bcondNotZero, "null oop not allowed (G1 post)", 0x322); // Checked by caller.
+#endif
+  } else {
+    __ z_ltgr(Rnew_val, Rnew_val);
+    __ z_bre(filtered);
+  }
+
+  Rnew_val = noreg; // end of lifetime
+
+  // Storing region crossing non-null, is card already dirty?
+  assert_different_registers(Rtmp1, Rtmp2, Rtmp3);
+  // Make sure not to use Z_R0 for any of these registers.
+  Register Rcard_addr = (Rtmp1 != Z_R0_scratch) ? Rtmp1 : Rtmp3;
+  Register Rbase      = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp3;
+
+  // calculate address of card
+  __ load_const_optimized(Rbase, (address)ct->card_table()->byte_map_base());      // Card table base.
+  __ z_srlg(Rcard_addr, Rstore_addr, CardTable::card_shift());         // Index into card table.
+  __ z_algr(Rcard_addr, Rbase);                                      // Explicit calculation needed for cli.
+  Rbase = noreg; // end of lifetime
+
+  // Filter young.
+  __ z_cli(0, Rcard_addr, G1CardTable::g1_young_card_val());
+  __ z_bre(filtered);
+
+  // Check the card value. If dirty, we're done.
+  // This also avoids false sharing of the (already dirty) card.
+  __ z_sync(); // Required to support concurrent cleaning.
+  __ z_cli(0, Rcard_addr, G1CardTable::dirty_card_val()); // Reload after membar.
+  __ z_bre(filtered);
+
+  // Storing a region crossing, non-null oop, card is clean.
+  // Dirty card and log.
+  __ z_mvi(0, Rcard_addr, G1CardTable::dirty_card_val());
+
+  Register Rcard_addr_x = Rcard_addr;
+  Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
+  if (Rcard_addr == Rqueue_index) {
+    Rcard_addr_x = Z_R0_scratch;  // Register shortage. We have to use Z_R0.
+  }
+  __ lgr_if_needed(Rcard_addr_x, Rcard_addr);
+
+  generate_queue_test_and_insertion(masm,
+                                    G1ThreadLocalData::dirty_card_queue_index_offset(),
+                                    G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                                    callRuntime,
+                                    Z_thread, Rcard_addr_x, Rqueue_index);
+  __ z_bru(filtered);
+
+  __ bind(callRuntime);
+
+  // TODO: do we need a frame? Introduced to be on the safe side.
+  bool needs_frame = true;
+  __ lgr_if_needed(Rcard_addr, Rcard_addr_x); // copy back asap. push_frame will destroy Z_R0_scratch!
+
+  // VM call need frame to access(write) O register.
+  if (needs_frame) {
+    __ save_return_pc();
+    __ push_frame_abi160(0); // Will use Z_R0 as tmp on old CPUs.
+  }
+
+  // Save the live input values.
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, Z_thread);
+
+  if (needs_frame) {
+    __ pop_frame();
+    __ restore_return_pc();
+  }
+
+  __ bind(filtered);
 
   BLOCK_COMMENT("} g1_write_barrier_post");
 }
@@ -491,19 +615,22 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
 }
 
-#undef __
-
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2) {
-   Label done;
-   generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
-   masm->bind(done);
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+  ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots.
+  assert(stub->addr()->is_register(), "Precondition.");
+  assert(stub->new_val()->is_register(), "Precondition.");
+  Register new_val_reg = stub->new_val()->as_register();
+  __ z_ltgr(new_val_reg, new_val_reg);
+  __ branch_optimized(Assembler::bcondZero, *stub->continuation());
+  __ z_lgr(Z_R1_scratch, stub->addr()->as_pointer_register());
+  ce->emit_call_c(bs->post_barrier_c1_runtime_code_blob()->code_begin());
+  __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
 }
 
+#undef __
+
 #define __ sasm->
 
 static OopMap* save_volatile_registers(StubAssembler* sasm, Register return_pc = Z_R14) {
@@ -578,6 +705,92 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ z_bru(restart);
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  // Z_R1_scratch: oop address, address of updated memory slot
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+  __ set_info("g1_post_barrier_slow_id", false);
+
+  Register addr_oop  = Z_R1_scratch;
+  Register addr_card = Z_R1_scratch;
+  Register r1        = Z_R6; // Must be saved/restored.
+  Register r2        = Z_R7; // Must be saved/restored.
+  Register cardtable = r1;   // Must be non-volatile, because it is used to save addr_card.
+  CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
+  CardTable* ct = ctbs->card_table();
+  CardTable::CardValue* byte_map_base = ct->byte_map_base();
+
+  // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
+  __ z_stg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
+
+  Label not_already_dirty, restart, refill, young_card;
+
+  // Calculate address of card corresponding to the updated oop slot.
+  AddressLiteral rs(byte_map_base);
+  __ z_srlg(addr_card, addr_oop, CardTable::card_shift());
+  addr_oop = noreg; // dead now
+  __ load_const_optimized(cardtable, rs); // cardtable := <card table base>
+  __ z_agr(addr_card, cardtable); // addr_card := addr_oop>>card_shift + cardtable
+
+  __ z_cli(0, addr_card, (int)G1CardTable::g1_young_card_val());
+  __ z_bre(young_card);
+
+  __ z_sync(); // Required to support concurrent cleaning.
+
+  __ z_cli(0, addr_card, (int)CardTable::dirty_card_val());
+  __ z_brne(not_already_dirty);
+
+  __ bind(young_card);
+  // We didn't take the branch, so we're already dirty: restore
+  // used registers and return.
+  __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
+  __ z_br(Z_R14);
+
+  // Not dirty.
+  __ bind(not_already_dirty);
+
+  // First, dirty it: [addr_card] := 0
+  __ z_mvi(0, addr_card, CardTable::dirty_card_val());
+
+  Register idx = cardtable; // Must be non-volatile, because it is used to save addr_card.
+  Register buf = r2;
+  cardtable = noreg; // now dead
+
+  // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
+  __ z_stg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
+
+  ByteSize dirty_card_q_index_byte_offset = G1ThreadLocalData::dirty_card_queue_index_offset();
+  ByteSize dirty_card_q_buf_byte_offset = G1ThreadLocalData::dirty_card_queue_buffer_offset();
+
+  __ bind(restart);
+
+  // Get the index into the update buffer. G1DirtyCardQueue::_index is
+  // a size_t so z_ltg is appropriate here.
+  __ z_ltg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
+
+  // index == 0?
+  __ z_brz(refill);
+
+  __ z_lg(buf, Address(Z_thread, dirty_card_q_buf_byte_offset));
+  __ add2reg(idx, -oopSize);
+
+  __ z_stg(addr_card, 0, idx, buf); // [_buf + index] := <address_of_card>
+  __ z_stg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
+  // Restore killed registers and return.
+  __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
+  __ z_lg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
+  __ z_br(Z_R14);
+
+  __ bind(refill);
+  save_volatile_registers(sasm);
+  __ z_lgr(idx, addr_card); // Save addr_card, tmp3 must be non-volatile.
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread),
+                                   Z_thread);
+  __ z_lgr(addr_card, idx);
+  restore_volatile_registers(sasm); // Restore addr_card.
+  __ z_bru(restart);
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, 2024 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -33,7 +33,9 @@
 class LIR_Assembler;
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
  protected:
@@ -58,16 +60,10 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
  public:
 #ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
-
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
-
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -85,7 +81,9 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
                                 Register thread,
                                 Register tmp1,
                                 Register tmp2,
-                                bool new_val_may_be_null);
+                                G1PostBarrierStubC2* c2_stub);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif // COMPILER2
 
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

src/hotspot/cpu/s390/gc/g1/g1_s390.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
 // Copyright 2024 IBM Corporation. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
@@ -62,13 +62,13 @@ static void write_barrier_post(MacroAssembler* masm,
                                Register new_val,
                                Register tmp1,
                                Register tmp2) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Z_thread, tmp1, tmp2, new_val_may_be_null);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Z_thread, tmp1, tmp2, stub);
 }
 
 %} // source

src/hotspot/cpu/s390/s390.ad

@@ -1947,7 +1947,6 @@ bool Matcher::is_spillable_arg(int reg) {
 uint Matcher::int_pressure_limit()
 {
   // Medium size register set, 6 special purpose regs, 3 SOE regs.
-  // 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
   return (INTPRESSURE == -1) ? 10 : INTPRESSURE;
 }
 

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -2347,11 +2347,13 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
+                                            AdapterHandlerEntry* handler) {
   __ align(CodeEntryAlignment);
-  entry_address[AdapterBlob::I2C] = __ pc();
+  address i2c_entry = __ pc();
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
+  address c2i_unverified_entry;
+
   Label skip_fixup;
   {
     Label ic_miss;
@@ -2361,7 +2363,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 
     // Unverified Entry Point UEP
     __ align(CodeEntryAlignment);
-    entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+    c2i_unverified_entry = __ pc();
 
     __ ic_check(2);
     __ z_lg(Z_method, Address(Z_inline_cache, CompiledICData::speculated_method_offset()));
@@ -2374,10 +2376,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     // Fallthru to VEP. Duplicate LTG, but saved taken branch.
   }
 
-  entry_address[AdapterBlob::C2I] = __ pc();
+  address c2i_entry = __ pc();
 
   // Class initialization barrier for static methods
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_no_clinit_check_entry = nullptr;
   if (VM_Version::supports_fast_class_init_checks()) {
     Label L_skip_barrier;
 
@@ -2394,10 +2396,12 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ z_br(klass);
 
     __ bind(L_skip_barrier);
-    entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
+    c2i_no_clinit_check_entry = __ pc();
   }
 
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
   return;
 }
 

src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp

@@ -207,10 +207,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
 
 void MonitorExitStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
-
-  // lock_reg was destroyed by fast unlocking attempt => recompute it
-  ce->monitor_address(_monitor_ix, _lock_reg);
-
+  if (_compute_lock) {
+    // lock_reg was destroyed by fast unlocking attempt => recompute it
+    ce->monitor_address(_monitor_ix, _lock_reg);
+  }
   ce->store_parameter(_lock_reg->as_register(), 0);
   // note: non-blocking leaf routine => no call info needed
   StubId exit_id;

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -412,7 +412,7 @@ int LIR_Assembler::emit_unwind_handler() {
   MonitorExitStub* stub = nullptr;
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::rax_opr);
-    stub = new MonitorExitStub(FrameMap::rax_opr, 0);
+    stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
     __ unlock_object(rdi, rsi, rax, *stub->entry());
     __ bind(*stub->continuation());
   }
@@ -2730,6 +2730,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register hdr = op->hdr_opr()->as_register();
   Register lock = op->lock_opr()->as_register();
   if (op->code() == lir_lock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     Register tmp = op->scratch_opr()->as_register();
     // add debug info for NullPointerException only if one is possible
     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
@@ -2738,6 +2739,7 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
     }
     // done
   } else if (op->code() == lir_unlock) {
+    assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
   } else {
     Unimplemented();

src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

@@ -41,32 +41,32 @@
 #include "utilities/checkedCast.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register tmp, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
   assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
-  assert_different_registers(hdr, obj, basic_lock, tmp);
+  assert_different_registers(hdr, obj, disp_hdr, tmp);
   int null_check_offset = -1;
 
   verify_oop(obj);
 
   // save object being locked into the BasicObjectLock
-  movptr(Address(basic_lock, BasicObjectLock::obj_offset()), obj);
+  movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
 
   null_check_offset = offset();
 
-  lightweight_lock(basic_lock, obj, hdr, tmp, slow_case);
+  lightweight_lock(disp_hdr, obj, hdr, tmp, slow_case);
 
   return null_check_offset;
 }
 
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
-  assert(basic_lock == rax, "basic_lock must be rax, for the cmpxchg instruction");
-  assert(hdr != obj && hdr != basic_lock && obj != basic_lock, "registers must be different");
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
+  assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
+  assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
 
   // load object
-  movptr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
+  movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
   verify_oop(obj);
 
-  lightweight_unlock(obj, rax, hdr, slow_case);
+  lightweight_unlock(obj, disp_hdr, hdr, slow_case);
 }
 
 

src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,17 +46,17 @@
   void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1);
 
   // locking
-  // hdr       : must be rax, contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must point to the basic lock, contents preserved
+  // hdr     : must be rax, contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must point to the displaced header location, contents preserved
   // returns code offset at which to add null check debug information
-  int lock_object  (Register swap, Register obj, Register basic_lock, Register tmp, Label& slow_case);
+  int lock_object  (Register swap, Register obj, Register disp_hdr, Register tmp, Label& slow_case);
 
   // unlocking
-  // hdr       : contents destroyed
-  // obj       : must point to the object to lock, contents preserved
-  // basic_lock: must be eax & must point to the basic lock, contents destroyed
-  void unlock_object(Register swap, Register obj, Register basic_lock, Label& slow_case);
+  // hdr     : contents destroyed
+  // obj     : must point to the object to lock, contents preserved
+  // disp_hdr: must be eax & must point to the displaced header location, contents destroyed
+  void unlock_object(Register swap, Register obj, Register lock, Label& slow_case);
 
   void initialize_object(
     Register obj,                      // result: pointer to object after successful allocation

src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp

@@ -89,53 +89,19 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
 
 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
                                                              Register addr, Register count, Register tmp) {
-  Label done;
-
-  __ testptr(count, count);
-  __ jcc(Assembler::zero, done);
-
-  // Calculate end address in "count".
-  Address::ScaleFactor scale = UseCompressedOops ? Address::times_4 : Address::times_8;
-  __ leaq(count, Address(addr, count, scale));
-
-  // Calculate start card address in "addr".
-  __ shrptr(addr, CardTable::card_shift());
-
-  Register thread = r15_thread;
-
-  __ movptr(tmp, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
-  __ addptr(addr, tmp);
-
-  // Calculate address of card of last word in the array.
-  __ subptr(count, 1);
-  __ shrptr(count, CardTable::card_shift());
-  __ addptr(count, tmp);
-
-  Label loop;
-  // Iterate from start card to end card (inclusive).
-  __ bind(loop);
-
-  Label is_clean_card;
-  if (UseCondCardMark) {
-    __ cmpb(Address(addr, 0), G1CardTable::clean_card_val());
-    __ jcc(Assembler::equal, is_clean_card);
+  __ push_call_clobbered_registers(false /* save_fpu */);
+  if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
+    assert_different_registers(c_rarg1, addr);
+    __ mov(c_rarg1, count);
+    __ mov(c_rarg0, addr);
   } else {
-   __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
+    assert_different_registers(c_rarg0, count);
+    __ mov(c_rarg0, addr);
+    __ mov(c_rarg1, count);
   }
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
+  __ pop_call_clobbered_registers(false /* save_fpu */);
 
-  Label next_card;
-  __ bind(next_card);
-  __ addptr(addr, sizeof(CardTable::CardValue));
-  __ cmpptr(addr, count);
-  __ jcc(Assembler::belowEqual, loop);
-  __ jmp(done);
-
-  __ bind(is_clean_card);
-  // Card was clean. Dirty card and go to next..
-  __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
-  __ jmp(next_card);
-
-  __ bind(done);
 }
 
 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -216,6 +182,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
   // If expand_call is true then we expand the call_VM_leaf macro
   // directly to skip generating the check by
   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
   const Register thread = r15_thread;
 
   Label done;
@@ -268,49 +235,76 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
   __ bind(done);
 }
 
-static void generate_post_barrier(MacroAssembler* masm,
-                                  const Register store_addr,
-                                  const Register new_val,
-                                  const Register tmp1,
-                                  Label& done,
-                                  bool new_val_may_be_null) {
-
-  assert_different_registers(store_addr, new_val, tmp1, noreg);
-
-  Register thread = r15_thread;
-
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+                                            const Register store_addr,
+                                            const Register new_val,
+                                            const Register tmp,
+                                            const Register tmp2,
+                                            Label& done,
+                                            bool new_val_may_be_null) {
+  CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
   // Does store cross heap regions?
-  __ movptr(tmp1, store_addr);                                    // tmp1 := store address
-  __ xorptr(tmp1, new_val);                                       // tmp1 := store address ^ new value
-  __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
+  __ movptr(tmp, store_addr);                                    // tmp := store address
+  __ xorptr(tmp, new_val);                                       // tmp := store address ^ new value
+  __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
   __ jcc(Assembler::equal, done);
-
   // Crosses regions, storing null?
   if (new_val_may_be_null) {
-    __ cmpptr(new_val, NULL_WORD);                                // new value == null?
+    __ cmpptr(new_val, NULL_WORD);                               // new value == null?
     __ jcc(Assembler::equal, done);
   }
+  // Storing region crossing non-null, is card young?
+  __ movptr(tmp, store_addr);                                    // tmp := store address
+  __ shrptr(tmp, CardTable::card_shift());                       // tmp := card address relative to card table base
+  // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
+  // a valid address and therefore is not properly handled by the relocation code.
+  __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base());  // tmp2 := card table base address
+  __ addptr(tmp, tmp2);                                          // tmp := card address
+  __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val());    // *(card address) == young_card_val?
+}
 
-  __ movptr(tmp1, store_addr);                                    // tmp1 := store address
-  __ shrptr(tmp1, CardTable::card_shift());                       // tmp1 := card address relative to card table base
-
-  Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
-  __ addptr(tmp1, card_table_addr);                               // tmp1 := card address
-  if (UseCondCardMark) {
-    __ cmpb(Address(tmp1, 0), G1CardTable::clean_card_val());     // *(card address) == clean_card_val?
-    __ jcc(Assembler::notEqual, done);
-  }
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+                                            const Register thread,
+                                            const Register tmp,
+                                            const Register tmp2,
+                                            Label& done,
+                                            Label& runtime) {
+  __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));  // StoreLoad membar
+  __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val());       // *(card address) == dirty_card_val?
+  __ jcc(Assembler::equal, done);
   // Storing a region crossing, non-null oop, card is clean.
-  // Dirty card.
-  __ movb(Address(tmp1, 0), G1CardTable::dirty_card_val());       // *(card address) := dirty_card_val
+  // Dirty card and log.
+  __ movb(Address(tmp, 0), G1CardTable::dirty_card_val());       // *(card address) := dirty_card_val
+  generate_queue_insertion(masm,
+                           G1ThreadLocalData::dirty_card_queue_index_offset(),
+                           G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+                           runtime,
+                           thread, tmp, tmp2);
+  __ jmp(done);
 }
 
 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
                                                   Register store_addr,
                                                   Register new_val,
-                                                  Register tmp) {
+                                                  Register tmp,
+                                                  Register tmp2) {
+  const Register thread = r15_thread;
+
   Label done;
-  generate_post_barrier(masm, store_addr, new_val, tmp, done, true /* new_val_may_be_null */);
+  Label runtime;
+
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
+  // If card is young, jump to done
+  __ jcc(Assembler::equal, done);
+  generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(store_addr);
+  __ push_set(saved);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
+  __ pop_set(saved);
+
   __ bind(done);
 }
 
@@ -373,10 +367,34 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
                                                      Register store_addr,
                                                      Register new_val,
                                                      Register tmp,
-                                                     bool new_val_may_be_null) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, tmp, done, new_val_may_be_null);
-  __ bind(done);
+                                                     Register tmp2,
+                                                     G1PostBarrierStubC2* stub) {
+  const Register thread = r15_thread;
+  stub->initialize_registers(thread, tmp, tmp2);
+
+  bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+  generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
+  // If card is not young, jump to stub (slow path)
+  __ jcc(Assembler::notEqual, *stub->entry());
+
+  __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                                          G1PostBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+  Label runtime;
+  Register thread = stub->thread();
+  Register tmp = stub->tmp1(); // tmp holds the card address.
+  Register tmp2 = stub->tmp2();
+  assert(stub->tmp3() == noreg, "not needed in this platform");
+
+  __ bind(*stub->entry());
+  generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
+
+  __ bind(runtime);
+  generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+  __ jmp(*stub->continuation());
 }
 
 #endif // COMPILER2
@@ -423,7 +441,8 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
       g1_write_barrier_post(masm /*masm*/,
                             tmp1 /* store_adr */,
                             new_val /* new_val */,
-                            tmp3 /* tmp */);
+                            tmp3 /* tmp */,
+                            tmp2 /* tmp2 */);
     }
   }
 }
@@ -457,19 +476,21 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
 
 }
 
-#undef __
-
-void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
-                                                     Register store_addr,
-                                                     Register new_val,
-                                                     Register thread,
-                                                     Register tmp1,
-                                                     Register tmp2 /* unused on x86 */) {
-  Label done;
-  generate_post_barrier(masm, store_addr, new_val, tmp1, done, true /* new_val_may_be_null */);
-  masm->bind(done);
+void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
+  G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  __ bind(*stub->entry());
+  assert(stub->addr()->is_register(), "Precondition.");
+  assert(stub->new_val()->is_register(), "Precondition.");
+  Register new_val_reg = stub->new_val()->as_register();
+  __ cmpptr(new_val_reg, NULL_WORD);
+  __ jcc(Assembler::equal, *stub->continuation());
+  ce->store_parameter(stub->addr()->as_pointer_register(), 0);
+  __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
+  __ jmp(*stub->continuation());
 }
 
+#undef __
+
 #define __ sasm->
 
 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -534,6 +555,78 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
   __ epilogue();
 }
 
+void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("g1_post_barrier", false);
+
+  CardTableBarrierSet* ct =
+    barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
+
+  Label done;
+  Label enqueued;
+  Label runtime;
+
+  // At this point we know new_value is non-null and the new_value crosses regions.
+  // Must check to see if card is already dirty
+
+  const Register thread = r15_thread;
+
+  Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
+  Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
+
+  __ push_ppx(rax);
+  __ push_ppx(rcx);
+
+  const Register cardtable = rax;
+  const Register card_addr = rcx;
+
+  __ load_parameter(0, card_addr);
+  __ shrptr(card_addr, CardTable::card_shift());
+  // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
+  // a valid address and therefore is not properly handled by the relocation code.
+  __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
+  __ addptr(card_addr, cardtable);
+
+  __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
+  __ jcc(Assembler::equal, done);
+
+  __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
+  __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
+  __ jcc(Assembler::equal, done);
+
+  // storing region crossing non-null, card is clean.
+  // dirty card and log.
+
+  __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
+
+  const Register tmp = rdx;
+  __ push_ppx(rdx);
+
+  __ movptr(tmp, queue_index);
+  __ testptr(tmp, tmp);
+  __ jcc(Assembler::zero, runtime);
+  __ subptr(tmp, wordSize);
+  __ movptr(queue_index, tmp);
+  __ addptr(tmp, buffer);
+  __ movptr(Address(tmp, 0), card_addr);
+  __ jmp(enqueued);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
+
+  __ pop_call_clobbered_registers();
+
+  __ bind(enqueued);
+  __ pop_ppx(rdx);
+
+  __ bind(done);
+  __ pop_ppx(rcx);
+  __ pop_ppx(rax);
+
+  __ epilogue();
+}
+
 #undef __
 
 #endif // COMPILER1

src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,8 +31,10 @@
 class LIR_Assembler;
 class StubAssembler;
 class G1PreBarrierStub;
+class G1PostBarrierStub;
 class G1BarrierStubC2;
 class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
 
 class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
  protected:
@@ -49,27 +51,21 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
   void g1_write_barrier_post(MacroAssembler* masm,
                              Register store_addr,
                              Register new_val,
-                             Register tmp);
+                             Register tmp,
+                             Register tmp2);
 
   virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                             Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
 
  public:
-  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                       Register dst, Address src, Register tmp1);
-
-#ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+  void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
 
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+  void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
 
-  void g1_write_barrier_post_c1(MacroAssembler* masm,
-                                Register store_addr,
-                                Register new_val,
-                                Register thread,
-                                Register tmp1,
-                                Register tmp2);
-#endif
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1);
 
 #ifdef COMPILER2
   void g1_write_barrier_pre_c2(MacroAssembler* masm,
@@ -83,7 +79,10 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
                                 Register store_addr,
                                 Register new_val,
                                 Register tmp,
-                                bool new_val_may_be_null);
+                                Register tmp2,
+                                G1PostBarrierStubC2* c2_stub);
+  void generate_c2_post_barrier_stub(MacroAssembler* masm,
+                                     G1PostBarrierStubC2* stub) const;
 #endif // COMPILER2
 };
 

src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -59,14 +59,15 @@ static void write_barrier_post(MacroAssembler* masm,
                                const MachNode* node,
                                Register store_addr,
                                Register new_val,
-                               Register tmp1) {
-  if (!G1BarrierStubC2::needs_post_barrier(node)) {
+                               Register tmp1,
+                               Register tmp2) {
+  if (!G1PostBarrierStubC2::needs_barrier(node)) {
     return;
   }
   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
-  bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
-  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
+  G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+  g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, stub);
 }
 
 %}
@@ -94,7 +95,8 @@ instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3,
     write_barrier_post(masm, this,
                        $tmp1$$Register /* store_addr */,
                        $src$$Register /* new_val */,
-                       $tmp3$$Register /* tmp1 */);
+                       $tmp3$$Register /* tmp1 */,
+                       $tmp2$$Register /* tmp2 */);
   %}
   ins_pipe(ialu_mem_reg);
 %}
@@ -125,7 +127,8 @@ instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFl
     write_barrier_post(masm, this,
                        $tmp1$$Register /* store_addr */,
                        $tmp2$$Register /* new_val */,
-                       $tmp3$$Register /* tmp1 */);
+                       $tmp3$$Register /* tmp1 */,
+                       $tmp2$$Register /* tmp2 */);
   %}
   ins_pipe(ialu_mem_reg);
 %}
@@ -155,7 +158,8 @@ instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rR
     write_barrier_post(masm, this,
                        $tmp1$$Register /* store_addr */,
                        $src$$Register /* new_val */,
-                       $tmp3$$Register /* tmp1 */);
+                       $tmp3$$Register /* tmp1 */,
+                       $tmp2$$Register /* tmp2 */);
   %}
   ins_pipe(ialu_mem_reg);
 %}
@@ -183,7 +187,8 @@ instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}
@@ -209,7 +214,8 @@ instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}
@@ -240,7 +246,8 @@ instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rR
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}
@@ -272,7 +279,8 @@ instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rR
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}
@@ -295,7 +303,8 @@ instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}
@@ -319,7 +328,8 @@ instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP
     write_barrier_post(masm, this,
                        $mem$$Register /* store_addr */,
                        $tmp1$$Register /* new_val */,
-                       $tmp2$$Register /* tmp1 */);
+                       $tmp2$$Register /* tmp1 */,
+                       $tmp3$$Register /* tmp2 */);
   %}
   ins_pipe(pipe_cmpxchg);
 %}

src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp

@@ -1410,9 +1410,11 @@ void ZBarrierSetAssembler::patch_barriers() {
   }
 }
 
+
 #undef __
 #define __ masm->
 
+
 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
   // C1 calls verfy_oop in the middle of barriers, before they have been uncolored
   // and after being colored. Therefore, we must deal with colored oops as well.

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -2897,8 +2897,7 @@ void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vecto
 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
   // WARN: Allow dst == (src1|src2), mask == scratch
-  bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
-                         !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
+  bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
   bool dst_available = dst != mask && (dst != src1 || dst != src2);
   if (blend_emulation && scratch_available && dst_available) {
@@ -2922,8 +2921,7 @@ void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister sr
 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
   // WARN: Allow dst == (src1|src2), mask == scratch
-  bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
-                         !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
+  bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
   bool dst_available = dst != mask && (dst != src1 || dst != src2);
   if (blend_emulation && scratch_available && dst_available) {

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -1007,8 +1007,8 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
-  entry_address[AdapterBlob::I2C] = __ pc();
+                                            AdapterHandlerEntry* handler) {
+  address i2c_entry = __ pc();
 
   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 
@@ -1021,7 +1021,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
   // On exit from the interpreter, the interpreter will restore our SP (lest the
   // compiled code, which relies solely on SP and not RBP, get sick).
 
-  entry_address[AdapterBlob::C2I_Unverified] = __ pc();
+  address c2i_unverified_entry = __ pc();
   Label skip_fixup;
 
   Register data = rax;
@@ -1039,10 +1039,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
   }
 
-  entry_address[AdapterBlob::C2I] = __ pc();
+  address c2i_entry = __ pc();
 
   // Class initialization barrier for static methods
-  entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
+  address c2i_no_clinit_check_entry = nullptr;
   if (VM_Version::supports_fast_class_init_checks()) {
     Label L_skip_barrier;
     Register method = rbx;
@@ -1061,13 +1061,15 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 
     __ bind(L_skip_barrier);
-    entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
+    c2i_no_clinit_check_entry = __ pc();
   }
 
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
   bs->c2i_entry_barrier(masm);
 
   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+
+  handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
   return;
 }
 

src/hotspot/cpu/x86/vm_version_x86.cpp

@@ -2093,10 +2093,6 @@ bool VM_Version::is_intel_cascade_lake() {
   return is_intel_skylake() && _stepping >= 5;
 }
 
-bool VM_Version::is_intel_darkmont() {
-  return is_intel() && is_intel_server_family() && (_model == 0xCC || _model == 0xDD);
-}
-
 // avx3_threshold() sets the threshold at which 64-byte instructions are used
 // for implementing the array copy and clear operations.
 // The Intel platforms that supports the serialize instruction

src/hotspot/cpu/x86/vm_version_x86.hpp

@@ -938,8 +938,6 @@ public:
 
   static bool is_intel_cascade_lake();
 
-  static bool is_intel_darkmont();
-
   static int avx3_threshold();
 
   static bool is_intel_tsc_synched_at_init();

src/hotspot/cpu/zero/sharedRuntime_zero.cpp

@@ -38,6 +38,12 @@
 #include "opto/runtime.hpp"
 #endif
 
+
+static address zero_null_code_stub() {
+  address start = ShouldNotCallThisStub();
+  return start;
+}
+
 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
                                            VMRegPair *regs,
                                            int total_args_passed) {
@@ -49,9 +55,12 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
                                             int comp_args_on_stack,
                                             const BasicType *sig_bt,
                                             const VMRegPair *regs,
-                                            address entry_address[AdapterBlob::ENTRY_COUNT]) {
-  ShouldNotCallThis();
-  return;
+                                            AdapterHandlerEntry* handler) {
+  // foil any attempt to call the i2c, c2i or unverified c2i entries
+  handler->set_entry_points(CAST_FROM_FN_PTR(address,zero_null_code_stub),
+                            CAST_FROM_FN_PTR(address,zero_null_code_stub),
+                            CAST_FROM_FN_PTR(address,zero_null_code_stub),
+                            nullptr);
 }
 
 nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,

src/hotspot/os/linux/os_linux.cpp

@@ -160,6 +160,7 @@ address   os::Linux::_initial_thread_stack_bottom = nullptr;
 uintptr_t os::Linux::_initial_thread_stack_size   = 0;
 
 int (*os::Linux::_pthread_getcpuclockid)(pthread_t, clockid_t *) = nullptr;
+int (*os::Linux::_pthread_setname_np)(pthread_t, const char*) = nullptr;
 pthread_t os::Linux::_main_thread;
 bool os::Linux::_supports_fast_thread_cpu_time = false;
 const char * os::Linux::_libc_version = nullptr;
@@ -859,9 +860,11 @@ static void *thread_native_entry(Thread *thread) {
   osthread->set_thread_id(checked_cast<pid_t>(os::current_thread_id()));
 
   if (UseNUMA) {
-    thread->update_lgrp_id();
+    int lgrp_id = os::numa_get_group_id();
+    if (lgrp_id != -1) {
+      thread->set_lgrp_id(lgrp_id);
+    }
   }
-
   // initialize signal mask for this thread
   PosixSignals::hotspot_sigmask(thread);
 
@@ -1188,7 +1191,10 @@ bool os::create_attached_thread(JavaThread* thread) {
   thread->set_osthread(osthread);
 
   if (UseNUMA) {
-    thread->update_lgrp_id();
+    int lgrp_id = os::numa_get_group_id();
+    if (lgrp_id != -1) {
+      thread->set_lgrp_id(lgrp_id);
+    }
   }
 
   if (os::is_primordial_thread()) {
@@ -4370,6 +4376,10 @@ void os::init(void) {
   // _main_thread points to the thread that created/loaded the JVM.
   Linux::_main_thread = pthread_self();
 
+  // retrieve entry point for pthread_setname_np
+  Linux::_pthread_setname_np =
+    (int(*)(pthread_t, const char*))dlsym(RTLD_DEFAULT, "pthread_setname_np");
+
   check_pax();
 
   // Check the availability of MADV_POPULATE_WRITE.
@@ -4846,24 +4856,14 @@ uint os::processor_id() {
 }
 
 void os::set_native_thread_name(const char *name) {
-  char buf[16]; // according to glibc manpage, 16 chars incl. '/0'
-  // We may need to truncate the thread name. Since a common pattern
-  // for thread names is to be both longer than 15 chars and have a
-  // trailing number ("DispatcherWorkerThread21", "C2 CompilerThread#54" etc),
-  // we preserve the end of the thread name by truncating the middle
-  // (e.g. "Dispatc..read21").
-  const size_t len = strlen(name);
-  if (len < sizeof(buf)) {
-    strcpy(buf, name);
-  } else {
-    (void) os::snprintf(buf, sizeof(buf), "%.7s..%.6s", name, name + len - 6);
+  if (Linux::_pthread_setname_np) {
+    char buf [16]; // according to glibc manpage, 16 chars incl. '/0'
+    (void) os::snprintf(buf, sizeof(buf), "%s", name);
+    buf[sizeof(buf) - 1] = '\0';
+    const int rc = Linux::_pthread_setname_np(pthread_self(), buf);
+    // ERANGE should not happen; all other errors should just be ignored.
+    assert(rc != ERANGE, "pthread_setname_np failed");
   }
-  // Note: we use the system call here instead of calling pthread_setname_np
-  // since this is the only way to make ASAN aware of our thread names. Even
-  // though ASAN intercepts both prctl and pthread_setname_np, it only processes
-  // the thread name given to the former.
-  int rc = prctl(PR_SET_NAME, buf);
-  assert(rc == 0, "prctl(PR_SET_NAME) failed");
 }
 
 ////////////////////////////////////////////////////////////////////////////////

src/hotspot/os/linux/os_linux.hpp

@@ -33,6 +33,7 @@ class os::Linux {
   friend class os;
 
   static int (*_pthread_getcpuclockid)(pthread_t, clockid_t *);
+  static int (*_pthread_setname_np)(pthread_t, const char*);
 
   static address   _initial_thread_stack_bottom;
   static uintptr_t _initial_thread_stack_size;

src/hotspot/os/posix/os_posix.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,9 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
+// Aix does not have NUMA support but need these for compilation.
+inline bool os::numa_has_group_homing()     { AIX_ONLY(ShouldNotReachHere();) return false;  }
+
 // Platform Mutex/Monitor implementation
 
 inline void PlatformMutex::lock() {

src/hotspot/os/windows/os_windows.cpp

@@ -534,6 +534,13 @@ static unsigned thread_native_entry(void* t) {
   OSThread* osthr = thread->osthread();
   assert(osthr->get_state() == RUNNABLE, "invalid os thread state");
 
+  if (UseNUMA) {
+    int lgrp_id = os::numa_get_group_id();
+    if (lgrp_id != -1) {
+      thread->set_lgrp_id(lgrp_id);
+    }
+  }
+
   // Diagnostic code to investigate JDK-6573254
   int res = 30115;  // non-java thread
   if (thread->is_Java_thread()) {
@@ -591,6 +598,13 @@ static OSThread* create_os_thread(Thread* thread, HANDLE thread_handle,
   osthread->set_thread_handle(thread_handle);
   osthread->set_thread_id(thread_id);
 
+  if (UseNUMA) {
+    int lgrp_id = os::numa_get_group_id();
+    if (lgrp_id != -1) {
+      thread->set_lgrp_id(lgrp_id);
+    }
+  }
+
   // Initial thread state is INITIALIZED, not SUSPENDED
   osthread->set_state(INITIALIZED);
 

src/hotspot/os/windows/os_windows.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -62,6 +62,8 @@ inline void os::map_stack_shadow_pages(address sp) {
   state->set_shadow_zone_growth_watermark(original_sp);
 }
 
+inline bool os::numa_has_group_homing()     { return false;  }
+
 // Platform Mutex/Monitor implementation
 
 inline void PlatformMutex::lock() {

src/hotspot/os_cpu/linux_ppc/gc/z/zSyscall_linux_ppc.hpp

@@ -31,6 +31,7 @@
 // Support for building on older Linux systems
 //
 
+
 #ifndef SYS_memfd_create
 #define SYS_memfd_create     360
 #endif

src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp

@@ -89,24 +89,7 @@
 #define   RISCV_HWPROBE_MISALIGNED_UNSUPPORTED  (4 << 0)
 #define   RISCV_HWPROBE_MISALIGNED_MASK         (7 << 0)
 
-#define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE      6
-
-#define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS   7
-
-#define RISCV_HWPROBE_KEY_TIME_CSR_FREQ          8
-
-#define RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF 9
-#define   RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN       0
-#define   RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED      1
-#define   RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW          2
-#define   RISCV_HWPROBE_MISALIGNED_SCALAR_FAST          3
-#define   RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED   4
-
-#define RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF 10
-#define   RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN       0
-#define   RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW          2
-#define   RISCV_HWPROBE_MISALIGNED_VECTOR_FAST          3
-#define   RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED   4
+#define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6
 
 #ifndef NR_riscv_hwprobe
 #ifndef NR_arch_specific_syscall
@@ -134,11 +117,7 @@ static struct riscv_hwprobe query[] = {{RISCV_HWPROBE_KEY_MVENDORID, 0},
                                        {RISCV_HWPROBE_KEY_BASE_BEHAVIOR, 0},
                                        {RISCV_HWPROBE_KEY_IMA_EXT_0,     0},
                                        {RISCV_HWPROBE_KEY_CPUPERF_0,     0},
-                                       {RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE,      0},
-                                       {RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS,   0},
-                                       {RISCV_HWPROBE_KEY_TIME_CSR_FREQ,          0},
-                                       {RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF, 0},
-                                       {RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF, 0}};
+                                       {RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE, 0}};
 
 bool RiscvHwprobe::probe_features() {
   assert(!rw_hwprobe_completed, "Called twice.");
@@ -267,16 +246,10 @@ void RiscvHwprobe::add_features_from_query_result() {
     VM_Version::ext_Zicond.enable_feature();
   }
 #endif
-  // RISCV_HWPROBE_KEY_CPUPERF_0 is deprecated. Keep it there for backward
-  // compatibility with old kernels.
   if (is_valid(RISCV_HWPROBE_KEY_CPUPERF_0)) {
-    VM_Version::unaligned_scalar.enable_feature(
+    VM_Version::unaligned_access.enable_feature(
        query[RISCV_HWPROBE_KEY_CPUPERF_0].value & RISCV_HWPROBE_MISALIGNED_MASK);
   }
-  if (is_valid(RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF)) {
-    VM_Version::unaligned_vector.enable_feature(
-       query[RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF].value);
-  }
   if (is_valid(RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE)) {
     VM_Version::zicboz_block_size.enable_feature(query[RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE].value);
   }

src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp

@@ -303,7 +303,7 @@ void VM_Version::rivos_features() {
 
   ext_Zvfh.enable_feature();
 
-  unaligned_scalar.enable_feature(MISALIGNED_SCALAR_FAST);
+  unaligned_access.enable_feature(MISALIGNED_FAST);
   satp_mode.enable_feature(VM_SV48);
 
   // Features dependent on march/mimpid.

src/hotspot/share/adlc/formsopt.cpp

@@ -162,24 +162,22 @@ bool   RegisterForm::verify() {
   return  valid;
 }
 
-// Compute the least number of words required for registers in register masks.
-int RegisterForm::words_for_regs() {
-  return (_reg_ctr + 31) >> 5;
-}
-
 // Compute RegMask size
 int RegisterForm::RegMask_Size() {
-  // The array of Register Mask bits should be large enough to cover all the
-  // machine registers, as well as a certain number of parameters that need to
-  // be passed on the stack (stack registers). The number of parameters that can
-  // fit in the mask should be dimensioned to cover most common cases.
+  // Need at least this many words
+  int words_for_regs = (_reg_ctr + 31)>>5;
+  // The array of Register Mask bits should be large enough to cover
+  // all the machine registers and all parameters that need to be passed
+  // on the stack (stack registers) up to some interesting limit.  Methods
+  // that need more parameters will NOT be compiled.  On Intel, the limit
+  // is something like 90+ parameters.
   // - Add a few (3 words == 96 bits) for incoming & outgoing arguments to
   //   calls.
   // - Round up to the next doubleword size.
   // - Add one more word to accommodate a reasonable number of stack locations
   //   in the register mask regardless of how much slack is created by rounding.
   //   This was found necessary after adding 16 new registers for APX.
-  return (words_for_regs() + 3 + 1 + 1) & ~1;
+  return (words_for_regs + 3 + 1 + 1) & ~1;
 }
 
 void RegisterForm::dump() {                  // Debug printer
@@ -371,14 +369,14 @@ void RegClass::build_register_masks(FILE* fp) {
   for(i = 0; i < len - 1; i++) {
     fprintf(fp," 0x%x,", regs_in_word(i, false));
   }
-  fprintf(fp, " 0x%x, false );\n", regs_in_word(i, false));
+  fprintf(fp," 0x%x );\n", regs_in_word(i, false));
 
   if (_stack_or_reg) {
     fprintf(fp, "const RegMask _%sSTACK_OR_%s_mask(", prefix, rc_name_to_upper);
     for(i = 0; i < len - 1; i++) {
       fprintf(fp," 0x%x,", regs_in_word(i, true));
     }
-    fprintf(fp, " 0x%x, true );\n", regs_in_word(i, true));
+    fprintf(fp," 0x%x );\n", regs_in_word(i, true));
   }
   delete[] rc_name_to_upper;
 }

src/hotspot/share/adlc/formsopt.hpp

@@ -93,8 +93,6 @@ public:
   Dict        _allocClass;         // Dictionary of allocation classes
 
   static int  _reg_ctr;         // Register counter
-  static int  words_for_regs(); // Compute the least number of words required for
-                                // registers in register masks.
   static int  RegMask_Size();   // Compute RegMask size
 
   // Public Methods

src/hotspot/share/adlc/output_h.cpp

@@ -99,8 +99,6 @@ void ArchDesc::buildMachRegisterNumbers(FILE *fp_hpp) {
 
   fprintf(fp_hpp, "\n// Size of register-mask in ints\n");
   fprintf(fp_hpp, "#define RM_SIZE_IN_INTS %d\n", RegisterForm::RegMask_Size());
-  fprintf(fp_hpp, "// Minimum size of register-mask in ints\n");
-  fprintf(fp_hpp, "#define RM_SIZE_IN_INTS_MIN %d\n", RegisterForm::words_for_regs());
   fprintf(fp_hpp, "// Unroll factor for loops over the data in a RegMask\n");
   fprintf(fp_hpp, "#define FORALL_BODY ");
   int len = RegisterForm::RegMask_Size();

src/hotspot/share/c1/c1_CodeStubs.hpp

@@ -371,16 +371,21 @@ class MonitorEnterStub: public MonitorAccessStub {
 
 class MonitorExitStub: public MonitorAccessStub {
  private:
+  bool _compute_lock;
   int  _monitor_ix;
 
  public:
-  MonitorExitStub(LIR_Opr lock_reg, int monitor_ix)
+  MonitorExitStub(LIR_Opr lock_reg, bool compute_lock, int monitor_ix)
     : MonitorAccessStub(LIR_OprFact::illegalOpr, lock_reg),
-      _monitor_ix(monitor_ix) { }
+      _compute_lock(compute_lock), _monitor_ix(monitor_ix) { }
   virtual void emit_code(LIR_Assembler* e);
   virtual void visit(LIR_OpVisitState* visitor) {
     assert(_obj_reg->is_illegal(), "unused");
-    visitor->do_temp(_lock_reg);
+    if (_compute_lock) {
+      visitor->do_temp(_lock_reg);
+    } else {
+      visitor->do_input(_lock_reg);
+    }
   }
 #ifndef PRODUCT
   virtual void print_name(outputStream* out) const { out->print("MonitorExitStub"); }

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -635,7 +635,7 @@ void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, L
   // setup registers
   LIR_Opr hdr = lock;
   lock = new_hdr;
-  CodeStub* slow_path = new MonitorExitStub(lock, monitor_no);
+  CodeStub* slow_path = new MonitorExitStub(lock, true, monitor_no);
   __ load_stack_address_monitor(monitor_no, lock);
   __ unlock_object(hdr, object, lock, scratch, slow_path);
 }

src/hotspot/share/cds/aotClassInitializer.cpp

@@ -23,7 +23,6 @@
  */
 
 #include "cds/aotClassInitializer.hpp"
-#include "cds/aotLinkedClassBulkLoader.hpp"
 #include "cds/archiveBuilder.hpp"
 #include "cds/cdsConfig.hpp"
 #include "cds/heapShared.hpp"
@@ -48,14 +47,7 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
     ik = RegeneratedClasses::get_original_object(ik);
   }
 
-  check_aot_annotations(ik);
-
   if (!ik->is_initialized() && !ik->is_being_initialized()) {
-    if (ik->has_aot_safe_initializer()) {
-      ResourceMark rm;
-      log_info(aot, init)("Class %s is annotated with @AOTSafeClassInitializer but has not been initialized",
-                          ik->external_name());
-    }
     return false;
   }
 
@@ -252,56 +244,6 @@ void AOTClassInitializer::call_runtime_setup(JavaThread* current, InstanceKlass*
   }
 }
 
-template <typename FUNCTION>
-void require_annotation_for_super_types(InstanceKlass* ik, const char* annotation, FUNCTION func) {
-  if (log_is_enabled(Info, aot, init)) {
-    ResourceMark rm;
-    log_info(aot, init)("Found %s class %s", annotation, ik->external_name());
-  }
-
-  // Since ik has this annotation, we require that
-  //   - all super classes must have this annotation
-  //   - all super interfaces that are interface_needs_clinit_execution_as_super()
-  //     must have this annotation
-  // This avoid the situation where in the production run, we run the <clinit>
-  // of a supertype but not the <clinit> of ik
-
-  InstanceKlass* super = ik->java_super();
-  if (super != nullptr && !func(super)) {
-    ResourceMark rm;
-    log_error(aot, init)("Missing %s in superclass %s for class %s",
-                         annotation, super->external_name(), ik->external_name());
-    AOTMetaspace::unrecoverable_writing_error();
-  }
-
-  int len = ik->local_interfaces()->length();
-  for (int i = 0; i < len; i++) {
-    InstanceKlass* intf = ik->local_interfaces()->at(i);
-    if (intf->interface_needs_clinit_execution_as_super() && !func(intf)) {
-      ResourceMark rm;
-      log_error(aot, init)("Missing %s in superinterface %s for class %s",
-                           annotation, intf->external_name(), ik->external_name());
-      AOTMetaspace::unrecoverable_writing_error();
-    }
-  }
-}
-
-void AOTClassInitializer::check_aot_annotations(InstanceKlass* ik) {
-  if (ik->has_aot_safe_initializer()) {
-    require_annotation_for_super_types(ik, "@AOTSafeClassInitializer", [&] (const InstanceKlass* supertype) {
-      return supertype->has_aot_safe_initializer();
-    });
-  } else {
-    // @AOTRuntimeSetup only meaningful in @AOTSafeClassInitializer
-    if (ik->is_runtime_setup_required()) {
-      ResourceMark rm;
-      log_error(aot, init)("@AOTRuntimeSetup meaningless in non-@AOTSafeClassInitializer class %s",
-                           ik->external_name());
-    }
-  }
-}
-
-
 #ifdef ASSERT
 void AOTClassInitializer::init_test_class(TRAPS) {
   // -XX:AOTInitTestClass is used in regression tests for adding additional AOT-initialized classes

src/hotspot/share/cds/aotClassInitializer.hpp

@@ -31,9 +31,6 @@
 class InstanceKlass;
 
 class AOTClassInitializer : AllStatic {
-
-  static void check_aot_annotations(InstanceKlass* ik);
-
 public:
   // Called by heapShared.cpp to see if src_ik->java_mirror() can be archived in
   // the initialized state.

src/hotspot/share/cds/aotClassLinker.cpp

@@ -192,7 +192,7 @@ void AOTClassLinker::write_to_archive() {
 
   if (CDSConfig::is_dumping_aot_linked_classes()) {
     AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-    table->set_boot1(write_classes(nullptr, true));
+    table->set_boot(write_classes(nullptr, true));
     table->set_boot2(write_classes(nullptr, false));
     table->set_platform(write_classes(SystemDictionary::java_platform_loader(), false));
     table->set_app(write_classes(SystemDictionary::java_system_loader(), false));

src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp

@@ -29,8 +29,6 @@
 #include "cds/cdsConfig.hpp"
 #include "cds/heapShared.hpp"
 #include "classfile/classLoaderData.hpp"
-#include "classfile/classLoaderDataShared.hpp"
-#include "classfile/javaClasses.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "classfile/systemDictionaryShared.hpp"
 #include "classfile/vmClasses.hpp"
@@ -43,197 +41,66 @@
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
 
+bool AOTLinkedClassBulkLoader::_boot2_completed = false;
+bool AOTLinkedClassBulkLoader::_platform_completed = false;
+bool AOTLinkedClassBulkLoader::_app_completed = false;
+bool AOTLinkedClassBulkLoader::_all_completed = false;
+
 void AOTLinkedClassBulkLoader::serialize(SerializeClosure* soc) {
   AOTLinkedClassTable::get()->serialize(soc);
 }
 
-// This function is called before the VM executes any Java code (include AOT-compiled Java methods).
-//
-// We populate the boot/platform/app class loaders with classes from the AOT cache. This is a fundamental
-// step in restoring the JVM's state from the snapshot recorded in the AOT cache: other AOT optimizations
-// such as AOT compiled methods can make direct references to the preloaded classes, knowing that
-// these classes are guaranteed to be in at least the "loaded" state.
-void AOTLinkedClassBulkLoader::preload_classes(JavaThread* current) {
-  preload_classes_impl(current);
-  if (current->has_pending_exception()) {
-    exit_on_exception(current);
+bool AOTLinkedClassBulkLoader::class_preloading_finished() {
+  if (!CDSConfig::is_using_aot_linked_classes()) {
+    return true;
+  } else {
+    // The ConstantPools of preloaded classes have references to other preloaded classes. We don't
+    // want any Java code (including JVMCI compiler) to use these classes until all of them
+    // are loaded.
+    return AtomicAccess::load_acquire(&_all_completed);
   }
 }
 
-void AOTLinkedClassBulkLoader::preload_classes_impl(TRAPS) {
-  precond(CDSConfig::is_using_aot_linked_classes());
-
-  ClassLoaderDataShared::restore_archived_modules_for_preloading_classes(THREAD);
-  Handle h_platform_loader(THREAD, SystemDictionary::java_platform_loader());
-  Handle h_system_loader(THREAD, SystemDictionary::java_system_loader());
-
-  AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-
-  preload_classes_in_table(table->boot1(), "boot1", Handle(), CHECK);
-  preload_classes_in_table(table->boot2(), "boot2", Handle(), CHECK);
-
-  initiate_loading(THREAD, "plat", h_platform_loader, table->boot1());
-  initiate_loading(THREAD, "plat", h_platform_loader, table->boot2());
-  preload_classes_in_table(table->platform(), "plat", h_platform_loader, CHECK);
-
-  initiate_loading(THREAD, "app", h_system_loader, table->boot1());
-  initiate_loading(THREAD, "app", h_system_loader, table->boot2());
-  initiate_loading(THREAD, "app", h_system_loader, table->platform());
-  preload_classes_in_table(table->app(), "app", h_system_loader, CHECK);
-}
-
-void AOTLinkedClassBulkLoader::preload_classes_in_table(Array<InstanceKlass*>* classes,
-                                                        const char* category_name, Handle loader, TRAPS) {
-  if (classes == nullptr) {
-    return;
-  }
-
-  for (int i = 0; i < classes->length(); i++) {
-    InstanceKlass* ik = classes->at(i);
-    if (log_is_enabled(Info, aot, load)) {
-      ResourceMark rm(THREAD);
-      log_info(aot, load)("%-5s %s%s", category_name, ik->external_name(),
-                          ik->is_hidden() ? " (hidden)" : "");
-    }
-
-    SystemDictionary::preload_class(loader, ik, CHECK);
-
-    if (ik->is_hidden()) {
-      DEBUG_ONLY({
-        // Make sure we don't make this hidden class available by name, even if we don't
-        // use any special ClassLoaderData.
-        ResourceMark rm(THREAD);
-        assert(SystemDictionary::find_instance_klass(THREAD, ik->name(), loader) == nullptr,
-               "hidden classes cannot be accessible by name: %s", ik->external_name());
-      });
-    } else {
-      precond(SystemDictionary::find_instance_klass(THREAD, ik->name(), loader) == ik);
-    }
-  }
-}
-
-#ifdef ASSERT
-void AOTLinkedClassBulkLoader::validate_module_of_preloaded_classes() {
-  oop javabase_module_oop = ModuleEntryTable::javabase_moduleEntry()->module_oop();
-  for (int i = T_BOOLEAN; i < T_LONG+1; i++) {
-    TypeArrayKlass* tak = Universe::typeArrayKlass((BasicType)i);
-    validate_module(tak, "boot1", javabase_module_oop);
-  }
-
-  JavaThread* current = JavaThread::current();
-  Handle h_platform_loader(current, SystemDictionary::java_platform_loader());
-  Handle h_system_loader(current, SystemDictionary::java_system_loader());
-  AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-
-  validate_module_of_preloaded_classes_in_table(table->boot1(), "boot1", Handle());
-  validate_module_of_preloaded_classes_in_table(table->boot2(), "boot2", Handle());
-  validate_module_of_preloaded_classes_in_table(table->platform(), "plat", h_platform_loader);
-  validate_module_of_preloaded_classes_in_table(table->app(), "app", h_system_loader);
-}
-
-void AOTLinkedClassBulkLoader::validate_module_of_preloaded_classes_in_table(Array<InstanceKlass*>* classes,
-                                                                             const char* category_name, Handle loader) {
-  if (classes == nullptr) {
-    return;
-  }
-
-  ClassLoaderData* loader_data = ClassLoaderData::class_loader_data(loader());
-  for (int i = 0; i < classes->length(); i++) {
-    InstanceKlass* ik = classes->at(i);
-    PackageEntry* pkg_entry = ik->package();
-    oop module_oop;
-    if (pkg_entry == nullptr) {
-      module_oop = loader_data->unnamed_module()->module_oop();
-    } else {
-      module_oop = pkg_entry->module()->module_oop();
-    }
-
-    validate_module(ik, category_name, module_oop);
-  }
-}
-
-void AOTLinkedClassBulkLoader::validate_module(Klass* k, const char* category_name, oop module_oop) {
-  assert(module_oop != nullptr, "module system must have been initialized");
-
-  if (log_is_enabled(Debug, aot, module)) {
-    ResourceMark rm;
-    log_debug(aot, module)("Validate module of %-5s %s", category_name, k->external_name());
-  }
-  precond(java_lang_Class::module(k->java_mirror()) == module_oop);
-
-  ArrayKlass* ak = k->array_klass_or_null();
-  while (ak != nullptr) {
-    if (log_is_enabled(Debug, aot, module)) {
-      ResourceMark rm;
-      log_debug(aot, module)("Validate module of %-5s %s", category_name, ak->external_name());
-    }
-    precond(java_lang_Class::module(ak->java_mirror()) == module_oop);
-    ak = ak->array_klass_or_null();
-  }
-}
-#endif
-
-// Link all java.base classes in the AOTLinkedClassTable. Of those classes,
-// move the ones that have been AOT-initialized to the "initialized" state.
-void AOTLinkedClassBulkLoader::link_or_init_javabase_classes(JavaThread* current) {
-  link_or_init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
-  if (current->has_pending_exception()) {
-    exit_on_exception(current);
-  }
-}
-
-// Do the same thing as link_or_init_javabase_classes(), but for the classes that are not
-// in the java.base module.
-void AOTLinkedClassBulkLoader::link_or_init_non_javabase_classes(JavaThread* current) {
-  link_or_init_non_javabase_classes_impl(current);
-  if (current->has_pending_exception()) {
-    exit_on_exception(current);
-  }
-}
-
-void AOTLinkedClassBulkLoader::link_or_init_non_javabase_classes_impl(TRAPS) {
+void AOTLinkedClassBulkLoader::load_javabase_classes(JavaThread* current) {
   assert(CDSConfig::is_using_aot_linked_classes(), "sanity");
+  load_classes_in_loader(current, AOTLinkedClassCategory::BOOT1, nullptr); // only java.base classes
+}
 
-  DEBUG_ONLY(validate_module_of_preloaded_classes());
+void AOTLinkedClassBulkLoader::load_non_javabase_classes(JavaThread* current) {
+  assert(CDSConfig::is_using_aot_linked_classes(), "sanity");
 
   // is_using_aot_linked_classes() requires is_using_full_module_graph(). As a result,
   // the platform/system class loader should already have been initialized as part
   // of the FMG support.
   assert(CDSConfig::is_using_full_module_graph(), "must be");
+  assert(SystemDictionary::java_platform_loader() != nullptr, "must be");
+  assert(SystemDictionary::java_system_loader() != nullptr,   "must be");
 
-  Handle h_platform_loader(THREAD, SystemDictionary::java_platform_loader());
-  Handle h_system_loader(THREAD, SystemDictionary::java_system_loader());
+  load_classes_in_loader(current, AOTLinkedClassCategory::BOOT2, nullptr); // all boot classes outside of java.base
+  _boot2_completed = true;
 
-  assert(h_platform_loader() != nullptr, "must be");
-  assert(h_system_loader() != nullptr,   "must be");
+  load_classes_in_loader(current, AOTLinkedClassCategory::PLATFORM, SystemDictionary::java_platform_loader());
+  _platform_completed = true;
 
-  AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-  link_or_init_classes_for_loader(Handle(), table->boot2(), CHECK);
-  link_or_init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
-  link_or_init_classes_for_loader(h_system_loader, table->app(), CHECK);
-
-  if (Universe::is_fully_initialized() && VerifyDuringStartup) {
-    // Make sure we're still in a clean state.
-    VM_Verify verify_op;
-    VMThread::execute(&verify_op);
-  }
+  load_classes_in_loader(current, AOTLinkedClassCategory::APP, SystemDictionary::java_system_loader());
 
   if (AOTPrintTrainingInfo) {
     tty->print_cr("==================== archived_training_data ** after all classes preloaded ====================");
     TrainingData::print_archived_training_data_on(tty);
   }
+
+  _app_completed = true;
+  AtomicAccess::release_store(&_all_completed, true);
 }
 
-// For the AOT cache to function properly, all classes in the AOTLinkedClassTable
-// must be loaded and linked. In addition, AOT-initialized classes must be moved to
-// the initialized state.
-//
-// We can encounter a failure during the loading, linking, or initialization of
-// classes in the AOTLinkedClassTable only if:
-//   - We ran out of memory,
-//   - There is a serious error in the VM implemenation
-// When this happens, the VM may be in an inconsistent state (e.g., we have a cached
-// heap object of class X, but X is not linked). We must exit the JVM now.
+void AOTLinkedClassBulkLoader::load_classes_in_loader(JavaThread* current, AOTLinkedClassCategory class_category, oop class_loader_oop) {
+  load_classes_in_loader_impl(class_category, class_loader_oop, current);
+  if (current->has_pending_exception()) {
+    // We cannot continue, as we might have loaded some of the aot-linked classes, which
+    // may have dangling C++ pointers to other aot-linked classes that we have failed to load.
+    exit_on_exception(current);
+  }
+}
 
 void AOTLinkedClassBulkLoader::exit_on_exception(JavaThread* current) {
   assert(current->has_pending_exception(), "precondition");
@@ -248,6 +115,117 @@ void AOTLinkedClassBulkLoader::exit_on_exception(JavaThread* current) {
   vm_exit_during_initialization("Unexpected exception when loading aot-linked classes.");
 }
 
+void AOTLinkedClassBulkLoader::load_classes_in_loader_impl(AOTLinkedClassCategory class_category, oop class_loader_oop, TRAPS) {
+  Handle h_loader(THREAD, class_loader_oop);
+  AOTLinkedClassTable* table = AOTLinkedClassTable::get();
+  load_table(table, class_category, h_loader, CHECK);
+
+  // Initialize the InstanceKlasses of all archived heap objects that are reachable from the
+  // archived java class mirrors.
+  switch (class_category) {
+  case AOTLinkedClassCategory::BOOT1:
+    // Delayed until finish_loading_javabase_classes(), as the VM is not ready to
+    // execute some of the <clinit> methods.
+    break;
+  case AOTLinkedClassCategory::BOOT2:
+    init_required_classes_for_loader(h_loader, table->boot2(), CHECK);
+    break;
+  case AOTLinkedClassCategory::PLATFORM:
+    init_required_classes_for_loader(h_loader, table->platform(), CHECK);
+    break;
+  case AOTLinkedClassCategory::APP:
+    init_required_classes_for_loader(h_loader, table->app(), CHECK);
+    break;
+  case AOTLinkedClassCategory::UNREGISTERED:
+    ShouldNotReachHere();
+    break;
+  }
+
+  if (Universe::is_fully_initialized() && VerifyDuringStartup) {
+    // Make sure we're still in a clean state.
+    VM_Verify verify_op;
+    VMThread::execute(&verify_op);
+  }
+}
+
+void AOTLinkedClassBulkLoader::load_table(AOTLinkedClassTable* table, AOTLinkedClassCategory class_category, Handle loader, TRAPS) {
+  if (class_category != AOTLinkedClassCategory::BOOT1) {
+    assert(Universe::is_module_initialized(), "sanity");
+  }
+
+  const char* category_name = AOTClassLinker::class_category_name(class_category);
+  switch (class_category) {
+  case AOTLinkedClassCategory::BOOT1:
+    load_classes_impl(class_category, table->boot(), category_name, loader, CHECK);
+    break;
+
+  case AOTLinkedClassCategory::BOOT2:
+    load_classes_impl(class_category, table->boot2(), category_name, loader, CHECK);
+    break;
+
+  case AOTLinkedClassCategory::PLATFORM:
+    {
+      initiate_loading(THREAD, category_name, loader, table->boot());
+      initiate_loading(THREAD, category_name, loader, table->boot2());
+      load_classes_impl(class_category, table->platform(), category_name, loader, CHECK);
+    }
+    break;
+  case AOTLinkedClassCategory::APP:
+    {
+      initiate_loading(THREAD, category_name, loader, table->boot());
+      initiate_loading(THREAD, category_name, loader, table->boot2());
+      initiate_loading(THREAD, category_name, loader, table->platform());
+      load_classes_impl(class_category, table->app(), category_name, loader, CHECK);
+    }
+    break;
+  case AOTLinkedClassCategory::UNREGISTERED:
+  default:
+    ShouldNotReachHere(); // Currently aot-linked classes are not supported for this category.
+    break;
+  }
+}
+
+void AOTLinkedClassBulkLoader::load_classes_impl(AOTLinkedClassCategory class_category, Array<InstanceKlass*>* classes,
+                                                 const char* category_name, Handle loader, TRAPS) {
+  if (classes == nullptr) {
+    return;
+  }
+
+  ClassLoaderData* loader_data = ClassLoaderData::class_loader_data(loader());
+
+  for (int i = 0; i < classes->length(); i++) {
+    InstanceKlass* ik = classes->at(i);
+    if (log_is_enabled(Info, aot, load)) {
+      ResourceMark rm(THREAD);
+      log_info(aot, load)("%-5s %s%s%s", category_name, ik->external_name(),
+                          ik->is_loaded() ? " (already loaded)" : "",
+                          ik->is_hidden() ? " (hidden)" : "");
+    }
+
+    if (!ik->is_loaded()) {
+      if (ik->is_hidden()) {
+        load_hidden_class(loader_data, ik, CHECK);
+      } else {
+        InstanceKlass* actual;
+        if (loader_data == ClassLoaderData::the_null_class_loader_data()) {
+          actual = SystemDictionary::load_instance_class(ik->name(), loader, CHECK);
+        } else {
+          actual = SystemDictionaryShared::find_or_load_shared_class(ik->name(), loader, CHECK);
+        }
+
+        if (actual != ik) {
+          ResourceMark rm(THREAD);
+          log_error(aot)("Unable to resolve %s class from %s: %s", category_name, CDSConfig::type_of_archive_being_loaded(), ik->external_name());
+          log_error(aot)("Expected: " INTPTR_FORMAT ", actual: " INTPTR_FORMAT, p2i(ik), p2i(actual));
+          log_error(aot)("JVMTI class retransformation is not supported when archive was generated with -XX:+AOTClassLinking.");
+          AOTMetaspace::unrecoverable_loading_error();
+        }
+        assert(actual->is_loaded(), "must be");
+      }
+    }
+  }
+}
+
 // Initiate loading of the <classes> in the <initiating_loader>. The <classes> should have already been loaded
 // by a parent loader of the <initiating_loader>. This is necessary for handling pre-resolved CP entries.
 //
@@ -277,7 +255,7 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
       if (log_is_enabled(Info, aot, load)) {
         ResourceMark rm(current);
         const char* defining_loader = (ik->class_loader() == nullptr ? "boot" : "plat");
-        log_info(aot, load)("%-5s %s (initiated, defined by %s)", category_name, ik->external_name(),
+        log_info(aot, load)("%s %s (initiated, defined by %s)", category_name, ik->external_name(),
                             defining_loader);
       }
       SystemDictionary::add_to_initiating_loader(current, ik, loader_data);
@@ -285,11 +263,81 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
   }
 }
 
+// Currently, we archive only three types of hidden classes:
+//    - LambdaForms
+//    - lambda proxy classes
+//    - StringConcat classes
+// See HeapShared::is_archivable_hidden_klass().
+//
+// LambdaForm classes (with names like java/lang/invoke/LambdaForm$MH+0x800000015) logically
+// belong to the boot loader, but they are usually stored in their own special ClassLoaderData to
+// facilitate class unloading, as a LambdaForm may refer to a class loaded by a custom loader
+// that may be unloaded.
+//
+// We only support AOT-resolution of indys in the boot/platform/app loader, so there's no need
+// to support class unloading. For simplicity, we put all archived LambdaForm classes in the
+// "main" ClassLoaderData of the boot loader.
+//
+// (Even if we were to support other loaders, we would still feel free to ignore any requirement
+// of class unloading, for any class asset in the AOT cache.  Anything that makes it into the AOT
+// cache has a lifetime dispensation from unloading.  After all, the AOT cache never grows, and
+// we can assume that the user is content with its size, and doesn't need its footprint to shrink.)
+//
+// Lambda proxy classes are normally stored in the same ClassLoaderData as their nest hosts, and
+// StringConcat are normally stored in the main ClassLoaderData of the boot class loader. We
+// do the same for the archived copies of such classes.
+void AOTLinkedClassBulkLoader::load_hidden_class(ClassLoaderData* loader_data, InstanceKlass* ik, TRAPS) {
+  assert(HeapShared::is_lambda_form_klass(ik) ||
+         HeapShared::is_lambda_proxy_klass(ik) ||
+         HeapShared::is_string_concat_klass(ik), "sanity");
+  DEBUG_ONLY({
+      assert(ik->super()->is_loaded(), "must be");
+      for (int i = 0; i < ik->local_interfaces()->length(); i++) {
+        assert(ik->local_interfaces()->at(i)->is_loaded(), "must be");
+      }
+    });
+
+  Handle pd;
+  PackageEntry* pkg_entry = nullptr;
+
+  // Since a hidden class does not have a name, it cannot be reloaded
+  // normally via the system dictionary. Instead, we have to finish the
+  // loading job here.
+
+  if (HeapShared::is_lambda_proxy_klass(ik)) {
+    InstanceKlass* nest_host = ik->nest_host_not_null();
+    assert(nest_host->is_loaded(), "must be");
+    pd = Handle(THREAD, nest_host->protection_domain());
+    pkg_entry = nest_host->package();
+  }
+
+  ik->restore_unshareable_info(loader_data, pd, pkg_entry, CHECK);
+  SystemDictionary::load_shared_class_misc(ik, loader_data);
+  ik->add_to_hierarchy(THREAD);
+  assert(ik->is_loaded(), "Must be in at least loaded state");
+
+  DEBUG_ONLY({
+      // Make sure we don't make this hidden class available by name, even if we don't
+      // use any special ClassLoaderData.
+      Handle loader(THREAD, loader_data->class_loader());
+      ResourceMark rm(THREAD);
+      assert(SystemDictionary::resolve_or_null(ik->name(), loader, THREAD) == nullptr,
+             "hidden classes cannot be accessible by name: %s", ik->external_name());
+      if (HAS_PENDING_EXCEPTION) {
+        CLEAR_PENDING_EXCEPTION;
+      }
+    });
+}
+
+void AOTLinkedClassBulkLoader::finish_loading_javabase_classes(TRAPS) {
+  init_required_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot(), CHECK);
+}
+
 // Some AOT-linked classes for <class_loader> must be initialized early. This includes
 // - classes that were AOT-initialized by AOTClassInitializer
 // - the classes of all objects that are reachable from the archived mirrors of
 //   the AOT-linked classes for <class_loader>.
-void AOTLinkedClassBulkLoader::link_or_init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS) {
+void AOTLinkedClassBulkLoader::init_required_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS) {
   if (classes != nullptr) {
     for (int i = 0; i < classes->length(); i++) {
       InstanceKlass* ik = classes->at(i);
@@ -313,6 +361,56 @@ void AOTLinkedClassBulkLoader::link_or_init_classes_for_loader(Handle class_load
   HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
 }
 
+bool AOTLinkedClassBulkLoader::is_pending_aot_linked_class(Klass* k) {
+  if (!CDSConfig::is_using_aot_linked_classes()) {
+    return false;
+  }
+
+  if (_all_completed) { // no more pending aot-linked classes
+    return false;
+  }
+
+  if (k->is_objArray_klass()) {
+    k = ObjArrayKlass::cast(k)->bottom_klass();
+  }
+  if (!k->is_instance_klass()) {
+    // type array klasses (and their higher dimensions),
+    // must have been loaded before a GC can ever happen.
+    return false;
+  }
+
+  // There's a small window during VM start-up where a not-yet loaded aot-linked
+  // class k may be discovered by the GC during VM initialization. This can happen
+  // when the heap contains an aot-cached instance of k, but k is not ready to be
+  // loaded yet. (TODO: JDK-8342429 eliminates this possibility)
+  //
+  // The following checks try to limit this window as much as possible for each of
+  // the four AOTLinkedClassCategory of classes that can be aot-linked.
+
+  InstanceKlass* ik = InstanceKlass::cast(k);
+  if (ik->defined_by_boot_loader()) {
+    if (ik->module() != nullptr && ik->in_javabase_module()) {
+      // AOTLinkedClassCategory::BOOT1 -- all aot-linked classes in
+      // java.base must have been loaded before a GC can ever happen.
+      return false;
+    } else {
+      // AOTLinkedClassCategory::BOOT2 classes cannot be loaded until
+      // module system is ready.
+      return !_boot2_completed;
+    }
+  } else if (ik->defined_by_platform_loader()) {
+    // AOTLinkedClassCategory::PLATFORM classes cannot be loaded until
+    // the platform class loader is initialized.
+    return !_platform_completed;
+  } else if (ik->defined_by_app_loader()) {
+    // AOTLinkedClassCategory::APP cannot be loaded until the app class loader
+    // is initialized.
+    return !_app_completed;
+  } else {
+    return false;
+  }
+}
+
 void AOTLinkedClassBulkLoader::replay_training_at_init(Array<InstanceKlass*>* classes, TRAPS) {
   if (classes != nullptr) {
     for (int i = 0; i < classes->length(); i++) {
@@ -327,7 +425,7 @@ void AOTLinkedClassBulkLoader::replay_training_at_init(Array<InstanceKlass*>* cl
 void AOTLinkedClassBulkLoader::replay_training_at_init_for_preloaded_classes(TRAPS) {
   if (CDSConfig::is_using_aot_linked_classes() && TrainingData::have_data()) {
     AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-    replay_training_at_init(table->boot1(),    CHECK);
+    replay_training_at_init(table->boot(),     CHECK);
     replay_training_at_init(table->boot2(),    CHECK);
     replay_training_at_init(table->platform(), CHECK);
     replay_training_at_init(table->app(),      CHECK);

src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp

@@ -39,40 +39,34 @@ template <typename T> class Array;
 enum class AOTLinkedClassCategory : int;
 
 // During a Production Run, the AOTLinkedClassBulkLoader loads all classes from
-// the AOTLinkedClassTable into their respective ClassLoaders. This happens very early
-// in the JVM bootstrap stage, before any Java bytecode is executed.
+// a AOTLinkedClassTable into their respective ClassLoaders. This happens very early
+// in the JVM bootstrap stage, before any application code is executed.
 //
-// IMPLEMENTATION NOTES:
-// We also proactively link all the classes in the AOTLinkedClassTable, and move
-// the AOT-initialized classes to the "initialized" state. Due to limitations
-// of the current JVM bootstrap sequence, link_or_init_javabase_classes() and
-// link_or_init_non_javabase_classes() need to be called after some Java bytecodes are
-// executed. Future RFEs will move these calls to earlier stages.
 class AOTLinkedClassBulkLoader :  AllStatic {
-  static void preload_classes_impl(TRAPS);
-  static void preload_classes_in_table(Array<InstanceKlass*>* classes,
-                                       const char* category_name, Handle loader, TRAPS);
-  static void initiate_loading(JavaThread* current, const char* category, Handle initiating_loader,
-                               Array<InstanceKlass*>* classes);
-  static void link_or_init_non_javabase_classes_impl(TRAPS);
-  static void link_or_init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS);
+  static bool _boot2_completed;
+  static bool _platform_completed;
+  static bool _app_completed;
+  static bool _all_completed;
+  static void load_classes_in_loader(JavaThread* current, AOTLinkedClassCategory class_category, oop class_loader_oop);
+  static void load_classes_in_loader_impl(AOTLinkedClassCategory class_category, oop class_loader_oop, TRAPS);
+  static void load_table(AOTLinkedClassTable* table, AOTLinkedClassCategory class_category, Handle loader, TRAPS);
+  static void initiate_loading(JavaThread* current, const char* category, Handle initiating_loader, Array<InstanceKlass*>* classes);
+  static void load_classes_impl(AOTLinkedClassCategory class_category, Array<InstanceKlass*>* classes,
+                                const char* category_name, Handle loader, TRAPS);
+  static void load_hidden_class(ClassLoaderData* loader_data, InstanceKlass* ik, TRAPS);
+  static void init_required_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS);
   static void replay_training_at_init(Array<InstanceKlass*>* classes, TRAPS) NOT_CDS_RETURN;
-
-#ifdef ASSERT
-  static void validate_module_of_preloaded_classes();
-  static void validate_module_of_preloaded_classes_in_table(Array<InstanceKlass*>* classes,
-                                                            const char* category_name, Handle loader);
-  static void validate_module(Klass* k, const char* category_name, oop module_oop);
-#endif
-
 public:
   static void serialize(SerializeClosure* soc) NOT_CDS_RETURN;
-  static void preload_classes(JavaThread* current);
-  static void link_or_init_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
-  static void link_or_init_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
+
+  static void load_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void load_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void finish_loading_javabase_classes(TRAPS) NOT_CDS_RETURN;
   static void exit_on_exception(JavaThread* current);
 
   static void replay_training_at_init_for_preloaded_classes(TRAPS) NOT_CDS_RETURN;
+  static bool class_preloading_finished();
+  static bool is_pending_aot_linked_class(Klass* k) NOT_CDS_RETURN_(false);
 };
 
 #endif // SHARE_CDS_AOTLINKEDCLASSBULKLOADER_HPP

src/hotspot/share/cds/aotLinkedClassTable.cpp

@@ -30,7 +30,7 @@
 AOTLinkedClassTable AOTLinkedClassTable::_instance;
 
 void AOTLinkedClassTable::serialize(SerializeClosure* soc) {
-  soc->do_ptr((void**)&_boot1);
+  soc->do_ptr((void**)&_boot);
   soc->do_ptr((void**)&_boot2);
   soc->do_ptr((void**)&_platform);
   soc->do_ptr((void**)&_app);

src/hotspot/share/cds/aotLinkedClassTable.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,27 +41,26 @@ class SerializeClosure;
 class AOTLinkedClassTable {
   static AOTLinkedClassTable _instance;
 
-  Array<InstanceKlass*>* _boot1; // boot classes in java.base module
-  Array<InstanceKlass*>* _boot2; // boot classes in all other (named and unnamed) modules,
-                                 // including classes from -Xbootclasspath/a
+  Array<InstanceKlass*>* _boot;  // only java.base classes
+  Array<InstanceKlass*>* _boot2; // boot classes in other modules
   Array<InstanceKlass*>* _platform;
   Array<InstanceKlass*>* _app;
 
 public:
   AOTLinkedClassTable() :
-    _boot1(nullptr), _boot2(nullptr),
+    _boot(nullptr), _boot2(nullptr),
     _platform(nullptr), _app(nullptr) {}
 
   static AOTLinkedClassTable* get() {
     return &_instance;
   }
 
-  Array<InstanceKlass*>* boot1()    const { return _boot1;    }
+  Array<InstanceKlass*>* boot()     const { return _boot;     }
   Array<InstanceKlass*>* boot2()    const { return _boot2;    }
   Array<InstanceKlass*>* platform() const { return _platform; }
   Array<InstanceKlass*>* app()      const { return _app;      }
 
-  void set_boot1   (Array<InstanceKlass*>* value) { _boot1    = value; }
+  void set_boot    (Array<InstanceKlass*>* value) { _boot     = value; }
   void set_boot2   (Array<InstanceKlass*>* value) { _boot2    = value; }
   void set_platform(Array<InstanceKlass*>* value) { _platform = value; }
   void set_app     (Array<InstanceKlass*>* value) { _app      = value; }

src/hotspot/share/cds/aotMetaspace.cpp

@@ -124,7 +124,7 @@ bool AOTMetaspace::_use_optimized_module_handling = true;
 // These regions are aligned with AOTMetaspace::core_region_alignment().
 //
 // These 2 regions are populated in the following steps:
-// [0] All classes are loaded in AOTMetaspace::load_classes(). All metadata are
+// [0] All classes are loaded in AOTMetaspace::preload_classes(). All metadata are
 //     temporarily allocated outside of the shared regions.
 // [1] We enter a safepoint and allocate a buffer for the rw/ro regions.
 // [2] C++ vtables are copied into the rw region.
@@ -804,23 +804,6 @@ void AOTMetaspace::link_shared_classes(TRAPS) {
   AOTClassLinker::initialize();
   AOTClassInitializer::init_test_class(CHECK);
 
-  if (CDSConfig::is_dumping_final_static_archive()) {
-    // - Load and link all classes used in the training run.
-    // - Initialize @AOTSafeClassInitializer classes that were
-    //   initialized in the training run.
-    // - Perform per-class optimization such as AOT-resolution of
-    //   constant pool entries that were resolved during the training run.
-    FinalImageRecipes::apply_recipes(CHECK);
-
-    // Because the AOT assembly phase does not run the same exact code as in the
-    // training run (e.g., we use different lambda form invoker classes;
-    // generated lambda form classes are not recorded in FinalImageRecipes),
-    // the recipes do not cover all classes that have been loaded so far. As
-    // a result, we might have some unlinked classes at this point. Since we
-    // require cached classes to be linked, all such classes will be linked
-    // by the following step.
-  }
-
   link_all_loaded_classes(THREAD);
 
   // Eargerly resolve all string constants in constant pools
@@ -834,12 +817,18 @@ void AOTMetaspace::link_shared_classes(TRAPS) {
       AOTConstantPoolResolver::preresolve_string_cp_entries(ik, CHECK);
     }
   }
+
+  if (CDSConfig::is_dumping_final_static_archive()) {
+    FinalImageRecipes::apply_recipes(CHECK);
+  }
 }
 
-void AOTMetaspace::dump_static_archive(TRAPS) {
+// Preload classes from a list, populate the shared spaces and dump to a
+// file.
+void AOTMetaspace::preload_and_dump(TRAPS) {
   CDSConfig::DumperThreadMark dumper_thread_mark(THREAD);
   ResourceMark rm(THREAD);
-  HandleMark hm(THREAD);
+ HandleMark hm(THREAD);
 
  if (CDSConfig::is_dumping_final_static_archive() && AOTPrintTrainingInfo) {
    tty->print_cr("==================== archived_training_data ** before dumping ====================");
@@ -847,7 +836,7 @@ void AOTMetaspace::dump_static_archive(TRAPS) {
  }
 
   StaticArchiveBuilder builder;
-  dump_static_archive_impl(builder, THREAD);
+  preload_and_dump_impl(builder, THREAD);
   if (HAS_PENDING_EXCEPTION) {
     if (PENDING_EXCEPTION->is_a(vmClasses::OutOfMemoryError_klass())) {
       aot_log_error(aot)("Out of memory. Please run with a larger Java heap, current MaxHeapSize = "
@@ -913,7 +902,7 @@ void AOTMetaspace::get_default_classlist(char* default_classlist, const size_t b
                Arguments::get_java_home(), filesep, filesep);
 }
 
-void AOTMetaspace::load_classes(TRAPS) {
+void AOTMetaspace::preload_classes(TRAPS) {
   char default_classlist[JVM_MAXPATHLEN];
   const char* classlist_path;
 
@@ -940,8 +929,8 @@ void AOTMetaspace::load_classes(TRAPS) {
     }
   }
 
-  // Some classes are used at CDS runtime but are not yet loaded at this point.
-  // We can perform dummmy calls to these classes at dumptime to ensure they
+  // Some classes are used at CDS runtime but are not loaded, and therefore archived, at
+  // dumptime. We can perform dummmy calls to these classes at dumptime to ensure they
   // are archived.
   exercise_runtime_cds_code(CHECK);
 
@@ -957,10 +946,10 @@ void AOTMetaspace::exercise_runtime_cds_code(TRAPS) {
   CDSProtectionDomain::to_file_URL("dummy.jar", Handle(), CHECK);
 }
 
-void AOTMetaspace::dump_static_archive_impl(StaticArchiveBuilder& builder, TRAPS) {
+void AOTMetaspace::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS) {
   if (CDSConfig::is_dumping_classic_static_archive()) {
     // We are running with -Xshare:dump
-    load_classes(CHECK);
+    preload_classes(CHECK);
 
     if (SharedArchiveConfigFile) {
       log_info(aot)("Reading extra data from %s ...", SharedArchiveConfigFile);

src/hotspot/share/cds/aotMetaspace.hpp

@@ -72,15 +72,15 @@ class AOTMetaspace : AllStatic {
     n_regions = 5              // total number of regions
   };
 
-  static void dump_static_archive(TRAPS) NOT_CDS_RETURN;
+  static void preload_and_dump(TRAPS) NOT_CDS_RETURN;
 #ifdef _LP64
   static void adjust_heap_sizes_for_dumping() NOT_CDS_JAVA_HEAP_RETURN;
 #endif
 
 private:
   static void exercise_runtime_cds_code(TRAPS) NOT_CDS_RETURN;
-  static void dump_static_archive_impl(StaticArchiveBuilder& builder, TRAPS) NOT_CDS_RETURN;
-  static void load_classes(TRAPS) NOT_CDS_RETURN;
+  static void preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS) NOT_CDS_RETURN;
+  static void preload_classes(TRAPS) NOT_CDS_RETURN;
 
 public:
   static Symbol* symbol_rs_base() {

src/hotspot/share/cds/aotOopChecker.cpp

@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "cds/aotMetaspace.hpp"
-#include "cds/aotOopChecker.hpp"
-#include "cds/heapShared.hpp"
-#include "classfile/javaClasses.hpp"
-#include "classfile/symbolTable.hpp"
-#include "classfile/vmClasses.hpp"
-#include "oops/instanceKlass.inline.hpp"
-#include "runtime/fieldDescriptor.inline.hpp"
-#include "utilities/debug.hpp"
-
-#if INCLUDE_CDS_JAVA_HEAP
-
-oop AOTOopChecker::get_oop_field(oop obj, const char* name, const char* sig) {
-  Symbol* name_sym = SymbolTable::probe(name, checked_cast<int>(strlen(name)));
-  assert(name_sym != nullptr, "Symbol must have been resolved for an existing field of this obj");
-  Symbol* sig_sym = SymbolTable::probe(sig, checked_cast<int>(strlen(sig)));
-  assert(sig_sym != nullptr, "Symbol must have been resolved for an existing field of this obj");
-
-  fieldDescriptor fd;
-  Klass* k = InstanceKlass::cast(obj->klass())->find_field(name_sym, sig_sym, &fd);
-  assert(k != nullptr, "field must exist");
-  precond(!fd.is_static());
-  precond(fd.field_type() == T_OBJECT || fd.field_type() == T_ARRAY);
-  return obj->obj_field(fd.offset());
-}
-
-// Make sure we are not caching objects with assumptions that can be violated in
-// the production run.
-void AOTOopChecker::check(oop obj) {
-  // Currently we only check URL objects, but more rules may be added in the future.
-
-  if (obj->klass()->is_subclass_of(vmClasses::URL_klass())) {
-    // If URL could be subclassed, obj may have new fields that we don't know about.
-    precond(vmClasses::URL_klass()->is_final());
-
-    // URLs are referenced by the CodeSources/ProtectDomains that are cached
-    // for AOT-linked classes loaded by the platform/app loaders.
-    //
-    // Do not cache any URLs whose URLStreamHandler can be overridden by the application.
-    // - "jrt" and "file" will always use the built-in URLStreamHandler. See
-    //   java.net.URL::isOverrideable().
-    // -  When an AOT-linked class is loaded from a JAR file, its URL is something
-    //    like file:HelloWorl.jar, and does NOT use the "jar" protocol.
-    oop protocol = get_oop_field(obj, "protocol", "Ljava/lang/String;");
-    if (!java_lang_String::equals(protocol, "jrt", 3) &&
-        !java_lang_String::equals(protocol, "file", 4)) {
-      ResourceMark rm;
-      log_error(aot)("Must cache only URLs with jrt/file protocols but got: %s",
-                     java_lang_String::as_quoted_ascii(protocol));
-      HeapShared::debug_trace();
-      AOTMetaspace::unrecoverable_writing_error();
-    }
-  }
-}
-
-#endif //INCLUDE_CDS_JAVA_HEAP

src/hotspot/share/cds/cdsConfig.cpp

@@ -67,8 +67,7 @@ JavaThread* CDSConfig::_dumper_thread = nullptr;
 
 int CDSConfig::get_status() {
   assert(Universe::is_fully_initialized(), "status is finalized only after Universe is initialized");
-  return (is_dumping_aot_linked_classes()   ? IS_DUMPING_AOT_LINKED_CLASSES : 0) |
-         (is_dumping_archive()              ? IS_DUMPING_ARCHIVE : 0) |
+  return (is_dumping_archive()              ? IS_DUMPING_ARCHIVE : 0) |
          (is_dumping_method_handles()       ? IS_DUMPING_METHOD_HANDLES : 0) |
          (is_dumping_static_archive()       ? IS_DUMPING_STATIC_ARCHIVE : 0) |
          (is_logging_lambda_form_invokers() ? IS_LOGGING_LAMBDA_FORM_INVOKERS : 0) |

src/hotspot/share/cds/cdsConfig.hpp

@@ -80,12 +80,11 @@ class CDSConfig : public AllStatic {
 
 public:
   // Used by jdk.internal.misc.CDS.getCDSConfigStatus();
-  static const int IS_DUMPING_AOT_LINKED_CLASSES   = 1 << 0;
-  static const int IS_DUMPING_ARCHIVE              = 1 << 1;
-  static const int IS_DUMPING_METHOD_HANDLES       = 1 << 2;
-  static const int IS_DUMPING_STATIC_ARCHIVE       = 1 << 3;
-  static const int IS_LOGGING_LAMBDA_FORM_INVOKERS = 1 << 4;
-  static const int IS_USING_ARCHIVE                = 1 << 5;
+  static const int IS_DUMPING_ARCHIVE              = 1 << 0;
+  static const int IS_DUMPING_METHOD_HANDLES       = 1 << 1;
+  static const int IS_DUMPING_STATIC_ARCHIVE       = 1 << 2;
+  static const int IS_LOGGING_LAMBDA_FORM_INVOKERS = 1 << 3;
+  static const int IS_USING_ARCHIVE                = 1 << 4;
 
   static int get_status() NOT_CDS_RETURN_(0);
 

src/hotspot/share/cds/cdsEnumKlass.cpp

@@ -40,9 +40,7 @@ bool CDSEnumKlass::is_enum_obj(oop orig_obj) {
          InstanceKlass::cast(k)->is_enum_subclass();
 }
 
-// !!! This is legacy support for enum classes before JEP 483. This file is not used when
-// !!! CDSConfig::is_initing_classes_at_dump_time()==true.
-//
+// -- Handling of Enum objects
 // Java Enum classes have synthetic <clinit> methods that look like this
 //     enum MyEnum {FOO, BAR}
 //     MyEnum::<clinint> {
@@ -64,7 +62,6 @@ bool CDSEnumKlass::is_enum_obj(oop orig_obj) {
 void CDSEnumKlass::handle_enum_obj(int level,
                                    KlassSubGraphInfo* subgraph_info,
                                    oop orig_obj) {
-  assert(!CDSConfig::is_initing_classes_at_dump_time(), "only for legacy support of enums");
   assert(level > 1, "must never be called at the first (outermost) level");
   assert(is_enum_obj(orig_obj), "must be");