Prevent contended monitor enters at CRIU single thread mode

objectMonitorEnterNonBlocking() returns
J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW instead of
J9_OBJECT_MONITOR_BLOCKING;
Updated the callers to handle this additional return value
J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW;
Add tests.

Signed-off-by: Jason Feng <fengj@ca.ibm.com>

Author: JasonFengJ9

runtime/codert_vm/cnathelp.cpp

@@ -417,6 +417,16 @@ setNativeOutOfMemoryErrorFromJIT(J9VMThread *currentThread, U_32 moduleName, U_3
 	return J9_JITHELPER_ACTION_THROW;
 }
 
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+static VMINLINE void*
+setCRIUSingleThreadModeExceptionFromJIT(J9VMThread *currentThread, U_32 moduleName, U_32 messageNumber)
+{
+	TIDY_BEFORE_THROW();
+	currentThread->javaVM->internalVMFunctions->setCRIUSingleThreadModeJVMCRIUException(currentThread, moduleName, messageNumber);
+	return J9_JITHELPER_ACTION_THROW;
+}
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+
 static VMINLINE void*
 setHeapOutOfMemoryErrorFromJIT(J9VMThread *currentThread)
 {
@@ -1725,8 +1735,9 @@ slow_jitMonitorEnterImpl(J9VMThread *currentThread, bool forMethod)
 				resolveFrame->specialFrameFlags = (resolveFrame->specialFrameFlags & ~J9_STACK_FLAGS_JIT_FRAME_SUB_TYPE_MASK) | J9_STACK_FLAGS_JIT_FAILED_METHOD_MONITOR_ENTER_RESOLVE;
 			}
 		}
+		switch (monstatus) {
 #if JAVA_SPEC_VERSION >= 16
-		if (J9_OBJECT_MONITOR_VALUE_TYPE_IMSE == monstatus) {
+		case J9_OBJECT_MONITOR_VALUE_TYPE_IMSE: {
 			j9object_t syncObject = (j9object_t)currentThread->floatTemp2;
 			J9Class* badClass = J9OBJECT_CLAZZ(currentThread, syncObject);
 			J9UTF8 *className = J9ROMCLASS_CLASSNAME(badClass->romClass);
@@ -1740,16 +1751,24 @@ slow_jitMonitorEnterImpl(J9VMThread *currentThread, bool forMethod)
 			{
 				Assert_CodertVM_true(J9_ARE_ALL_BITS_SET(currentThread->javaVM->extendedRuntimeFlags2, J9_EXTENDED_RUNTIME2_VALUE_BASED_EXCEPTION));
 				currentThread->javaVM->internalVMFunctions->setCurrentExceptionNLSWithArgs(currentThread, J9NLS_VM_ERROR_BYTECODE_OBJECTREF_CANNOT_BE_VALUE_BASED, 
-						J9VMCONSTANTPOOL_JAVALANGVIRTUALMACHINEERROR, J9UTF8_LENGTH(className), J9UTF8_DATA(className));
+					J9VMCONSTANTPOOL_JAVALANGVIRTUALMACHINEERROR, J9UTF8_LENGTH(className), J9UTF8_DATA(className));
 			}
 			addr = J9_JITHELPER_ACTION_THROW;
-			goto done;
+			break;
 		}
 #endif /* JAVA_SPEC_VERSION >= 16 */
-		if (J9_OBJECT_MONITOR_OOM == monstatus) {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+		case J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW:
+			addr = setCRIUSingleThreadModeExceptionFromJIT(currentThread, J9NLS_VM_CRIU_SINGLETHREADMODE_JVMCRIUEXCEPTION);
+			break;
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+		case J9_OBJECT_MONITOR_OOM:
 			addr = setNativeOutOfMemoryErrorFromJIT(currentThread, J9NLS_VM_FAILED_TO_ALLOCATE_MONITOR);
-			goto done;
+			break;
+		default:
+			Assert_CodertVM_unreachable();
 		}
+		goto done;
 	} else {
 		currentThread->javaVM->internalVMFunctions->objectMonitorEnterBlocking(currentThread);
 		/* Do not check asyncs for synchronized block (non-method) entry, since we now have the monitor,

runtime/jcl/common/sun_misc_Unsafe.cpp

@@ -405,9 +405,16 @@ Java_sun_misc_Unsafe_monitorEnter(JNIEnv *env, jobject receiver, jobject obj)
 		vmFuncs->setCurrentExceptionUTF(currentThread, J9VMCONSTANTPOOL_JAVALANGNULLPOINTEREXCEPTION, NULL);
 	} else {
 		UDATA monresult = vmFuncs->objectMonitorEnter(currentThread, J9_JNI_UNWRAP_REFERENCE(obj));
-		if (0 == monresult) {
+		if (J9_OBJECT_MONITOR_ENTER_FAILED(monresult)) {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+			if (J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW == monresult) {
+				vmFuncs->setCRIUSingleThreadModeJVMCRIUException(currentThread, 0, 0);
+			} else
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+			if (J9_OBJECT_MONITOR_OOM == monresult) {
 oom:
-			vmFuncs->setNativeOutOfMemoryError(currentThread, 0, 0);
+				vmFuncs->setNativeOutOfMemoryError(currentThread, 0, 0);
+			}
 		} else if (J9_UNEXPECTED(!VM_ObjectMonitor::recordJNIMonitorEnter(currentThread, (j9object_t)monresult))) {
 			vmFuncs->objectMonitorExit(currentThread, (j9object_t)monresult);
 			goto oom;

runtime/jvmti/jvmtiThreadGroup.c

@@ -183,8 +183,16 @@ jvmtiGetThreadGroupChildren(jvmtiEnv* env,
 			childrenGroupsLock = (j9object_t)vmFuncs->objectMonitorEnter(currentThread, childrenGroupsLock);
 			/* The threadGroupObject has to be reobtained as it might have been GC'ed while waiting for the lock */
 			threadGroupObject = *((j9object_t*)group);
-			if (J9_OBJECT_MONITOR_ENTER_FAILED((UDATA)childrenGroupsLock)) {
-				rc = JVMTI_ERROR_OUT_OF_MEMORY;
+			if (J9_OBJECT_MONITOR_ENTER_FAILED(childrenGroupsLock)) {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+				if (J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW == (UDATA)childrenGroupsLock) {
+					vmFuncs->setCRIUSingleThreadModeJVMCRIUException(currentThread, 0, 0);
+					rc = JVMTI_ERROR_THREAD_SUSPENDED;
+				} else
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+				if (J9_OBJECT_MONITOR_OOM == (UDATA)childrenGroupsLock) {
+					rc = JVMTI_ERROR_OUT_OF_MEMORY;
+				}
 				goto done;
 			}
 
@@ -211,8 +219,16 @@ jvmtiGetThreadGroupChildren(jvmtiEnv* env,
 			childrenThreadsLock = (j9object_t)vmFuncs->objectMonitorEnter(currentThread, childrenThreadsLock);
 			/* The threadGroupObject has to be reobtained as it might have been GC'ed while waiting for the lock */
 			threadGroupObject = *((j9object_t*)group);
-			if (J9_OBJECT_MONITOR_ENTER_FAILED((UDATA)childrenThreadsLock)) {
-				rc = JVMTI_ERROR_OUT_OF_MEMORY;
+			if (J9_OBJECT_MONITOR_ENTER_FAILED(childrenThreadsLock)) {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+				if (J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW == (UDATA)childrenThreadsLock) {
+					vmFuncs->setCRIUSingleThreadModeJVMCRIUException(currentThread, 0, 0);
+					rc = JVMTI_ERROR_THREAD_SUSPENDED;
+				} else
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+				if (J9_OBJECT_MONITOR_OOM == (UDATA)childrenThreadsLock) {
+					rc = JVMTI_ERROR_OUT_OF_MEMORY;
+				}
 				j9mem_free_memory(groups);
 				goto done;
 			}

runtime/nls/j9vm/j9vm.nls

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2000, 2021 IBM Corp. and others
+# Copyright (c) 2000, 2022 IBM Corp. and others
 #
 # This program and the accompanying materials are made available under
 # the terms of the Eclipse Public License 2.0 which accompanies this
@@ -2097,3 +2097,11 @@ J9NLS_ENSUREHASHED_CONFIG_OUT_OF_MEM.explanation=There was a failure to allocate
 J9NLS_ENSUREHASHED_CONFIG_OUT_OF_MEM.system_action=The JVM prints this message and terminates.
 J9NLS_ENSUREHASHED_CONFIG_OUT_OF_MEM.user_response=Check the system memory configuration.
 # END NON-TRANSLATABLE
+
+# JVMCRIUException is thrown to indicate that blocking operation is not allowed in CRIU single thread mode
+J9NLS_VM_CRIU_SINGLETHREADMODE_JVMCRIUEXCEPTION=Blocking operation is not allowed in CRIU single thread mode.
+# START NON-TRANSLATABLE
+J9NLS_VM_CRIU_SINGLETHREADMODE_JVMCRIUEXCEPTION.explanation=Blocking operation is not allowed in CRIU single thread mode.
+J9NLS_VM_CRIU_SINGLETHREADMODE_JVMCRIUEXCEPTION.system_action=The JVM will throw a JVMCRIUException.
+J9NLS_VM_CRIU_SINGLETHREADMODE_JVMCRIUEXCEPTION.user_response=View CRIU documentation to determine how to resolve the exception.
+# END NON-TRANSLATABLE

runtime/oti/j9nonbuilder.h

@@ -4867,6 +4867,7 @@ typedef struct J9InternalVMFunctions {
 	BOOLEAN (*runInternalJVMCheckpointHooks)(struct J9VMThread *currentThread);
 	BOOLEAN (*runInternalJVMRestoreHooks)(struct J9VMThread *currentThread);
 	BOOLEAN (*runDelayedLockRelatedOperations)(struct J9VMThread *currentThread);
+	void (*setCRIUSingleThreadModeJVMCRIUException)(struct J9VMThread *vmThread, U_32 moduleName, U_32 messageNumber);
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
 	j9object_t (*getClassNameString)(struct J9VMThread *currentThread, j9object_t classObject, jboolean internAndAssign);
 	j9object_t* (*getDefaultValueSlotAddress)(struct J9Class *clazz);
@@ -5692,18 +5693,16 @@ typedef struct J9JavaVM {
 /* objectMonitorEnterNonBlocking return codes */
 #define J9_OBJECT_MONITOR_OOM 0
 #define J9_OBJECT_MONITOR_VALUE_TYPE_IMSE 1
-/*
- * Currently, not needed but reserving it for future use
- *
- * #define J9_OBJECT_MONITOR_PRIMITIVE_WRAPPER_IMSE 2
- */
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+#define J9_OBJECT_MONITOR_CRIU_SINGLE_THREAD_MODE_THROW 2
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
 #define J9_OBJECT_MONITOR_BLOCKING 3
 
-#if JAVA_SPEC_VERSION >= 16
-#define J9_OBJECT_MONITOR_ENTER_FAILED(rc) ((rc) < J9_OBJECT_MONITOR_BLOCKING)
-#else /* JAVA_SPEC_VERSION >= 16 */
-#define J9_OBJECT_MONITOR_ENTER_FAILED(rc) ((rc) == J9_OBJECT_MONITOR_OOM)
-#endif /* JAVA_SPEC_VERSION >= 16 */
+#if (JAVA_SPEC_VERSION >= 16) || defined(J9VM_OPT_CRIU_SUPPORT)
+#define J9_OBJECT_MONITOR_ENTER_FAILED(rc) ((UDATA)(rc) < J9_OBJECT_MONITOR_BLOCKING)
+#else /* (JAVA_SPEC_VERSION >= 16) || defined(J9VM_OPT_CRIU_SUPPORT) */
+#define J9_OBJECT_MONITOR_ENTER_FAILED(rc) ((UDATA)(rc) == J9_OBJECT_MONITOR_OOM)
+#endif /* (JAVA_SPEC_VERSION >= 16) || defined(J9VM_OPT_CRIU_SUPPORT) */
 #define J9JAVAVM_REFERENCE_SIZE(vm) (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm) ? sizeof(U_32) : sizeof(UDATA))
 #define J9JAVAVM_OBJECT_HEADER_SIZE(vm) (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm) ? sizeof(J9ObjectCompressed) : sizeof(J9ObjectFull))
 #define J9JAVAVM_CONTIGUOUS_HEADER_SIZE(vm) (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm) ? sizeof(J9IndexableObjectContiguousCompressed) : sizeof(J9IndexableObjectContiguousFull))

runtime/oti/vm_api.h

@@ -968,6 +968,19 @@ setClassLoadingConstraintSignatureError(J9VMThread *currentThread, J9ClassLoader
 void
 setClassLoadingConstraintOverrideError(J9VMThread *currentThread, J9UTF8 *newClassNameUTF, J9ClassLoader *loader1, J9UTF8 *class1NameUTF, J9ClassLoader *loader2, J9UTF8 *class2NameUTF, J9UTF8 *exceptionClassNameUTF, U_8 *methodName, UDATA methodNameLength, U_8 *signature, UDATA signatureLength);
 
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+/**
+ * @brief Set JVMCRIUException to indicate that current operation is not allowed in CRIU single thread mode.
+ * @param vmThread[in] the current J9VMThread
+ * @param moduleName[in] the NLS module
+ * @param messageNumber[in] the NLS message
+ *
+ * @return void
+ */
+void
+setCRIUSingleThreadModeJVMCRIUException(J9VMThread *vmThread, U_32 moduleName, U_32 messageNumber);
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+
 /* ---------------- extendedMessageNPE.cpp ---------------- */
 
 /**

runtime/oti/vmconstantpool.xml

@@ -151,6 +151,9 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti
 
 	<classref name="jdk/internal/vm/Continuation" versions="19-"/>
 
+	<classref name="org/eclipse/openj9/criu/JVMCheckpointException" flags="opt_criuSupport"/>
+	<classref name="org/eclipse/openj9/criu/RestoreException" flags="opt_criuSupport"/>
+
 <!--
 	NOTE: the resolution code in jclcinit.c only looks at the J9ROMClassRef->runtimeFlags to determine
 	whether or not to attempt the class load.  The flags sepcified on the fields are ignored.

runtime/vm/BytecodeAction.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2001, 2020 IBM Corp. and others
+ * Copyright (c) 2001, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -57,6 +57,9 @@ typedef enum {
 #if JAVA_SPEC_VERSION >= 16
 	THROW_VALUE_TYPE_ILLEGAL_MONITOR_STATE,
 #endif /* JAVA_SPEC_VERSION >= 16 */
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+	THROW_CRIU_SINGLE_THREAD_MODE,
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
 	/* All values after this line are for the debug interpreter only - add general values above this line */
 	GOTO_EXECUTE_BREAKPOINTED_BYTECODE,
 	HANDLE_POP_FRAMES,