GC update for supporting yield of pinned VirtualThread(JEP491)

In order to supporting yield of pinned VirtualThread
a couple of heap references have been added in chain of continuation
structures, GC need to maintain(scan/update) these references.

1, new ContunuationSlotIterator to iterate all of the references
 list of monitorRecord
 list of jniMonitorRecord
 vthread reference
ContunuationSlotIterator also contains state and related vmThread
state (can be state_monitor_records or state_vthread) for assert check
in doContinuationSlot() (monitor_records can be non heap reference),
vmThread is for retrieving related carrier context during rootscanning.

2, two new methods doSlot() and doContinuationSlot() are introduced
doSlot() would handle basic processing, which can be shared between
doContinuationSlot() and doStackSlot().
doContinuationSlot() would have condition check for heap reference and
also get ContunuationSlotIterator as a parameter for the extra assert
checks.

3, mounted continuations are scanned during rootscan/scanOneThread
unmounted continuations are scanned during ContinuationNativeSlots

4, for doContinuationSlot in CopyForwardScheme case
the reservingContext (numa node related) can be retrieved via
ContunuationSlotIterator (carrier thread context) for root scanning
phase, and retrieved via the region of related Continuation Object
for heap scanning phase.

Signed-off-by: lhu <linhu@ca.ibm.com>

Author: LinHu2016

runtime/gc_base/ReferenceChainWalker.cpp

@@ -32,6 +32,9 @@
 
 #include "ClassIteratorClassSlots.hpp"
 #include "ClassIteratorDeclarationOrder.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "EnvironmentBase.hpp"
 #include "Forge.hpp"
 #include "GCExtensions.hpp"
@@ -418,6 +421,16 @@ MM_ReferenceChainWalker::scanContinuationNativeSlots(J9Object *objectPtr)
 		}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 		GC_VMThreadStackSlotIterator::scanContinuationSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForReferenceChainWalker, false, _trackVisibleStackFrameDepth);
+
+#if JAVA_SPEC_VERSION >= 24
+		J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, objectPtr);
+		GC_ContinuationSlotIterator continuationSlotIterator(currentThread, continuation);
+
+		while (J9Object **slotPtr = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(slotPtr, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	}
 }
 
@@ -651,6 +664,31 @@ MM_ReferenceChainWalker::doVMClassSlot(J9Class *classPtr)
 	doClassSlot(classPtr, J9GC_ROOT_TYPE_VM_CLASS_SLOT, -1, NULL);
 }
 
+#if JAVA_SPEC_VERSION >= 24
+/**
+ * @todo Provide function documentation
+ */
+void
+MM_ReferenceChainWalker::doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	J9Object *slotValue = *slotPtr;
+	/* Only report heap objects */
+	if (isHeapObject(slotValue) && !_heap->objectIsInGap(slotValue)) {
+		switch(continuationSlotIterator->getState()) {
+		case GC_ContinuationSlotIterator::state_monitor_records:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_CONTINUATION_MONITOR, -1, NULL);
+			break;
+		case GC_ContinuationSlotIterator::state_vthread:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_CONTINUATION_VTHREAD, -1, NULL);
+			break;
+		default:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_UNKNOWN, -1, NULL);
+			break;
+		}
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 /**
  * @todo Provide function documentation
  */

runtime/gc_base/ReferenceChainWalker.hpp

@@ -127,6 +127,9 @@ class MM_ReferenceChainWalker : public MM_RootScanner
 	virtual void doStringTableSlot(J9Object **slotPtr, GC_StringTableIterator *stringTableIterator);
 	virtual void doVMClassSlot(J9Class *classPtr);
 	virtual void doVMThreadSlot(J9Object **slotPtr, GC_VMThreadIterator *vmThreadIterator);
+#if JAVA_SPEC_VERSION >= 24
+	virtual void doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	virtual void doStackSlot(J9Object **slotPtr, void *walkState, const void* stackLocation);
 	virtual void doSlot(J9Object **slotPtr);
 	virtual void doClassSlot(J9Class *classPtr);

runtime/gc_base/RootScanner.cpp

@@ -356,6 +356,20 @@ MM_RootScanner::doClassSlot(J9Class *classPtr)
 	/* ignore class slots by default */
 }
 
+#if JAVA_SPEC_VERSION >= 24
+/**
+ * @todo Provide function documentation
+ */
+void
+MM_RootScanner::doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	/* ensure that this isn't a slot pointing into the gap (only matters for split heap VMs) */
+	if (!_extensions->heap->objectIsInGap(*slotPtr)) {
+		doSlot(slotPtr);
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 /**
  * @todo Provide function documentation
  */
@@ -552,6 +566,14 @@ MM_RootScanner::scanOneThread(MM_EnvironmentBase *env, J9VMThread *walkThread, v
 		/* At this point we know that a virtual thread is mounted. We previously scanned its stack,
 		 * and now we will scan carrier's stack, that continuation struct is currently pointing to. */
 		GC_VMThreadStackSlotIterator::scanSlots(currentThread, walkThread, walkThread->currentContinuation, localData, stackSlotIterator, isStackFrameClassWalkNeeded(), _trackVisibleStackFrameDepth);
+#if JAVA_SPEC_VERSION >= 24
+		GC_ContinuationSlotIterator continuationSlotIterator(walkThread, walkThread->currentContinuation);
+
+		while (J9Object **slot = continuationSlotIterator.nextSlot()) {
+			/* do current continuation slot (mounted vthread case, the slot for saved carrier thread) */
+			doContinuationSlot(slot, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 	return false;

runtime/gc_base/RootScanner.hpp

@@ -35,6 +35,9 @@
 
 #include "BaseVirtual.hpp"
 
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "EnvironmentBase.hpp"
 #include "GCExtensions.hpp"
 #include "JVMTIObjectTagTableIterator.hpp"
@@ -582,6 +585,10 @@ class MM_RootScanner : public MM_BaseVirtual
 	virtual void doDoubleMappedObjectSlot(J9Object *objectPtr, struct J9PortVmemIdentifier *identifier);
 #endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 
+#if JAVA_SPEC_VERSION >= 24
+	virtual void doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	/**
 	 * Called for each object stack slot. Subclasses may override.
 	 * 

runtime/gc_glue_java/CompactSchemeFixupObject.cpp

@@ -26,17 +26,21 @@
 #include "objectdescription.h"
 
 #include "CollectorLanguageInterfaceImpl.hpp"
-#include "EnvironmentStandard.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "Debug.hpp"
+#include "EnvironmentStandard.hpp"
+#include "FlattenedContiguousArrayIterator.hpp"
 #include "HeapRegionIteratorStandard.hpp"
 #include "MixedObjectIterator.hpp"
 #include "ObjectAccessBarrier.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
-#include "VMHelpers.hpp"
 #include "ParallelDispatcher.hpp"
 #include "PointerContiguousArrayIterator.hpp"
-#include "FlattenedContiguousArrayIterator.hpp"
+#include "StackSlotValidator.hpp"
 #include "Task.hpp"
+#include "VMHelpers.hpp"
 
 void
 MM_CompactSchemeFixupObject::fixupMixedObject(omrobjectptr_t objectPtr)
@@ -50,19 +54,38 @@ MM_CompactSchemeFixupObject::fixupMixedObject(omrobjectptr_t objectPtr)
 }
 
 void
-MM_CompactSchemeFixupObject::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot)
+MM_CompactSchemeFixupObject::doSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr)
+{
+	*slotPtr = _compactScheme->getForwardingPtr(*slotPtr);
+}
+
+#if JAVA_SPEC_VERSION >= 24
+void
+MM_CompactSchemeFixupObject::doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
 {
-	*slot = _compactScheme->getForwardingPtr(*slot);
+	if (isHeapObject(*slotPtr) && !_extensions->heap->objectIsInGap(*slotPtr)) {
+		doSlot(env, fromObject, slotPtr);
+	} else if (NULL != *slotPtr) {
+		Assert_MM_true(GC_ContinuationSlotIterator::state_monitor_records == continuationSlotIterator->getState());
+	}
 }
+#endif /* JAVA_SPEC_VERSION >= 24 */
 
+void
+MM_CompactSchemeFixupObject::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation)
+{
+	if (isHeapObject(*slotPtr) && !_extensions->heap->objectIsInGap(*slotPtr)) {
+		doSlot(env, fromObject, slotPtr);
+	}
+}
 /**
  * @todo Provide function documentation
  */
 void
 stackSlotIteratorForCompactScheme(J9JavaVM *javaVM, J9Object **slotPtr, void *localData, J9StackWalkState *walkState, const void *stackLocation)
 {
 	StackIteratorData4CompactSchemeFixupObject *data = (StackIteratorData4CompactSchemeFixupObject *)localData;
-	data->compactSchemeFixupObject->doStackSlot(data->env, data->fromObject, slotPtr);
+	data->compactSchemeFixupObject->doStackSlot(data->env, data->fromObject, slotPtr, walkState, stackLocation);
 }
 
 
@@ -85,6 +108,16 @@ MM_CompactSchemeFixupObject::fixupContinuationNativeSlots(MM_EnvironmentStandard
 		localData.fromObject = objectPtr;
 
 		GC_VMThreadStackSlotIterator::scanContinuationSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForCompactScheme, false, false);
+
+#if JAVA_SPEC_VERSION >= 24
+		J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, objectPtr);
+		GC_ContinuationSlotIterator continuationSlotIterator(currentThread, continuation);
+
+		while (J9Object **slotPtr = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(env, objectPtr, slotPtr, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	}
 }
 

runtime/gc_glue_java/CompactSchemeFixupObject.hpp

@@ -29,6 +29,10 @@
 #include "CompactScheme.hpp"
 #include "GCExtensions.hpp"
 
+#if JAVA_SPEC_VERSION >= 24
+class GC_ContinuationSlotIterator;
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 class MM_CompactSchemeFixupObject {
 public:
 protected:
@@ -38,7 +42,11 @@ class MM_CompactSchemeFixupObject {
 	MM_CompactScheme *_compactScheme;
 public:
 
-	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot);
+	MMINLINE void doSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr);
+#if JAVA_SPEC_VERSION >= 24
+	void doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation);
 	/**
 	 * Perform fixup for a single object
 	 * @param env[in] the current thread
@@ -82,6 +90,15 @@ class MM_CompactSchemeFixupObject {
 	 * @param object -- The object of type or subclass of java.util.concurrent.locks.AbstractOwnableSynchronizer.
 	 */
 	MMINLINE void addOwnableSynchronizerObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
+
+	/**
+	 * Determine whether the object pointer is found within the heap proper.
+	 * @return Boolean indicating if the object pointer is within the heap boundaries.
+	 */
+	MMINLINE bool isHeapObject(J9Object *objectPtr)
+	{
+		return (_extensions->heap->getHeapBase() <= (uint8_t *)objectPtr) && (_extensions->heap->getHeapTop() > (uint8_t *)objectPtr);
+	}
 };
 
 typedef struct StackIteratorData4CompactSchemeFixupObject {

runtime/gc_glue_java/ConcurrentMarkingDelegate.cpp

@@ -25,6 +25,9 @@
 #include "AsyncCallbackHandler.hpp"
 #include "ClassLoaderIterator.hpp"
 #include "ConfigurationDelegate.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "FinalizeListManager.hpp"
 #include "Heap.hpp"
 #include "HeapRegionDescriptorStandard.hpp"
@@ -35,6 +38,32 @@
 #include "VMInterface.hpp"
 #include "VMThreadListIterator.hpp"
 
+void
+MM_ConcurrentMarkingDelegate::doSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr)
+{
+	_markingScheme->markObject(env, *slotPtr);
+}
+
+#if JAVA_SPEC_VERSION >= 24
+void
+MM_ConcurrentMarkingDelegate::doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	if (_markingScheme->isHeapObject(*slotPtr) && !env->getExtensions()->heap->objectIsInGap(*slotPtr)) {
+		doSlot(env, slotPtr);
+	} else if (NULL != *slotPtr) {
+		Assert_MM_true(GC_ContinuationSlotIterator::state_monitor_records == continuationSlotIterator->getState());
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
+void
+MM_ConcurrentMarkingDelegate::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation)
+{
+	omrobjectptr_t object = *slotPtr;
+	if (_markingScheme->isHeapObject(object) && !env->getExtensions()->heap->objectIsInGap(object)) {
+		doSlot(env, slotPtr);
+	}
+}
 /**
  * Concurrents stack slot iterator.
  * Called for each slot in a threads active stack frames which contains a object reference.
@@ -48,19 +77,7 @@ void
 concurrentStackSlotIterator(J9JavaVM *javaVM, omrobjectptr_t *objectIndirect, void *localData, J9StackWalkState *walkState, const void *stackLocation)
 {
 	MM_ConcurrentMarkingDelegate::markSchemeStackIteratorData *data = (MM_ConcurrentMarkingDelegate::markSchemeStackIteratorData *)localData;
-
-	omrobjectptr_t object = *objectIndirect;
-	if (data->env->getExtensions()->heap->objectIsInGap(object)) {
-		/* CMVC 136483:  Ensure that the object is not in the gap of a split heap (stack-allocated object) since we can't mark that part of the address space */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(data->env));
-	} else if (data->markingScheme->isHeapObject(object)) {
-		/* heap object - validate and mark */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(0, object, stackLocation, walkState).validate(data->env));
-		data->markingScheme->markObject(data->env, object);
-	} else if (NULL != object) {
-		/* stack object - just validate */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(data->env));
-	}
+	data->concurrentMarkingDelegate->doStackSlot(data->env, objectIndirect, walkState, stackLocation);
 }
 
 bool
@@ -150,14 +167,21 @@ MM_ConcurrentMarkingDelegate::scanThreadRoots(MM_EnvironmentBase *env)
 	}
 
 	markSchemeStackIteratorData localData;
-	localData.markingScheme = _markingScheme;
+	localData.concurrentMarkingDelegate = this;
 	localData.env = env;
 	/* In a case this thread is a carrier thread, and a virtual thread is mounted, we will scan virtual thread's stack. */
 	GC_VMThreadStackSlotIterator::scanSlots(vmThread, vmThread, (void *)&localData, concurrentStackSlotIterator, true, false);
 
 #if JAVA_SPEC_VERSION >= 19
 	if (NULL != vmThread->currentContinuation) {
 		GC_VMThreadStackSlotIterator::scanSlots(vmThread, vmThread, vmThread->currentContinuation, (void *)&localData, concurrentStackSlotIterator, true, false);
+#if JAVA_SPEC_VERSION >= 24
+		GC_ContinuationSlotIterator continuationSlotIterator(vmThread, vmThread->currentContinuation);
+
+		while (J9Object **slot = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(env, slot, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 

runtime/gc_glue_java/ConcurrentMarkingDelegate.hpp

@@ -46,6 +46,9 @@
 #include "ScanClassesMode.hpp"
 #endif /* J9VM_GC_DYNAMIC_CLASS_UNLOADING */
 
+#if JAVA_SPEC_VERSION >= 24
+class GC_ContinuationSlotIterator;
+#endif /* JAVA_SPEC_VERSION >= 24 */
 class GC_VMThreadIterator;
 class MM_ConcurrentGC;
 class MM_MarkingScheme;
@@ -87,7 +90,7 @@ class MM_ConcurrentMarkingDelegate
 	};
 
 	typedef struct markSchemeStackIteratorData {
-		MM_MarkingScheme *markingScheme;
+		MM_ConcurrentMarkingDelegate *concurrentMarkingDelegate;
 		MM_EnvironmentBase *env;
 	} markSchemeStackIteratorData;
 
@@ -114,6 +117,11 @@ class MM_ConcurrentMarkingDelegate
 	 */
 	bool initialize(MM_EnvironmentBase *env, MM_ConcurrentGC *collector);
 
+	MMINLINE void doSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr);
+#if JAVA_SPEC_VERSION >= 24
+	void doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation);
 	/**
 	 * In the case of Weak Consistency platforms we require this method to bring mutator threads to a safe point. A safe
 	 * point is a point at which a GC may occur.

runtime/gc_glue_java/HeapWalkerDelegate.cpp

@@ -24,6 +24,9 @@
 #include "HeapWalkerDelegate.hpp"
 
 #include "j9.h"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "EnvironmentBase.hpp"
 #include "GCExtensions.hpp"
 #include "ObjectModel.hpp"
@@ -72,5 +75,15 @@ MM_HeapWalkerDelegate::doContinuationNativeSlots(MM_EnvironmentBase *env, omrobj
 		localData.userData = userData;
 
 		GC_VMThreadStackSlotIterator::scanContinuationSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForHeapWalker, false, false);
+
+#if JAVA_SPEC_VERSION >= 24
+		J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, objectPtr);
+		GC_ContinuationSlotIterator continuationSlotIterator(currentThread, continuation);
+
+		while (J9Object **slotPtr = continuationSlotIterator.nextSlot()) {
+			_heapWalker->heapWalkerSlotCallback(env, slotPtr, function, userData);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	}
 }