[wip] Pass opline as argument to opcode handlers in CALL VM

Zend/zend_portability.h

@@ -59,6 +59,7 @@
 
 #include <stdarg.h>
 #include <stddef.h>
+#include <stdint.h>
 
 #ifdef HAVE_DLFCN_H
 # include <dlfcn.h>
@@ -864,4 +865,20 @@ static zend_always_inline uint64_t ZEND_BYTES_SWAP64(uint64_t u)
 # define ZEND_OPCACHE_SHM_REATTACHMENT 1
 #endif
 
+#ifndef CHAR_BITS
+# define CHAR_BITS 8
+#endif
+
+#ifndef UINTPTR_WIDTH
+# define UINTPTR_WIDTH (CHAR_BITS * sizeof(uintptr_t))
+#endif
+
+#if (defined(__linux__) && defined(__x86_64__)) \
+		|| (defined(ZEND_WIN32) && defined(_M_AMD64))
+/* The kernel reserves the higher part of the address space for itself.
+ * Therefore, we can assume that the higher bit of user space addresses is
+ * never set. */
+# define ZEND_HIGH_HALF_KERNEL
+#endif
+
 #endif /* ZEND_PORTABILITY_H */

Zend/zend_vm_def.h

@@ -5602,11 +5602,10 @@ ZEND_VM_HOT_HANDLER(199, ZEND_CHECK_UNDEF_ARGS, UNUSED, UNUSED)
 
 ZEND_VM_COLD_HELPER(zend_missing_arg_helper, ANY, ANY)
 {
-#ifdef ZEND_VM_IP_GLOBAL_REG
 	USE_OPLINE
 
 	SAVE_OPLINE();
-#endif
+
 	zend_missing_arg_error(execute_data);
 	HANDLE_EXCEPTION();
 }

Zend/zend_vm_execute.skl

@@ -1,3 +1,5 @@
+#include "Zend/zend_vm_opcodes.h"
+
 {%DEFINES%}
 
 #if (ZEND_VM_KIND != ZEND_VM_KIND_CALL) && (ZEND_GCC_VERSION >= 4000) && !defined(__clang__)

ext/opcache/jit/zend_jit.c

@@ -98,7 +98,7 @@ static const void *zend_jit_func_trace_counter_handler = NULL;
 static const void *zend_jit_ret_trace_counter_handler = NULL;
 static const void *zend_jit_loop_trace_counter_handler = NULL;
 
-static int ZEND_FASTCALL zend_runtime_jit(void);
+static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
 
 static int zend_jit_trace_op_len(const zend_op *opline);
 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
@@ -2871,7 +2871,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
 							if (GCC_GLOBAL_REGS) {
 								ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
 							} else {
-								ir_RETURN(ir_CONST_I32(1)); /* ZEND_VM_ENTER */
+								zend_jit_vm_enter(jit, jit_IP(jit));
 							}
 							ir_IF_TRUE(if_hook_enter);
 						}
@@ -3074,11 +3074,18 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons
 }
 
 /* Run-time JIT handler */
-static int ZEND_FASTCALL zend_runtime_jit(void)
+static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
 {
-	zend_execute_data *execute_data = EG(current_execute_data);
+#if GCC_GLOBAL_REGS
+	zend_execute_data *execute_data;
+	zend_op *opline;
+#else
+	const zend_op *orig_opline = opline;
+#endif
+
+	execute_data = EG(current_execute_data);
 	zend_op_array *op_array = &EX(func)->op_array;
-	zend_op *opline = op_array->opcodes;
+	opline = op_array->opcodes;
 	zend_jit_op_array_extension *jit_extension;
 	bool do_bailout = 0;
 
@@ -3097,7 +3104,7 @@ static int ZEND_FASTCALL zend_runtime_jit(void)
 				}
 			}
 			jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
-			opline->handler = jit_extension->orig_handler;
+			((zend_op*)opline)->handler = jit_extension->orig_handler;
 
 			/* perform real JIT for this function */
 			zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);
@@ -3116,7 +3123,11 @@ static int ZEND_FASTCALL zend_runtime_jit(void)
 	}
 
 	/* JIT-ed code is going to be called by VM */
-	return 0;
+#if GCC_GLOBAL_REGS
+	return; // ZEND_VM_CONTINUE
+#else
+	return orig_opline; // ZEND_VM_CONTINUE
+#endif
 }
 
 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {

ext/opcache/jit/zend_jit_internal.h

@@ -21,6 +21,12 @@
 #ifndef ZEND_JIT_INTERNAL_H
 #define ZEND_JIT_INTERNAL_H
 
+#include "Zend/zend_types.h"
+#include "Zend/zend_compile.h"
+#include "Zend/zend_constants.h"
+#include "Zend/Optimizer/zend_func_info.h"
+#include "Zend/Optimizer/zend_call_graph.h"
+
 /* Address Encoding */
 typedef uintptr_t zend_jit_addr;
 
@@ -183,17 +189,19 @@ extern const zend_op *zend_jit_halt_op;
 # define ZEND_OPCODE_HANDLER_RET              void
 # define ZEND_OPCODE_HANDLER_ARGS             EXECUTE_DATA_D
 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU
-# define ZEND_OPCODE_HANDLER_ARGS_DC
-# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC
+# define ZEND_OPCODE_HANDLER_ARGS_EX
+# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX
 # define ZEND_OPCODE_RETURN()                 return
 # define ZEND_OPCODE_TAIL_CALL(handler)       do { \
 		handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
 		return; \
 	} while(0)
 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
-		handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
+		handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX arg); \
 		return; \
 	} while(0)
+# define ZEND_VM_ENTER_BIT 0
+# define ZEND_VM_RETURN_VAL 0
 #else
 # define EXECUTE_DATA_D                       zend_execute_data* execute_data
 # define EXECUTE_DATA_C                       execute_data
@@ -203,27 +211,34 @@ extern const zend_op *zend_jit_halt_op;
 # define OPLINE_C                             opline
 # define OPLINE_DC                            , OPLINE_D
 # define OPLINE_CC                            , OPLINE_C
-# define ZEND_OPCODE_HANDLER_RET              int
-# define ZEND_OPCODE_HANDLER_ARGS             EXECUTE_DATA_D
-# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU    EXECUTE_DATA_C
-# define ZEND_OPCODE_HANDLER_ARGS_DC          EXECUTE_DATA_DC
-# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC EXECUTE_DATA_CC
-# define ZEND_OPCODE_RETURN()                 return 0
+# define ZEND_OPCODE_HANDLER_RET              const zend_op *
+# define ZEND_OPCODE_HANDLER_ARGS             EXECUTE_DATA_D OPLINE_DC
+# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU    EXECUTE_DATA_C OPLINE_CC
+# define ZEND_OPCODE_HANDLER_ARGS_EX          EXECUTE_DATA_D OPLINE_DC,
+# define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX EXECUTE_DATA_C OPLINE_CC,
+# define ZEND_OPCODE_RETURN()                 return opline
 # define ZEND_OPCODE_TAIL_CALL(handler)       do { \
 		return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
 	} while(0)
 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
-		return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
+		return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX arg); \
 	} while(0)
+# ifdef ZEND_HIGH_HALF_KERNEL
+#  define ZEND_VM_ENTER_BIT (1ULL<<(UINTPTR_WIDTH-1))
+#  define ZEND_VM_RETURN_VAL 0
+# else
+#  define ZEND_VM_ENTER_BIT 1ULL
+#  define ZEND_VM_RETURN_VAL ZEND_VM_ENTER_BIT
+# endif
 #endif
 
 /* VM handlers */
 typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS);
 
 /* VM helpers */
-ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC);
-ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC);
-ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info);
+ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(ZEND_OPCODE_HANDLER_ARGS);
 
 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);