Add test case for interpreter generator w/ overridden opcodes

Author: DinoV

Include/internal/pycore_code.h

@@ -563,7 +563,7 @@ _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co)
 
 // Return a pointer to the thread-local bytecode for the current thread,
 // creating it if necessary.
-extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co);
+PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co);
 
 // Reserve an index for the current thread into thread-local bytecode
 // arrays

Include/internal/pycore_instruments.h

@@ -67,8 +67,8 @@ _Py_call_instrumentation_exc2(PyThreadState *tstate, int event,
 extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, int index);
 
-extern PyObject _PyInstrumentation_MISSING;
-extern PyObject _PyInstrumentation_DISABLE;
+PyAPI_DATA(PyObject) _PyInstrumentation_MISSING;
+PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE;
 
 
 /* Total tool ids available */

Include/internal/pycore_interpframe.h

@@ -279,7 +279,7 @@ _PyThreadState_GetFrame(PyThreadState *tstate)
 
 /* For use by _PyFrame_GetFrameObject
   Do not call directly. */
-PyFrameObject *
+PyAPI_FUNC(PyFrameObject *)
 _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame);
 
 /* Gets the PyFrameObject for this frame, lazily

Include/internal/pycore_list.h

@@ -17,12 +17,12 @@ PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*);
 extern void _PyList_DebugMallocStats(FILE *out);
 // _PyList_GetItemRef should be used only when the object is known as a list
 // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does.
-extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i);
+PyAPI_FUNC(PyObject *) _PyList_GetItemRef(PyListObject *, Py_ssize_t i);
 
 
 #ifdef Py_GIL_DISABLED
 // Returns -1 in case of races with other threads.
-extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *);
+PyAPI_FUNC(int) _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *);
 #endif
 
 #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)

Include/internal/pycore_opcode_metadata.h

@@ -11,8 +11,8 @@ extern "C" {
 
 /* Error handling definitions */
 
-extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate);
-extern PyObject* _PyErr_GetHandledException(PyThreadState *);
+PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate);
+PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *);
 extern void _PyErr_SetHandledException(PyThreadState *, PyObject *);
 extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **);
 
@@ -108,7 +108,7 @@ extern void _PyErr_Restore(
     PyObject *value,
     PyObject *traceback);
 
-extern void _PyErr_SetObject(
+PyAPI_FUNC(void) _PyErr_SetObject(
     PyThreadState *tstate,
     PyObject *type,
     PyObject *value);

Include/internal/pycore_pyerrors.h

@@ -2857,6 +2857,24 @@ def func():
         names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"]
         self.do_test(func, names)
 
+    def test_replaced_interpreter(self):
+        def inner():
+            yield 'abc'
+        def outer():
+            yield from inner()
+        def func():
+            list(outer())
+        _testinternalcapi.set_eval_frame_interp()
+        try:
+            func()
+        finally:
+            _testinternalcapi.set_eval_frame_default()
+
+        stats = _testinternalcapi.get_eval_frame_stats()
+
+        self.assertEqual(stats["resumes"], 5)
+        self.assertEqual(stats["loads"], 5)
+
 
 @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED')
 class TestPyThreadId(unittest.TestCase):

Lib/test/test_capi/test_misc.py

@@ -2150,7 +2150,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c
 regen-cases: \
         regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \
 		regen-generated-cases regen-executor-cases regen-optimizer-cases \
-		regen-opcode-metadata regen-uop-metadata
+		regen-opcode-metadata regen-uop-metadata regen-test-cases regen-test-opcode-targets
 
 .PHONY: regen-opcode-ids
 regen-opcode-ids:
@@ -2182,6 +2182,20 @@ regen-generated-cases:
 	    -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
 
+.PHONY: regen-test-cases
+regen-test-cases:
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \
+	    -o $(srcdir)/Modules/_testinternalcapi/test_cases.c.h.new $(srcdir)/Python/bytecodes.c \
+		$(srcdir)/Modules/_testinternalcapi/testbytecodes.c
+	$(UPDATE_FILE) $(srcdir)/Modules/_testinternalcapi/test_cases.c.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h.new
+
+.PHONY: regen-test-opcode-targets
+regen-test-opcode-targets:
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \
+	    -o $(srcdir)/Modules/_testinternalcapi/test_targets.h.new $(srcdir)/Python/bytecodes.c \
+		$(srcdir)/Modules/_testinternalcapi/testbytecodes.c
+	$(UPDATE_FILE) $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_targets.h.new
+
 .PHONY: regen-executor-cases
 regen-executor-cases:
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \
@@ -3428,7 +3442,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 

Makefile.pre.in

@@ -174,7 +174,7 @@
 @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
-@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c
+@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c

Modules/Setup.stdlib.in

@@ -693,6 +693,39 @@ set_eval_frame_record(PyObject *self, PyObject *list)
     Py_RETURN_NONE;
 }
 
+// Defined in interpreter.c
+extern PyObject*
+Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag);
+extern int Test_EvalFrame_Resumes, Test_EvalFrame_Loads;
+
+static PyObject *
+get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args))
+{
+    PyObject *res = PyDict_New();
+    if (res == NULL) {
+        return NULL;
+    }
+    PyObject *resumes = PyLong_FromLong(Test_EvalFrame_Resumes);
+    if (resumes == NULL || PyDict_SetItemString(res, "resumes", resumes) < 0) {
+        Py_XDECREF(resumes);
+        return NULL;
+    }
+    PyObject *loads = PyLong_FromLong(Test_EvalFrame_Loads);
+    if (loads == NULL || PyDict_SetItemString(res, "loads", loads) < 0) {
+        Py_XDECREF(loads);
+        return NULL;
+    }
+    Test_EvalFrame_Resumes = Test_EvalFrame_Loads = 0;
+    return res;
+}
+
+static PyObject *
+set_eval_frame_interp(PyObject *self, PyObject *Py_UNUSED(args))
+{
+    _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame);
+    Py_RETURN_NONE;
+}
+
 /*[clinic input]
 
 _testinternalcapi.compiler_cleandoc -> object
@@ -2531,6 +2564,7 @@ test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args))
 
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
+    {"get_eval_frame_stats", get_eval_frame_stats, METH_NOARGS, NULL},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
     {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS},
     {"get_stack_pointer", get_stack_pointer, METH_NOARGS},
@@ -2547,6 +2581,7 @@ static PyMethodDef module_functions[] = {
     {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
     {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
     {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
+    {"set_eval_frame_interp", set_eval_frame_interp, METH_NOARGS, NULL},
     {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
     _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF
     _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF