gh-116968: Reimplement Tier 2 counters

Include/cpython/optimizer.h

@@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 
 typedef struct _exit_data {
     uint32_t target;
-    int16_t temperature;
+    uint16_t temperature;
     const struct _PyExecutorObject *executor;
 } _PyExitData;
 
@@ -115,11 +115,9 @@ typedef int (*optimize_func)(
 struct _PyOptimizerObject {
     PyObject_HEAD
     optimize_func optimize;
-    /* These thresholds are treated as signed so do not exceed INT16_MAX
-     * Use INT16_MAX to indicate that the optimizer should never be called */
-    uint16_t resume_threshold;
-    uint16_t side_threshold;
+    /* Initial values for adaptive-style counters */
     uint16_t backedge_threshold;
+    uint16_t side_threshold;
     /* Data needed by the optimizer goes here, but is opaque to the VM */
 };
 
@@ -151,14 +149,6 @@ extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_inval
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
 
-#define OPTIMIZER_BITS_IN_COUNTER 4
-/* Minimum of 16 additional executions before retry */
-#define MIN_TIER2_BACKOFF 4
-#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER)
-#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1)
-/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */
-#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX
-
 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
 

Include/internal/pycore_backoff.h

@@ -0,0 +1,117 @@
+
+#ifndef Py_INTERNAL_BACKOFF_H
+#define Py_INTERNAL_BACKOFF_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+/* 16-bit countdown counters using exponential backoff.
+
+   These are used by the adaptive specializer to count down until
+   it is time to specialize an instruction. If specialization fails
+   the counter is reset using exponential backoff.
+
+   Another use is for the Tier 2 optimizer to decide when to create
+   a new Tier 2 trace (executor). Again, exponential backoff is used.
+
+   The 16-bit counter is structured as a 12-bit unsigned 'value'
+   and a 4-bit 'backoff' field. When resetting the counter, the
+   backoff field is incremented (until it reaches a limit) and the
+   value is set to a bit mask representing the value 2**backoff - 1.
+   The maximum backoff is 12 (the number of value bits).
+
+   There is an exceptional value which must not be updated, 0xFFFF.
+*/
+
+typedef struct {
+    union {
+        uint16_t counter;
+        struct {
+            uint16_t value : 12;
+            uint16_t backoff : 4;
+        };
+    };
+} backoff_counter_t;
+
+static_assert(sizeof(backoff_counter_t) == 2, "backoff counter size should be 2 bytes");
+
+#define UNREACHABLE_BACKOFF 0xFFFF
+#define UNREACHABLE_BACKOFF_COUNTER ((backoff_counter_t){.counter = UNREACHABLE_BACKOFF})
+
+/* Alias used by optimizer */
+#define OPTIMIZER_UNREACHABLE_THRESHOLD UNREACHABLE_BACKOFF
+
+static inline bool
+is_unreachable_backoff_counter(backoff_counter_t counter)
+{
+    return counter.counter == 0xFFFF;
+}
+
+static inline backoff_counter_t
+make_backoff_counter(uint16_t value, uint16_t backoff)
+{
+    assert(backoff <= 12);
+    assert(value <= 0xFFF);
+    return (backoff_counter_t){.value = value, .backoff = backoff};
+}
+
+static inline backoff_counter_t
+forge_backoff_counter(uint16_t counter)
+{
+    return (backoff_counter_t){.counter = counter};
+}
+
+static inline backoff_counter_t
+reset_backoff_counter(backoff_counter_t counter)
+{
+    assert(!is_unreachable_backoff_counter(counter));
+    if (counter.backoff < 12) {
+        return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1);
+    }
+    else {
+        return make_backoff_counter((1 << 12) - 1, 12);
+    }
+}
+
+static inline backoff_counter_t
+increment_backoff_counter(backoff_counter_t counter)
+{
+    if (!is_unreachable_backoff_counter(counter)) {
+        assert(counter.value != 0xFFF);
+        return make_backoff_counter(counter.value + 1, counter.backoff);
+    }
+    else {
+        return counter;
+    }
+}
+
+static inline backoff_counter_t
+decrement_backoff_counter(backoff_counter_t counter)
+{
+    if (!is_unreachable_backoff_counter(counter)) {
+        assert(counter.value != 0);
+        return make_backoff_counter(counter.value - 1, counter.backoff);
+    }
+    else {
+        return counter;
+    }
+}
+
+static inline bool
+backoff_counter_is_zero(backoff_counter_t counter)
+{
+    return counter.value == 0;
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_BACKOFF_H */

Include/internal/pycore_code.h

@@ -448,18 +448,14 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 
 /** Counters
  * The first 16-bit value in each inline cache is a counter.
- * When counting misses, the counter is treated as a simple unsigned value.
  *
  * When counting executions until the next specialization attempt,
  * exponential backoff is used to reduce the number of specialization failures.
- * The high 12 bits store the counter, the low 4 bits store the backoff exponent.
- * On a specialization failure, the backoff exponent is incremented and the
- * counter set to (2**backoff - 1).
- * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023.
+ * See pycore_backoff.h for more details.
+ * On a specialization failure, the backoff counter is reset.
  */
 
-/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
-#define ADAPTIVE_BACKOFF_BITS 4
+#include "pycore_backoff.h"
 
 // A value of 1 means that we attempt to specialize the *second* time each
 // instruction is executed. Executing twice is a much better indicator of
@@ -477,13 +473,9 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 #define ADAPTIVE_COOLDOWN_VALUE 52
 #define ADAPTIVE_COOLDOWN_BACKOFF 0
 
-#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
-
-
 static inline uint16_t
 adaptive_counter_bits(uint16_t value, uint16_t backoff) {
-    return ((value << ADAPTIVE_BACKOFF_BITS)
-            | (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1)));
+    return make_backoff_counter(value, backoff).counter;
 }
 
 static inline uint16_t
@@ -500,13 +492,7 @@ adaptive_counter_cooldown(void) {
 
 static inline uint16_t
 adaptive_counter_backoff(uint16_t counter) {
-    uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1);
-    backoff++;
-    if (backoff > MAX_BACKOFF_VALUE) {
-        backoff = MAX_BACKOFF_VALUE;
-    }
-    uint16_t value = (uint16_t)(1 << backoff) - 1;
-    return adaptive_counter_bits(value, backoff);
+    return reset_backoff_counter(forge_backoff_counter(counter)).counter;
 }
 
 

Include/internal/pycore_interp.h

@@ -239,10 +239,8 @@ struct _is {
     _PyOptimizerObject *optimizer;
     _PyExecutorObject *executor_list_head;
 
-    /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */
-    uint32_t optimizer_resume_threshold;
+    /* These three values are shifted and offset to speed up check in JUMP_BACKWARD */
     uint32_t optimizer_backedge_threshold;
-
     uint16_t optimizer_side_threshold;
 
     _rare_events rare_events;

Makefile.pre.in

@@ -1117,6 +1117,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_ast.h \
 		$(srcdir)/Include/internal/pycore_ast_state.h \
 		$(srcdir)/Include/internal/pycore_atexit.h \
+		$(srcdir)/Include/internal/pycore_backoff.h \
 		$(srcdir)/Include/internal/pycore_bitutils.h \
 		$(srcdir)/Include/internal/pycore_blocks_output_buffer.h \
 		$(srcdir)/Include/internal/pycore_brc.h \

PCbuild/pythoncore.vcxproj

@@ -204,6 +204,7 @@
     <ClInclude Include="..\Include\internal\pycore_ast.h" />
     <ClInclude Include="..\Include\internal\pycore_ast_state.h" />
     <ClInclude Include="..\Include\internal\pycore_atexit.h" />
+    <ClInclude Include="..\Include\internal\pycore_backoff.h" />
     <ClInclude Include="..\Include\internal\pycore_bitutils.h" />
     <ClInclude Include="..\Include\internal\pycore_brc.h" />
     <ClInclude Include="..\Include\internal\pycore_bytes_methods.h" />

Python/bytecodes.c

@@ -8,6 +8,7 @@
 
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
+#include "pycore_backoff.h"
 #include "pycore_code.h"
 #include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
 #include "pycore_function.h"
@@ -2348,41 +2349,35 @@ dummy_func(
             JUMPBY(-oparg);
             #if ENABLE_SPECIALIZATION
             uint16_t counter = this_instr[1].cache;
-            this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER);
-            /* We are using unsigned values, but we really want signed values, so
-             * do the 2s complement adjustment manually */
-            uint32_t offset_counter = counter ^ (1 << 15);
-            uint32_t threshold = tstate->interp->optimizer_backedge_threshold;
-            assert((threshold & OPTIMIZER_BITS_MASK) == 0);
-            // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result.
-            // Double-check that the opcode isn't instrumented or something:
-            if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) {
-                _Py_CODEUNIT *start = this_instr;
-                /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
-                while (oparg > 255) {
-                    oparg >>= 8;
-                    start--;
-                }
-                _PyExecutorObject *executor;
-                int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
-                ERROR_IF(optimized < 0, error);
-                if (optimized) {
-                    assert(tstate->previous_executor == NULL);
-                    tstate->previous_executor = Py_None;
-                    GOTO_TIER_TWO(executor);
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                if (counter == 0) {
+                    // Dynamically initialize the counter
+                    counter = tstate->interp->optimizer_backedge_threshold;
+                    this_instr[1].cache = counter;
                 }
-                else {
-                    int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK;
-                    backoff++;
-                    if (backoff < MIN_TIER2_BACKOFF) {
-                        backoff = MIN_TIER2_BACKOFF;
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter) && this_instr->op.code == JUMP_BACKWARD) {
+                    _Py_CODEUNIT *start = this_instr;
+                    /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
+                    while (oparg > 255) {
+                        oparg >>= 8;
+                        start--;
+                    }
+                    _PyExecutorObject *executor;
+                    int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
+                    ERROR_IF(optimized < 0, error);
+                    if (optimized) {
+                        assert(tstate->previous_executor == NULL);
+                        tstate->previous_executor = Py_None;
+                        GOTO_TIER_TWO(executor);
                     }
-                    else if (backoff > MAX_TIER2_BACKOFF) {
-                        backoff = MAX_TIER2_BACKOFF;
+                    else {
+                        this_instr[1].cache = adaptive_counter_backoff(counter);
                     }
-                    this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff;
                 }
             }
+            else {
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+            }
             #endif  /* ENABLE_SPECIALIZATION */
         }
 
@@ -3973,7 +3968,12 @@ dummy_func(
             ERROR_IF(next_opcode < 0, error);
             next_instr = this_instr;
             if (_PyOpcode_Caches[next_opcode]) {
-                INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                if (next_opcode != POP_JUMP_IF_FALSE &&
+                    next_opcode != POP_JUMP_IF_TRUE &&
+                    next_opcode != POP_JUMP_IF_NOT_NONE &&
+                    next_opcode != POP_JUMP_IF_NONE) {
+                    INCREMENT_ADAPTIVE_COUNTER(next_instr[1].cache);
+                }
             }
             assert(next_opcode > 0 && next_opcode < 256);
             opcode = next_opcode;
@@ -4159,34 +4159,36 @@ dummy_func(
         tier2 op(_COLD_EXIT, (--)) {
             _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
             _PyExitData *exit = &previous->exits[oparg];
-            exit->temperature++;
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
-            if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) {
-                GOTO_TIER_ONE(target);
-            }
-            _PyExecutorObject *executor;
-            if (target->op.code == ENTER_EXECUTOR) {
-                executor = code->co_executors->executors[target->op.arg];
-                Py_INCREF(executor);
-            } else {
-                int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
-                if (optimized <= 0) {
-                    int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold;
-                    exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
-                    if (optimized < 0) {
-                        Py_DECREF(previous);
-                        tstate->previous_executor = Py_None;
-                        GOTO_UNWIND();
+            #if ENABLE_SPECIALIZATION
+            if (ADAPTIVE_COUNTER_IS_ZERO(exit->temperature)) {
+                _PyExecutorObject *executor;
+                if (target->op.code == ENTER_EXECUTOR) {
+                    executor = code->co_executors->executors[target->op.arg];
+                    Py_INCREF(executor);
+                }
+                else {
+                    int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
+                    if (optimized <= 0) {
+                        exit->temperature = adaptive_counter_backoff(exit->temperature);
+                        if (optimized < 0) {
+                            Py_DECREF(previous);
+                            tstate->previous_executor = Py_None;
+                            GOTO_UNWIND();
+                        }
+                        GOTO_TIER_ONE(target);
                     }
-                    GOTO_TIER_ONE(target);
                 }
+                /* We need two references. One to store in exit->executor and
+                * one to keep the executor alive when executing. */
+                Py_INCREF(executor);
+                exit->executor = executor;
+                GOTO_TIER_TWO(executor);
             }
-            /* We need two references. One to store in exit->executor and
-             * one to keep the executor alive when executing. */
-            Py_INCREF(executor);
-            exit->executor = executor;
-            GOTO_TIER_TWO(executor);
+            DECREMENT_ADAPTIVE_COUNTER(exit->temperature);
+            #endif
+            GOTO_TIER_ONE(target);
         }
 
         tier2 op(_START_EXECUTOR, (executor/4 --)) {