ceval.c - cs107-lecture-examples - Example codes used during Harvard CS107 lectures

ceval.c (153098B)
      1 // XXX: [fabianw@seas.harvard.edu; 2022-11-08] This code has been shortened to
      2 // fit the context of the lecture.  The full code for this file can be found
      3 // here:
      4 //
      5 // https://github.com/python/cpython/blob/v3.11.0rc2/Python/ceval.c
      6 
      7 PyObject* _Py_HOT_FUNCTION
      8 _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
      9 {
     10     _Py_EnsureTstateNotNULL(tstate);
     11     CALL_STAT_INC(pyeval_calls);
     12 
     13 #if USE_COMPUTED_GOTOS
     14 /* Import the static jump table */
     15 #include "opcode_targets.h"
     16 #endif
     17 
     18 #ifdef Py_STATS
     19     int lastopcode = 0;
     20 #endif
     21     // opcode is an 8-bit value to improve the code generated by MSVC
     22     // for the big switch below (in combination with the EXTRA_CASES macro).
     23     uint8_t opcode;        /* Current opcode */
     24     int oparg;         /* Current opcode argument, if any */
     25     _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
     26 #ifdef LLTRACE
     27     int lltrace = 0;
     28 #endif
     29 
     30     _PyCFrame cframe;
     31     CallShape call_shape;
     32     call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions.
     33 
     34     /* WARNING: Because the _PyCFrame lives on the C stack,
     35      * but can be accessed from a heap allocated object (tstate)
     36      * strict stack discipline must be maintained.
     37      */
     38     _PyCFrame *prev_cframe = tstate->cframe;
     39     cframe.use_tracing = prev_cframe->use_tracing;
     40     cframe.previous = prev_cframe;
     41     tstate->cframe = &cframe;
     42 
     43     frame->is_entry = true;
     44     /* Push frame */
     45     frame->previous = prev_cframe->current_frame;
     46     cframe.current_frame = frame;
     47 
     48     /* support for generator.throw() */
     49     if (throwflag) {
     50         if (_Py_EnterRecursiveCallTstate(tstate, "")) {
     51             tstate->recursion_remaining--;
     52             goto exit_unwind;
     53         }
     54         TRACE_FUNCTION_THROW_ENTRY();
     55         DTRACE_FUNCTION_ENTRY();
     56         goto resume_with_error;
     57     }
     58 
     59     /* Local "register" variables.
     60      * These are cached values from the frame and code object.  */
     61 
     62     PyObject *names;
     63     PyObject *consts;
     64     _Py_CODEUNIT *first_instr;
     65     _Py_CODEUNIT *next_instr;
     66     PyObject **stack_pointer;
     67 
     68 /* Sets the above local variables from the frame */
     69 #define SET_LOCALS_FROM_FRAME() \
     70     { \
     71         PyCodeObject *co = frame->f_code; \
     72         names = co->co_names; \
     73         consts = co->co_consts; \
     74         first_instr = _PyCode_CODE(co); \
     75     } \
     76     assert(_PyInterpreterFrame_LASTI(frame) >= -1); \
     77     /* Jump back to the last instruction executed... */ \
     78     next_instr = frame->prev_instr + 1; \
     79     stack_pointer = _PyFrame_GetStackPointer(frame); \
     80     /* Set stackdepth to -1. \
     81         Update when returning or calling trace function. \
     82         Having stackdepth <= 0 ensures that invalid \
     83         values are not visible to the cycle GC. \
     84         We choose -1 rather than 0 to assist debugging. \
     85         */ \
     86     frame->stacktop = -1;
     87 
     88 
     89 start_frame:
     90     if (_Py_EnterRecursiveCallTstate(tstate, "")) {
     91         tstate->recursion_remaining--;
     92         goto exit_unwind;
     93     }
     94 
     95 resume_frame:
     96     SET_LOCALS_FROM_FRAME();
     97 
     98 #ifdef LLTRACE
     99     {
    100         int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
    101         if (r < 0) {
    102             goto exit_unwind;
    103         }
    104         lltrace = r;
    105     }
    106     if (lltrace) {
    107         lltrace_resume_frame(frame);
    108     }
    109 #endif
    110 
    111 #ifdef Py_DEBUG
    112     /* _PyEval_EvalFrameDefault() must not be called with an exception set,
    113        because it can clear it (directly or indirectly) and so the
    114        caller loses its exception */
    115     assert(!_PyErr_Occurred(tstate));
    116 #endif
    117 
    118     DISPATCH();
    119 
    120 handle_eval_breaker:
    121 
    122     /* Do periodic things, like check for signals and async I/0.
    123      * We need to do reasonably frequently, but not too frequently.
    124      * All loops should include a check of the eval breaker.
    125      * We also check on return from any builtin function.
    126      */
    127     if (eval_frame_handle_pending(tstate) != 0) {
    128         goto error;
    129     }
    130     DISPATCH();
    131 
    132     {
    133     /* Start instructions */
    134 #if USE_COMPUTED_GOTOS
    135     {
    136 #else
    137     dispatch_opcode:
    138         switch (opcode) {
    139 #endif
    140 
    141         /* BEWARE!
    142            It is essential that any operation that fails must goto error
    143            and that all operation that succeed call DISPATCH() ! */
    144 
    145         TARGET(NOP) {
    146             DISPATCH();
    147         }
    148 
    149         TARGET(RESUME) {
    150             _PyCode_Warmup(frame->f_code);
    151             JUMP_TO_INSTRUCTION(RESUME_QUICK);
    152         }
    153 
    154         TARGET(RESUME_QUICK) {
    155             PREDICTED(RESUME_QUICK);
    156             assert(tstate->cframe == &cframe);
    157             assert(frame == cframe.current_frame);
    158             if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
    159                 goto handle_eval_breaker;
    160             }
    161             DISPATCH();
    162         }
    163 
    164         TARGET(LOAD_CLOSURE) {
    165             /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
    166             PyObject *value = GETLOCAL(oparg);
    167             if (value == NULL) {
    168                 goto unbound_local_error;
    169             }
    170             Py_INCREF(value);
    171             PUSH(value);
    172             DISPATCH();
    173         }
    174 
    175         TARGET(LOAD_FAST) {
    176             PyObject *value = GETLOCAL(oparg);
    177             if (value == NULL) {
    178                 goto unbound_local_error;
    179             }
    180             Py_INCREF(value);
    181             PUSH(value);
    182             DISPATCH();
    183         }
    184 
    185         TARGET(LOAD_CONST) {
    186             PREDICTED(LOAD_CONST);
    187             PyObject *value = GETITEM(consts, oparg);
    188             Py_INCREF(value);
    189             PUSH(value);
    190             DISPATCH();
    191         }
    192 
    193         TARGET(STORE_FAST) {
    194             PREDICTED(STORE_FAST);
    195             PyObject *value = POP();
    196             SETLOCAL(oparg, value);
    197             DISPATCH();
    198         }
    199 
    200         TARGET(LOAD_FAST__LOAD_FAST) {
    201             PyObject *value = GETLOCAL(oparg);
    202             if (value == NULL) {
    203                 goto unbound_local_error;
    204             }
    205             NEXTOPARG();
    206             next_instr++;
    207             Py_INCREF(value);
    208             PUSH(value);
    209             value = GETLOCAL(oparg);
    210             if (value == NULL) {
    211                 goto unbound_local_error;
    212             }
    213             Py_INCREF(value);
    214             PUSH(value);
    215             DISPATCH();
    216         }
    217 
    218         TARGET(LOAD_FAST__LOAD_CONST) {
    219             PyObject *value = GETLOCAL(oparg);
    220             if (value == NULL) {
    221                 goto unbound_local_error;
    222             }
    223             NEXTOPARG();
    224             next_instr++;
    225             Py_INCREF(value);
    226             PUSH(value);
    227             value = GETITEM(consts, oparg);
    228             Py_INCREF(value);
    229             PUSH(value);
    230             DISPATCH();
    231         }
    232 
    233         TARGET(STORE_FAST__LOAD_FAST) {
    234             PyObject *value = POP();
    235             SETLOCAL(oparg, value);
    236             NEXTOPARG();
    237             next_instr++;
    238             value = GETLOCAL(oparg);
    239             if (value == NULL) {
    240                 goto unbound_local_error;
    241             }
    242             Py_INCREF(value);
    243             PUSH(value);
    244             DISPATCH();
    245         }
    246 
    247         TARGET(STORE_FAST__STORE_FAST) {
    248             PyObject *value = POP();
    249             SETLOCAL(oparg, value);
    250             NEXTOPARG();
    251             next_instr++;
    252             value = POP();
    253             SETLOCAL(oparg, value);
    254             DISPATCH();
    255         }
    256 
    257         TARGET(LOAD_CONST__LOAD_FAST) {
    258             PyObject *value = GETITEM(consts, oparg);
    259             NEXTOPARG();
    260             next_instr++;
    261             Py_INCREF(value);
    262             PUSH(value);
    263             value = GETLOCAL(oparg);
    264             if (value == NULL) {
    265                 goto unbound_local_error;
    266             }
    267             Py_INCREF(value);
    268             PUSH(value);
    269             DISPATCH();
    270         }
    271 
    272         TARGET(POP_TOP) {
    273             PyObject *value = POP();
    274             Py_DECREF(value);
    275             DISPATCH();
    276         }
    277 
    278         TARGET(PUSH_NULL) {
    279             /* Use BASIC_PUSH as NULL is not a valid object pointer */
    280             BASIC_PUSH(NULL);
    281             DISPATCH();
    282         }
    283 
    284         TARGET(UNARY_POSITIVE) {
    285             PyObject *value = TOP();
    286             PyObject *res = PyNumber_Positive(value);
    287             Py_DECREF(value);
    288             SET_TOP(res);
    289             if (res == NULL)
    290                 goto error;
    291             DISPATCH();
    292         }
    293 
    294         TARGET(UNARY_NEGATIVE) {
    295             PyObject *value = TOP();
    296             PyObject *res = PyNumber_Negative(value);
    297             Py_DECREF(value);
    298             SET_TOP(res);
    299             if (res == NULL)
    300                 goto error;
    301             DISPATCH();
    302         }
    303 
    304         TARGET(UNARY_NOT) {
    305             PyObject *value = TOP();
    306             int err = PyObject_IsTrue(value);
    307             Py_DECREF(value);
    308             if (err == 0) {
    309                 Py_INCREF(Py_True);
    310                 SET_TOP(Py_True);
    311                 DISPATCH();
    312             }
    313             else if (err > 0) {
    314                 Py_INCREF(Py_False);
    315                 SET_TOP(Py_False);
    316                 DISPATCH();
    317             }
    318             STACK_SHRINK(1);
    319             goto error;
    320         }
    321 
    322         TARGET(UNARY_INVERT) {
    323             PyObject *value = TOP();
    324             PyObject *res = PyNumber_Invert(value);
    325             Py_DECREF(value);
    326             SET_TOP(res);
    327             if (res == NULL)
    328                 goto error;
    329             DISPATCH();
    330         }
    331 
    332         TARGET(BINARY_OP_MULTIPLY_INT) {
    333             assert(cframe.use_tracing == 0);
    334             PyObject *left = SECOND();
    335             PyObject *right = TOP();
    336             DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
    337             DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
    338             STAT_INC(BINARY_OP, hit);
    339             PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
    340             SET_SECOND(prod);
    341             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
    342             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
    343             STACK_SHRINK(1);
    344             if (prod == NULL) {
    345                 goto error;
    346             }
    347             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    348             DISPATCH();
    349         }
    350 
    351         TARGET(BINARY_OP_MULTIPLY_FLOAT) {
    352             assert(cframe.use_tracing == 0);
    353             PyObject *left = SECOND();
    354             PyObject *right = TOP();
    355             DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
    356             DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
    357             STAT_INC(BINARY_OP, hit);
    358             double dprod = ((PyFloatObject *)left)->ob_fval *
    359                 ((PyFloatObject *)right)->ob_fval;
    360             PyObject *prod = PyFloat_FromDouble(dprod);
    361             SET_SECOND(prod);
    362             _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
    363             _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
    364             STACK_SHRINK(1);
    365             if (prod == NULL) {
    366                 goto error;
    367             }
    368             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    369             DISPATCH();
    370         }
    371 
    372         TARGET(BINARY_OP_SUBTRACT_INT) {
    373             assert(cframe.use_tracing == 0);
    374             PyObject *left = SECOND();
    375             PyObject *right = TOP();
    376             DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
    377             DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
    378             STAT_INC(BINARY_OP, hit);
    379             PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
    380             SET_SECOND(sub);
    381             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
    382             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
    383             STACK_SHRINK(1);
    384             if (sub == NULL) {
    385                 goto error;
    386             }
    387             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    388             DISPATCH();
    389         }
    390 
    391         TARGET(BINARY_OP_SUBTRACT_FLOAT) {
    392             assert(cframe.use_tracing == 0);
    393             PyObject *left = SECOND();
    394             PyObject *right = TOP();
    395             DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
    396             DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
    397             STAT_INC(BINARY_OP, hit);
    398             double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;
    399             PyObject *sub = PyFloat_FromDouble(dsub);
    400             SET_SECOND(sub);
    401             _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
    402             _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
    403             STACK_SHRINK(1);
    404             if (sub == NULL) {
    405                 goto error;
    406             }
    407             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    408             DISPATCH();
    409         }
    410 
    411         TARGET(BINARY_OP_ADD_UNICODE) {
    412             assert(cframe.use_tracing == 0);
    413             PyObject *left = SECOND();
    414             PyObject *right = TOP();
    415             DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
    416             DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
    417             STAT_INC(BINARY_OP, hit);
    418             PyObject *res = PyUnicode_Concat(left, right);
    419             STACK_SHRINK(1);
    420             SET_TOP(res);
    421             _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
    422             _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
    423             if (TOP() == NULL) {
    424                 goto error;
    425             }
    426             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    427             DISPATCH();
    428         }
    429 
    430         TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
    431             assert(cframe.use_tracing == 0);
    432             PyObject *left = SECOND();
    433             PyObject *right = TOP();
    434             DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
    435             DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
    436             _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
    437             assert(_Py_OPCODE(true_next) == STORE_FAST ||
    438                    _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST);
    439             PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next));
    440             DEOPT_IF(*target_local != left, BINARY_OP);
    441             STAT_INC(BINARY_OP, hit);
    442             /* Handle `left = left + right` or `left += right` for str.
    443              *
    444              * When possible, extend `left` in place rather than
    445              * allocating a new PyUnicodeObject. This attempts to avoid
    446              * quadratic behavior when one neglects to use str.join().
    447              *
    448              * If `left` has only two references remaining (one from
    449              * the stack, one in the locals), DECREFing `left` leaves
    450              * only the locals reference, so PyUnicode_Append knows
    451              * that the string is safe to mutate.
    452              */
    453             assert(Py_REFCNT(left) >= 2);
    454             _Py_DECREF_NO_DEALLOC(left);
    455             STACK_SHRINK(2);
    456             PyUnicode_Append(target_local, right);
    457             _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
    458             if (*target_local == NULL) {
    459                 goto error;
    460             }
    461             // The STORE_FAST is already done.
    462             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
    463             DISPATCH();
    464         }
    465 
    466         TARGET(BINARY_OP_ADD_FLOAT) {
    467             assert(cframe.use_tracing == 0);
    468             PyObject *left = SECOND();
    469             PyObject *right = TOP();
    470             DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
    471             DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
    472             STAT_INC(BINARY_OP, hit);
    473             double dsum = ((PyFloatObject *)left)->ob_fval +
    474                 ((PyFloatObject *)right)->ob_fval;
    475             PyObject *sum = PyFloat_FromDouble(dsum);
    476             SET_SECOND(sum);
    477             _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
    478             _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
    479             STACK_SHRINK(1);
    480             if (sum == NULL) {
    481                 goto error;
    482             }
    483             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    484             DISPATCH();
    485         }
    486 
    487         TARGET(BINARY_OP_ADD_INT) {
    488             assert(cframe.use_tracing == 0);
    489             PyObject *left = SECOND();
    490             PyObject *right = TOP();
    491             DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
    492             DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
    493             STAT_INC(BINARY_OP, hit);
    494             PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
    495             SET_SECOND(sum);
    496             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
    497             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
    498             STACK_SHRINK(1);
    499             if (sum == NULL) {
    500                 goto error;
    501             }
    502             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
    503             DISPATCH();
    504         }
    505 
    506         TARGET(BINARY_SUBSCR) {
    507             PREDICTED(BINARY_SUBSCR);
    508             PyObject *sub = POP();
    509             PyObject *container = TOP();
    510             PyObject *res = PyObject_GetItem(container, sub);
    511             Py_DECREF(container);
    512             Py_DECREF(sub);
    513             SET_TOP(res);
    514             if (res == NULL)
    515                 goto error;
    516             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
    517             DISPATCH();
    518         }
    519 
    520         TARGET(BINARY_SUBSCR_ADAPTIVE) {
    521             _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
    522             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
    523                 PyObject *sub = TOP();
    524                 PyObject *container = SECOND();
    525                 next_instr--;
    526                 if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
    527                     goto error;
    528                 }
    529                 DISPATCH_SAME_OPARG();
    530             }
    531             else {
    532                 STAT_INC(BINARY_SUBSCR, deferred);
    533                 DECREMENT_ADAPTIVE_COUNTER(cache);
    534                 JUMP_TO_INSTRUCTION(BINARY_SUBSCR);
    535             }
    536         }
    537 
    538         TARGET(BINARY_SUBSCR_LIST_INT) {
    539             assert(cframe.use_tracing == 0);
    540             PyObject *sub = TOP();
    541             PyObject *list = SECOND();
    542             DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
    543             DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
    544 
    545             // Deopt unless 0 <= sub < PyList_Size(list)
    546             Py_ssize_t signed_magnitude = Py_SIZE(sub);
    547             DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
    548             assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
    549             Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
    550             DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
    551             STAT_INC(BINARY_SUBSCR, hit);
    552             PyObject *res = PyList_GET_ITEM(list, index);
    553             assert(res != NULL);
    554             Py_INCREF(res);
    555             STACK_SHRINK(1);
    556             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
    557             SET_TOP(res);
    558             Py_DECREF(list);
    559             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
    560             DISPATCH();
    561         }
    562 
    563         TARGET(BINARY_SUBSCR_TUPLE_INT) {
    564             assert(cframe.use_tracing == 0);
    565             PyObject *sub = TOP();
    566             PyObject *tuple = SECOND();
    567             DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
    568             DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
    569 
    570             // Deopt unless 0 <= sub < PyTuple_Size(list)
    571             Py_ssize_t signed_magnitude = Py_SIZE(sub);
    572             DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
    573             assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
    574             Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
    575             DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
    576             STAT_INC(BINARY_SUBSCR, hit);
    577             PyObject *res = PyTuple_GET_ITEM(tuple, index);
    578             assert(res != NULL);
    579             Py_INCREF(res);
    580             STACK_SHRINK(1);
    581             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
    582             SET_TOP(res);
    583             Py_DECREF(tuple);
    584             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
    585             DISPATCH();
    586         }
    587 
    588         TARGET(BINARY_SUBSCR_DICT) {
    589             assert(cframe.use_tracing == 0);
    590             PyObject *dict = SECOND();
    591             DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR);
    592             STAT_INC(BINARY_SUBSCR, hit);
    593             PyObject *sub = TOP();
    594             PyObject *res = PyDict_GetItemWithError(dict, sub);
    595             if (res == NULL) {
    596                 goto binary_subscr_dict_error;
    597             }
    598             Py_INCREF(res);
    599             STACK_SHRINK(1);
    600             Py_DECREF(sub);
    601             SET_TOP(res);
    602             Py_DECREF(dict);
    603             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
    604             DISPATCH();
    605         }
    606 
    607         TARGET(BINARY_SUBSCR_GETITEM) {
    608             PyObject *sub = TOP();
    609             PyObject *container = SECOND();
    610             _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
    611             uint32_t type_version = read_u32(cache->type_version);
    612             PyTypeObject *tp = Py_TYPE(container);
    613             DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR);
    614             assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
    615             PyObject *cached = ((PyHeapTypeObject *)tp)->_spec_cache.getitem;
    616             assert(PyFunction_Check(cached));
    617             PyFunctionObject *getitem = (PyFunctionObject *)cached;
    618             DEOPT_IF(getitem->func_version != cache->func_version, BINARY_SUBSCR);
    619             PyCodeObject *code = (PyCodeObject *)getitem->func_code;
    620             size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
    621             assert(code->co_argcount == 2);
    622             _PyInterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
    623             if (new_frame == NULL) {
    624                 goto error;
    625             }
    626             CALL_STAT_INC(frames_pushed);
    627             Py_INCREF(getitem);
    628             _PyFrame_InitializeSpecials(new_frame, getitem,
    629                                         NULL, code->co_nlocalsplus);
    630             STACK_SHRINK(2);
    631             new_frame->localsplus[0] = container;
    632             new_frame->localsplus[1] = sub;
    633             for (int i = 2; i < code->co_nlocalsplus; i++) {
    634                 new_frame->localsplus[i] = NULL;
    635             }
    636             _PyFrame_SetStackPointer(frame, stack_pointer);
    637             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
    638             frame->prev_instr = next_instr - 1;
    639             new_frame->previous = frame;
    640             frame = cframe.current_frame = new_frame;
    641             CALL_STAT_INC(inlined_py_calls);
    642             goto start_frame;
    643         }
    644 
    645         TARGET(LIST_APPEND) {
    646             PyObject *v = POP();
    647             PyObject *list = PEEK(oparg);
    648             if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
    649                 goto error;
    650             PREDICT(JUMP_BACKWARD_QUICK);
    651             DISPATCH();
    652         }
    653 
    654         TARGET(SET_ADD) {
    655             PyObject *v = POP();
    656             PyObject *set = PEEK(oparg);
    657             int err;
    658             err = PySet_Add(set, v);
    659             Py_DECREF(v);
    660             if (err != 0)
    661                 goto error;
    662             PREDICT(JUMP_BACKWARD_QUICK);
    663             DISPATCH();
    664         }
    665 
    666         TARGET(STORE_SUBSCR) {
    667             PREDICTED(STORE_SUBSCR);
    668             PyObject *sub = TOP();
    669             PyObject *container = SECOND();
    670             PyObject *v = THIRD();
    671             int err;
    672             STACK_SHRINK(3);
    673             /* container[sub] = v */
    674             err = PyObject_SetItem(container, sub, v);
    675             Py_DECREF(v);
    676             Py_DECREF(container);
    677             Py_DECREF(sub);
    678             if (err != 0) {
    679                 goto error;
    680             }
    681             JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
    682             DISPATCH();
    683         }
    684 
    685         TARGET(STORE_SUBSCR_ADAPTIVE) {
    686             _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
    687             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
    688                 PyObject *sub = TOP();
    689                 PyObject *container = SECOND();
    690                 next_instr--;
    691                 if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
    692                     goto error;
    693                 }
    694                 DISPATCH_SAME_OPARG();
    695             }
    696             else {
    697                 STAT_INC(STORE_SUBSCR, deferred);
    698                 DECREMENT_ADAPTIVE_COUNTER(cache);
    699                 JUMP_TO_INSTRUCTION(STORE_SUBSCR);
    700             }
    701         }
    702 
    703         TARGET(STORE_SUBSCR_LIST_INT) {
    704             assert(cframe.use_tracing == 0);
    705             PyObject *sub = TOP();
    706             PyObject *list = SECOND();
    707             PyObject *value = THIRD();
    708             DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
    709             DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
    710 
    711             // Ensure nonnegative, zero-or-one-digit ints.
    712             DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR);
    713             Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
    714             // Ensure index < len(list)
    715             DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
    716             STAT_INC(STORE_SUBSCR, hit);
    717 
    718             PyObject *old_value = PyList_GET_ITEM(list, index);
    719             PyList_SET_ITEM(list, index, value);
    720             STACK_SHRINK(3);
    721             assert(old_value != NULL);
    722             Py_DECREF(old_value);
    723             _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
    724             Py_DECREF(list);
    725             JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
    726             DISPATCH();
    727         }
    728 
    729         TARGET(STORE_SUBSCR_DICT) {
    730             assert(cframe.use_tracing == 0);
    731             PyObject *sub = TOP();
    732             PyObject *dict = SECOND();
    733             PyObject *value = THIRD();
    734             DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
    735             STACK_SHRINK(3);
    736             STAT_INC(STORE_SUBSCR, hit);
    737             int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
    738             Py_DECREF(dict);
    739             if (err != 0) {
    740                 goto error;
    741             }
    742             JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
    743             DISPATCH();
    744         }
    745 
    746         TARGET(DELETE_SUBSCR) {
    747             PyObject *sub = TOP();
    748             PyObject *container = SECOND();
    749             int err;
    750             STACK_SHRINK(2);
    751             /* del container[sub] */
    752             err = PyObject_DelItem(container, sub);
    753             Py_DECREF(container);
    754             Py_DECREF(sub);
    755             if (err != 0)
    756                 goto error;
    757             DISPATCH();
    758         }
    759 
    760         TARGET(PRINT_EXPR) {
    761             PyObject *value = POP();
    762             PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook));
    763             PyObject *res;
    764             if (hook == NULL) {
    765                 _PyErr_SetString(tstate, PyExc_RuntimeError,
    766                                  "lost sys.displayhook");
    767                 Py_DECREF(value);
    768                 goto error;
    769             }
    770             res = PyObject_CallOneArg(hook, value);
    771             Py_DECREF(value);
    772             if (res == NULL)
    773                 goto error;
    774             Py_DECREF(res);
    775             DISPATCH();
    776         }
    777 
    778         TARGET(RAISE_VARARGS) {
    779             PyObject *cause = NULL, *exc = NULL;
    780             switch (oparg) {
    781             case 2:
    782                 cause = POP(); /* cause */
    783                 /* fall through */
    784             case 1:
    785                 exc = POP(); /* exc */
    786                 /* fall through */
    787             case 0:
    788                 if (do_raise(tstate, exc, cause)) {
    789                     goto exception_unwind;
    790                 }
    791                 break;
    792             default:
    793                 _PyErr_SetString(tstate, PyExc_SystemError,
    794                                  "bad RAISE_VARARGS oparg");
    795                 break;
    796             }
    797             goto error;
    798         }
    799 
    800         TARGET(RETURN_VALUE) {
    801             PyObject *retval = POP();
    802             assert(EMPTY());
    803             _PyFrame_SetStackPointer(frame, stack_pointer);
    804             TRACE_FUNCTION_EXIT();
    805             DTRACE_FUNCTION_EXIT();
    806             _Py_LeaveRecursiveCallTstate(tstate);
    807             if (!frame->is_entry) {
    808                 frame = cframe.current_frame = pop_frame(tstate, frame);
    809                 _PyFrame_StackPush(frame, retval);
    810                 goto resume_frame;
    811             }
    812             /* Restore previous cframe and return. */
    813             tstate->cframe = cframe.previous;
    814             tstate->cframe->use_tracing = cframe.use_tracing;
    815             assert(tstate->cframe->current_frame == frame->previous);
    816             assert(!_PyErr_Occurred(tstate));
    817             return retval;
    818         }
    819 
    820         TARGET(GET_AITER) {
    821             unaryfunc getter = NULL;
    822             PyObject *iter = NULL;
    823             PyObject *obj = TOP();
    824             PyTypeObject *type = Py_TYPE(obj);
    825 
    826             if (type->tp_as_async != NULL) {
    827                 getter = type->tp_as_async->am_aiter;
    828             }
    829 
    830             if (getter != NULL) {
    831                 iter = (*getter)(obj);
    832                 Py_DECREF(obj);
    833                 if (iter == NULL) {
    834                     SET_TOP(NULL);
    835                     goto error;
    836                 }
    837             }
    838             else {
    839                 SET_TOP(NULL);
    840                 _PyErr_Format(tstate, PyExc_TypeError,
    841                               "'async for' requires an object with "
    842                               "__aiter__ method, got %.100s",
    843                               type->tp_name);
    844                 Py_DECREF(obj);
    845                 goto error;
    846             }
    847 
    848             if (Py_TYPE(iter)->tp_as_async == NULL ||
    849                     Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
    850 
    851                 SET_TOP(NULL);
    852                 _PyErr_Format(tstate, PyExc_TypeError,
    853                               "'async for' received an object from __aiter__ "
    854                               "that does not implement __anext__: %.100s",
    855                               Py_TYPE(iter)->tp_name);
    856                 Py_DECREF(iter);
    857                 goto error;
    858             }
    859 
    860             SET_TOP(iter);
    861             DISPATCH();
    862         }
    863 
    864         TARGET(GET_ANEXT) {
    865             unaryfunc getter = NULL;
    866             PyObject *next_iter = NULL;
    867             PyObject *awaitable = NULL;
    868             PyObject *aiter = TOP();
    869             PyTypeObject *type = Py_TYPE(aiter);
    870 
    871             if (PyAsyncGen_CheckExact(aiter)) {
    872                 awaitable = type->tp_as_async->am_anext(aiter);
    873                 if (awaitable == NULL) {
    874                     goto error;
    875                 }
    876             } else {
    877                 if (type->tp_as_async != NULL){
    878                     getter = type->tp_as_async->am_anext;
    879                 }
    880 
    881                 if (getter != NULL) {
    882                     next_iter = (*getter)(aiter);
    883                     if (next_iter == NULL) {
    884                         goto error;
    885                     }
    886                 }
    887                 else {
    888                     _PyErr_Format(tstate, PyExc_TypeError,
    889                                   "'async for' requires an iterator with "
    890                                   "__anext__ method, got %.100s",
    891                                   type->tp_name);
    892                     goto error;
    893                 }
    894 
    895                 awaitable = _PyCoro_GetAwaitableIter(next_iter);
    896                 if (awaitable == NULL) {
    897                     _PyErr_FormatFromCause(
    898                         PyExc_TypeError,
    899                         "'async for' received an invalid object "
    900                         "from __anext__: %.100s",
    901                         Py_TYPE(next_iter)->tp_name);
    902 
    903                     Py_DECREF(next_iter);
    904                     goto error;
    905                 } else {
    906                     Py_DECREF(next_iter);
    907                 }
    908             }
    909 
    910             PUSH(awaitable);
    911             PREDICT(LOAD_CONST);
    912             DISPATCH();
    913         }
    914 
    915         TARGET(GET_AWAITABLE) {
    916             PREDICTED(GET_AWAITABLE);
    917             PyObject *iterable = TOP();
    918             PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
    919 
    920             if (iter == NULL) {
    921                 format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
    922             }
    923 
    924             Py_DECREF(iterable);
    925 
    926             if (iter != NULL && PyCoro_CheckExact(iter)) {
    927                 PyObject *yf = _PyGen_yf((PyGenObject*)iter);
    928                 if (yf != NULL) {
    929                     /* `iter` is a coroutine object that is being
    930                        awaited, `yf` is a pointer to the current awaitable
    931                        being awaited on. */
    932                     Py_DECREF(yf);
    933                     Py_CLEAR(iter);
    934                     _PyErr_SetString(tstate, PyExc_RuntimeError,
    935                                      "coroutine is being awaited already");
    936                     /* The code below jumps to `error` if `iter` is NULL. */
    937                 }
    938             }
    939 
    940             SET_TOP(iter); /* Even if it's NULL */
    941 
    942             if (iter == NULL) {
    943                 goto error;
    944             }
    945 
    946             PREDICT(LOAD_CONST);
    947             DISPATCH();
    948         }
    949 
    950         TARGET(SEND) {
    951             assert(frame->is_entry);
    952             assert(STACK_LEVEL() >= 2);
    953             PyObject *v = POP();
    954             PyObject *receiver = TOP();
    955             PySendResult gen_status;
    956             PyObject *retval;
    957             if (tstate->c_tracefunc == NULL) {
    958                 gen_status = PyIter_Send(receiver, v, &retval);
    959             } else {
    960                 if (Py_IsNone(v) && PyIter_Check(receiver)) {
    961                     retval = Py_TYPE(receiver)->tp_iternext(receiver);
    962                 }
    963                 else {
    964                     retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
    965                 }
    966                 if (retval == NULL) {
    967                     if (tstate->c_tracefunc != NULL
    968                             && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
    969                         call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
    970                     if (_PyGen_FetchStopIterationValue(&retval) == 0) {
    971                         gen_status = PYGEN_RETURN;
    972                     }
    973                     else {
    974                         gen_status = PYGEN_ERROR;
    975                     }
    976                 }
    977                 else {
    978                     gen_status = PYGEN_NEXT;
    979                 }
    980             }
    981             Py_DECREF(v);
    982             if (gen_status == PYGEN_ERROR) {
    983                 assert(retval == NULL);
    984                 goto error;
    985             }
    986             if (gen_status == PYGEN_RETURN) {
    987                 assert(retval != NULL);
    988                 Py_DECREF(receiver);
    989                 SET_TOP(retval);
    990                 JUMPBY(oparg);
    991                 DISPATCH();
    992             }
    993             assert(gen_status == PYGEN_NEXT);
    994             assert(retval != NULL);
    995             PUSH(retval);
    996             DISPATCH();
    997         }
    998 
    999         TARGET(ASYNC_GEN_WRAP) {
   1000             PyObject *v = TOP();
   1001             assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR);
   1002             PyObject *w = _PyAsyncGenValueWrapperNew(v);
   1003             if (w == NULL) {
   1004                 goto error;
   1005             }
   1006             SET_TOP(w);
   1007             Py_DECREF(v);
   1008             DISPATCH();
   1009         }
   1010 
   1011         TARGET(YIELD_VALUE) {
   1012             assert(frame->is_entry);
   1013             PyObject *retval = POP();
   1014             _PyFrame_GetGenerator(frame)->gi_frame_state = FRAME_SUSPENDED;
   1015             _PyFrame_SetStackPointer(frame, stack_pointer);
   1016             TRACE_FUNCTION_EXIT();
   1017             DTRACE_FUNCTION_EXIT();
   1018             _Py_LeaveRecursiveCallTstate(tstate);
   1019             /* Restore previous cframe and return. */
   1020             tstate->cframe = cframe.previous;
   1021             tstate->cframe->use_tracing = cframe.use_tracing;
   1022             assert(tstate->cframe->current_frame == frame->previous);
   1023             assert(!_PyErr_Occurred(tstate));
   1024             return retval;
   1025         }
   1026 
   1027         TARGET(POP_EXCEPT) {
   1028             _PyErr_StackItem *exc_info = tstate->exc_info;
   1029             PyObject *value = exc_info->exc_value;
   1030             exc_info->exc_value = POP();
   1031             Py_XDECREF(value);
   1032             DISPATCH();
   1033         }
   1034 
   1035         TARGET(RERAISE) {
   1036             if (oparg) {
   1037                 PyObject *lasti = PEEK(oparg + 1);
   1038                 if (PyLong_Check(lasti)) {
   1039                     frame->prev_instr = first_instr + PyLong_AsLong(lasti);
   1040                     assert(!_PyErr_Occurred(tstate));
   1041                 }
   1042                 else {
   1043                     assert(PyLong_Check(lasti));
   1044                     _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
   1045                     goto error;
   1046                 }
   1047             }
   1048             PyObject *val = POP();
   1049             assert(val && PyExceptionInstance_Check(val));
   1050             PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
   1051             PyObject *tb = PyException_GetTraceback(val);
   1052             _PyErr_Restore(tstate, exc, val, tb);
   1053             goto exception_unwind;
   1054         }
   1055 
   1056         TARGET(PREP_RERAISE_STAR) {
   1057             PyObject *excs = POP();
   1058             assert(PyList_Check(excs));
   1059             PyObject *orig = POP();
   1060 
   1061             PyObject *val = _PyExc_PrepReraiseStar(orig, excs);
   1062             Py_DECREF(excs);
   1063             Py_DECREF(orig);
   1064 
   1065             if (val == NULL) {
   1066                 goto error;
   1067             }
   1068 
   1069             PUSH(val);
   1070             DISPATCH();
   1071         }
   1072 
   1073         TARGET(END_ASYNC_FOR) {
   1074             PyObject *val = POP();
   1075             assert(val && PyExceptionInstance_Check(val));
   1076             if (PyErr_GivenExceptionMatches(val, PyExc_StopAsyncIteration)) {
   1077                 Py_DECREF(val);
   1078                 Py_DECREF(POP());
   1079                 DISPATCH();
   1080             }
   1081             else {
   1082                 PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
   1083                 PyObject *tb = PyException_GetTraceback(val);
   1084                 _PyErr_Restore(tstate, exc, val, tb);
   1085                 goto exception_unwind;
   1086             }
   1087         }
   1088 
   1089         TARGET(LOAD_ASSERTION_ERROR) {
   1090             PyObject *value = PyExc_AssertionError;
   1091             Py_INCREF(value);
   1092             PUSH(value);
   1093             DISPATCH();
   1094         }
   1095 
   1096         TARGET(LOAD_BUILD_CLASS) {
   1097             PyObject *bc;
   1098             if (PyDict_CheckExact(BUILTINS())) {
   1099                 bc = _PyDict_GetItemWithError(BUILTINS(),
   1100                                               &_Py_ID(__build_class__));
   1101                 if (bc == NULL) {
   1102                     if (!_PyErr_Occurred(tstate)) {
   1103                         _PyErr_SetString(tstate, PyExc_NameError,
   1104                                          "__build_class__ not found");
   1105                     }
   1106                     goto error;
   1107                 }
   1108                 Py_INCREF(bc);
   1109             }
   1110             else {
   1111                 bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__));
   1112                 if (bc == NULL) {
   1113                     if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
   1114                         _PyErr_SetString(tstate, PyExc_NameError,
   1115                                          "__build_class__ not found");
   1116                     goto error;
   1117                 }
   1118             }
   1119             PUSH(bc);
   1120             DISPATCH();
   1121         }
   1122 
   1123         TARGET(STORE_NAME) {
   1124             PyObject *name = GETITEM(names, oparg);
   1125             PyObject *v = POP();
   1126             PyObject *ns = LOCALS();
   1127             int err;
   1128             if (ns == NULL) {
   1129                 _PyErr_Format(tstate, PyExc_SystemError,
   1130                               "no locals found when storing %R", name);
   1131                 Py_DECREF(v);
   1132                 goto error;
   1133             }
   1134             if (PyDict_CheckExact(ns))
   1135                 err = PyDict_SetItem(ns, name, v);
   1136             else
   1137                 err = PyObject_SetItem(ns, name, v);
   1138             Py_DECREF(v);
   1139             if (err != 0)
   1140                 goto error;
   1141             DISPATCH();
   1142         }
   1143 
   1144         TARGET(DELETE_NAME) {
   1145             PyObject *name = GETITEM(names, oparg);
   1146             PyObject *ns = LOCALS();
   1147             int err;
   1148             if (ns == NULL) {
   1149                 _PyErr_Format(tstate, PyExc_SystemError,
   1150                               "no locals when deleting %R", name);
   1151                 goto error;
   1152             }
   1153             err = PyObject_DelItem(ns, name);
   1154             if (err != 0) {
   1155                 format_exc_check_arg(tstate, PyExc_NameError,
   1156                                      NAME_ERROR_MSG,
   1157                                      name);
   1158                 goto error;
   1159             }
   1160             DISPATCH();
   1161         }
   1162 
   1163         TARGET(UNPACK_SEQUENCE) {
   1164             PREDICTED(UNPACK_SEQUENCE);
   1165             PyObject *seq = POP();
   1166             PyObject **top = stack_pointer + oparg;
   1167             if (!unpack_iterable(tstate, seq, oparg, -1, top)) {
   1168                 Py_DECREF(seq);
   1169                 goto error;
   1170             }
   1171             STACK_GROW(oparg);
   1172             Py_DECREF(seq);
   1173             JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
   1174             DISPATCH();
   1175         }
   1176 
   1177         TARGET(UNPACK_SEQUENCE_ADAPTIVE) {
   1178             assert(cframe.use_tracing == 0);
   1179             _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
   1180             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   1181                 PyObject *seq = TOP();
   1182                 next_instr--;
   1183                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
   1184                 DISPATCH_SAME_OPARG();
   1185             }
   1186             else {
   1187                 STAT_INC(UNPACK_SEQUENCE, deferred);
   1188                 DECREMENT_ADAPTIVE_COUNTER(cache);
   1189                 JUMP_TO_INSTRUCTION(UNPACK_SEQUENCE);
   1190             }
   1191         }
   1192 
   1193         TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
   1194             PyObject *seq = TOP();
   1195             DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
   1196             DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
   1197             STAT_INC(UNPACK_SEQUENCE, hit);
   1198             SET_TOP(Py_NewRef(PyTuple_GET_ITEM(seq, 1)));
   1199             PUSH(Py_NewRef(PyTuple_GET_ITEM(seq, 0)));
   1200             Py_DECREF(seq);
   1201             JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
   1202             DISPATCH();
   1203         }
   1204 
   1205         TARGET(UNPACK_SEQUENCE_TUPLE) {
   1206             PyObject *seq = TOP();
   1207             DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
   1208             DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
   1209             STAT_INC(UNPACK_SEQUENCE, hit);
   1210             STACK_SHRINK(1);
   1211             PyObject **items = _PyTuple_ITEMS(seq);
   1212             while (oparg--) {
   1213                 PUSH(Py_NewRef(items[oparg]));
   1214             }
   1215             Py_DECREF(seq);
   1216             JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
   1217             DISPATCH();
   1218         }
   1219 
   1220         TARGET(UNPACK_SEQUENCE_LIST) {
   1221             PyObject *seq = TOP();
   1222             DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
   1223             DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
   1224             STAT_INC(UNPACK_SEQUENCE, hit);
   1225             STACK_SHRINK(1);
   1226             PyObject **items = _PyList_ITEMS(seq);
   1227             while (oparg--) {
   1228                 PUSH(Py_NewRef(items[oparg]));
   1229             }
   1230             Py_DECREF(seq);
   1231             JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
   1232             DISPATCH();
   1233         }
   1234 
   1235         TARGET(UNPACK_EX) {
   1236             int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
   1237             PyObject *seq = POP();
   1238             PyObject **top = stack_pointer + totalargs;
   1239             if (!unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top)) {
   1240                 Py_DECREF(seq);
   1241                 goto error;
   1242             }
   1243             STACK_GROW(totalargs);
   1244             Py_DECREF(seq);
   1245             DISPATCH();
   1246         }
   1247 
   1248         TARGET(STORE_ATTR) {
   1249             PREDICTED(STORE_ATTR);
   1250             PyObject *name = GETITEM(names, oparg);
   1251             PyObject *owner = TOP();
   1252             PyObject *v = SECOND();
   1253             int err;
   1254             STACK_SHRINK(2);
   1255             err = PyObject_SetAttr(owner, name, v);
   1256             Py_DECREF(v);
   1257             Py_DECREF(owner);
   1258             if (err != 0) {
   1259                 goto error;
   1260             }
   1261             JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
   1262             DISPATCH();
   1263         }
   1264 
   1265         TARGET(DELETE_ATTR) {
   1266             PyObject *name = GETITEM(names, oparg);
   1267             PyObject *owner = POP();
   1268             int err;
   1269             err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
   1270             Py_DECREF(owner);
   1271             if (err != 0)
   1272                 goto error;
   1273             DISPATCH();
   1274         }
   1275 
   1276         TARGET(STORE_GLOBAL) {
   1277             PyObject *name = GETITEM(names, oparg);
   1278             PyObject *v = POP();
   1279             int err;
   1280             err = PyDict_SetItem(GLOBALS(), name, v);
   1281             Py_DECREF(v);
   1282             if (err != 0)
   1283                 goto error;
   1284             DISPATCH();
   1285         }
   1286 
   1287         TARGET(DELETE_GLOBAL) {
   1288             PyObject *name = GETITEM(names, oparg);
   1289             int err;
   1290             err = PyDict_DelItem(GLOBALS(), name);
   1291             if (err != 0) {
   1292                 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1293                     format_exc_check_arg(tstate, PyExc_NameError,
   1294                                          NAME_ERROR_MSG, name);
   1295                 }
   1296                 goto error;
   1297             }
   1298             DISPATCH();
   1299         }
   1300 
   1301         TARGET(LOAD_NAME) {
   1302             PyObject *name = GETITEM(names, oparg);
   1303             PyObject *locals = LOCALS();
   1304             PyObject *v;
   1305             if (locals == NULL) {
   1306                 _PyErr_Format(tstate, PyExc_SystemError,
   1307                               "no locals when loading %R", name);
   1308                 goto error;
   1309             }
   1310             if (PyDict_CheckExact(locals)) {
   1311                 v = PyDict_GetItemWithError(locals, name);
   1312                 if (v != NULL) {
   1313                     Py_INCREF(v);
   1314                 }
   1315                 else if (_PyErr_Occurred(tstate)) {
   1316                     goto error;
   1317                 }
   1318             }
   1319             else {
   1320                 v = PyObject_GetItem(locals, name);
   1321                 if (v == NULL) {
   1322                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
   1323                         goto error;
   1324                     _PyErr_Clear(tstate);
   1325                 }
   1326             }
   1327             if (v == NULL) {
   1328                 v = PyDict_GetItemWithError(GLOBALS(), name);
   1329                 if (v != NULL) {
   1330                     Py_INCREF(v);
   1331                 }
   1332                 else if (_PyErr_Occurred(tstate)) {
   1333                     goto error;
   1334                 }
   1335                 else {
   1336                     if (PyDict_CheckExact(BUILTINS())) {
   1337                         v = PyDict_GetItemWithError(BUILTINS(), name);
   1338                         if (v == NULL) {
   1339                             if (!_PyErr_Occurred(tstate)) {
   1340                                 format_exc_check_arg(
   1341                                         tstate, PyExc_NameError,
   1342                                         NAME_ERROR_MSG, name);
   1343                             }
   1344                             goto error;
   1345                         }
   1346                         Py_INCREF(v);
   1347                     }
   1348                     else {
   1349                         v = PyObject_GetItem(BUILTINS(), name);
   1350                         if (v == NULL) {
   1351                             if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1352                                 format_exc_check_arg(
   1353                                             tstate, PyExc_NameError,
   1354                                             NAME_ERROR_MSG, name);
   1355                             }
   1356                             goto error;
   1357                         }
   1358                     }
   1359                 }
   1360             }
   1361             PUSH(v);
   1362             DISPATCH();
   1363         }
   1364 
   1365         TARGET(LOAD_GLOBAL) {
   1366             PREDICTED(LOAD_GLOBAL);
   1367             int push_null = oparg & 1;
   1368             PEEK(0) = NULL;
   1369             PyObject *name = GETITEM(names, oparg>>1);
   1370             PyObject *v;
   1371             if (PyDict_CheckExact(GLOBALS())
   1372                 && PyDict_CheckExact(BUILTINS()))
   1373             {
   1374                 v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
   1375                                        (PyDictObject *)BUILTINS(),
   1376                                        name);
   1377                 if (v == NULL) {
   1378                     if (!_PyErr_Occurred(tstate)) {
   1379                         /* _PyDict_LoadGlobal() returns NULL without raising
   1380                          * an exception if the key doesn't exist */
   1381                         format_exc_check_arg(tstate, PyExc_NameError,
   1382                                              NAME_ERROR_MSG, name);
   1383                     }
   1384                     goto error;
   1385                 }
   1386                 Py_INCREF(v);
   1387             }
   1388             else {
   1389                 /* Slow-path if globals or builtins is not a dict */
   1390 
   1391                 /* namespace 1: globals */
   1392                 v = PyObject_GetItem(GLOBALS(), name);
   1393                 if (v == NULL) {
   1394                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1395                         goto error;
   1396                     }
   1397                     _PyErr_Clear(tstate);
   1398 
   1399                     /* namespace 2: builtins */
   1400                     v = PyObject_GetItem(BUILTINS(), name);
   1401                     if (v == NULL) {
   1402                         if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1403                             format_exc_check_arg(
   1404                                         tstate, PyExc_NameError,
   1405                                         NAME_ERROR_MSG, name);
   1406                         }
   1407                         goto error;
   1408                     }
   1409                 }
   1410             }
   1411             /* Skip over inline cache */
   1412             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
   1413             STACK_GROW(push_null);
   1414             PUSH(v);
   1415             DISPATCH();
   1416         }
   1417 
   1418         TARGET(LOAD_GLOBAL_ADAPTIVE) {
   1419             assert(cframe.use_tracing == 0);
   1420             _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
   1421             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   1422                 PyObject *name = GETITEM(names, oparg>>1);
   1423                 next_instr--;
   1424                 if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) {
   1425                     goto error;
   1426                 }
   1427                 DISPATCH_SAME_OPARG();
   1428             }
   1429             else {
   1430                 STAT_INC(LOAD_GLOBAL, deferred);
   1431                 DECREMENT_ADAPTIVE_COUNTER(cache);
   1432                 JUMP_TO_INSTRUCTION(LOAD_GLOBAL);
   1433             }
   1434         }
   1435 
   1436         TARGET(LOAD_GLOBAL_MODULE) {
   1437             assert(cframe.use_tracing == 0);
   1438             DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
   1439             PyDictObject *dict = (PyDictObject *)GLOBALS();
   1440             _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
   1441             uint32_t version = read_u32(cache->module_keys_version);
   1442             DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
   1443             assert(DK_IS_UNICODE(dict->ma_keys));
   1444             PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
   1445             PyObject *res = entries[cache->index].me_value;
   1446             DEOPT_IF(res == NULL, LOAD_GLOBAL);
   1447             int push_null = oparg & 1;
   1448             PEEK(0) = NULL;
   1449             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
   1450             STAT_INC(LOAD_GLOBAL, hit);
   1451             STACK_GROW(push_null+1);
   1452             Py_INCREF(res);
   1453             SET_TOP(res);
   1454             DISPATCH();
   1455         }
   1456 
   1457         TARGET(LOAD_GLOBAL_BUILTIN) {
   1458             assert(cframe.use_tracing == 0);
   1459             DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
   1460             DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL);
   1461             PyDictObject *mdict = (PyDictObject *)GLOBALS();
   1462             PyDictObject *bdict = (PyDictObject *)BUILTINS();
   1463             _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
   1464             uint32_t mod_version = read_u32(cache->module_keys_version);
   1465             uint16_t bltn_version = cache->builtin_keys_version;
   1466             DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL);
   1467             DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL);
   1468             assert(DK_IS_UNICODE(bdict->ma_keys));
   1469             PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
   1470             PyObject *res = entries[cache->index].me_value;
   1471             DEOPT_IF(res == NULL, LOAD_GLOBAL);
   1472             int push_null = oparg & 1;
   1473             PEEK(0) = NULL;
   1474             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
   1475             STAT_INC(LOAD_GLOBAL, hit);
   1476             STACK_GROW(push_null+1);
   1477             Py_INCREF(res);
   1478             SET_TOP(res);
   1479             DISPATCH();
   1480         }
   1481 
   1482         TARGET(DELETE_FAST) {
   1483             PyObject *v = GETLOCAL(oparg);
   1484             if (v != NULL) {
   1485                 SETLOCAL(oparg, NULL);
   1486                 DISPATCH();
   1487             }
   1488             goto unbound_local_error;
   1489         }
   1490 
   1491         TARGET(MAKE_CELL) {
   1492             // "initial" is probably NULL but not if it's an arg (or set
   1493             // via PyFrame_LocalsToFast() before MAKE_CELL has run).
   1494             PyObject *initial = GETLOCAL(oparg);
   1495             PyObject *cell = PyCell_New(initial);
   1496             if (cell == NULL) {
   1497                 goto resume_with_error;
   1498             }
   1499             SETLOCAL(oparg, cell);
   1500             DISPATCH();
   1501         }
   1502 
   1503         TARGET(DELETE_DEREF) {
   1504             PyObject *cell = GETLOCAL(oparg);
   1505             PyObject *oldobj = PyCell_GET(cell);
   1506             if (oldobj != NULL) {
   1507                 PyCell_SET(cell, NULL);
   1508                 Py_DECREF(oldobj);
   1509                 DISPATCH();
   1510             }
   1511             format_exc_unbound(tstate, frame->f_code, oparg);
   1512             goto error;
   1513         }
   1514 
   1515         TARGET(LOAD_CLASSDEREF) {
   1516             PyObject *name, *value, *locals = LOCALS();
   1517             assert(locals);
   1518             assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
   1519             name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
   1520             if (PyDict_CheckExact(locals)) {
   1521                 value = PyDict_GetItemWithError(locals, name);
   1522                 if (value != NULL) {
   1523                     Py_INCREF(value);
   1524                 }
   1525                 else if (_PyErr_Occurred(tstate)) {
   1526                     goto error;
   1527                 }
   1528             }
   1529             else {
   1530                 value = PyObject_GetItem(locals, name);
   1531                 if (value == NULL) {
   1532                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1533                         goto error;
   1534                     }
   1535                     _PyErr_Clear(tstate);
   1536                 }
   1537             }
   1538             if (!value) {
   1539                 PyObject *cell = GETLOCAL(oparg);
   1540                 value = PyCell_GET(cell);
   1541                 if (value == NULL) {
   1542                     format_exc_unbound(tstate, frame->f_code, oparg);
   1543                     goto error;
   1544                 }
   1545                 Py_INCREF(value);
   1546             }
   1547             PUSH(value);
   1548             DISPATCH();
   1549         }
   1550 
   1551         TARGET(LOAD_DEREF) {
   1552             PyObject *cell = GETLOCAL(oparg);
   1553             PyObject *value = PyCell_GET(cell);
   1554             if (value == NULL) {
   1555                 format_exc_unbound(tstate, frame->f_code, oparg);
   1556                 goto error;
   1557             }
   1558             Py_INCREF(value);
   1559             PUSH(value);
   1560             DISPATCH();
   1561         }
   1562 
   1563         TARGET(STORE_DEREF) {
   1564             PyObject *v = POP();
   1565             PyObject *cell = GETLOCAL(oparg);
   1566             PyObject *oldobj = PyCell_GET(cell);
   1567             PyCell_SET(cell, v);
   1568             Py_XDECREF(oldobj);
   1569             DISPATCH();
   1570         }
   1571 
   1572         TARGET(COPY_FREE_VARS) {
   1573             /* Copy closure variables to free variables */
   1574             PyCodeObject *co = frame->f_code;
   1575             PyObject *closure = frame->f_func->func_closure;
   1576             int offset = co->co_nlocals + co->co_nplaincellvars;
   1577             assert(oparg == co->co_nfreevars);
   1578             for (int i = 0; i < oparg; ++i) {
   1579                 PyObject *o = PyTuple_GET_ITEM(closure, i);
   1580                 Py_INCREF(o);
   1581                 frame->localsplus[offset + i] = o;
   1582             }
   1583             DISPATCH();
   1584         }
   1585 
   1586         TARGET(BUILD_STRING) {
   1587             PyObject *str;
   1588             str = _PyUnicode_JoinArray(&_Py_STR(empty),
   1589                                        stack_pointer - oparg, oparg);
   1590             if (str == NULL)
   1591                 goto error;
   1592             while (--oparg >= 0) {
   1593                 PyObject *item = POP();
   1594                 Py_DECREF(item);
   1595             }
   1596             PUSH(str);
   1597             DISPATCH();
   1598         }
   1599 
   1600         TARGET(BUILD_TUPLE) {
   1601             PyObject *tup = PyTuple_New(oparg);
   1602             if (tup == NULL)
   1603                 goto error;
   1604             while (--oparg >= 0) {
   1605                 PyObject *item = POP();
   1606                 PyTuple_SET_ITEM(tup, oparg, item);
   1607             }
   1608             PUSH(tup);
   1609             DISPATCH();
   1610         }
   1611 
   1612         TARGET(BUILD_LIST) {
   1613             PyObject *list =  PyList_New(oparg);
   1614             if (list == NULL)
   1615                 goto error;
   1616             while (--oparg >= 0) {
   1617                 PyObject *item = POP();
   1618                 PyList_SET_ITEM(list, oparg, item);
   1619             }
   1620             PUSH(list);
   1621             DISPATCH();
   1622         }
   1623 
   1624         TARGET(LIST_TO_TUPLE) {
   1625             PyObject *list = POP();
   1626             PyObject *tuple = PyList_AsTuple(list);
   1627             Py_DECREF(list);
   1628             if (tuple == NULL) {
   1629                 goto error;
   1630             }
   1631             PUSH(tuple);
   1632             DISPATCH();
   1633         }
   1634 
   1635         TARGET(LIST_EXTEND) {
   1636             PyObject *iterable = POP();
   1637             PyObject *list = PEEK(oparg);
   1638             PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
   1639             if (none_val == NULL) {
   1640                 if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
   1641                    (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
   1642                 {
   1643                     _PyErr_Clear(tstate);
   1644                     _PyErr_Format(tstate, PyExc_TypeError,
   1645                           "Value after * must be an iterable, not %.200s",
   1646                           Py_TYPE(iterable)->tp_name);
   1647                 }
   1648                 Py_DECREF(iterable);
   1649                 goto error;
   1650             }
   1651             Py_DECREF(none_val);
   1652             Py_DECREF(iterable);
   1653             DISPATCH();
   1654         }
   1655 
   1656         TARGET(SET_UPDATE) {
   1657             PyObject *iterable = POP();
   1658             PyObject *set = PEEK(oparg);
   1659             int err = _PySet_Update(set, iterable);
   1660             Py_DECREF(iterable);
   1661             if (err < 0) {
   1662                 goto error;
   1663             }
   1664             DISPATCH();
   1665         }
   1666 
   1667         TARGET(BUILD_SET) {
   1668             PyObject *set = PySet_New(NULL);
   1669             int err = 0;
   1670             int i;
   1671             if (set == NULL)
   1672                 goto error;
   1673             for (i = oparg; i > 0; i--) {
   1674                 PyObject *item = PEEK(i);
   1675                 if (err == 0)
   1676                     err = PySet_Add(set, item);
   1677                 Py_DECREF(item);
   1678             }
   1679             STACK_SHRINK(oparg);
   1680             if (err != 0) {
   1681                 Py_DECREF(set);
   1682                 goto error;
   1683             }
   1684             PUSH(set);
   1685             DISPATCH();
   1686         }
   1687 
   1688         TARGET(BUILD_MAP) {
   1689             PyObject *map = _PyDict_FromItems(
   1690                     &PEEK(2*oparg), 2,
   1691                     &PEEK(2*oparg - 1), 2,
   1692                     oparg);
   1693             if (map == NULL)
   1694                 goto error;
   1695 
   1696             while (oparg--) {
   1697                 Py_DECREF(POP());
   1698                 Py_DECREF(POP());
   1699             }
   1700             PUSH(map);
   1701             DISPATCH();
   1702         }
   1703 
   1704         TARGET(SETUP_ANNOTATIONS) {
   1705             int err;
   1706             PyObject *ann_dict;
   1707             if (LOCALS() == NULL) {
   1708                 _PyErr_Format(tstate, PyExc_SystemError,
   1709                               "no locals found when setting up annotations");
   1710                 goto error;
   1711             }
   1712             /* check if __annotations__ in locals()... */
   1713             if (PyDict_CheckExact(LOCALS())) {
   1714                 ann_dict = _PyDict_GetItemWithError(LOCALS(),
   1715                                                     &_Py_ID(__annotations__));
   1716                 if (ann_dict == NULL) {
   1717                     if (_PyErr_Occurred(tstate)) {
   1718                         goto error;
   1719                     }
   1720                     /* ...if not, create a new one */
   1721                     ann_dict = PyDict_New();
   1722                     if (ann_dict == NULL) {
   1723                         goto error;
   1724                     }
   1725                     err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__),
   1726                                          ann_dict);
   1727                     Py_DECREF(ann_dict);
   1728                     if (err != 0) {
   1729                         goto error;
   1730                     }
   1731                 }
   1732             }
   1733             else {
   1734                 /* do the same if locals() is not a dict */
   1735                 ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__));
   1736                 if (ann_dict == NULL) {
   1737                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
   1738                         goto error;
   1739                     }
   1740                     _PyErr_Clear(tstate);
   1741                     ann_dict = PyDict_New();
   1742                     if (ann_dict == NULL) {
   1743                         goto error;
   1744                     }
   1745                     err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
   1746                                            ann_dict);
   1747                     Py_DECREF(ann_dict);
   1748                     if (err != 0) {
   1749                         goto error;
   1750                     }
   1751                 }
   1752                 else {
   1753                     Py_DECREF(ann_dict);
   1754                 }
   1755             }
   1756             DISPATCH();
   1757         }
   1758 
   1759         TARGET(BUILD_CONST_KEY_MAP) {
   1760             PyObject *map;
   1761             PyObject *keys = TOP();
   1762             if (!PyTuple_CheckExact(keys) ||
   1763                 PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
   1764                 _PyErr_SetString(tstate, PyExc_SystemError,
   1765                                  "bad BUILD_CONST_KEY_MAP keys argument");
   1766                 goto error;
   1767             }
   1768             map = _PyDict_FromItems(
   1769                     &PyTuple_GET_ITEM(keys, 0), 1,
   1770                     &PEEK(oparg + 1), 1, oparg);
   1771             if (map == NULL) {
   1772                 goto error;
   1773             }
   1774 
   1775             Py_DECREF(POP());
   1776             while (oparg--) {
   1777                 Py_DECREF(POP());
   1778             }
   1779             PUSH(map);
   1780             DISPATCH();
   1781         }
   1782 
   1783         TARGET(DICT_UPDATE) {
   1784             PyObject *update = POP();
   1785             PyObject *dict = PEEK(oparg);
   1786             if (PyDict_Update(dict, update) < 0) {
   1787                 if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
   1788                     _PyErr_Format(tstate, PyExc_TypeError,
   1789                                     "'%.200s' object is not a mapping",
   1790                                     Py_TYPE(update)->tp_name);
   1791                 }
   1792                 Py_DECREF(update);
   1793                 goto error;
   1794             }
   1795             Py_DECREF(update);
   1796             DISPATCH();
   1797         }
   1798 
   1799         TARGET(DICT_MERGE) {
   1800             PyObject *update = POP();
   1801             PyObject *dict = PEEK(oparg);
   1802 
   1803             if (_PyDict_MergeEx(dict, update, 2) < 0) {
   1804                 format_kwargs_error(tstate, PEEK(2 + oparg), update);
   1805                 Py_DECREF(update);
   1806                 goto error;
   1807             }
   1808             Py_DECREF(update);
   1809             PREDICT(CALL_FUNCTION_EX);
   1810             DISPATCH();
   1811         }
   1812 
   1813         TARGET(MAP_ADD) {
   1814             PyObject *value = TOP();
   1815             PyObject *key = SECOND();
   1816             PyObject *map;
   1817             STACK_SHRINK(2);
   1818             map = PEEK(oparg);                      /* dict */
   1819             assert(PyDict_CheckExact(map));
   1820             /* map[key] = value */
   1821             if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
   1822                 goto error;
   1823             }
   1824             PREDICT(JUMP_BACKWARD_QUICK);
   1825             DISPATCH();
   1826         }
   1827 
   1828         TARGET(LOAD_ATTR) {
   1829             PREDICTED(LOAD_ATTR);
   1830             PyObject *name = GETITEM(names, oparg);
   1831             PyObject *owner = TOP();
   1832             PyObject *res = PyObject_GetAttr(owner, name);
   1833             if (res == NULL) {
   1834                 goto error;
   1835             }
   1836             Py_DECREF(owner);
   1837             SET_TOP(res);
   1838             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
   1839             DISPATCH();
   1840         }
   1841 
   1842         TARGET(LOAD_ATTR_ADAPTIVE) {
   1843             assert(cframe.use_tracing == 0);
   1844             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1845             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   1846                 PyObject *owner = TOP();
   1847                 PyObject *name = GETITEM(names, oparg);
   1848                 next_instr--;
   1849                 if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) {
   1850                     goto error;
   1851                 }
   1852                 DISPATCH_SAME_OPARG();
   1853             }
   1854             else {
   1855                 STAT_INC(LOAD_ATTR, deferred);
   1856                 DECREMENT_ADAPTIVE_COUNTER(cache);
   1857                 JUMP_TO_INSTRUCTION(LOAD_ATTR);
   1858             }
   1859         }
   1860 
   1861         TARGET(LOAD_ATTR_INSTANCE_VALUE) {
   1862             assert(cframe.use_tracing == 0);
   1863             PyObject *owner = TOP();
   1864             PyObject *res;
   1865             PyTypeObject *tp = Py_TYPE(owner);
   1866             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1867             uint32_t type_version = read_u32(cache->version);
   1868             assert(type_version != 0);
   1869             DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
   1870             assert(tp->tp_dictoffset < 0);
   1871             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
   1872             PyDictValues *values = *_PyObject_ValuesPointer(owner);
   1873             DEOPT_IF(values == NULL, LOAD_ATTR);
   1874             res = values->values[cache->index];
   1875             DEOPT_IF(res == NULL, LOAD_ATTR);
   1876             STAT_INC(LOAD_ATTR, hit);
   1877             Py_INCREF(res);
   1878             SET_TOP(res);
   1879             Py_DECREF(owner);
   1880             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
   1881             DISPATCH();
   1882         }
   1883 
   1884         TARGET(LOAD_ATTR_MODULE) {
   1885             assert(cframe.use_tracing == 0);
   1886             // shared with LOAD_METHOD_MODULE
   1887             PyObject *owner = TOP();
   1888             PyObject *res;
   1889             LOAD_MODULE_ATTR_OR_METHOD(ATTR);
   1890             SET_TOP(res);
   1891             Py_DECREF(owner);
   1892             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
   1893             DISPATCH();
   1894         }
   1895 
   1896         TARGET(LOAD_ATTR_WITH_HINT) {
   1897             assert(cframe.use_tracing == 0);
   1898             PyObject *owner = TOP();
   1899             PyObject *res;
   1900             PyTypeObject *tp = Py_TYPE(owner);
   1901             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1902             uint32_t type_version = read_u32(cache->version);
   1903             assert(type_version != 0);
   1904             DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
   1905             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
   1906             PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
   1907             DEOPT_IF(dict == NULL, LOAD_ATTR);
   1908             assert(PyDict_CheckExact((PyObject *)dict));
   1909             PyObject *name = GETITEM(names, oparg);
   1910             uint16_t hint = cache->index;
   1911             DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
   1912             if (DK_IS_UNICODE(dict->ma_keys)) {
   1913                 PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
   1914                 DEOPT_IF(ep->me_key != name, LOAD_ATTR);
   1915                 res = ep->me_value;
   1916             }
   1917             else {
   1918                 PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
   1919                 DEOPT_IF(ep->me_key != name, LOAD_ATTR);
   1920                 res = ep->me_value;
   1921             }
   1922             DEOPT_IF(res == NULL, LOAD_ATTR);
   1923             STAT_INC(LOAD_ATTR, hit);
   1924             Py_INCREF(res);
   1925             SET_TOP(res);
   1926             Py_DECREF(owner);
   1927             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
   1928             DISPATCH();
   1929         }
   1930 
   1931         TARGET(LOAD_ATTR_SLOT) {
   1932             assert(cframe.use_tracing == 0);
   1933             PyObject *owner = TOP();
   1934             PyObject *res;
   1935             PyTypeObject *tp = Py_TYPE(owner);
   1936             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1937             uint32_t type_version = read_u32(cache->version);
   1938             assert(type_version != 0);
   1939             DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
   1940             char *addr = (char *)owner + cache->index;
   1941             res = *(PyObject **)addr;
   1942             DEOPT_IF(res == NULL, LOAD_ATTR);
   1943             STAT_INC(LOAD_ATTR, hit);
   1944             Py_INCREF(res);
   1945             SET_TOP(res);
   1946             Py_DECREF(owner);
   1947             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
   1948             DISPATCH();
   1949         }
   1950 
   1951             DISPATCH();
   1952         TARGET(STORE_ATTR_ADAPTIVE) {
   1953             assert(cframe.use_tracing == 0);
   1954             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1955             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   1956                 PyObject *owner = TOP();
   1957                 PyObject *name = GETITEM(names, oparg);
   1958                 next_instr--;
   1959                 if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) {
   1960                     goto error;
   1961                 }
   1962                 DISPATCH_SAME_OPARG();
   1963             }
   1964             else {
   1965                 STAT_INC(STORE_ATTR, deferred);
   1966                 DECREMENT_ADAPTIVE_COUNTER(cache);
   1967                 JUMP_TO_INSTRUCTION(STORE_ATTR);
   1968             }
   1969         }
   1970 
   1971         TARGET(STORE_ATTR_INSTANCE_VALUE) {
   1972             assert(cframe.use_tracing == 0);
   1973             PyObject *owner = TOP();
   1974             PyTypeObject *tp = Py_TYPE(owner);
   1975             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   1976             uint32_t type_version = read_u32(cache->version);
   1977             assert(type_version != 0);
   1978             DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
   1979             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
   1980             PyDictValues *values = *_PyObject_ValuesPointer(owner);
   1981             DEOPT_IF(values == NULL, STORE_ATTR);
   1982             STAT_INC(STORE_ATTR, hit);
   1983             Py_ssize_t index = cache->index;
   1984             STACK_SHRINK(1);
   1985             PyObject *value = POP();
   1986             PyObject *old_value = values->values[index];
   1987             values->values[index] = value;
   1988             if (old_value == NULL) {
   1989                 _PyDictValues_AddToInsertionOrder(values, index);
   1990             }
   1991             else {
   1992                 Py_DECREF(old_value);
   1993             }
   1994             Py_DECREF(owner);
   1995             JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
   1996             DISPATCH();
   1997         }
   1998 
   1999         TARGET(STORE_ATTR_WITH_HINT) {
   2000             assert(cframe.use_tracing == 0);
   2001             PyObject *owner = TOP();
   2002             PyTypeObject *tp = Py_TYPE(owner);
   2003             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   2004             uint32_t type_version = read_u32(cache->version);
   2005             assert(type_version != 0);
   2006             DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
   2007             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
   2008             PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
   2009             DEOPT_IF(dict == NULL, STORE_ATTR);
   2010             assert(PyDict_CheckExact((PyObject *)dict));
   2011             PyObject *name = GETITEM(names, oparg);
   2012             uint16_t hint = cache->index;
   2013             DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
   2014             PyObject *value, *old_value;
   2015             if (DK_IS_UNICODE(dict->ma_keys)) {
   2016                 PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
   2017                 DEOPT_IF(ep->me_key != name, STORE_ATTR);
   2018                 old_value = ep->me_value;
   2019                 DEOPT_IF(old_value == NULL, STORE_ATTR);
   2020                 STACK_SHRINK(1);
   2021                 value = POP();
   2022                 ep->me_value = value;
   2023             }
   2024             else {
   2025                 PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
   2026                 DEOPT_IF(ep->me_key != name, STORE_ATTR);
   2027                 old_value = ep->me_value;
   2028                 DEOPT_IF(old_value == NULL, STORE_ATTR);
   2029                 STACK_SHRINK(1);
   2030                 value = POP();
   2031                 ep->me_value = value;
   2032             }
   2033             Py_DECREF(old_value);
   2034             STAT_INC(STORE_ATTR, hit);
   2035             /* Ensure dict is GC tracked if it needs to be */
   2036             if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
   2037                 _PyObject_GC_TRACK(dict);
   2038             }
   2039             /* PEP 509 */
   2040             dict->ma_version_tag = DICT_NEXT_VERSION();
   2041             Py_DECREF(owner);
   2042             JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
   2043             DISPATCH();
   2044         }
   2045 
   2046         TARGET(STORE_ATTR_SLOT) {
   2047             assert(cframe.use_tracing == 0);
   2048             PyObject *owner = TOP();
   2049             PyTypeObject *tp = Py_TYPE(owner);
   2050             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
   2051             uint32_t type_version = read_u32(cache->version);
   2052             assert(type_version != 0);
   2053             DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
   2054             char *addr = (char *)owner + cache->index;
   2055             STAT_INC(STORE_ATTR, hit);
   2056             STACK_SHRINK(1);
   2057             PyObject *value = POP();
   2058             PyObject *old_value = *(PyObject **)addr;
   2059             *(PyObject **)addr = value;
   2060             Py_XDECREF(old_value);
   2061             Py_DECREF(owner);
   2062             JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
   2063             DISPATCH();
   2064         }
   2065 
   2066         TARGET(COMPARE_OP) {
   2067             PREDICTED(COMPARE_OP);
   2068             assert(oparg <= Py_GE);
   2069             PyObject *right = POP();
   2070             PyObject *left = TOP();
   2071             PyObject *res = PyObject_RichCompare(left, right, oparg);
   2072             SET_TOP(res);
   2073             Py_DECREF(left);
   2074             Py_DECREF(right);
   2075             if (res == NULL) {
   2076                 goto error;
   2077             }
   2078             JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
   2079             DISPATCH();
   2080         }
   2081 
   2082         TARGET(COMPARE_OP_ADAPTIVE) {
   2083             assert(cframe.use_tracing == 0);
   2084             _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
   2085             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   2086                 PyObject *right = TOP();
   2087                 PyObject *left = SECOND();
   2088                 next_instr--;
   2089                 _Py_Specialize_CompareOp(left, right, next_instr, oparg);
   2090                 DISPATCH_SAME_OPARG();
   2091             }
   2092             else {
   2093                 STAT_INC(COMPARE_OP, deferred);
   2094                 DECREMENT_ADAPTIVE_COUNTER(cache);
   2095                 JUMP_TO_INSTRUCTION(COMPARE_OP);
   2096             }
   2097         }
   2098 
   2099         TARGET(COMPARE_OP_FLOAT_JUMP) {
   2100             assert(cframe.use_tracing == 0);
   2101             // Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false)
   2102             _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
   2103             int when_to_jump_mask = cache->mask;
   2104             PyObject *right = TOP();
   2105             PyObject *left = SECOND();
   2106             DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
   2107             DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
   2108             double dleft = PyFloat_AS_DOUBLE(left);
   2109             double dright = PyFloat_AS_DOUBLE(right);
   2110             int sign = (dleft > dright) - (dleft < dright);
   2111             DEOPT_IF(isnan(dleft), COMPARE_OP);
   2112             DEOPT_IF(isnan(dright), COMPARE_OP);
   2113             STAT_INC(COMPARE_OP, hit);
   2114             JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
   2115             NEXTOPARG();
   2116             STACK_SHRINK(2);
   2117             _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
   2118             _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
   2119             assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
   2120                    opcode == POP_JUMP_BACKWARD_IF_FALSE ||
   2121                    opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2122                    opcode == POP_JUMP_BACKWARD_IF_TRUE);
   2123             int jump = (9 << (sign + 1)) & when_to_jump_mask;
   2124             if (!jump) {
   2125                 next_instr++;
   2126             }
   2127             else if (jump >= 8) {
   2128                 assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
   2129                        opcode == POP_JUMP_BACKWARD_IF_FALSE);
   2130                 JUMPBY(1 - oparg);
   2131                 CHECK_EVAL_BREAKER();
   2132             }
   2133             else {
   2134                 assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2135                        opcode == POP_JUMP_FORWARD_IF_FALSE);
   2136                 JUMPBY(1 + oparg);
   2137             }
   2138             DISPATCH();
   2139         }
   2140 
   2141         TARGET(COMPARE_OP_INT_JUMP) {
   2142             assert(cframe.use_tracing == 0);
   2143             // Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false)
   2144             _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
   2145             int when_to_jump_mask = cache->mask;
   2146             PyObject *right = TOP();
   2147             PyObject *left = SECOND();
   2148             DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
   2149             DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
   2150             DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
   2151             DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
   2152             STAT_INC(COMPARE_OP, hit);
   2153             assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
   2154             Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
   2155             Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
   2156             int sign = (ileft > iright) - (ileft < iright);
   2157             JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
   2158             NEXTOPARG();
   2159             STACK_SHRINK(2);
   2160             _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
   2161             _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
   2162             assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
   2163                    opcode == POP_JUMP_BACKWARD_IF_FALSE ||
   2164                    opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2165                    opcode == POP_JUMP_BACKWARD_IF_TRUE);
   2166             int jump = (9 << (sign + 1)) & when_to_jump_mask;
   2167             if (!jump) {
   2168                 next_instr++;
   2169             }
   2170             else if (jump >= 8) {
   2171                 assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
   2172                        opcode == POP_JUMP_BACKWARD_IF_FALSE);
   2173                 JUMPBY(1 - oparg);
   2174                 CHECK_EVAL_BREAKER();
   2175             }
   2176             else {
   2177                 assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2178                        opcode == POP_JUMP_FORWARD_IF_FALSE);
   2179                 JUMPBY(1 + oparg);
   2180             }
   2181             DISPATCH();
   2182         }
   2183 
   2184         TARGET(COMPARE_OP_STR_JUMP) {
   2185             assert(cframe.use_tracing == 0);
   2186             // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false)
   2187             _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
   2188             int when_to_jump_mask = cache->mask;
   2189             PyObject *right = TOP();
   2190             PyObject *left = SECOND();
   2191             DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
   2192             DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
   2193             STAT_INC(COMPARE_OP, hit);
   2194             int res = _PyUnicode_Equal(left, right);
   2195             if (res < 0) {
   2196                 goto error;
   2197             }
   2198             assert(oparg == Py_EQ || oparg == Py_NE);
   2199             JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
   2200             NEXTOPARG();
   2201             assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
   2202                    opcode == POP_JUMP_BACKWARD_IF_FALSE ||
   2203                    opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2204                    opcode == POP_JUMP_BACKWARD_IF_TRUE);
   2205             STACK_SHRINK(2);
   2206             _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
   2207             _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
   2208             assert(res == 0 || res == 1);
   2209             int sign = 1 - res;
   2210             int jump = (9 << (sign + 1)) & when_to_jump_mask;
   2211             if (!jump) {
   2212                 next_instr++;
   2213             }
   2214             else if (jump >= 8) {
   2215                 assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
   2216                        opcode == POP_JUMP_BACKWARD_IF_FALSE);
   2217                 JUMPBY(1 - oparg);
   2218                 CHECK_EVAL_BREAKER();
   2219             }
   2220             else {
   2221                 assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
   2222                        opcode == POP_JUMP_FORWARD_IF_FALSE);
   2223                 JUMPBY(1 + oparg);
   2224             }
   2225             DISPATCH();
   2226         }
   2227 
   2228         TARGET(IS_OP) {
   2229             PyObject *right = POP();
   2230             PyObject *left = TOP();
   2231             int res = Py_Is(left, right) ^ oparg;
   2232             PyObject *b = res ? Py_True : Py_False;
   2233             Py_INCREF(b);
   2234             SET_TOP(b);
   2235             Py_DECREF(left);
   2236             Py_DECREF(right);
   2237             DISPATCH();
   2238         }
   2239 
   2240         TARGET(CONTAINS_OP) {
   2241             PyObject *right = POP();
   2242             PyObject *left = POP();
   2243             int res = PySequence_Contains(right, left);
   2244             Py_DECREF(left);
   2245             Py_DECREF(right);
   2246             if (res < 0) {
   2247                 goto error;
   2248             }
   2249             PyObject *b = (res^oparg) ? Py_True : Py_False;
   2250             Py_INCREF(b);
   2251             PUSH(b);
   2252             DISPATCH();
   2253         }
   2254 
   2255         TARGET(CHECK_EG_MATCH) {
   2256             PyObject *match_type = POP();
   2257             if (check_except_star_type_valid(tstate, match_type) < 0) {
   2258                 Py_DECREF(match_type);
   2259                 goto error;
   2260             }
   2261 
   2262             PyObject *exc_value = TOP();
   2263             PyObject *match = NULL, *rest = NULL;
   2264             int res = exception_group_match(exc_value, match_type,
   2265                                             &match, &rest);
   2266             Py_DECREF(match_type);
   2267             if (res < 0) {
   2268                 goto error;
   2269             }
   2270 
   2271             if (match == NULL || rest == NULL) {
   2272                 assert(match == NULL);
   2273                 assert(rest == NULL);
   2274                 goto error;
   2275             }
   2276             if (Py_IsNone(match)) {
   2277                 PUSH(match);
   2278                 Py_XDECREF(rest);
   2279             }
   2280             else {
   2281                 /* Total or partial match - update the stack from
   2282                  * [val]
   2283                  * to
   2284                  * [rest, match]
   2285                  * (rest can be Py_None)
   2286                  */
   2287 
   2288                 SET_TOP(rest);
   2289                 PUSH(match);
   2290                 PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL);
   2291                 Py_DECREF(exc_value);
   2292             }
   2293             DISPATCH();
   2294         }
   2295 
   2296         TARGET(CHECK_EXC_MATCH) {
   2297             PyObject *right = POP();
   2298             PyObject *left = TOP();
   2299             assert(PyExceptionInstance_Check(left));
   2300             if (check_except_type_valid(tstate, right) < 0) {
   2301                  Py_DECREF(right);
   2302                  goto error;
   2303             }
   2304 
   2305             int res = PyErr_GivenExceptionMatches(left, right);
   2306             Py_DECREF(right);
   2307             PUSH(Py_NewRef(res ? Py_True : Py_False));
   2308             DISPATCH();
   2309         }
   2310 
   2311         TARGET(IMPORT_NAME) {
   2312             PyObject *name = GETITEM(names, oparg);
   2313             PyObject *fromlist = POP();
   2314             PyObject *level = TOP();
   2315             PyObject *res;
   2316             res = import_name(tstate, frame, name, fromlist, level);
   2317             Py_DECREF(level);
   2318             Py_DECREF(fromlist);
   2319             SET_TOP(res);
   2320             if (res == NULL)
   2321                 goto error;
   2322             DISPATCH();
   2323         }
   2324 
   2325         TARGET(IMPORT_STAR) {
   2326             PyObject *from = POP(), *locals;
   2327             int err;
   2328             if (_PyFrame_FastToLocalsWithError(frame) < 0) {
   2329                 Py_DECREF(from);
   2330                 goto error;
   2331             }
   2332 
   2333             locals = LOCALS();
   2334             if (locals == NULL) {
   2335                 _PyErr_SetString(tstate, PyExc_SystemError,
   2336                                  "no locals found during 'import *'");
   2337                 Py_DECREF(from);
   2338                 goto error;
   2339             }
   2340             err = import_all_from(tstate, locals, from);
   2341             _PyFrame_LocalsToFast(frame, 0);
   2342             Py_DECREF(from);
   2343             if (err != 0)
   2344                 goto error;
   2345             DISPATCH();
   2346         }
   2347 
   2348         TARGET(IMPORT_FROM) {
   2349             PyObject *name = GETITEM(names, oparg);
   2350             PyObject *from = TOP();
   2351             PyObject *res;
   2352             res = import_from(tstate, from, name);
   2353             PUSH(res);
   2354             if (res == NULL)
   2355                 goto error;
   2356             DISPATCH();
   2357         }
   2358 
   2359         TARGET(JUMP_FORWARD) {
   2360             JUMPBY(oparg);
   2361             DISPATCH();
   2362         }
   2363 
   2364         TARGET(JUMP_BACKWARD) {
   2365             _PyCode_Warmup(frame->f_code);
   2366             JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
   2367         }
   2368 
   2369         TARGET(POP_JUMP_BACKWARD_IF_FALSE) {
   2370             PREDICTED(POP_JUMP_BACKWARD_IF_FALSE);
   2371             PyObject *cond = POP();
   2372             if (Py_IsTrue(cond)) {
   2373                 _Py_DECREF_NO_DEALLOC(cond);
   2374                 DISPATCH();
   2375             }
   2376             if (Py_IsFalse(cond)) {
   2377                 _Py_DECREF_NO_DEALLOC(cond);
   2378                 JUMPBY(-oparg);
   2379                 CHECK_EVAL_BREAKER();
   2380                 DISPATCH();
   2381             }
   2382             int err = PyObject_IsTrue(cond);
   2383             Py_DECREF(cond);
   2384             if (err > 0)
   2385                 ;
   2386             else if (err == 0) {
   2387                 JUMPBY(-oparg);
   2388                 CHECK_EVAL_BREAKER();
   2389             }
   2390             else
   2391                 goto error;
   2392             DISPATCH();
   2393         }
   2394 
   2395         TARGET(POP_JUMP_FORWARD_IF_FALSE) {
   2396             PREDICTED(POP_JUMP_FORWARD_IF_FALSE);
   2397             PyObject *cond = POP();
   2398             if (Py_IsTrue(cond)) {
   2399                 _Py_DECREF_NO_DEALLOC(cond);
   2400             }
   2401             else if (Py_IsFalse(cond)) {
   2402                 _Py_DECREF_NO_DEALLOC(cond);
   2403                 JUMPBY(oparg);
   2404             }
   2405             else {
   2406                 int err = PyObject_IsTrue(cond);
   2407                 Py_DECREF(cond);
   2408                 if (err > 0)
   2409                     ;
   2410                 else if (err == 0) {
   2411                     JUMPBY(oparg);
   2412                 }
   2413                 else
   2414                     goto error;
   2415             }
   2416             DISPATCH();
   2417         }
   2418 
   2419         TARGET(POP_JUMP_BACKWARD_IF_TRUE) {
   2420             PyObject *cond = POP();
   2421             if (Py_IsFalse(cond)) {
   2422                 _Py_DECREF_NO_DEALLOC(cond);
   2423                 DISPATCH();
   2424             }
   2425             if (Py_IsTrue(cond)) {
   2426                 _Py_DECREF_NO_DEALLOC(cond);
   2427                 JUMPBY(-oparg);
   2428                 CHECK_EVAL_BREAKER();
   2429                 DISPATCH();
   2430             }
   2431             int err = PyObject_IsTrue(cond);
   2432             Py_DECREF(cond);
   2433             if (err > 0) {
   2434                 JUMPBY(-oparg);
   2435                 CHECK_EVAL_BREAKER();
   2436             }
   2437             else if (err == 0)
   2438                 ;
   2439             else
   2440                 goto error;
   2441             DISPATCH();
   2442         }
   2443 
   2444         TARGET(POP_JUMP_FORWARD_IF_TRUE) {
   2445             PyObject *cond = POP();
   2446             if (Py_IsFalse(cond)) {
   2447                 _Py_DECREF_NO_DEALLOC(cond);
   2448             }
   2449             else if (Py_IsTrue(cond)) {
   2450                 _Py_DECREF_NO_DEALLOC(cond);
   2451                 JUMPBY(oparg);
   2452             }
   2453             else {
   2454                 int err = PyObject_IsTrue(cond);
   2455                 Py_DECREF(cond);
   2456                 if (err > 0) {
   2457                     JUMPBY(oparg);
   2458                 }
   2459                 else if (err == 0)
   2460                     ;
   2461                 else
   2462                     goto error;
   2463             }
   2464             DISPATCH();
   2465         }
   2466 
   2467         TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) {
   2468             PyObject *value = POP();
   2469             if (!Py_IsNone(value)) {
   2470                 Py_DECREF(value);
   2471                 JUMPBY(-oparg);
   2472                 CHECK_EVAL_BREAKER();
   2473                 DISPATCH();
   2474             }
   2475             _Py_DECREF_NO_DEALLOC(value);
   2476             DISPATCH();
   2477         }
   2478 
   2479         TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) {
   2480             PyObject *value = POP();
   2481             if (!Py_IsNone(value)) {
   2482                 JUMPBY(oparg);
   2483             }
   2484             Py_DECREF(value);
   2485             DISPATCH();
   2486         }
   2487 
   2488         TARGET(POP_JUMP_BACKWARD_IF_NONE) {
   2489             PyObject *value = POP();
   2490             if (Py_IsNone(value)) {
   2491                 _Py_DECREF_NO_DEALLOC(value);
   2492                 JUMPBY(-oparg);
   2493                 CHECK_EVAL_BREAKER();
   2494             }
   2495             else {
   2496                 Py_DECREF(value);
   2497             }
   2498             DISPATCH();
   2499         }
   2500 
   2501         TARGET(POP_JUMP_FORWARD_IF_NONE) {
   2502             PyObject *value = POP();
   2503             if (Py_IsNone(value)) {
   2504                 _Py_DECREF_NO_DEALLOC(value);
   2505                 JUMPBY(oparg);
   2506             }
   2507             else {
   2508                 Py_DECREF(value);
   2509             }
   2510             DISPATCH();
   2511         }
   2512 
   2513         TARGET(JUMP_IF_FALSE_OR_POP) {
   2514             PyObject *cond = TOP();
   2515             int err;
   2516             if (Py_IsTrue(cond)) {
   2517                 STACK_SHRINK(1);
   2518                 _Py_DECREF_NO_DEALLOC(cond);
   2519                 DISPATCH();
   2520             }
   2521             if (Py_IsFalse(cond)) {
   2522                 JUMPBY(oparg);
   2523                 DISPATCH();
   2524             }
   2525             err = PyObject_IsTrue(cond);
   2526             if (err > 0) {
   2527                 STACK_SHRINK(1);
   2528                 Py_DECREF(cond);
   2529             }
   2530             else if (err == 0)
   2531                 JUMPBY(oparg);
   2532             else
   2533                 goto error;
   2534             DISPATCH();
   2535         }
   2536 
   2537         TARGET(JUMP_IF_TRUE_OR_POP) {
   2538             PyObject *cond = TOP();
   2539             int err;
   2540             if (Py_IsFalse(cond)) {
   2541                 STACK_SHRINK(1);
   2542                 _Py_DECREF_NO_DEALLOC(cond);
   2543                 DISPATCH();
   2544             }
   2545             if (Py_IsTrue(cond)) {
   2546                 JUMPBY(oparg);
   2547                 DISPATCH();
   2548             }
   2549             err = PyObject_IsTrue(cond);
   2550             if (err > 0) {
   2551                 JUMPBY(oparg);
   2552             }
   2553             else if (err == 0) {
   2554                 STACK_SHRINK(1);
   2555                 Py_DECREF(cond);
   2556             }
   2557             else
   2558                 goto error;
   2559             DISPATCH();
   2560         }
   2561 
   2562         TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
   2563             /* This bytecode is used in the `yield from` or `await` loop.
   2564              * If there is an interrupt, we want it handled in the innermost
   2565              * generator or coroutine, so we deliberately do not check it here.
   2566              * (see bpo-30039).
   2567              */
   2568             JUMPBY(-oparg);
   2569             DISPATCH();
   2570         }
   2571 
   2572         TARGET(JUMP_BACKWARD_QUICK) {
   2573             PREDICTED(JUMP_BACKWARD_QUICK);
   2574             assert(oparg < INSTR_OFFSET());
   2575             JUMPBY(-oparg);
   2576             CHECK_EVAL_BREAKER();
   2577             DISPATCH();
   2578         }
   2579 
   2580         TARGET(GET_LEN) {
   2581             // PUSH(len(TOS))
   2582             Py_ssize_t len_i = PyObject_Length(TOP());
   2583             if (len_i < 0) {
   2584                 goto error;
   2585             }
   2586             PyObject *len_o = PyLong_FromSsize_t(len_i);
   2587             if (len_o == NULL) {
   2588                 goto error;
   2589             }
   2590             PUSH(len_o);
   2591             DISPATCH();
   2592         }
   2593 
   2594         TARGET(MATCH_CLASS) {
   2595             // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
   2596             // None on failure.
   2597             PyObject *names = POP();
   2598             PyObject *type = POP();
   2599             PyObject *subject = TOP();
   2600             assert(PyTuple_CheckExact(names));
   2601             PyObject *attrs = match_class(tstate, subject, type, oparg, names);
   2602             Py_DECREF(names);
   2603             Py_DECREF(type);
   2604             if (attrs) {
   2605                 // Success!
   2606                 assert(PyTuple_CheckExact(attrs));
   2607                 SET_TOP(attrs);
   2608             }
   2609             else if (_PyErr_Occurred(tstate)) {
   2610                 // Error!
   2611                 goto error;
   2612             }
   2613             else {
   2614                 // Failure!
   2615                 Py_INCREF(Py_None);
   2616                 SET_TOP(Py_None);
   2617             }
   2618             Py_DECREF(subject);
   2619             DISPATCH();
   2620         }
   2621 
   2622         TARGET(MATCH_MAPPING) {
   2623             PyObject *subject = TOP();
   2624             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
   2625             PyObject *res = match ? Py_True : Py_False;
   2626             Py_INCREF(res);
   2627             PUSH(res);
   2628             PREDICT(POP_JUMP_FORWARD_IF_FALSE);
   2629             PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
   2630             DISPATCH();
   2631         }
   2632 
   2633         TARGET(MATCH_SEQUENCE) {
   2634             PyObject *subject = TOP();
   2635             int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
   2636             PyObject *res = match ? Py_True : Py_False;
   2637             Py_INCREF(res);
   2638             PUSH(res);
   2639             PREDICT(POP_JUMP_FORWARD_IF_FALSE);
   2640             PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
   2641             DISPATCH();
   2642         }
   2643 
   2644         TARGET(MATCH_KEYS) {
   2645             // On successful match, PUSH(values). Otherwise, PUSH(None).
   2646             PyObject *keys = TOP();
   2647             PyObject *subject = SECOND();
   2648             PyObject *values_or_none = match_keys(tstate, subject, keys);
   2649             if (values_or_none == NULL) {
   2650                 goto error;
   2651             }
   2652             PUSH(values_or_none);
   2653             DISPATCH();
   2654         }
   2655 
   2656         TARGET(GET_ITER) {
   2657             /* before: [obj]; after [getiter(obj)] */
   2658             PyObject *iterable = TOP();
   2659             PyObject *iter = PyObject_GetIter(iterable);
   2660             Py_DECREF(iterable);
   2661             SET_TOP(iter);
   2662             if (iter == NULL)
   2663                 goto error;
   2664             PREDICT(FOR_ITER);
   2665             DISPATCH();
   2666         }
   2667 
   2668         TARGET(GET_YIELD_FROM_ITER) {
   2669             /* before: [obj]; after [getiter(obj)] */
   2670             PyObject *iterable = TOP();
   2671             PyObject *iter;
   2672             if (PyCoro_CheckExact(iterable)) {
   2673                 /* `iterable` is a coroutine */
   2674                 if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
   2675                     /* and it is used in a 'yield from' expression of a
   2676                        regular generator. */
   2677                     Py_DECREF(iterable);
   2678                     SET_TOP(NULL);
   2679                     _PyErr_SetString(tstate, PyExc_TypeError,
   2680                                      "cannot 'yield from' a coroutine object "
   2681                                      "in a non-coroutine generator");
   2682                     goto error;
   2683                 }
   2684             }
   2685             else if (!PyGen_CheckExact(iterable)) {
   2686                 /* `iterable` is not a generator. */
   2687                 iter = PyObject_GetIter(iterable);
   2688                 Py_DECREF(iterable);
   2689                 SET_TOP(iter);
   2690                 if (iter == NULL)
   2691                     goto error;
   2692             }
   2693             PREDICT(LOAD_CONST);
   2694             DISPATCH();
   2695         }
   2696 
   2697         TARGET(FOR_ITER) {
   2698             PREDICTED(FOR_ITER);
   2699             /* before: [iter]; after: [iter, iter()] *or* [] */
   2700             PyObject *iter = TOP();
   2701 #ifdef Py_STATS
   2702             extern int _PySpecialization_ClassifyIterator(PyObject *);
   2703             _py_stats.opcode_stats[FOR_ITER].specialization.failure++;
   2704             _py_stats.opcode_stats[FOR_ITER].specialization.failure_kinds[_PySpecialization_ClassifyIterator(iter)]++;
   2705 #endif
   2706             PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
   2707             if (next != NULL) {
   2708                 PUSH(next);
   2709                 PREDICT(STORE_FAST);
   2710                 PREDICT(UNPACK_SEQUENCE);
   2711                 DISPATCH();
   2712             }
   2713             if (_PyErr_Occurred(tstate)) {
   2714                 if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
   2715                     goto error;
   2716                 }
   2717                 else if (tstate->c_tracefunc != NULL) {
   2718                     call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
   2719                 }
   2720                 _PyErr_Clear(tstate);
   2721             }
   2722             /* iterator ended normally */
   2723             STACK_SHRINK(1);
   2724             Py_DECREF(iter);
   2725             JUMPBY(oparg);
   2726             DISPATCH();
   2727         }
   2728 
   2729         TARGET(BEFORE_ASYNC_WITH) {
   2730             PyObject *mgr = TOP();
   2731             PyObject *res;
   2732             PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__));
   2733             if (enter == NULL) {
   2734                 if (!_PyErr_Occurred(tstate)) {
   2735                     _PyErr_Format(tstate, PyExc_TypeError,
   2736                                   "'%.200s' object does not support the "
   2737                                   "asynchronous context manager protocol",
   2738                                   Py_TYPE(mgr)->tp_name);
   2739                 }
   2740                 goto error;
   2741             }
   2742             PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
   2743             if (exit == NULL) {
   2744                 if (!_PyErr_Occurred(tstate)) {
   2745                     _PyErr_Format(tstate, PyExc_TypeError,
   2746                                   "'%.200s' object does not support the "
   2747                                   "asynchronous context manager protocol "
   2748                                   "(missed __aexit__ method)",
   2749                                   Py_TYPE(mgr)->tp_name);
   2750                 }
   2751                 Py_DECREF(enter);
   2752                 goto error;
   2753             }
   2754             SET_TOP(exit);
   2755             Py_DECREF(mgr);
   2756             res = _PyObject_CallNoArgs(enter);
   2757             Py_DECREF(enter);
   2758             if (res == NULL)
   2759                 goto error;
   2760             PUSH(res);
   2761             PREDICT(GET_AWAITABLE);
   2762             DISPATCH();
   2763         }
   2764 
   2765         TARGET(BEFORE_WITH) {
   2766             PyObject *mgr = TOP();
   2767             PyObject *res;
   2768             PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__));
   2769             if (enter == NULL) {
   2770                 if (!_PyErr_Occurred(tstate)) {
   2771                     _PyErr_Format(tstate, PyExc_TypeError,
   2772                                   "'%.200s' object does not support the "
   2773                                   "context manager protocol",
   2774                                   Py_TYPE(mgr)->tp_name);
   2775                 }
   2776                 goto error;
   2777             }
   2778             PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
   2779             if (exit == NULL) {
   2780                 if (!_PyErr_Occurred(tstate)) {
   2781                     _PyErr_Format(tstate, PyExc_TypeError,
   2782                                   "'%.200s' object does not support the "
   2783                                   "context manager protocol "
   2784                                   "(missed __exit__ method)",
   2785                                   Py_TYPE(mgr)->tp_name);
   2786                 }
   2787                 Py_DECREF(enter);
   2788                 goto error;
   2789             }
   2790             SET_TOP(exit);
   2791             Py_DECREF(mgr);
   2792             res = _PyObject_CallNoArgs(enter);
   2793             Py_DECREF(enter);
   2794             if (res == NULL) {
   2795                 goto error;
   2796             }
   2797             PUSH(res);
   2798             DISPATCH();
   2799         }
   2800 
   2801         TARGET(WITH_EXCEPT_START) {
   2802             /* At the top of the stack are 4 values:
   2803                - TOP = exc_info()
   2804                - SECOND = previous exception
   2805                - THIRD: lasti of exception in exc_info()
   2806                - FOURTH: the context.__exit__ bound method
   2807                We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
   2808                Then we push the __exit__ return value.
   2809             */
   2810             PyObject *exit_func;
   2811             PyObject *exc, *val, *tb, *res;
   2812 
   2813             val = TOP();
   2814             assert(val && PyExceptionInstance_Check(val));
   2815             exc = PyExceptionInstance_Class(val);
   2816             tb = PyException_GetTraceback(val);
   2817             Py_XDECREF(tb);
   2818             assert(PyLong_Check(PEEK(3)));
   2819             exit_func = PEEK(4);
   2820             PyObject *stack[4] = {NULL, exc, val, tb};
   2821             res = PyObject_Vectorcall(exit_func, stack + 1,
   2822                     3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
   2823             if (res == NULL)
   2824                 goto error;
   2825 
   2826             PUSH(res);
   2827             DISPATCH();
   2828         }
   2829 
   2830         TARGET(PUSH_EXC_INFO) {
   2831             PyObject *value = TOP();
   2832 
   2833             _PyErr_StackItem *exc_info = tstate->exc_info;
   2834             if (exc_info->exc_value != NULL) {
   2835                 SET_TOP(exc_info->exc_value);
   2836             }
   2837             else {
   2838                 Py_INCREF(Py_None);
   2839                 SET_TOP(Py_None);
   2840             }
   2841 
   2842             Py_INCREF(value);
   2843             PUSH(value);
   2844             assert(PyExceptionInstance_Check(value));
   2845             exc_info->exc_value = value;
   2846 
   2847             DISPATCH();
   2848         }
   2849 
   2850         TARGET(LOAD_METHOD) {
   2851             PREDICTED(LOAD_METHOD);
   2852             /* Designed to work in tandem with PRECALL. */
   2853             PyObject *name = GETITEM(names, oparg);
   2854             PyObject *obj = TOP();
   2855             PyObject *meth = NULL;
   2856 
   2857             int meth_found = _PyObject_GetMethod(obj, name, &meth);
   2858 
   2859             if (meth == NULL) {
   2860                 /* Most likely attribute wasn't found. */
   2861                 goto error;
   2862             }
   2863 
   2864             if (meth_found) {
   2865                 /* We can bypass temporary bound method object.
   2866                    meth is unbound method and obj is self.
   2867 
   2868                    meth | self | arg1 | ... | argN
   2869                  */
   2870                 SET_TOP(meth);
   2871                 PUSH(obj);  // self
   2872             }
   2873             else {
   2874                 /* meth is not an unbound method (but a regular attr, or
   2875                    something was returned by a descriptor protocol).  Set
   2876                    the second element of the stack to NULL, to signal
   2877                    PRECALL that it's not a method call.
   2878 
   2879                    NULL | meth | arg1 | ... | argN
   2880                 */
   2881                 SET_TOP(NULL);
   2882                 Py_DECREF(obj);
   2883                 PUSH(meth);
   2884             }
   2885             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   2886             DISPATCH();
   2887         }
   2888 
   2889         TARGET(LOAD_METHOD_ADAPTIVE) {
   2890             assert(cframe.use_tracing == 0);
   2891             _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
   2892             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   2893                 PyObject *owner = TOP();
   2894                 PyObject *name = GETITEM(names, oparg);
   2895                 next_instr--;
   2896                 if (_Py_Specialize_LoadMethod(owner, next_instr, name) < 0) {
   2897                     goto error;
   2898                 }
   2899                 DISPATCH_SAME_OPARG();
   2900             }
   2901             else {
   2902                 STAT_INC(LOAD_METHOD, deferred);
   2903                 DECREMENT_ADAPTIVE_COUNTER(cache);
   2904                 JUMP_TO_INSTRUCTION(LOAD_METHOD);
   2905             }
   2906         }
   2907 
   2908         TARGET(LOAD_METHOD_WITH_VALUES) {
   2909             /* LOAD_METHOD, with cached method object */
   2910             assert(cframe.use_tracing == 0);
   2911             PyObject *self = TOP();
   2912             PyTypeObject *self_cls = Py_TYPE(self);
   2913             _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
   2914             uint32_t type_version = read_u32(cache->type_version);
   2915             assert(type_version != 0);
   2916             DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_METHOD);
   2917             assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
   2918             PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self);
   2919             DEOPT_IF(dict != NULL, LOAD_METHOD);
   2920             PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls;
   2921             DEOPT_IF(self_heap_type->ht_cached_keys->dk_version !=
   2922                      read_u32(cache->keys_version), LOAD_METHOD);
   2923             STAT_INC(LOAD_METHOD, hit);
   2924             PyObject *res = read_obj(cache->descr);
   2925             assert(res != NULL);
   2926             assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
   2927             Py_INCREF(res);
   2928             SET_TOP(res);
   2929             PUSH(self);
   2930             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   2931             DISPATCH();
   2932         }
   2933 
   2934         TARGET(LOAD_METHOD_WITH_DICT) {
   2935             /* LOAD_METHOD, with a dict
   2936              Can be either a managed dict, or a tp_dictoffset offset.*/
   2937             assert(cframe.use_tracing == 0);
   2938             PyObject *self = TOP();
   2939             PyTypeObject *self_cls = Py_TYPE(self);
   2940             _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
   2941 
   2942             DEOPT_IF(self_cls->tp_version_tag != read_u32(cache->type_version),
   2943                      LOAD_METHOD);
   2944             /* Treat index as a signed 16 bit value */
   2945             int dictoffset = *(int16_t *)&cache->dict_offset;
   2946             PyDictObject **dictptr = (PyDictObject**)(((char *)self)+dictoffset);
   2947             assert(
   2948                 dictoffset == MANAGED_DICT_OFFSET ||
   2949                 (dictoffset == self_cls->tp_dictoffset && dictoffset > 0)
   2950             );
   2951             PyDictObject *dict = *dictptr;
   2952             DEOPT_IF(dict == NULL, LOAD_METHOD);
   2953             DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->keys_version),
   2954                      LOAD_METHOD);
   2955             STAT_INC(LOAD_METHOD, hit);
   2956             PyObject *res = read_obj(cache->descr);
   2957             assert(res != NULL);
   2958             assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
   2959             Py_INCREF(res);
   2960             SET_TOP(res);
   2961             PUSH(self);
   2962             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   2963             DISPATCH();
   2964         }
   2965 
   2966         TARGET(LOAD_METHOD_NO_DICT) {
   2967             assert(cframe.use_tracing == 0);
   2968             PyObject *self = TOP();
   2969             PyTypeObject *self_cls = Py_TYPE(self);
   2970             _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
   2971             uint32_t type_version = read_u32(cache->type_version);
   2972             DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_METHOD);
   2973             assert(self_cls->tp_dictoffset == 0);
   2974             STAT_INC(LOAD_METHOD, hit);
   2975             PyObject *res = read_obj(cache->descr);
   2976             assert(res != NULL);
   2977             assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
   2978             Py_INCREF(res);
   2979             SET_TOP(res);
   2980             PUSH(self);
   2981             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   2982             DISPATCH();
   2983         }
   2984 
   2985         TARGET(LOAD_METHOD_MODULE) {
   2986             /* LOAD_METHOD, for module methods */
   2987             assert(cframe.use_tracing == 0);
   2988             PyObject *owner = TOP();
   2989             PyObject *res;
   2990             LOAD_MODULE_ATTR_OR_METHOD(METHOD);
   2991             SET_TOP(NULL);
   2992             Py_DECREF(owner);
   2993             PUSH(res);
   2994             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   2995             DISPATCH();
   2996         }
   2997 
   2998         TARGET(LOAD_METHOD_CLASS) {
   2999             /* LOAD_METHOD, for class methods */
   3000             assert(cframe.use_tracing == 0);
   3001             _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
   3002 
   3003             PyObject *cls = TOP();
   3004             DEOPT_IF(!PyType_Check(cls), LOAD_METHOD);
   3005             uint32_t type_version = read_u32(cache->type_version);
   3006             DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version,
   3007                      LOAD_METHOD);
   3008             assert(type_version != 0);
   3009 
   3010             STAT_INC(LOAD_METHOD, hit);
   3011             PyObject *res = read_obj(cache->descr);
   3012             assert(res != NULL);
   3013             Py_INCREF(res);
   3014             SET_TOP(NULL);
   3015             Py_DECREF(cls);
   3016             PUSH(res);
   3017             JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD);
   3018             DISPATCH();
   3019         }
   3020 
   3021         TARGET(PRECALL) {
   3022             PREDICTED(PRECALL);
   3023             /* Designed to work in tamdem with LOAD_METHOD. */
   3024             /* `meth` is NULL when LOAD_METHOD thinks that it's not
   3025                 a method call.
   3026 
   3027                 Stack layout:
   3028 
   3029                        ... | NULL | callable | arg1 | ... | argN
   3030                                                             ^- TOP()
   3031                                                ^- (-oparg)
   3032                                     ^- (-oparg-1)
   3033                              ^- (-oparg-2)
   3034 
   3035                 `callable` will be POPed by call_function.
   3036                 NULL will will be POPed manually later.
   3037                 If `meth` isn't NULL, it's a method call.  Stack layout:
   3038 
   3039                      ... | method | self | arg1 | ... | argN
   3040                                                         ^- TOP()
   3041                                            ^- (-oparg)
   3042                                     ^- (-oparg-1)
   3043                            ^- (-oparg-2)
   3044 
   3045                `self` and `method` will be POPed by call_function.
   3046                We'll be passing `oparg + 1` to call_function, to
   3047                make it accept the `self` as a first argument.
   3048             */
   3049             int is_meth = is_method(stack_pointer, oparg);
   3050             int nargs = oparg + is_meth;
   3051             /* Move ownership of reference from stack to call_shape
   3052              * and make sure that NULL is cleared from stack */
   3053             PyObject *function = PEEK(nargs + 1);
   3054             if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
   3055                 PyObject *meth = ((PyMethodObject *)function)->im_func;
   3056                 PyObject *self = ((PyMethodObject *)function)->im_self;
   3057                 Py_INCREF(meth);
   3058                 Py_INCREF(self);
   3059                 PEEK(oparg+1) = self;
   3060                 PEEK(oparg+2) = meth;
   3061                 Py_DECREF(function);
   3062             }
   3063             JUMPBY(INLINE_CACHE_ENTRIES_PRECALL);
   3064             DISPATCH();
   3065         }
   3066 
   3067         TARGET(PRECALL_BOUND_METHOD) {
   3068             DEOPT_IF(is_method(stack_pointer, oparg), PRECALL);
   3069             PyObject *function = PEEK(oparg + 1);
   3070             DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, PRECALL);
   3071             STAT_INC(PRECALL, hit);
   3072             PyObject *meth = ((PyMethodObject *)function)->im_func;
   3073             PyObject *self = ((PyMethodObject *)function)->im_self;
   3074             Py_INCREF(meth);
   3075             Py_INCREF(self);
   3076             PEEK(oparg + 1) = self;
   3077             PEEK(oparg + 2) = meth;
   3078             Py_DECREF(function);
   3079             JUMPBY(INLINE_CACHE_ENTRIES_PRECALL);
   3080             DISPATCH();
   3081         }
   3082 
   3083         TARGET(PRECALL_PYFUNC) {
   3084             int nargs = oparg + is_method(stack_pointer, oparg);
   3085             PyObject *function = PEEK(nargs + 1);
   3086             DEOPT_IF(Py_TYPE(function) != &PyFunction_Type, PRECALL);
   3087             STAT_INC(PRECALL, hit);
   3088             JUMPBY(INLINE_CACHE_ENTRIES_PRECALL);
   3089             DISPATCH();
   3090         }
   3091 
   3092         TARGET(KW_NAMES) {
   3093             assert(call_shape.kwnames == NULL);
   3094             assert(oparg < PyTuple_GET_SIZE(consts));
   3095             call_shape.kwnames = GETITEM(consts, oparg);
   3096             DISPATCH();
   3097         }
   3098 
   3099         TARGET(CALL) {
   3100             int is_meth;
   3101         call_function:
   3102             is_meth = is_method(stack_pointer, oparg);
   3103             int total_args = oparg + is_meth;
   3104             PyObject *function = PEEK(total_args + 1);
   3105             int positional_args = total_args - KWNAMES_LEN();
   3106             // Check if the call can be inlined or not
   3107             if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
   3108                 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
   3109                 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
   3110                 STACK_SHRINK(total_args);
   3111                 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
   3112                     tstate, (PyFunctionObject *)function, locals,
   3113                     stack_pointer, positional_args, call_shape.kwnames
   3114                 );
   3115                 call_shape.kwnames = NULL;
   3116                 STACK_SHRINK(2-is_meth);
   3117                 // The frame has stolen all the arguments from the stack,
   3118                 // so there is no need to clean them up.
   3119                 if (new_frame == NULL) {
   3120                     goto error;
   3121                 }
   3122                 _PyFrame_SetStackPointer(frame, stack_pointer);
   3123                 JUMPBY(INLINE_CACHE_ENTRIES_CALL);
   3124                 frame->prev_instr = next_instr - 1;
   3125                 new_frame->previous = frame;
   3126                 cframe.current_frame = frame = new_frame;
   3127                 CALL_STAT_INC(inlined_py_calls);
   3128                 goto start_frame;
   3129             }
   3130             /* Callable is not a normal Python function */
   3131             PyObject *res;
   3132             if (cframe.use_tracing) {
   3133                 res = trace_call_function(
   3134                     tstate, function, stack_pointer-total_args,
   3135                     positional_args, call_shape.kwnames);
   3136             }
   3137             else {
   3138                 res = PyObject_Vectorcall(
   3139                     function, stack_pointer-total_args,
   3140                     positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
   3141                     call_shape.kwnames);
   3142             }
   3143             call_shape.kwnames = NULL;
   3144             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3145             Py_DECREF(function);
   3146             /* Clear the stack */
   3147             STACK_SHRINK(total_args);
   3148             for (int i = 0; i < total_args; i++) {
   3149                 Py_DECREF(stack_pointer[i]);
   3150             }
   3151             STACK_SHRINK(2-is_meth);
   3152             PUSH(res);
   3153             if (res == NULL) {
   3154                 goto error;
   3155             }
   3156             JUMPBY(INLINE_CACHE_ENTRIES_CALL);
   3157             CHECK_EVAL_BREAKER();
   3158             DISPATCH();
   3159         }
   3160 
   3161         TARGET(PRECALL_ADAPTIVE) {
   3162             _PyPrecallCache *cache = (_PyPrecallCache *)next_instr;
   3163             if (cache->counter == 0) {
   3164                 next_instr--;
   3165                 int is_meth = is_method(stack_pointer, oparg);
   3166                 int nargs = oparg + is_meth;
   3167                 PyObject *callable = PEEK(nargs + 1);
   3168                 int err = _Py_Specialize_Precall(callable, next_instr, nargs,
   3169                                                  call_shape.kwnames, oparg);
   3170                 if (err < 0) {
   3171                     goto error;
   3172                 }
   3173                 DISPATCH_SAME_OPARG();
   3174             }
   3175             else {
   3176                 STAT_INC(PRECALL, deferred);
   3177                 cache->counter--;
   3178                 JUMP_TO_INSTRUCTION(PRECALL);
   3179             }
   3180         }
   3181 
   3182         TARGET(CALL_ADAPTIVE) {
   3183             _PyCallCache *cache = (_PyCallCache *)next_instr;
   3184             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   3185                 next_instr--;
   3186                 int is_meth = is_method(stack_pointer, oparg);
   3187                 int nargs = oparg + is_meth;
   3188                 PyObject *callable = PEEK(nargs + 1);
   3189                 int err = _Py_Specialize_Call(callable, next_instr, nargs,
   3190                                               call_shape.kwnames);
   3191                 if (err < 0) {
   3192                     goto error;
   3193                 }
   3194                 DISPATCH_SAME_OPARG();
   3195             }
   3196             else {
   3197                 STAT_INC(CALL, deferred);
   3198                 DECREMENT_ADAPTIVE_COUNTER(cache);
   3199                 goto call_function;
   3200             }
   3201         }
   3202 
   3203         TARGET(CALL_PY_EXACT_ARGS) {
   3204             assert(call_shape.kwnames == NULL);
   3205             DEOPT_IF(tstate->interp->eval_frame, CALL);
   3206             _PyCallCache *cache = (_PyCallCache *)next_instr;
   3207             int is_meth = is_method(stack_pointer, oparg);
   3208             int argcount = oparg + is_meth;
   3209             PyObject *callable = PEEK(argcount + 1);
   3210             DEOPT_IF(!PyFunction_Check(callable), CALL);
   3211             PyFunctionObject *func = (PyFunctionObject *)callable;
   3212             DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
   3213             PyCodeObject *code = (PyCodeObject *)func->func_code;
   3214             DEOPT_IF(code->co_argcount != argcount, CALL);
   3215             STAT_INC(CALL, hit);
   3216             _PyInterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
   3217             if (new_frame == NULL) {
   3218                 goto error;
   3219             }
   3220             CALL_STAT_INC(inlined_py_calls);
   3221             STACK_SHRINK(argcount);
   3222             for (int i = 0; i < argcount; i++) {
   3223                 new_frame->localsplus[i] = stack_pointer[i];
   3224             }
   3225             for (int i = argcount; i < code->co_nlocalsplus; i++) {
   3226                 new_frame->localsplus[i] = NULL;
   3227             }
   3228             STACK_SHRINK(2-is_meth);
   3229             _PyFrame_SetStackPointer(frame, stack_pointer);
   3230             JUMPBY(INLINE_CACHE_ENTRIES_CALL);
   3231             frame->prev_instr = next_instr - 1;
   3232             new_frame->previous = frame;
   3233             frame = cframe.current_frame = new_frame;
   3234             goto start_frame;
   3235         }
   3236 
   3237         TARGET(CALL_PY_WITH_DEFAULTS) {
   3238             assert(call_shape.kwnames == NULL);
   3239             DEOPT_IF(tstate->interp->eval_frame, CALL);
   3240             _PyCallCache *cache = (_PyCallCache *)next_instr;
   3241             int is_meth = is_method(stack_pointer, oparg);
   3242             int argcount = oparg + is_meth;
   3243             PyObject *callable = PEEK(argcount + 1);
   3244             DEOPT_IF(!PyFunction_Check(callable), CALL);
   3245             PyFunctionObject *func = (PyFunctionObject *)callable;
   3246             DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
   3247             PyCodeObject *code = (PyCodeObject *)func->func_code;
   3248             DEOPT_IF(argcount > code->co_argcount, CALL);
   3249             int minargs = cache->min_args;
   3250             DEOPT_IF(argcount < minargs, CALL);
   3251             STAT_INC(CALL, hit);
   3252             _PyInterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
   3253             if (new_frame == NULL) {
   3254                 goto error;
   3255             }
   3256             CALL_STAT_INC(inlined_py_calls);
   3257             STACK_SHRINK(argcount);
   3258             for (int i = 0; i < argcount; i++) {
   3259                 new_frame->localsplus[i] = stack_pointer[i];
   3260             }
   3261             for (int i = argcount; i < code->co_argcount; i++) {
   3262                 PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
   3263                                                  i - minargs);
   3264                 Py_INCREF(def);
   3265                 new_frame->localsplus[i] = def;
   3266             }
   3267             for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) {
   3268                 new_frame->localsplus[i] = NULL;
   3269             }
   3270             STACK_SHRINK(2-is_meth);
   3271             _PyFrame_SetStackPointer(frame, stack_pointer);
   3272             JUMPBY(INLINE_CACHE_ENTRIES_CALL);
   3273             frame->prev_instr = next_instr - 1;
   3274             new_frame->previous = frame;
   3275             frame = cframe.current_frame = new_frame;
   3276             goto start_frame;
   3277         }
   3278 
   3279         TARGET(PRECALL_NO_KW_TYPE_1) {
   3280             assert(call_shape.kwnames == NULL);
   3281             assert(cframe.use_tracing == 0);
   3282             assert(oparg == 1);
   3283             DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
   3284             PyObject *obj = TOP();
   3285             PyObject *callable = SECOND();
   3286             DEOPT_IF(callable != (PyObject *)&PyType_Type, PRECALL);
   3287             STAT_INC(PRECALL, hit);
   3288             SKIP_CALL();
   3289             PyObject *res = Py_NewRef(Py_TYPE(obj));
   3290             Py_DECREF(callable);
   3291             Py_DECREF(obj);
   3292             STACK_SHRINK(2);
   3293             SET_TOP(res);
   3294             DISPATCH();
   3295         }
   3296 
   3297         TARGET(PRECALL_NO_KW_STR_1) {
   3298             assert(call_shape.kwnames == NULL);
   3299             assert(cframe.use_tracing == 0);
   3300             assert(oparg == 1);
   3301             DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
   3302             PyObject *callable = PEEK(2);
   3303             DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, PRECALL);
   3304             STAT_INC(PRECALL, hit);
   3305             SKIP_CALL();
   3306             PyObject *arg = TOP();
   3307             PyObject *res = PyObject_Str(arg);
   3308             Py_DECREF(arg);
   3309             Py_DECREF(&PyUnicode_Type);
   3310             STACK_SHRINK(2);
   3311             SET_TOP(res);
   3312             if (res == NULL) {
   3313                 goto error;
   3314             }
   3315             CHECK_EVAL_BREAKER();
   3316             DISPATCH();
   3317         }
   3318 
   3319         TARGET(PRECALL_NO_KW_TUPLE_1) {
   3320             assert(call_shape.kwnames == NULL);
   3321             assert(oparg == 1);
   3322             DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
   3323             PyObject *callable = PEEK(2);
   3324             DEOPT_IF(callable != (PyObject *)&PyTuple_Type, PRECALL);
   3325             STAT_INC(PRECALL, hit);
   3326             SKIP_CALL();
   3327             PyObject *arg = TOP();
   3328             PyObject *res = PySequence_Tuple(arg);
   3329             Py_DECREF(arg);
   3330             Py_DECREF(&PyTuple_Type);
   3331             STACK_SHRINK(2);
   3332             SET_TOP(res);
   3333             if (res == NULL) {
   3334                 goto error;
   3335             }
   3336             CHECK_EVAL_BREAKER();
   3337             DISPATCH();
   3338         }
   3339 
   3340         TARGET(PRECALL_BUILTIN_CLASS) {
   3341             int is_meth = is_method(stack_pointer, oparg);
   3342             int total_args = oparg + is_meth;
   3343             int kwnames_len = KWNAMES_LEN();
   3344             PyObject *callable = PEEK(total_args + 1);
   3345             DEOPT_IF(!PyType_Check(callable), PRECALL);
   3346             PyTypeObject *tp = (PyTypeObject *)callable;
   3347             DEOPT_IF(tp->tp_vectorcall == NULL, PRECALL);
   3348             STAT_INC(PRECALL, hit);
   3349             SKIP_CALL();
   3350             STACK_SHRINK(total_args);
   3351             PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
   3352                                               total_args-kwnames_len, call_shape.kwnames);
   3353             call_shape.kwnames = NULL;
   3354             /* Free the arguments. */
   3355             for (int i = 0; i < total_args; i++) {
   3356                 Py_DECREF(stack_pointer[i]);
   3357             }
   3358             Py_DECREF(tp);
   3359             STACK_SHRINK(1-is_meth);
   3360             SET_TOP(res);
   3361             if (res == NULL) {
   3362                 goto error;
   3363             }
   3364             CHECK_EVAL_BREAKER();
   3365             DISPATCH();
   3366         }
   3367 
   3368         TARGET(PRECALL_NO_KW_BUILTIN_O) {
   3369             assert(cframe.use_tracing == 0);
   3370             /* Builtin METH_O functions */
   3371             assert(call_shape.kwnames == NULL);
   3372             int is_meth = is_method(stack_pointer, oparg);
   3373             int total_args = oparg + is_meth;
   3374             DEOPT_IF(total_args != 1, PRECALL);
   3375             PyObject *callable = PEEK(total_args + 1);
   3376             DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
   3377             DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, PRECALL);
   3378             STAT_INC(PRECALL, hit);
   3379             SKIP_CALL();
   3380             PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
   3381             // This is slower but CPython promises to check all non-vectorcall
   3382             // function calls.
   3383             if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
   3384                 goto error;
   3385             }
   3386             PyObject *arg = TOP();
   3387             PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg);
   3388             _Py_LeaveRecursiveCallTstate(tstate);
   3389             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3390 
   3391             Py_DECREF(arg);
   3392             Py_DECREF(callable);
   3393             STACK_SHRINK(2-is_meth);
   3394             SET_TOP(res);
   3395             if (res == NULL) {
   3396                 goto error;
   3397             }
   3398             CHECK_EVAL_BREAKER();
   3399             DISPATCH();
   3400         }
   3401 
   3402         TARGET(PRECALL_NO_KW_BUILTIN_FAST) {
   3403             assert(cframe.use_tracing == 0);
   3404             /* Builtin METH_FASTCALL functions, without keywords */
   3405             assert(call_shape.kwnames == NULL);
   3406             int is_meth = is_method(stack_pointer, oparg);
   3407             int total_args = oparg + is_meth;
   3408             PyObject *callable = PEEK(total_args + 1);
   3409             DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
   3410             DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
   3411                 PRECALL);
   3412             STAT_INC(PRECALL, hit);
   3413             SKIP_CALL();
   3414             PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
   3415             STACK_SHRINK(total_args);
   3416             /* res = func(self, args, nargs) */
   3417             PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
   3418                 PyCFunction_GET_SELF(callable),
   3419                 stack_pointer,
   3420                 total_args);
   3421             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3422 
   3423             /* Free the arguments. */
   3424             for (int i = 0; i < total_args; i++) {
   3425                 Py_DECREF(stack_pointer[i]);
   3426             }
   3427             STACK_SHRINK(2-is_meth);
   3428             PUSH(res);
   3429             Py_DECREF(callable);
   3430             if (res == NULL) {
   3431                 /* Not deopting because this doesn't mean our optimization was
   3432                    wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
   3433                    'invalid'). In those cases an exception is set, so we must
   3434                    handle it.
   3435                 */
   3436                 goto error;
   3437             }
   3438             CHECK_EVAL_BREAKER();
   3439             DISPATCH();
   3440         }
   3441 
   3442         TARGET(PRECALL_BUILTIN_FAST_WITH_KEYWORDS) {
   3443             assert(cframe.use_tracing == 0);
   3444             /* Builtin METH_FASTCALL | METH_KEYWORDS functions */
   3445             int is_meth = is_method(stack_pointer, oparg);
   3446             int total_args = oparg + is_meth;
   3447             PyObject *callable = PEEK(total_args + 1);
   3448             DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
   3449             DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
   3450                 (METH_FASTCALL | METH_KEYWORDS), PRECALL);
   3451             STAT_INC(PRECALL, hit);
   3452             SKIP_CALL();
   3453             STACK_SHRINK(total_args);
   3454             /* res = func(self, args, nargs, kwnames) */
   3455             _PyCFunctionFastWithKeywords cfunc =
   3456                 (_PyCFunctionFastWithKeywords)(void(*)(void))
   3457                 PyCFunction_GET_FUNCTION(callable);
   3458             PyObject *res = cfunc(
   3459                 PyCFunction_GET_SELF(callable),
   3460                 stack_pointer,
   3461                 total_args - KWNAMES_LEN(),
   3462                 call_shape.kwnames
   3463             );
   3464             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3465             call_shape.kwnames = NULL;
   3466 
   3467             /* Free the arguments. */
   3468             for (int i = 0; i < total_args; i++) {
   3469                 Py_DECREF(stack_pointer[i]);
   3470             }
   3471             STACK_SHRINK(2-is_meth);
   3472             PUSH(res);
   3473             Py_DECREF(callable);
   3474             if (res == NULL) {
   3475                 goto error;
   3476             }
   3477             CHECK_EVAL_BREAKER();
   3478             DISPATCH();
   3479         }
   3480 
   3481         TARGET(PRECALL_NO_KW_LEN) {
   3482             assert(cframe.use_tracing == 0);
   3483             assert(call_shape.kwnames == NULL);
   3484             /* len(o) */
   3485             int is_meth = is_method(stack_pointer, oparg);
   3486             int total_args = oparg + is_meth;
   3487             DEOPT_IF(total_args != 1, PRECALL);
   3488             PyObject *callable = PEEK(total_args + 1);
   3489             PyInterpreterState *interp = _PyInterpreterState_GET();
   3490             DEOPT_IF(callable != interp->callable_cache.len, PRECALL);
   3491             STAT_INC(PRECALL, hit);
   3492             SKIP_CALL();
   3493             PyObject *arg = TOP();
   3494             Py_ssize_t len_i = PyObject_Length(arg);
   3495             if (len_i < 0) {
   3496                 goto error;
   3497             }
   3498             PyObject *res = PyLong_FromSsize_t(len_i);
   3499             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3500 
   3501             STACK_SHRINK(2-is_meth);
   3502             SET_TOP(res);
   3503             Py_DECREF(callable);
   3504             Py_DECREF(arg);
   3505             if (res == NULL) {
   3506                 goto error;
   3507             }
   3508             DISPATCH();
   3509         }
   3510 
   3511         TARGET(PRECALL_NO_KW_ISINSTANCE) {
   3512             assert(cframe.use_tracing == 0);
   3513             assert(call_shape.kwnames == NULL);
   3514             /* isinstance(o, o2) */
   3515             int is_meth = is_method(stack_pointer, oparg);
   3516             int total_args = oparg + is_meth;
   3517             PyObject *callable = PEEK(total_args + 1);
   3518             DEOPT_IF(total_args != 2, PRECALL);
   3519             PyInterpreterState *interp = _PyInterpreterState_GET();
   3520             DEOPT_IF(callable != interp->callable_cache.isinstance, PRECALL);
   3521             STAT_INC(PRECALL, hit);
   3522             SKIP_CALL();
   3523             PyObject *cls = POP();
   3524             PyObject *inst = TOP();
   3525             int retval = PyObject_IsInstance(inst, cls);
   3526             if (retval < 0) {
   3527                 Py_DECREF(cls);
   3528                 goto error;
   3529             }
   3530             PyObject *res = PyBool_FromLong(retval);
   3531             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3532 
   3533             STACK_SHRINK(2-is_meth);
   3534             SET_TOP(res);
   3535             Py_DECREF(inst);
   3536             Py_DECREF(cls);
   3537             Py_DECREF(callable);
   3538             if (res == NULL) {
   3539                 goto error;
   3540             }
   3541             DISPATCH();
   3542         }
   3543 
   3544         TARGET(PRECALL_NO_KW_LIST_APPEND) {
   3545             assert(cframe.use_tracing == 0);
   3546             assert(call_shape.kwnames == NULL);
   3547             assert(oparg == 1);
   3548             PyObject *callable = PEEK(3);
   3549             PyInterpreterState *interp = _PyInterpreterState_GET();
   3550             DEOPT_IF(callable != interp->callable_cache.list_append, PRECALL);
   3551             PyObject *list = SECOND();
   3552             DEOPT_IF(!PyList_Check(list), PRECALL);
   3553             STAT_INC(PRECALL, hit);
   3554             // PRECALL + CALL + POP_TOP
   3555             JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL + 1);
   3556             assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
   3557             PyObject *arg = POP();
   3558             if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
   3559                 goto error;
   3560             }
   3561             STACK_SHRINK(2);
   3562             Py_DECREF(list);
   3563             Py_DECREF(callable);
   3564             DISPATCH();
   3565         }
   3566 
   3567         TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_O) {
   3568             assert(call_shape.kwnames == NULL);
   3569             int is_meth = is_method(stack_pointer, oparg);
   3570             int total_args = oparg + is_meth;
   3571             PyMethodDescrObject *callable =
   3572                 (PyMethodDescrObject *)PEEK(total_args + 1);
   3573             DEOPT_IF(total_args != 2, PRECALL);
   3574             DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
   3575             PyMethodDef *meth = callable->d_method;
   3576             DEOPT_IF(meth->ml_flags != METH_O, PRECALL);
   3577             PyObject *arg = TOP();
   3578             PyObject *self = SECOND();
   3579             DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL);
   3580             STAT_INC(PRECALL, hit);
   3581             SKIP_CALL();
   3582             PyCFunction cfunc = meth->ml_meth;
   3583             // This is slower but CPython promises to check all non-vectorcall
   3584             // function calls.
   3585             if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
   3586                 goto error;
   3587             }
   3588             PyObject *res = cfunc(self, arg);
   3589             _Py_LeaveRecursiveCallTstate(tstate);
   3590             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3591             Py_DECREF(self);
   3592             Py_DECREF(arg);
   3593             STACK_SHRINK(oparg + 1);
   3594             SET_TOP(res);
   3595             Py_DECREF(callable);
   3596             if (res == NULL) {
   3597                 goto error;
   3598             }
   3599             CHECK_EVAL_BREAKER();
   3600             DISPATCH();
   3601         }
   3602 
   3603         TARGET(PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
   3604             int is_meth = is_method(stack_pointer, oparg);
   3605             int total_args = oparg + is_meth;
   3606             PyMethodDescrObject *callable =
   3607                 (PyMethodDescrObject *)PEEK(total_args + 1);
   3608             DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
   3609             PyMethodDef *meth = callable->d_method;
   3610             DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), PRECALL);
   3611             PyTypeObject *d_type = callable->d_common.d_type;
   3612             PyObject *self = PEEK(total_args);
   3613             DEOPT_IF(!Py_IS_TYPE(self, d_type), PRECALL);
   3614             STAT_INC(PRECALL, hit);
   3615             SKIP_CALL();
   3616             int nargs = total_args-1;
   3617             STACK_SHRINK(nargs);
   3618             _PyCFunctionFastWithKeywords cfunc =
   3619                 (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
   3620             PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
   3621                                   call_shape.kwnames);
   3622             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3623             call_shape.kwnames = NULL;
   3624 
   3625             /* Free the arguments. */
   3626             for (int i = 0; i < nargs; i++) {
   3627                 Py_DECREF(stack_pointer[i]);
   3628             }
   3629             Py_DECREF(self);
   3630             STACK_SHRINK(2-is_meth);
   3631             SET_TOP(res);
   3632             Py_DECREF(callable);
   3633             if (res == NULL) {
   3634                 goto error;
   3635             }
   3636             CHECK_EVAL_BREAKER();
   3637             DISPATCH();
   3638         }
   3639 
   3640         TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
   3641             assert(call_shape.kwnames == NULL);
   3642             assert(oparg == 0 || oparg == 1);
   3643             int is_meth = is_method(stack_pointer, oparg);
   3644             int total_args = oparg + is_meth;
   3645             DEOPT_IF(total_args != 1, PRECALL);
   3646             PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
   3647             DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
   3648             PyMethodDef *meth = callable->d_method;
   3649             PyObject *self = TOP();
   3650             DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL);
   3651             DEOPT_IF(meth->ml_flags != METH_NOARGS, PRECALL);
   3652             STAT_INC(PRECALL, hit);
   3653             SKIP_CALL();
   3654             PyCFunction cfunc = meth->ml_meth;
   3655             // This is slower but CPython promises to check all non-vectorcall
   3656             // function calls.
   3657             if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
   3658                 goto error;
   3659             }
   3660             PyObject *res = cfunc(self, NULL);
   3661             _Py_LeaveRecursiveCallTstate(tstate);
   3662             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3663             Py_DECREF(self);
   3664             STACK_SHRINK(oparg + 1);
   3665             SET_TOP(res);
   3666             Py_DECREF(callable);
   3667             if (res == NULL) {
   3668                 goto error;
   3669             }
   3670             CHECK_EVAL_BREAKER();
   3671             DISPATCH();
   3672         }
   3673 
   3674         TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
   3675             assert(call_shape.kwnames == NULL);
   3676             int is_meth = is_method(stack_pointer, oparg);
   3677             int total_args = oparg + is_meth;
   3678             PyMethodDescrObject *callable =
   3679                 (PyMethodDescrObject *)PEEK(total_args + 1);
   3680             /* Builtin METH_FASTCALL methods, without keywords */
   3681             DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
   3682             PyMethodDef *meth = callable->d_method;
   3683             DEOPT_IF(meth->ml_flags != METH_FASTCALL, PRECALL);
   3684             PyObject *self = PEEK(total_args);
   3685             DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL);
   3686             STAT_INC(PRECALL, hit);
   3687             SKIP_CALL();
   3688             _PyCFunctionFast cfunc =
   3689                 (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
   3690             int nargs = total_args-1;
   3691             STACK_SHRINK(nargs);
   3692             PyObject *res = cfunc(self, stack_pointer, nargs);
   3693             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
   3694             /* Clear the stack of the arguments. */
   3695             for (int i = 0; i < nargs; i++) {
   3696                 Py_DECREF(stack_pointer[i]);
   3697             }
   3698             Py_DECREF(self);
   3699             STACK_SHRINK(2-is_meth);
   3700             SET_TOP(res);
   3701             Py_DECREF(callable);
   3702             if (res == NULL) {
   3703                 goto error;
   3704             }
   3705             CHECK_EVAL_BREAKER();
   3706             DISPATCH();
   3707         }
   3708 
   3709         TARGET(CALL_FUNCTION_EX) {
   3710             PREDICTED(CALL_FUNCTION_EX);
   3711             PyObject *func, *callargs, *kwargs = NULL, *result;
   3712             if (oparg & 0x01) {
   3713                 kwargs = POP();
   3714                 if (!PyDict_CheckExact(kwargs)) {
   3715                     PyObject *d = PyDict_New();
   3716                     if (d == NULL)
   3717                         goto error;
   3718                     if (_PyDict_MergeEx(d, kwargs, 2) < 0) {
   3719                         Py_DECREF(d);
   3720                         format_kwargs_error(tstate, SECOND(), kwargs);
   3721                         Py_DECREF(kwargs);
   3722                         goto error;
   3723                     }
   3724                     Py_DECREF(kwargs);
   3725                     kwargs = d;
   3726                 }
   3727                 assert(PyDict_CheckExact(kwargs));
   3728             }
   3729             callargs = POP();
   3730             func = TOP();
   3731             if (!PyTuple_CheckExact(callargs)) {
   3732                 if (check_args_iterable(tstate, func, callargs) < 0) {
   3733                     Py_DECREF(callargs);
   3734                     goto error;
   3735                 }
   3736                 Py_SETREF(callargs, PySequence_Tuple(callargs));
   3737                 if (callargs == NULL) {
   3738                     goto error;
   3739                 }
   3740             }
   3741             assert(PyTuple_CheckExact(callargs));
   3742 
   3743             result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing);
   3744             Py_DECREF(func);
   3745             Py_DECREF(callargs);
   3746             Py_XDECREF(kwargs);
   3747 
   3748             STACK_SHRINK(1);
   3749             assert(TOP() == NULL);
   3750             SET_TOP(result);
   3751             if (result == NULL) {
   3752                 goto error;
   3753             }
   3754             CHECK_EVAL_BREAKER();
   3755             DISPATCH();
   3756         }
   3757 
   3758         TARGET(MAKE_FUNCTION) {
   3759             PyObject *codeobj = POP();
   3760             PyFunctionObject *func = (PyFunctionObject *)
   3761                 PyFunction_New(codeobj, GLOBALS());
   3762 
   3763             Py_DECREF(codeobj);
   3764             if (func == NULL) {
   3765                 goto error;
   3766             }
   3767 
   3768             if (oparg & 0x08) {
   3769                 assert(PyTuple_CheckExact(TOP()));
   3770                 func->func_closure = POP();
   3771             }
   3772             if (oparg & 0x04) {
   3773                 assert(PyTuple_CheckExact(TOP()));
   3774                 func->func_annotations = POP();
   3775             }
   3776             if (oparg & 0x02) {
   3777                 assert(PyDict_CheckExact(TOP()));
   3778                 func->func_kwdefaults = POP();
   3779             }
   3780             if (oparg & 0x01) {
   3781                 assert(PyTuple_CheckExact(TOP()));
   3782                 func->func_defaults = POP();
   3783             }
   3784 
   3785             PUSH((PyObject *)func);
   3786             DISPATCH();
   3787         }
   3788 
   3789         TARGET(RETURN_GENERATOR) {
   3790             PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(frame->f_func);
   3791             if (gen == NULL) {
   3792                 goto error;
   3793             }
   3794             assert(EMPTY());
   3795             _PyFrame_SetStackPointer(frame, stack_pointer);
   3796             _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
   3797             _PyFrame_Copy(frame, gen_frame);
   3798             assert(frame->frame_obj == NULL);
   3799             gen->gi_frame_state = FRAME_CREATED;
   3800             gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
   3801             _Py_LeaveRecursiveCallTstate(tstate);
   3802             if (!frame->is_entry) {
   3803                 _PyInterpreterFrame *prev = frame->previous;
   3804                 _PyThreadState_PopFrame(tstate, frame);
   3805                 frame = cframe.current_frame = prev;
   3806                 _PyFrame_StackPush(frame, (PyObject *)gen);
   3807                 goto resume_frame;
   3808             }
   3809             /* Make sure that frame is in a valid state */
   3810             frame->stacktop = 0;
   3811             frame->f_locals = NULL;
   3812             Py_INCREF(frame->f_func);
   3813             Py_INCREF(frame->f_code);
   3814             /* Restore previous cframe and return. */
   3815             tstate->cframe = cframe.previous;
   3816             tstate->cframe->use_tracing = cframe.use_tracing;
   3817             assert(tstate->cframe->current_frame == frame->previous);
   3818             assert(!_PyErr_Occurred(tstate));
   3819             return (PyObject *)gen;
   3820         }
   3821 
   3822         TARGET(BUILD_SLICE) {
   3823             PyObject *start, *stop, *step, *slice;
   3824             if (oparg == 3)
   3825                 step = POP();
   3826             else
   3827                 step = NULL;
   3828             stop = POP();
   3829             start = TOP();
   3830             slice = PySlice_New(start, stop, step);
   3831             Py_DECREF(start);
   3832             Py_DECREF(stop);
   3833             Py_XDECREF(step);
   3834             SET_TOP(slice);
   3835             if (slice == NULL)
   3836                 goto error;
   3837             DISPATCH();
   3838         }
   3839 
   3840         TARGET(FORMAT_VALUE) {
   3841             /* Handles f-string value formatting. */
   3842             PyObject *result;
   3843             PyObject *fmt_spec;
   3844             PyObject *value;
   3845             PyObject *(*conv_fn)(PyObject *);
   3846             int which_conversion = oparg & FVC_MASK;
   3847             int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC;
   3848 
   3849             fmt_spec = have_fmt_spec ? POP() : NULL;
   3850             value = POP();
   3851 
   3852             /* See if any conversion is specified. */
   3853             switch (which_conversion) {
   3854             case FVC_NONE:  conv_fn = NULL;           break;
   3855             case FVC_STR:   conv_fn = PyObject_Str;   break;
   3856             case FVC_REPR:  conv_fn = PyObject_Repr;  break;
   3857             case FVC_ASCII: conv_fn = PyObject_ASCII; break;
   3858             default:
   3859                 _PyErr_Format(tstate, PyExc_SystemError,
   3860                               "unexpected conversion flag %d",
   3861                               which_conversion);
   3862                 goto error;
   3863             }
   3864 
   3865             /* If there's a conversion function, call it and replace
   3866                value with that result. Otherwise, just use value,
   3867                without conversion. */
   3868             if (conv_fn != NULL) {
   3869                 result = conv_fn(value);
   3870                 Py_DECREF(value);
   3871                 if (result == NULL) {
   3872                     Py_XDECREF(fmt_spec);
   3873                     goto error;
   3874                 }
   3875                 value = result;
   3876             }
   3877 
   3878             /* If value is a unicode object, and there's no fmt_spec,
   3879                then we know the result of format(value) is value
   3880                itself. In that case, skip calling format(). I plan to
   3881                move this optimization in to PyObject_Format()
   3882                itself. */
   3883             if (PyUnicode_CheckExact(value) && fmt_spec == NULL) {
   3884                 /* Do nothing, just transfer ownership to result. */
   3885                 result = value;
   3886             } else {
   3887                 /* Actually call format(). */
   3888                 result = PyObject_Format(value, fmt_spec);
   3889                 Py_DECREF(value);
   3890                 Py_XDECREF(fmt_spec);
   3891                 if (result == NULL) {
   3892                     goto error;
   3893                 }
   3894             }
   3895 
   3896             PUSH(result);
   3897             DISPATCH();
   3898         }
   3899 
   3900         TARGET(COPY) {
   3901             assert(oparg != 0);
   3902             PyObject *peek = PEEK(oparg);
   3903             Py_INCREF(peek);
   3904             PUSH(peek);
   3905             DISPATCH();
   3906         }
   3907 
   3908         TARGET(BINARY_OP) {
   3909             PREDICTED(BINARY_OP);
   3910             PyObject *rhs = POP();
   3911             PyObject *lhs = TOP();
   3912             assert(0 <= oparg);
   3913             assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
   3914             assert(binary_ops[oparg]);
   3915             PyObject *res = binary_ops[oparg](lhs, rhs);
   3916             Py_DECREF(lhs);
   3917             Py_DECREF(rhs);
   3918             SET_TOP(res);
   3919             if (res == NULL) {
   3920                 goto error;
   3921             }
   3922             JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
   3923             DISPATCH();
   3924         }
   3925 
   3926         TARGET(BINARY_OP_ADAPTIVE) {
   3927             assert(cframe.use_tracing == 0);
   3928             _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
   3929             if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
   3930                 PyObject *lhs = SECOND();
   3931                 PyObject *rhs = TOP();
   3932                 next_instr--;
   3933                 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
   3934                 DISPATCH_SAME_OPARG();
   3935             }
   3936             else {
   3937                 STAT_INC(BINARY_OP, deferred);
   3938                 DECREMENT_ADAPTIVE_COUNTER(cache);
   3939                 JUMP_TO_INSTRUCTION(BINARY_OP);
   3940             }
   3941         }
   3942 
   3943         TARGET(SWAP) {
   3944             assert(oparg != 0);
   3945             PyObject *top = TOP();
   3946             SET_TOP(PEEK(oparg));
   3947             PEEK(oparg) = top;
   3948             DISPATCH();
   3949         }
   3950 
   3951         TARGET(EXTENDED_ARG) {
   3952             assert(oparg);
   3953             oparg <<= 8;
   3954             oparg |= _Py_OPARG(*next_instr);
   3955             // We might be tracing. To avoid breaking tracing guarantees in
   3956             // quickened instructions, always deoptimize the next opcode:
   3957             opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)];
   3958             PRE_DISPATCH_GOTO();
   3959             // CPython hasn't traced the following instruction historically
   3960             // (DO_TRACING would clobber our extended oparg anyways), so just
   3961             // skip our usual cframe.use_tracing check before dispatch. Also,
   3962             // make sure the next instruction isn't a RESUME, since that needs
   3963             // to trace properly (and shouldn't have an extended arg anyways):
   3964             assert(opcode != RESUME);
   3965             DISPATCH_GOTO();
   3966         }
   3967 
   3968         TARGET(EXTENDED_ARG_QUICK) {
   3969             assert(cframe.use_tracing == 0);
   3970             assert(oparg);
   3971             int oldoparg = oparg;
   3972             NEXTOPARG();
   3973             oparg |= oldoparg << 8;
   3974             DISPATCH_GOTO();
   3975         }
   3976 
   3977         TARGET(CACHE) {
   3978             Py_UNREACHABLE();
   3979         }
   3980 
   3981 #if USE_COMPUTED_GOTOS
   3982         TARGET_DO_TRACING:
   3983 #else
   3984         case DO_TRACING:
   3985 #endif
   3986     {
   3987         assert(cframe.use_tracing);
   3988         assert(tstate->tracing == 0);
   3989         if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) {
   3990             int instr_prev = _PyInterpreterFrame_LASTI(frame);
   3991             frame->prev_instr = next_instr;
   3992             TRACING_NEXTOPARG();
   3993             if (opcode == RESUME) {
   3994                 if (oparg < 2) {
   3995                     CHECK_EVAL_BREAKER();
   3996                 }
   3997                 /* Call tracing */
   3998                 TRACE_FUNCTION_ENTRY();
   3999                 DTRACE_FUNCTION_ENTRY();
   4000             }
   4001             else {
   4002                 /* line-by-line tracing support */
   4003                 if (PyDTrace_LINE_ENABLED()) {
   4004                     maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
   4005                 }
   4006 
   4007                 if (cframe.use_tracing &&
   4008                     tstate->c_tracefunc != NULL && !tstate->tracing) {
   4009                     int err;
   4010                     /* see maybe_call_line_trace()
   4011                     for expository comments */
   4012                     _PyFrame_SetStackPointer(frame, stack_pointer);
   4013 
   4014                     err = maybe_call_line_trace(tstate->c_tracefunc,
   4015                                                 tstate->c_traceobj,
   4016                                                 tstate, frame, instr_prev);
   4017                     // Reload possibly changed frame fields:
   4018                     stack_pointer = _PyFrame_GetStackPointer(frame);
   4019                     frame->stacktop = -1;
   4020                     // next_instr is only reloaded if tracing *does not* raise.
   4021                     // This is consistent with the behavior of older Python
   4022                     // versions. If a trace function sets a new f_lineno and
   4023                     // *then* raises, we use the *old* location when searching
   4024                     // for an exception handler, displaying the traceback, and
   4025                     // so on:
   4026                     if (err) {
   4027                         // next_instr wasn't incremented at the start of this
   4028                         // instruction. Increment it before handling the error,
   4029                         // so that it looks the same as a "normal" instruction:
   4030                         next_instr++;
   4031                         goto error;
   4032                     }
   4033                     // Reload next_instr. Don't increment it, though, since
   4034                     // we're going to re-dispatch to the "true" instruction now:
   4035                     next_instr = frame->prev_instr;
   4036                 }
   4037             }
   4038         }
   4039         TRACING_NEXTOPARG();
   4040         PRE_DISPATCH_GOTO();
   4041         DISPATCH_GOTO();
   4042     }
   4043 
   4044 #if USE_COMPUTED_GOTOS
   4045         _unknown_opcode:
   4046 #else
   4047         EXTRA_CASES  // From opcode.h, a 'case' for each unused opcode
   4048 #endif
   4049             /* Tell C compilers not to hold the opcode variable in the loop.
   4050                next_instr points the current instruction without TARGET(). */
   4051             opcode = _Py_OPCODE(*next_instr);
   4052             fprintf(stderr, "XXX lineno: %d, opcode: %d\n",
   4053                     _PyInterpreterFrame_GetLine(frame),  opcode);
   4054             _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode");
   4055             goto error;
   4056 
   4057         } /* End instructions */
   4058 
   4059         /* This should never be reached. Every opcode should end with DISPATCH()
   4060            or goto error. */
   4061         Py_UNREACHABLE();
   4062 
   4063 /* Specialization misses */
   4064 
   4065 miss:
   4066     {
   4067         STAT_INC(opcode, miss);
   4068         opcode = _PyOpcode_Deopt[opcode];
   4069         STAT_INC(opcode, miss);
   4070         /* The counter is always the first cache entry: */
   4071         _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr;
   4072         *counter -= 1;
   4073         if (*counter == 0) {
   4074             int adaptive_opcode = _PyOpcode_Adaptive[opcode];
   4075             assert(adaptive_opcode);
   4076             _Py_SET_OPCODE(next_instr[-1], adaptive_opcode);
   4077             STAT_INC(opcode, deopt);
   4078             *counter = adaptive_counter_start();
   4079         }
   4080         next_instr--;
   4081         DISPATCH_GOTO();
   4082     }
   4083 
   4084 binary_subscr_dict_error:
   4085         {
   4086             PyObject *sub = POP();
   4087             if (!_PyErr_Occurred(tstate)) {
   4088                 _PyErr_SetKeyError(sub);
   4089             }
   4090             Py_DECREF(sub);
   4091             goto error;
   4092         }
   4093 
   4094 unbound_local_error:
   4095         {
   4096             format_exc_check_arg(tstate, PyExc_UnboundLocalError,
   4097                 UNBOUNDLOCAL_ERROR_MSG,
   4098                 PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
   4099             );
   4100             goto error;
   4101         }
   4102 
   4103 error:
   4104         call_shape.kwnames = NULL;
   4105         /* Double-check exception status. */
   4106 #ifdef NDEBUG
   4107         if (!_PyErr_Occurred(tstate)) {
   4108             _PyErr_SetString(tstate, PyExc_SystemError,
   4109                              "error return without exception set");
   4110         }
   4111 #else
   4112         assert(_PyErr_Occurred(tstate));
   4113 #endif
   4114 
   4115         /* Log traceback info. */
   4116         PyFrameObject *f = _PyFrame_GetFrameObject(frame);
   4117         if (f != NULL) {
   4118             PyTraceBack_Here(f);
   4119         }
   4120 
   4121         if (tstate->c_tracefunc != NULL) {
   4122             /* Make sure state is set to FRAME_UNWINDING for tracing */
   4123             call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
   4124                            tstate, frame);
   4125         }
   4126 
   4127 exception_unwind:
   4128         {
   4129             /* We can't use frame->f_lasti here, as RERAISE may have set it */
   4130             int offset = INSTR_OFFSET()-1;
   4131             int level, handler, lasti;
   4132             if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
   4133                 // No handlers, so exit.
   4134                 assert(_PyErr_Occurred(tstate));
   4135 
   4136                 /* Pop remaining stack entries. */
   4137                 PyObject **stackbase = _PyFrame_Stackbase(frame);
   4138                 while (stack_pointer > stackbase) {
   4139                     PyObject *o = POP();
   4140                     Py_XDECREF(o);
   4141                 }
   4142                 assert(STACK_LEVEL() == 0);
   4143                 _PyFrame_SetStackPointer(frame, stack_pointer);
   4144                 TRACE_FUNCTION_UNWIND();
   4145                 DTRACE_FUNCTION_EXIT();
   4146                 goto exit_unwind;
   4147             }
   4148 
   4149             assert(STACK_LEVEL() >= level);
   4150             PyObject **new_top = _PyFrame_Stackbase(frame) + level;
   4151             while (stack_pointer > new_top) {
   4152                 PyObject *v = POP();
   4153                 Py_XDECREF(v);
   4154             }
   4155             PyObject *exc, *val, *tb;
   4156             if (lasti) {
   4157                 int frame_lasti = _PyInterpreterFrame_LASTI(frame);
   4158                 PyObject *lasti = PyLong_FromLong(frame_lasti);
   4159                 if (lasti == NULL) {
   4160                     goto exception_unwind;
   4161                 }
   4162                 PUSH(lasti);
   4163             }
   4164             _PyErr_Fetch(tstate, &exc, &val, &tb);
   4165             /* Make the raw exception data
   4166                 available to the handler,
   4167                 so a program can emulate the
   4168                 Python main loop. */
   4169             _PyErr_NormalizeException(tstate, &exc, &val, &tb);
   4170             if (tb != NULL)
   4171                 PyException_SetTraceback(val, tb);
   4172             else
   4173                 PyException_SetTraceback(val, Py_None);
   4174             Py_XDECREF(tb);
   4175             Py_XDECREF(exc);
   4176             PUSH(val);
   4177             JUMPTO(handler);
   4178             /* Resume normal execution */
   4179             DISPATCH();
   4180         }
   4181     }
   4182 
   4183 exit_unwind:
   4184     assert(_PyErr_Occurred(tstate));
   4185     _Py_LeaveRecursiveCallTstate(tstate);
   4186     if (frame->is_entry) {
   4187         /* Restore previous cframe and exit */
   4188         tstate->cframe = cframe.previous;
   4189         tstate->cframe->use_tracing = cframe.use_tracing;
   4190         assert(tstate->cframe->current_frame == frame->previous);
   4191         return NULL;
   4192     }
   4193     frame = cframe.current_frame = pop_frame(tstate, frame);
   4194 
   4195 resume_with_error:
   4196     SET_LOCALS_FROM_FRAME();
   4197     goto error;
   4198 
   4199 }
   4200 
   4201 
   4202 PyObject *
   4203 _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
   4204                PyObject *locals,
   4205                PyObject* const* args, size_t argcount,
   4206                PyObject *kwnames)
   4207 {
   4208     /* _PyEvalFramePushAndInit consumes the references
   4209      * to func and all its arguments */
   4210     Py_INCREF(func);
   4211     for (size_t i = 0; i < argcount; i++) {
   4212         Py_INCREF(args[i]);
   4213     }
   4214     if (kwnames) {
   4215         Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
   4216         for (Py_ssize_t i = 0; i < kwcount; i++) {
   4217             Py_INCREF(args[i+argcount]);
   4218         }
   4219     }
   4220     _PyInterpreterFrame *frame = _PyEvalFramePushAndInit(
   4221         tstate, func, locals, args, argcount, kwnames);
   4222     if (frame == NULL) {
   4223         return NULL;
   4224     }
   4225     PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
   4226     assert(
   4227         _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) ||
   4228         _PyFrame_GetStackPointer(frame) == frame->localsplus
   4229     );
   4230     _PyEvalFrameClearAndPop(tstate, frame);
   4231     return retval;
   4232 }
	cs107-lecture-examples Example codes used during Harvard CS107 lectures
	git clone https://git.0xfab.ch/cs107-lecture-examples.git
	Log \| Files \| Refs \| README \| LICENSE