gh-107015: Remove async_hacks from the tokenizer (#107018)

2023-07-26 16:34:15 +01:00
parent b0202a4e5d
commit da8f87b7ea
20 changed files with 404 additions and 499 deletions
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -237,9 +237,6 @@ tokenizeriter_next(tokenizeriterobject *it)
        if (type > DEDENT && type < OP) {
            type = OP;
        }
-        else if (type == ASYNC || type == AWAIT) {
-            type = NAME;
-        }
        else if (type == NEWLINE) {
            Py_DECREF(str);
            if (!it->tok->implicit_newline) {