gh-105564: Don't include artificial newlines in the line attribute of tokens (#105565)

2023-06-09 17:01:26 +01:00
parent 1dd267af64
commit d7f46bcd98
4 changed files with 30 additions and 25 deletions
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -206,6 +206,9 @@ tokenizeriter_next(tokenizeriterobject *it)
        line = PyUnicode_FromString("");
    } else {
        Py_ssize_t size = it->tok->inp - line_start;
+        if (size >= 1 && it->tok->implicit_newline) {
+            size -= 1;
+        }
        line = PyUnicode_DecodeUTF8(line_start, size, "replace");
    }
    if (line == NULL) {