greg/cpython
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

gh-97973: Return all necessary information from the tokenizer (GH-97984)

Browse Source 
Right now, the tokenizer only returns type and two pointers to the start and end of the token.
This PR modifies the tokenizer to return the type and set all of the necessary information,
so that the parser does not have to this.
This commit is contained in:
Lysandros Nikolaou
2022-10-06 16:07:17 -07:00
committed by GitHub
parent b9d2e81716
commit cbf0afd8a1
6 changed files with 161 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Python/Python-tokenize.c
View File

@@ -60,9 +60,8 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
static PyObject *
tokenizeriter_next(tokenizeriterobject *it)
{
const char *start;
const char *end;
int type = _PyTokenizer_Get(it->tok, &start, &end);
struct token token;
int type = _PyTokenizer_Get(it->tok, &token);
if (type == ERRORTOKEN && PyErr_Occurred()) {
return NULL;
}
@@ -71,11 +70,11 @@ tokenizeriter_next(tokenizeriterobject *it)
return NULL;
}
PyObject *str = NULL;
if (start == NULL || end == NULL) {
if (token.start == NULL || token.end == NULL) {
str = PyUnicode_FromString("");
}
else {
str = PyUnicode_FromStringAndSize(start, end - start);
str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
}
if (str == NULL) {
return NULL;
@@ -92,11 +91,11 @@ tokenizeriter_next(tokenizeriterobject *it)
int end_lineno = it->tok->lineno;
int col_offset = -1;
int end_col_offset = -1;
if (start != NULL && start >= line_start) {
col_offset = (int)(start - line_start);
if (token.start != NULL && token.start >= line_start) {
col_offset = (int)(token.start - line_start);
}
if (end != NULL && end >= it->tok->line_start) {
end_col_offset = (int)(end - it->tok->line_start);
if (token.end != NULL && token.end >= it->tok->line_start) {
end_col_offset = (int)(token.end - it->tok->line_start);
}
return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);