From 3a207cf7a789bbed5bbac87c19155687d82c27ce Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 14 Jun 2019 00:44:54 -0700 Subject: [PATCH] Make TokenHandler use an array and index Instead of a peekable iterator, so I can implement LL(k) parsing --- schala-lang/language/src/parsing.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/schala-lang/language/src/parsing.rs b/schala-lang/language/src/parsing.rs index 6cdee72..74c773f 100644 --- a/schala-lang/language/src/parsing.rs +++ b/schala-lang/language/src/parsing.rs @@ -143,8 +143,6 @@ //! use std::rc::Rc; -use std::iter::Peekable; -use std::vec::IntoIter; use crate::tokenizing::*; use crate::tokenizing::Kw::*; @@ -190,8 +188,7 @@ struct ParserRestrictions { } struct TokenHandler { - tokens: Peekable>, - token_array: Vec, + tokens: Vec, idx: usize, end_of_file: (usize, usize), } @@ -202,22 +199,18 @@ impl TokenHandler { None => (0, 0), Some(t) => (t.line_num, t.char_num) }; - let token_array = tokens.clone(); - let tokens = tokens.into_iter().peekable(); - TokenHandler { idx: 0, token_array, tokens, end_of_file } + TokenHandler { idx: 0, tokens, end_of_file } } fn peek_kind(&mut self) -> TokenKind { - let old = self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF); - old + self.peek().kind } fn peek(&mut self) -> Token { - let old = self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}); - old + self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) } fn next(&mut self) -> Token { - let old = self.tokens.next().unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}); - old + self.idx += 1; + self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) } }