Refactor parsing structure

TokenHandler should contain all the methods for actually manipulating
tokens, Parser should only contain the recursive descent methods
This commit is contained in:
greg 2019-01-05 18:38:18 -08:00
parent 2b338fd3c9
commit 5491169d55
2 changed files with 53 additions and 57 deletions

View File

@ -19,7 +19,7 @@ impl Fold for RecursiveDescentFn {
let new_block: syn::Block = parse_quote! { let new_block: syn::Block = parse_quote! {
{ {
let next_token_before_parse = self.token_handler.peek_token(); let next_token_before_parse = self.token_handler.peek();
let record = ParseRecord { let record = ParseRecord {
production_name: stringify!(#ident).to_string(), production_name: stringify!(#ident).to_string(),
next_token: format!("{}", next_token_before_parse.to_string_with_metadata()), next_token: format!("{}", next_token_before_parse.to_string_with_metadata()),
@ -34,7 +34,7 @@ impl Fold for RecursiveDescentFn {
} }
match result { match result {
Err(ParseError { token: None, msg }) => { Err(ParseError { token: None, msg }) => {
let next_token_after_parse = self.token_handler.peek_token(); let next_token_after_parse = self.token_handler.peek();
println!("HERE? {:?}", next_token_after_parse); println!("HERE? {:?}", next_token_after_parse);
Err(ParseError { token: Some(next_token_after_parse), msg }) Err(ParseError { token: Some(next_token_after_parse), msg })
}, },

View File

@ -67,10 +67,10 @@ impl TokenHandler {
TokenHandler { tokens, end_of_file } TokenHandler { tokens, end_of_file }
} }
fn peek(&mut self) -> TokenKind { fn peek_kind(&mut self) -> TokenKind {
self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF) self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF)
} }
fn peek_token(&mut self) -> Token { fn peek(&mut self) -> Token {
self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file}) self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file})
} }
fn next(&mut self) -> Token { fn next(&mut self) -> Token {
@ -88,10 +88,6 @@ impl Parser {
} }
} }
fn peek(&mut self) -> TokenKind {
self.token_handler.peek()
}
pub fn parse(&mut self) -> ParseResult<AST> { pub fn parse(&mut self) -> ParseResult<AST> {
self.program() self.program()
} }
@ -121,7 +117,7 @@ macro_rules! expect {
($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) }; ($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) };
($self:expr, $expected_kind:pat if $cond:expr) => { ($self:expr, $expected_kind:pat if $cond:expr) => {
{ {
let tok = $self.token_handler.peek_token(); let tok = $self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
$expected_kind if $cond => $self.token_handler.next(), $expected_kind if $cond => $self.token_handler.next(),
actual_kind => { actual_kind => {
@ -145,19 +141,19 @@ macro_rules! delimited {
expect!($self, $start); expect!($self, $start);
let mut acc = vec![]; let mut acc = vec![];
loop { loop {
let peek = $self.peek(); let peek = $self.token_handler.peek();
match peek { match peek.get_kind() {
$end | EOF => break, $end | EOF => break,
_ => (), _ => (),
} }
if !$strictness { if !$strictness {
match peek { match peek.get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue }, $( $delim )|* => { $self.token_handler.next(); continue },
_ => () _ => ()
} }
} }
acc.push($self.$parse_fn()?); acc.push($self.$parse_fn()?);
match $self.peek() { match $self.token_handler.peek().get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue }, $( $delim )|* => { $self.token_handler.next(); continue },
_ if $strictness => break, _ if $strictness => break,
_ => continue, _ => continue,
@ -296,7 +292,7 @@ impl Parser {
fn program(&mut self) -> ParseResult<AST> { fn program(&mut self) -> ParseResult<AST> {
let mut statements = Vec::new(); let mut statements = Vec::new();
loop { loop {
match self.peek() { match self.token_handler.peek().get_kind() {
EOF => break, EOF => break,
Newline | Semicolon => { Newline | Semicolon => {
self.token_handler.next(); self.token_handler.next();
@ -313,7 +309,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn statement(&mut self) -> ParseResult<Statement> { fn statement(&mut self) -> ParseResult<Statement> {
//TODO handle error recovery here //TODO handle error recovery here
match self.peek() { match self.token_handler.peek().get_kind() {
Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }), Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }),
Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }), Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }),
Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)), Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)),
@ -331,10 +327,10 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn type_declaration_body(&mut self) -> ParseResult<Declaration> { fn type_declaration_body(&mut self) -> ParseResult<Declaration> {
if let Keyword(Alias) = self.peek() { if let Keyword(Alias) = self.token_handler.peek_kind() {
self.type_alias() self.type_alias()
} else { } else {
let mutable = if let Keyword(Mut) = self.peek() { let mutable = if let Keyword(Mut) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
true true
} else { } else {
@ -361,7 +357,7 @@ impl Parser {
let mut variants = Vec::new(); let mut variants = Vec::new();
variants.push(self.variant_specifier()?); variants.push(self.variant_specifier()?);
loop { loop {
if let Pipe = self.peek() { if let Pipe = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
variants.push(self.variant_specifier()?); variants.push(self.variant_specifier()?);
} else { } else {
@ -376,7 +372,7 @@ impl Parser {
use self::Variant::*; use self::Variant::*;
let name = self.identifier()?; let name = self.identifier()?;
match self.peek() { match self.token_handler.peek_kind() {
LParen => { LParen => {
let tuple_members = delimited!(self, LParen, type_name, Comma, RParen); let tuple_members = delimited!(self, LParen, type_name, Comma, RParen);
Ok(TupleStruct(name, tuple_members)) Ok(TupleStruct(name, tuple_members))
@ -400,7 +396,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn func_declaration(&mut self) -> ParseResult<Declaration> { fn func_declaration(&mut self) -> ParseResult<Declaration> {
let signature = self.func_signature()?; let signature = self.func_signature()?;
if let LCurlyBrace = self.peek() { if let LCurlyBrace = self.token_handler.peek_kind() {
let statements = self.nonempty_func_body()?; let statements = self.nonempty_func_body()?;
Ok(Declaration::FuncDecl(signature, statements)) Ok(Declaration::FuncDecl(signature, statements))
} else { } else {
@ -411,7 +407,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn func_signature(&mut self) -> ParseResult<Signature> { fn func_signature(&mut self) -> ParseResult<Signature> {
expect!(self, Keyword(Func)); expect!(self, Keyword(Func));
let (name, operator) = match self.peek() { let (name, operator) = match self.token_handler.peek_kind() {
Operator(s) => { Operator(s) => {
let name = s.clone(); let name = s.clone();
self.token_handler.next(); self.token_handler.next();
@ -420,7 +416,7 @@ impl Parser {
_ => (self.identifier()?, false) _ => (self.identifier()?, false)
}; };
let params = self.formal_param_list()?; let params = self.formal_param_list()?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None, _ => None,
}; };
@ -441,7 +437,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn formal_param(&mut self) -> ParseResult<FormalParam> { fn formal_param(&mut self) -> ParseResult<FormalParam> {
let name = self.identifier()?; let name = self.identifier()?;
let ty = match self.peek() { let ty = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None _ => None
}; };
@ -451,7 +447,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn binding_declaration(&mut self) -> ParseResult<Declaration> { fn binding_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Kw::Let)); expect!(self, Keyword(Kw::Let));
let constant = match self.peek() { let constant = match self.token_handler.peek_kind() {
Keyword(Kw::Mut) => { Keyword(Kw::Mut) => {
self.token_handler.next(); self.token_handler.next();
false false
@ -482,7 +478,7 @@ impl Parser {
fn impl_declaration(&mut self) -> ParseResult<Declaration> { fn impl_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Impl)); expect!(self, Keyword(Impl));
let first = self.type_name()?; let first = self.type_name()?;
let second = if let Keyword(For) = self.peek() { let second = if let Keyword(For) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
Some(self.type_name()?) Some(self.type_name()?)
} else { } else {
@ -512,7 +508,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn expression(&mut self) -> ParseResult<Expression> { fn expression(&mut self) -> ParseResult<Expression> {
let mut expr_body = self.precedence_expr(BinOp::min_precedence())?; let mut expr_body = self.precedence_expr(BinOp::min_precedence())?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None _ => None
}; };
@ -532,7 +528,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn type_name(&mut self) -> ParseResult<TypeIdentifier> { fn type_name(&mut self) -> ParseResult<TypeIdentifier> {
use self::TypeIdentifier::*; use self::TypeIdentifier::*;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)), LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)),
_ => Singleton(self.type_singleton_name()?), _ => Singleton(self.type_singleton_name()?),
}) })
@ -542,7 +538,7 @@ impl Parser {
fn type_singleton_name(&mut self) -> ParseResult<TypeSingletonName> { fn type_singleton_name(&mut self) -> ParseResult<TypeSingletonName> {
Ok(TypeSingletonName { Ok(TypeSingletonName {
name: self.identifier()?, name: self.identifier()?,
params: match self.peek() { params: match self.token_handler.peek_kind() {
LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket), LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket),
_ => vec![], _ => vec![],
} }
@ -553,7 +549,7 @@ impl Parser {
fn precedence_expr(&mut self, precedence: i32) -> ParseResult<Expression> { fn precedence_expr(&mut self, precedence: i32) -> ParseResult<Expression> {
let record = ParseRecord { let record = ParseRecord {
production_name: "precedence_expr".to_string(), production_name: "precedence_expr".to_string(),
next_token: format!("{}", self.token_handler.peek_token().to_string_with_metadata()), next_token: format!("{}", self.token_handler.peek().to_string_with_metadata()),
level: self.parse_level, level: self.parse_level,
}; };
self.parse_level += 1; self.parse_level += 1;
@ -561,7 +557,7 @@ impl Parser {
let mut lhs = self.prefix_expr()?; let mut lhs = self.prefix_expr()?;
loop { loop {
let new_precedence = match BinOp::get_precedence_from_token(&self.peek()) { let new_precedence = match BinOp::get_precedence_from_token(&self.token_handler.peek_kind()) {
Some(p) => p, Some(p) => p,
None => break, None => break,
}; };
@ -583,7 +579,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn prefix_expr(&mut self) -> ParseResult<Expression> { fn prefix_expr(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
Operator(ref op) if PrefixOp::is_prefix(&*op) => { Operator(ref op) if PrefixOp::is_prefix(&*op) => {
let sigil = match self.token_handler.next().kind { let sigil = match self.token_handler.next().kind {
Operator(op) => op, Operator(op) => op,
@ -602,7 +598,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn call_expr(&mut self) -> ParseResult<Expression> { fn call_expr(&mut self) -> ParseResult<Expression> {
let mut expr = self.index_expr()?; let mut expr = self.index_expr()?;
while let LParen = self.peek() { while let LParen = self.token_handler.peek_kind() {
let arguments = delimited!(self, LParen, expression, Comma, RParen); let arguments = delimited!(self, LParen, expression, Comma, RParen);
let arguments = arguments.into_iter().map(|s| Node::new(s)).collect(); let arguments = arguments.into_iter().map(|s| Node::new(s)).collect();
expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect
@ -614,7 +610,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn index_expr(&mut self) -> ParseResult<Expression> { fn index_expr(&mut self) -> ParseResult<Expression> {
let primary = self.primary()?; let primary = self.primary()?;
Ok(if let LSquareBracket = self.peek() { Ok(if let LSquareBracket = self.token_handler.peek_kind() {
let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket); let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket);
Expression(ExpressionType::Index { Expression(ExpressionType::Index {
indexee: bx!(Expression(primary.0, None)), indexee: bx!(Expression(primary.0, None)),
@ -627,7 +623,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn primary(&mut self) -> ParseResult<Expression> { fn primary(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => self.curly_brace_expr(), LCurlyBrace => self.curly_brace_expr(),
Backslash => self.lambda_expr(), Backslash => self.lambda_expr(),
LParen => self.paren_expr(), LParen => self.paren_expr(),
@ -655,7 +651,7 @@ impl Parser {
fn lambda_expr(&mut self) -> ParseResult<Expression> { fn lambda_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Backslash); expect!(self, Backslash);
let params = self.lambda_param_list()?; let params = self.lambda_param_list()?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None, _ => None,
}; };
@ -665,7 +661,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn lambda_param_list(&mut self) -> ParseResult<Vec<FormalParam>> { fn lambda_param_list(&mut self) -> ParseResult<Vec<FormalParam>> {
if let LParen = self.peek() { if let LParen = self.token_handler.peek_kind() {
self.formal_param_list() self.formal_param_list()
} else { } else {
let single_param = self.formal_param()?; let single_param = self.formal_param()?;
@ -697,7 +693,7 @@ impl Parser {
fn identifier_expr(&mut self) -> ParseResult<Expression> { fn identifier_expr(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let identifier = self.identifier()?; let identifier = self.identifier()?;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LCurlyBrace if !self.restrictions.no_struct_literal => { LCurlyBrace if !self.restrictions.no_struct_literal => {
let fields = self.record_block()?; let fields = self.record_block()?;
Expression(NamedStruct { name: identifier, fields }, None) Expression(NamedStruct { name: identifier, fields }, None)
@ -729,7 +725,7 @@ impl Parser {
x? x?
}); });
let body = Box::new(match self.peek() { let body = Box::new(match self.token_handler.peek_kind() {
Keyword(Kw::Then) => self.conditional()?, Keyword(Kw::Then) => self.conditional()?,
Keyword(Kw::Is) => self.simple_pattern_match()? , Keyword(Kw::Is) => self.simple_pattern_match()? ,
_ => self.guard_block()? _ => self.guard_block()?
@ -741,7 +737,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn discriminator(&mut self) -> ParseResult<Discriminator> { fn discriminator(&mut self) -> ParseResult<Discriminator> {
let lhs = self.prefix_expr()?; let lhs = self.prefix_expr()?;
let ref next = self.peek(); let ref next = self.token_handler.peek_kind();
Ok(if let Some(op) = BinOp::from_sigil_token(next) { Ok(if let Some(op) = BinOp::from_sigil_token(next) {
Discriminator::BinOp(lhs, op) Discriminator::BinOp(lhs, op)
} else { } else {
@ -769,7 +765,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn else_clause(&mut self) -> ParseResult<Option<Block>> { fn else_clause(&mut self) -> ParseResult<Option<Block>> {
Ok(if let Keyword(Kw::Else) = self.peek() { Ok(if let Keyword(Kw::Else) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
Some(self.expr_or_block()?) Some(self.expr_or_block()?)
} else { } else {
@ -785,19 +781,19 @@ impl Parser {
let mut guards = vec![]; let mut guards = vec![];
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
RCurlyBrace | EOF => break, RCurlyBrace | EOF => break,
Semicolon | Newline => { self.token_handler.next(); continue}, Semicolon | Newline => { self.token_handler.next(); continue},
_ => { _ => {
let guard_arm = self.guard_arm()?; let guard_arm = self.guard_arm()?;
guards.push(guard_arm); guards.push(guard_arm);
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
Semicolon | Newline => { self.token_handler.next(); continue; }, Semicolon | Newline => { self.token_handler.next(); continue; },
_ => break, _ => break,
} }
} }
if let RCurlyBrace = self.peek() { if let RCurlyBrace = self.token_handler.peek_kind() {
break; break;
} }
expect!(self, Comma); expect!(self, Comma);
@ -818,7 +814,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn guard(&mut self) -> ParseResult<Guard> { fn guard(&mut self) -> ParseResult<Guard> {
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
Keyword(Kw::Is) => { Keyword(Kw::Is) => {
self.token_handler.next(); self.token_handler.next();
let pat = self.pattern()?; let pat = self.pattern()?;
@ -840,7 +836,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn pattern(&mut self) -> ParseResult<Pattern> { fn pattern(&mut self) -> ParseResult<Pattern> {
if let LParen = self.peek() { if let LParen = self.token_handler.peek_kind() {
let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen); let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen);
Ok(Pattern::TuplePattern(tuple_pattern_variants)) Ok(Pattern::TuplePattern(tuple_pattern_variants))
} else { } else {
@ -851,11 +847,11 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn simple_pattern(&mut self) -> ParseResult<Pattern> { fn simple_pattern(&mut self) -> ParseResult<Pattern> {
Ok({ Ok({
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
Identifier(_) => { Identifier(_) => {
let id = self.identifier()?; let id = self.identifier()?;
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => { LCurlyBrace => {
let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace); let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace);
Pattern::Record(id, members) Pattern::Record(id, members)
@ -892,7 +888,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn signed_number_literal(&mut self) -> ParseResult<Pattern> { fn signed_number_literal(&mut self) -> ParseResult<Pattern> {
let neg = match self.peek() { let neg = match self.token_handler.peek_kind() {
Operator(ref op) if **op == "-" => { Operator(ref op) if **op == "-" => {
self.token_handler.next(); self.token_handler.next();
true true
@ -906,7 +902,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn record_pattern_entry(&mut self) -> ParseResult<(Rc<String>, Pattern)> { fn record_pattern_entry(&mut self) -> ParseResult<(Rc<String>, Pattern)> {
let name = self.identifier()?; let name = self.identifier()?;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
Colon => { Colon => {
expect!(self, Colon); expect!(self, Colon);
let pat = self.pattern()?; let pat = self.pattern()?;
@ -924,7 +920,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn expr_or_block(&mut self) -> ParseResult<Block> { fn expr_or_block(&mut self) -> ParseResult<Block> {
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => self.block(), LCurlyBrace => self.block(),
_ => { _ => {
let expr = self.expression()?; let expr = self.expression()?;
@ -949,7 +945,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn while_cond(&mut self) -> ParseResult<Option<Expression>> { fn while_cond(&mut self) -> ParseResult<Option<Expression>> {
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LCurlyBrace => None, LCurlyBrace => None,
_ => Some(self.expression()?), _ => Some(self.expression()?),
}) })
@ -958,7 +954,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn for_expr(&mut self) -> ParseResult<Expression> { fn for_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Keyword(Kw::For)); expect!(self, Keyword(Kw::For));
let enumerators = if let LCurlyBrace = self.peek() { let enumerators = if let LCurlyBrace = self.token_handler.peek_kind() {
delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace) delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace)
} else { } else {
let single_enum = { let single_enum = {
@ -984,7 +980,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn for_expr_body(&mut self) -> ParseResult<ForBody> { fn for_expr_body(&mut self) -> ParseResult<ForBody> {
use self::ForBody::*; use self::ForBody::*;
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
Ok(match tok.get_kind() { Ok(match tok.get_kind() {
LCurlyBrace => { LCurlyBrace => {
let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict); let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict);
@ -1011,7 +1007,7 @@ impl Parser {
fn literal(&mut self) -> ParseResult<Expression> { fn literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(), DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(),
Keyword(Kw::True) => { Keyword(Kw::True) => {
@ -1032,7 +1028,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn number_literal(&mut self) -> ParseResult<Expression> { fn number_literal(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
HexLiteral(_) | BinNumberSigil => self.int_literal(), HexLiteral(_) | BinNumberSigil => self.int_literal(),
_ => self.float_literal(), _ => self.float_literal(),
} }
@ -1061,7 +1057,7 @@ impl Parser {
fn float_literal(&mut self) -> ParseResult<Expression> { fn float_literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let mut digits = self.digits()?; let mut digits = self.digits()?;
if let Period = self.peek() { if let Period = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
digits.push_str("."); digits.push_str(".");
digits.push_str(&self.digits()?); digits.push_str(&self.digits()?);
@ -1082,7 +1078,7 @@ impl Parser {
fn digits(&mut self) -> ParseResult<String> { fn digits(&mut self) -> ParseResult<String> {
let mut ds = String::new(); let mut ds = String::new();
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
Underscore => { self.token_handler.next(); continue; }, Underscore => { self.token_handler.next(); continue; },
DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)}, DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)},
_ => break, _ => break,