From ec5165945262d2e3ff3734b43b60284fa38a75eb Mon Sep 17 00:00:00 2001 From: Greg Shuflin Date: Thu, 18 Nov 2021 22:21:52 -0800 Subject: [PATCH] Make identifier a token --- schala-lang/src/parsing/combinator.rs | 43 +++++++++++++++------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/schala-lang/src/parsing/combinator.rs b/schala-lang/src/parsing/combinator.rs index ba910e5..bf28072 100644 --- a/schala-lang/src/parsing/combinator.rs +++ b/schala-lang/src/parsing/combinator.rs @@ -154,7 +154,7 @@ fn decl_block(input: Span) -> ParseResult> { } fn interface(input: Span) -> ParseResult { - map(preceded(kw("interface"), pair(tok(identifier), signature_block)), |(name, signatures)| { + map(preceded(kw("interface"), pair(identifier, signature_block)), |(name, signatures)| { Declaration::Interface { name: rc_string(name.fragment()), signatures } })(input) } @@ -190,7 +190,7 @@ fn func_decl(input: Span) -> ParseResult { //TODO handle operators fn func_signature(input: Span) -> ParseResult { - map(tuple((kw("fn"), tok(identifier), formal_params, opt(type_anno))), |(_, name, params, type_anno)| { + map(tuple((kw("fn"), identifier, formal_params, opt(type_anno))), |(_, name, params, type_anno)| { Signature { name: rc_string(name.fragment()), operator: false, params, type_anno } })(input) } @@ -202,7 +202,7 @@ fn formal_params(input: Span) -> ParseResult> { //TODO support 256-limit fn formal_param(input: Span) -> ParseResult { map( - tuple((tok(identifier), opt(type_anno), opt(preceded(tok(char('=')), expression)))), + tuple((identifier, opt(type_anno), opt(preceded(tok(char('=')), expression)))), |(name, anno, default)| FormalParam { name: rc_string(name.fragment()), anno, default }, )(input) } @@ -210,7 +210,7 @@ fn formal_param(input: Span) -> ParseResult { fn type_decl(input: Span) -> ParseResult { alt(( map( - tuple((kw("type"), kw("alias"), tok(identifier), tok(char('=')), tok(identifier))), + tuple((kw("type"), kw("alias"), identifier, tok(char('=')), identifier)), |(_, _, alias, _, name)| Declaration::TypeAlias { alias: rc_string(alias.fragment()), original: rc_string(name.fragment()), @@ -251,23 +251,22 @@ fn variant_spec(input: Span) -> ParseResult { let id = fresh_id(&input); let (rest, (name, kind)) = alt(( - pair(tok(identifier), record_variant), - pair(tok(identifier), tuple_variant), - map(tok(identifier), |ident| (ident, VariantKind::UnitStruct)), + pair(identifier, record_variant), + pair(identifier, tuple_variant), + map(identifier, |ident| (ident, VariantKind::UnitStruct)), ))(input)?; Ok((rest, Variant { id, name: rc_string(name.fragment()), kind })) } fn record_variant_item(input: Span) -> ParseResult<(Rc, TypeIdentifier)> { - map(tuple((tok(identifier), tok(char(':')), type_identifier)), |(name, _, ty)| { + map(tuple((identifier, tok(char(':')), type_identifier)), |(name, _, ty)| { (rc_string(name.fragment()), ty) })(input) } fn binding(input: Span) -> ParseResult { - let parser = - tuple((kw("let"), opt(kw("mut")), tok(identifier), opt(type_anno), tok(char('=')), expression)); + let parser = tuple((kw("let"), opt(kw("mut")), identifier, opt(type_anno), tok(char('=')), expression)); map(parser, |(_, maybe_mut, ident, type_anno, _, expr)| Declaration::Binding { name: rc_string(ident.fragment()), constant: maybe_mut.is_none(), @@ -277,7 +276,7 @@ fn binding(input: Span) -> ParseResult { } fn module(input: Span) -> ParseResult { - map(tuple((kw("module"), tok(identifier), block)), |(_, name, items)| Declaration::Module { + map(tuple((kw("module"), identifier, block)), |(_, name, items)| Declaration::Module { name: rc_string(name.fragment()), items, })(input) @@ -316,7 +315,7 @@ fn type_identifier(input: Span) -> ParseResult { } fn type_singleton_name(input: Span) -> ParseResult { - map(pair(tok(identifier), opt(type_params)), |(name, params)| TypeSingletonName { + map(pair(identifier, opt(type_params)), |(name, params)| TypeSingletonName { name: rc_string(name.fragment()), params: if let Some(params) = params { params } else { vec![] }, })(input) @@ -426,8 +425,9 @@ fn extended_expr_part(input: Span) -> ParseResult { fn invocation_argument(input: Span) -> ParseResult { alt(( map(tok(char('_')), |_| InvocationArgument::Ignored), - map(tuple((tok(identifier), tok(char('=')), expression)), |(name, _, expr)| { - InvocationArgument::Keyword { name: rc_string(name.fragment()), expr } + map(tuple((identifier, tok(char('=')), expression)), |(name, _, expr)| InvocationArgument::Keyword { + name: rc_string(name.fragment()), + expr, }), map(expression, InvocationArgument::Positional), ))(input) @@ -470,7 +470,7 @@ fn named_struct(input: Span) -> ParseResult { //TODO support anonymous structs and Elm-style update syntax for structs fn record_block(input: Span) -> ParseResult, Expression)>> { let record_entry = - separated_pair(map(tok(identifier), |span| rc_string(span.fragment())), tok(char(':')), expression); + separated_pair(map(identifier, |span| rc_string(span.fragment())), tok(char(':')), expression); delimited(tok(char('{')), separated_list0(tok(char(',')), record_entry), tok(char('}')))(input) } @@ -495,12 +495,13 @@ fn for_expr(input: Span) -> ParseResult { //TODO add guards, etc. fn enumerator(input: Span) -> ParseResult { alt(( - map(separated_pair(tok(identifier), kw("<-"), expression_no_struct), |(ident, generator)| { + map(separated_pair(identifier, kw("<-"), expression_no_struct), |(ident, generator)| { Enumerator { id: rc_string(ident.fragment()), generator } }), //TODO distinguish these two cases in AST - map(separated_pair(tok(identifier), kw("="), expression_no_struct), |(ident, generator)| { - Enumerator { id: rc_string(ident.fragment()), generator } + map(separated_pair(identifier, kw("="), expression_no_struct), |(ident, generator)| Enumerator { + id: rc_string(ident.fragment()), + generator, }), ))(input) } @@ -563,12 +564,16 @@ fn identifier_expr(input: Span) -> ParseResult { fn qualified_identifier(input: Span) -> ParseResult { let id = fresh_id(&input); - tok(map(separated_list1(tag("::"), map(identifier, |x| rc_string(x.fragment()))), move |items| { + tok(map(separated_list1(tag("::"), map(identifier_span, |x| rc_string(x.fragment()))), move |items| { QualifiedName { id, components: items } }))(input) } fn identifier(input: Span) -> ParseResult { + tok(identifier_span)(input) +} + +fn identifier_span(input: Span) -> ParseResult { recognize(pair(alt((tag("_"), alpha1)), take_while(|ch: char| is_alphanumeric(ch as u8) || ch == '_')))( input, )