Add benchmarking code for json parser

cf. https://www.greyblake.com/blog/property-based-testing-in-rust-with-arbitrary/

Cargo.toml

@@ -5,4 +5,21 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+testutil = []
+
 [dependencies]
+arbitrary = "1.2.0"
+proptest = "1.0.0"
+
+[dev-dependencies]
+criterion = "0.4.0"
+rstest = "0.16.0"
+# see https://github.com/rust-lang/cargo/issues/2911#issuecomment-749580481
+parser-combinator = { path = ".", features = ["testutil"] }
+
+[[bench]]
+name = "json-benchmark"
+harness = false
+
+

README.md

@@ -0,0 +1,10 @@
+# Rust Parser Combinator
+
+This is a super-basic Rust parser combinator library I wrote mostly
+as an exercise for myself. Inspired by [nom](https://github.com/rust-bakery/nom)
+and [chumsky](https://github.com/zesterer/chumsky)
+
+## Ideas for future work
+
+* See if some of the ideas in [Efficient Parsing with Parser Combinators](https://research.rug.nl/en/publications/efficient-parsing-with-parser-combinators)
+can be incorporated here.

benches/json-benchmark.rs

@@ -0,0 +1,14 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use parser_combinator::testutil::json_object;
+use parser_combinator::Parser;
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    let test_json = include_str!("../tests/joplin-cfg.json");
+
+    c.bench_function("parse_json", |b| {
+        b.iter(|| json_object().parse(black_box(test_json)))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

justfile

@@ -1,6 +0,0 @@
-_default:
-    just --list
-
-
-test *args:
-    cargo nextest run {{args}}

src/annotated.rs

@@ -1,55 +0,0 @@
-use std::marker::PhantomData;
-
-use crate::{representation::Representation, ParseResult, Parser};
-
-pub struct AnnotatedParser<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    inner: P,
-    name: Option<String>,
-    repr: Representation,
-    phantom: PhantomData<(I, O, E)>,
-}
-
-impl<P, I, O, E> Parser<I, O, E> for AnnotatedParser<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    fn parse(&self, input: I) -> ParseResult<I, O, E> {
-        self.inner.parse(input)
-    }
-
-    fn name(&self) -> Option<String> {
-        self.name.clone()
-    }
-
-    fn representation(&self) -> Representation {
-        self.repr.clone()
-    }
-}
-
-impl<P, I, O, E> AnnotatedParser<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    pub fn new(inner: P) -> Self {
-        Self {
-            inner,
-            name: None,
-            repr: Representation::new(),
-            phantom: PhantomData,
-        }
-    }
-
-    pub fn with_name(self, name: &str) -> Self {
-        Self {
-            name: Some(name.to_string()),
-            ..self
-        }
-    }
-
-    pub fn with_repr(self, repr: Representation) -> Self {
-        Self { repr, ..self }
-    }
-}

src/choice.rs

@@ -1,74 +0,0 @@
-use crate::{ParseResult, Parser};
-
-pub trait Choice<I, O, E> {
-    fn parse_choice(&self, input: I) -> Result<(O, I), (E, I)>;
-}
-
-pub fn choice<C: Choice<I, O, E>, I, O, E>(choices: C) -> impl Parser<I, O, E> {
-    move |input| choices.parse_choice(input)
-}
-
-fn choice_loop<'a, I, O, E>(
-    mut input: I,
-    parsers: &'a [&'a dyn Parser<I, O, E>],
-) -> ParseResult<I, O, E> {
-    //TODO need a more principled way to return an error when no choices work
-    let mut err = None;
-
-    for parser in parsers.iter() {
-        match parser.parse(input) {
-            Ok(res) => return Ok(res),
-            Err((e, rest)) => {
-                err = Some(e);
-                input = rest;
-            }
-        }
-    }
-    Err((err.unwrap(), input))
-}
-
-impl<P1, P2, I, O, E> Choice<I, O, E> for (P1, P2)
-where
-    P1: Parser<I, O, E>,
-    P2: Parser<I, O, E>,
-{
-    fn parse_choice(&self, input: I) -> Result<(O, I), (E, I)> {
-        choice_loop(input, &[&self.0, &self.1])
-    }
-}
-
-impl<P1, P2, P3, I, O, E> Choice<I, O, E> for (P1, P2, P3)
-where
-    P1: Parser<I, O, E>,
-    P2: Parser<I, O, E>,
-    P3: Parser<I, O, E>,
-{
-    fn parse_choice(&self, input: I) -> Result<(O, I), (E, I)> {
-        choice_loop(input, &[&self.0, &self.1, &self.2])
-    }
-}
-
-impl<P1, P2, P3, P4, I, O, E> Choice<I, O, E> for (P1, P2, P3, P4)
-where
-    P1: Parser<I, O, E>,
-    P2: Parser<I, O, E>,
-    P3: Parser<I, O, E>,
-    P4: Parser<I, O, E>,
-{
-    fn parse_choice(&self, input: I) -> Result<(O, I), (E, I)> {
-        choice_loop(input, &[&self.0, &self.1, &self.2, &self.3])
-    }
-}
-
-impl<P1, P2, P3, P4, P5, I, O, E> Choice<I, O, E> for (P1, P2, P3, P4, P5)
-where
-    P1: Parser<I, O, E>,
-    P2: Parser<I, O, E>,
-    P3: Parser<I, O, E>,
-    P4: Parser<I, O, E>,
-    P5: Parser<I, O, E>,
-{
-    fn parse_choice(&self, input: I) -> Result<(O, I), (E, I)> {
-        choice_loop(input, &[&self.0, &self.1, &self.2, &self.3, &self.4])
-    }
-}

src/choice/mod.rs

@@ -0,0 +1,198 @@
+use crate::parser::{ParseResult, Parser, ParserInput, Representation};
+
+pub fn choice2<P1, P2, I, O, E>(parser1: P1, parser2: P2) -> impl Parser<I, O, E>
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    choice((parser1, parser2))
+}
+
+pub fn choice<C, I, O, E>(choices: C) -> impl Parser<I, O, E>
+where
+    C: Choice<I, O, E>,
+    I: ParserInput + Clone,
+{
+    let rep = choices.representation();
+    (move |input| choices.parse(input), rep)
+}
+
+pub trait Choice<I: Clone, O, E> {
+    fn parse(&self, input: I) -> ParseResult<I, O, E>;
+    fn representation(&self) -> Representation;
+}
+
+impl<I, O, E, P1, P2> Choice<I, O, E> for (P1, P2)
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1];
+        choice_loop(input, parsers)
+    }
+
+    fn representation(&self) -> Representation {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1];
+        repr_loop(parsers)
+    }
+}
+
+impl<I, O, E, P1, P2, P3> Choice<I, O, E> for (P1, P2, P3)
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    P3: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1, &self.2];
+        choice_loop(input, parsers)
+    }
+
+    fn representation(&self) -> Representation {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1, &self.2];
+        repr_loop(parsers)
+    }
+}
+
+impl<I, O, E, P1, P2, P3, P4> Choice<I, O, E> for (P1, P2, P3, P4)
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    P3: Parser<I, O, E>,
+    P4: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1, &self.2, &self.3];
+        choice_loop(input, parsers)
+    }
+
+    fn representation(&self) -> Representation {
+        let parsers = vec![&self.0 as &dyn Parser<I, O, E>, &self.1, &self.2, &self.3];
+        repr_loop(parsers)
+    }
+}
+
+impl<I, O, E, P1, P2, P3, P4, P5> Choice<I, O, E> for (P1, P2, P3, P4, P5)
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    P3: Parser<I, O, E>,
+    P4: Parser<I, O, E>,
+    P5: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        let parsers = vec![
+            &self.0 as &dyn Parser<I, O, E>,
+            &self.1,
+            &self.2,
+            &self.3,
+            &self.4,
+        ];
+        choice_loop(input, parsers)
+    }
+
+    fn representation(&self) -> Representation {
+        let parsers = vec![
+            &self.0 as &dyn Parser<I, O, E>,
+            &self.1,
+            &self.2,
+            &self.3,
+            &self.4,
+        ];
+        repr_loop(parsers)
+    }
+}
+
+impl<I, O, E, P1, P2, P3, P4, P5, P6> Choice<I, O, E> for (P1, P2, P3, P4, P5, P6)
+where
+    P1: Parser<I, O, E>,
+    P2: Parser<I, O, E>,
+    P3: Parser<I, O, E>,
+    P4: Parser<I, O, E>,
+    P5: Parser<I, O, E>,
+    P6: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        let parsers = vec![
+            &self.0 as &dyn Parser<I, O, E>,
+            &self.1,
+            &self.2,
+            &self.3,
+            &self.4,
+            &self.5,
+        ];
+        choice_loop(input, parsers)
+    }
+    fn representation(&self) -> Representation {
+        let parsers = vec![
+            &self.0 as &dyn Parser<I, O, E>,
+            &self.1,
+            &self.2,
+            &self.3,
+            &self.4,
+            &self.5,
+        ];
+        repr_loop(parsers)
+    }
+}
+
+fn choice_loop<I, O, E>(input: I, parsers: Vec<&dyn Parser<I, O, E>>) -> ParseResult<I, O, E>
+where
+    I: ParserInput + Clone,
+{
+    //TODO need a more principled way to return an error when no choices work
+    let mut err = None;
+
+    for parser in parsers.iter() {
+        match parser.parse(input.clone()) {
+            Ok(result) => return Ok(result),
+            Err(e) => {
+                err = Some(e);
+            }
+        }
+    }
+    Err(err.unwrap())
+}
+
+fn repr_loop<I, O, E>(parsers: Vec<&dyn Parser<I, O, E>>) -> Representation
+where
+    I: ParserInput + Clone,
+{
+    let mut iter = parsers.iter().map(|p| p.representation());
+    Representation::from_choice(&mut iter)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::combinators::repeated;
+    use crate::primitives::literal;
+
+    #[test]
+    fn test_choice() {
+        let p = choice2(
+            literal("gnostika").to(1),
+            repeated(literal(" ")).at_least(1).to(2),
+        );
+        assert_eq!(p.parse("gnostika twentynine"), Ok((1, " twentynine")));
+    }
+
+    #[test]
+    fn test_several_choices() {
+        let p = choice((
+            literal("a").to(1),
+            literal("q").to(10),
+            repeated(literal("chutney")).to(200),
+            literal("banana").to(10000),
+        ));
+
+        assert_eq!(p.parse("q drugs").unwrap(), (10, " drugs"));
+    }
+}

src/combinators.rs

@@ -1,234 +0,0 @@
-use std::marker::PhantomData;
-
-use crate::{
-    representation::{Representation, EBNF},
-    ParseResult, Parser,
-};
-
-pub fn repeated<P, I, O, E>(parser: P) -> Repeated<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    Repeated::new(parser)
-}
-
-pub struct Repeated<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    inner_parser: P,
-    phantom: PhantomData<(I, O, E)>,
-    at_least: Option<u32>,
-    at_most: Option<u32>,
-}
-
-impl<P, I, O, E> Repeated<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-{
-    fn new(inner_parser: P) -> Self {
-        Self {
-            inner_parser,
-            phantom: PhantomData,
-            at_least: None,
-            at_most: None,
-        }
-    }
-    pub fn at_least(self, at_least: u32) -> Self {
-        Self {
-            at_least: Some(at_least),
-            ..self
-        }
-    }
-    pub fn at_most(self, at_most: u32) -> Self {
-        Self {
-            at_most: Some(at_most),
-            ..self
-        }
-    }
-    pub fn separated_by<D>(self, delimiter: D) -> SeparatedBy<D, P, I, O, E>
-    where
-        D: Parser<I, (), E>,
-        E: Default,
-    {
-        SeparatedBy {
-            inner_repeated: self,
-            delimiter,
-            allow_trailing: false,
-        }
-    }
-}
-
-impl<P, I, O, E> Parser<I, Vec<O>, E> for Repeated<P, I, O, E>
-where
-    P: Parser<I, O, E>,
-    E: Default,
-{
-    fn parse(&self, mut input: I) -> ParseResult<I, Vec<O>, E> {
-        let at_least = self.at_least.unwrap_or(0);
-        let at_most = self.at_most.unwrap_or(u32::MAX);
-
-        let mut results = vec![];
-        let mut count = 0;
-
-        if at_most == 0 {
-            return Ok((vec![], input));
-        }
-
-        loop {
-            match self.inner_parser.parse(input) {
-                Ok((item, rest)) => {
-                    results.push(item);
-                    input = rest;
-                    count += 1;
-                    if count >= at_most {
-                        break;
-                    }
-                }
-                Err((_err, rest)) => {
-                    input = rest;
-                    break;
-                }
-            }
-        }
-
-        if count < at_least {
-            return Err((Default::default(), input));
-        }
-        Ok((results, input))
-    }
-
-    fn name(&self) -> Option<String> {
-        self.inner_parser.name()
-    }
-
-    fn representation(&self) -> Representation {
-        let at_least = self.at_least.unwrap_or(0);
-        //TODO flesh this out better
-        let _at_most = self.at_most.unwrap_or(u32::MAX);
-        let production = EBNF::Repeated {
-            inner: Box::new(self.inner_parser.representation().production()),
-            more_than_once: at_least >= 1,
-        };
-        Representation::new().with_production(production)
-    }
-}
-
-pub struct SeparatedBy<D, P, I, O, E>
-where
-    D: Parser<I, (), E>,
-    P: Parser<I, O, E>,
-    E: Default,
-{
-    inner_repeated: Repeated<P, I, O, E>,
-    delimiter: D,
-    allow_trailing: bool,
-}
-
-impl<D, P, I, O, E> SeparatedBy<D, P, I, O, E>
-where
-    D: Parser<I, (), E>,
-    P: Parser<I, O, E>,
-    E: Default,
-{
-    pub fn allow_trailing(self, allow_trailing: bool) -> Self {
-        Self {
-            allow_trailing,
-            ..self
-        }
-    }
-}
-impl<D, P, I, O, E> Parser<I, Vec<O>, E> for SeparatedBy<D, P, I, O, E>
-where
-    D: Parser<I, (), E>,
-    P: Parser<I, O, E>,
-    E: Default,
-{
-    fn parse(&self, mut input: I) -> ParseResult<I, Vec<O>, E> {
-        let at_least = self.inner_repeated.at_least.unwrap_or(0);
-        let at_most = self.inner_repeated.at_most.unwrap_or(u32::MAX);
-        let inner = &self.inner_repeated.inner_parser;
-        let delimiter = &self.delimiter;
-
-        if at_most == 0 {
-            return Ok((vec![], input));
-        }
-
-        let mut results = Vec::new();
-        let mut count: u32 = 0;
-
-        match inner.parse(input) {
-            Ok((item, rest)) => {
-                results.push(item);
-                input = rest;
-            }
-            Err((err, rest)) => {
-                if at_least > 0 {
-                    return Err((err, rest));
-                } else {
-                    return Ok((vec![], rest));
-                }
-            }
-        }
-
-        loop {
-            match delimiter.parse(input) {
-                Ok(((), rest)) => {
-                    input = rest;
-                    count += 1;
-                }
-                Err((_err, rest)) => {
-                    input = rest;
-                    break;
-                }
-            }
-            match inner.parse(input) {
-                Ok((item, rest)) => {
-                    input = rest;
-                    results.push(item);
-                }
-                Err((err, rest)) => {
-                    if self.allow_trailing {
-                        input = rest;
-                        break;
-                    } else {
-                        return Err((err, rest));
-                    }
-                }
-            }
-            if count >= at_most {
-                break;
-            }
-        }
-
-        if count < at_least {
-            //return Err(??, rest) <- need to handle errors better
-            unimplemented!();
-        }
-
-        Ok((results, input))
-    }
-
-    fn representation(&self) -> Representation {
-        let inner = &self.inner_repeated.inner_parser;
-        let at_least = self.inner_repeated.at_least.unwrap_or(0);
-        let inner_production = inner.representation().production();
-        let delimiter_production = self.delimiter.representation().production();
-
-        let production = EBNF::Repeated {
-            inner: Box::new(EBNF::Sequence(vec![inner_production, delimiter_production])),
-            more_than_once: at_least >= 1,
-        };
-        Representation::new().with_production(production)
-    }
-}
-
-pub fn optional<I, O, E>(parser: impl Parser<I, O, E>) -> impl Parser<I, Option<O>, E>
-where
-    I: Clone,
-{
-    move |input: I| match parser.parse(input.clone()) {
-        Ok((output, rest)) => Ok((Some(output), rest)),
-        Err(_e) => Ok((None, input)),
-    }
-}

src/combinators/map.rs

@@ -0,0 +1,16 @@
+use crate::parser::{Parser, ParserInput};
+
+pub fn map<P, F, I, O1, O2, E>(parser: P, map_fn: F) -> impl Parser<I, O2, E>
+where
+    I: ParserInput,
+    P: Parser<I, O1, E>,
+    F: Fn(O1) -> O2,
+{
+    let rep = parser.representation();
+    let p = move |input| {
+        parser
+            .parse(input)
+            .map(|(result, rest)| (map_fn(result), rest))
+    };
+    (p, rep)
+}

src/combinators/mod.rs

@@ -0,0 +1,66 @@
+mod map;
+mod optional;
+mod repeated;
+mod separated_by;
+
+pub use map::map;
+pub use optional::optional;
+pub use repeated::repeated;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::parser::Parser;
+    use crate::primitives::literal;
+
+    #[test]
+    fn test_map() {
+        let lit_a = literal("a");
+        let output = lit_a.map(|s| s.to_uppercase()).parse("a yolo");
+        assert_eq!(output.unwrap(), ("A".to_string(), " yolo"));
+    }
+
+    #[test]
+    fn test_one_or_more() {
+        let p = repeated(literal("bongo ")).at_least(1);
+        let input = "bongo bongo bongo bongo bongo ";
+
+        let (output, rest) = p.parse(input).unwrap();
+        assert_eq!(rest, "");
+        assert_eq!(output.len(), 5);
+
+        let (output, rest) = p.parse("bongo ecks").unwrap();
+        assert_eq!(output.len(), 1);
+        assert_eq!(rest, "ecks");
+    }
+
+    #[test]
+    fn test_separated_by() {
+        let p = repeated(literal("garb").to(20))
+            .separated_by(repeated(literal(" ")).at_least(1), false);
+
+        assert_eq!(
+            p.parse("garb garb      garb     garb").unwrap(),
+            (vec![20, 20, 20, 20], "")
+        );
+
+        assert!(p.parse("garb garb      garb     garb ").is_err());
+
+        let p =
+            repeated(literal("garb").to(20)).separated_by(repeated(literal(" ")).at_least(1), true);
+
+        assert_eq!(
+            p.parse("garb garb      garb     garb").unwrap(),
+            (vec![20, 20, 20, 20], "")
+        );
+
+        assert_eq!(
+            p.parse("garb garb      garb     garb     ").unwrap(),
+            (vec![20, 20, 20, 20], "")
+        );
+        assert_eq!(
+            p.parse("garb garb      garb     garb     q").unwrap(),
+            (vec![20, 20, 20, 20], "q")
+        );
+    }
+}

src/combinators/optional.rs

@@ -0,0 +1,17 @@
+use crate::parser::{Parser, ParserInput, Representation};
+
+pub fn optional<P, I, O, E>(parser: P) -> impl Parser<I, Option<O>, E>
+where
+    P: Parser<I, O, E>,
+    I: ParserInput + Clone,
+{
+    let rep = Representation::from_choice(
+        &mut [parser.representation(), Representation::new("ε")].into_iter(),
+    );
+    let p = move |input: I| match parser.parse(input.clone()) {
+        Ok((output, rest)) => Ok((Some(output), rest)),
+        Err(_e) => Ok((None, input)),
+    };
+
+    (p, rep)
+}

src/combinators/repeated.rs

@@ -0,0 +1,94 @@
+use crate::combinators::separated_by::SeparatedBy;
+use crate::parser::{BoxedParser, ParseResult, Parser, ParserInput, Representation};
+
+pub fn repeated<'a, P, I, O>(parser: P) -> Repeated<'a, I, O>
+where
+    P: Parser<I, O, I> + 'a,
+    I: ParserInput + Clone + 'a,
+{
+    Repeated {
+        inner_parser: BoxedParser::new(parser),
+        at_least: None,
+        at_most: None,
+    }
+}
+
+pub struct Repeated<'a, I, O>
+where
+    I: ParserInput + Clone,
+{
+    pub(super) inner_parser: BoxedParser<'a, I, O, I>,
+    pub(super) at_least: Option<u16>,
+    pub(super) at_most: Option<u16>,
+}
+
+impl<'a, I, O> Repeated<'a, I, O>
+where
+    I: ParserInput + Clone,
+{
+    pub fn at_least(self, n: u16) -> Self {
+        Self {
+            at_least: Some(n),
+            ..self
+        }
+    }
+    pub fn at_most(self, n: u16) -> Self {
+        Self {
+            at_most: Some(n),
+            ..self
+        }
+    }
+
+    pub fn separated_by<D, O2>(self, delimiter: D, allow_trailing: bool) -> SeparatedBy<'a, I, O>
+    where
+        D: Parser<I, O2, I> + 'a,
+        O2: 'a,
+        I: 'a,
+    {
+        SeparatedBy {
+            inner_repeated: self,
+            delimiter: BoxedParser::new(delimiter.to(())),
+            allow_trailing,
+        }
+    }
+}
+
+impl<'a, I, O> Parser<I, Vec<O>, I> for Repeated<'a, I, O>
+where
+    I: ParserInput + Clone + 'a,
+{
+    fn parse(&self, input: I) -> ParseResult<I, Vec<O>, I> {
+        let at_least = self.at_least.unwrap_or(0);
+        let at_most = self.at_most.unwrap_or(u16::MAX);
+
+        if at_most == 0 {
+            return Ok((vec![], input));
+        }
+
+        let mut results = Vec::new();
+        let mut count: u16 = 0;
+        let mut further_input = input.clone();
+
+        while let Ok((item, rest)) = self.inner_parser.parse(further_input.clone()) {
+            results.push(item);
+            further_input = rest;
+            count += 1;
+            if count >= at_most {
+                break;
+            }
+        }
+        if count < at_least {
+            return Err(input);
+        }
+
+        Ok((results, further_input))
+    }
+
+    fn representation(&self) -> Representation {
+        Representation::repeated(
+            self.inner_parser.representation(),
+            self.at_least.unwrap_or(0),
+            self.at_most.unwrap_or(u16::MAX),
+        )
+    }
+}

src/combinators/separated_by.rs

@@ -0,0 +1,84 @@
+use crate::combinators::repeated::Repeated;
+use crate::parser::{BoxedParser, ParseResult, Parser, ParserInput, Representation};
+
+pub struct SeparatedBy<'a, I, O>
+where
+    I: ParserInput + Clone,
+{
+    pub(super) inner_repeated: Repeated<'a, I, O>,
+    pub(super) delimiter: BoxedParser<'a, I, (), I>,
+    pub(super) allow_trailing: bool,
+}
+
+impl<'a, I, O> Parser<I, Vec<O>, I> for SeparatedBy<'a, I, O>
+where
+    I: ParserInput + Clone + 'a,
+{
+    fn representation(&self) -> Representation {
+        Representation::new("sepby")
+    }
+
+    fn parse(&self, input: I) -> ParseResult<I, Vec<O>, I> {
+        let at_least = self.inner_repeated.at_least.unwrap_or(0);
+        let at_most = self.inner_repeated.at_most.unwrap_or(u16::MAX);
+        let parser = &self.inner_repeated.inner_parser;
+        let delimiter = &self.delimiter;
+
+        if at_most == 0 {
+            return Ok((vec![], input));
+        }
+
+        let mut results = Vec::new();
+        let mut count: u16 = 0;
+        let mut further_input;
+
+        match parser.parse(input.clone()) {
+            Ok((item, rest)) => {
+                results.push(item);
+                further_input = rest;
+            }
+            Err(_e) => {
+                if at_least > 0 {
+                    return Err(input);
+                } else {
+                    return Ok((vec![], input));
+                }
+            }
+        }
+
+        loop {
+            match delimiter.parse(further_input.clone()) {
+                Ok(((), rest)) => {
+                    further_input = rest;
+                }
+                Err(_e) => {
+                    break;
+                }
+            }
+
+            match parser.parse(further_input.clone()) {
+                Ok((item, rest)) => {
+                    results.push(item);
+                    further_input = rest;
+                    count += 1;
+                }
+                Err(_e) if self.allow_trailing => {
+                    break;
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            }
+
+            if count >= at_most {
+                break;
+            }
+        }
+
+        if count < at_least {
+            return Err(input);
+        }
+
+        Ok((results, further_input))
+    }
+}

src/lib.rs

@@ -1,20 +1,10 @@
-#![allow(dead_code)] //TODO eventually turn this off
-mod annotated;
-mod choice;
-mod combinators;
-mod map;
+pub mod choice;
+pub mod combinators;
 mod parser;
-mod primitives;
-mod representation;
-mod sequence;
-mod util;
+pub mod primitives;
+pub mod sequence;
 
-#[cfg(test)]
-mod test;
+#[cfg(feature = "testutil")]
+pub mod testutil;
 
-pub use choice::*;
-pub use combinators::*;
-pub use map::*;
-pub use parser::{ParseResult, Parser, ParserExtension};
-pub use primitives::*;
-pub use sequence::*;
+pub use parser::{ParseResult, Parser, ParserInput, Representation};

src/map.rs

@@ -1,16 +0,0 @@
-use crate::{representation::Representation, Parser, ParserExtension};
-
-pub fn map<P, F, I, O1, O2, E>(parser: P, map_fn: F) -> impl Parser<I, O2, E>
-where
-    P: Parser<I, O1, E>,
-    F: Fn(O1) -> O2,
-{
-    let production = parser.representation().production();
-    (move |input| {
-        parser
-            .parse(input)
-            .map(|(result, rest)| (map_fn(result), rest))
-    })
-    .to_anno()
-    .with_repr(Representation::new().with_production(production))
-}

src/parser.rs

@@ -1,76 +0,0 @@
-use crate::{annotated::AnnotatedParser, map, representation::Representation, seq2, surrounded_by};
-
-pub type ParseResult<I, O, E> = Result<(O, I), (E, I)>;
-
-pub trait Parser<I, O, E> {
-    fn parse(&self, input: I) -> ParseResult<I, O, E>;
-    fn name(&self) -> Option<String> {
-        None
-    }
-    fn representation(&self) -> Representation {
-        Representation::new()
-    }
-}
-
-impl<I, O, E, F> Parser<I, O, E> for F
-where
-    F: Fn(I) -> ParseResult<I, O, E>,
-{
-    fn parse(&self, input: I) -> ParseResult<I, O, E> {
-        self(input)
-    }
-}
-
-pub trait ParserExtension<I, O, E>: Parser<I, O, E> {
-    fn map<F, O2>(self, map_fn: F) -> impl Parser<I, O2, E>
-    where
-        F: Fn(O) -> O2;
-
-    fn to<O2: Clone>(self, item: O2) -> impl Parser<I, O2, E>;
-    fn then<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, (O, O2), E>;
-    fn then_ignore<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, O, E>;
-    fn ignore_then<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, O2, E>;
-    fn surrounded_by<O2>(self, surrounding: impl Parser<I, O2, E>) -> impl Parser<I, O, E>;
-    fn to_anno(self) -> AnnotatedParser<Self, I, O, E>
-    where
-        Self: Sized,
-    {
-        AnnotatedParser::new(self)
-    }
-    fn to_named(self, name: &str) -> AnnotatedParser<Self, I, O, E>
-    where
-        Self: Sized,
-    {
-        AnnotatedParser::new(self).with_name(name)
-    }
-}
-
-impl<T, I, O, E> ParserExtension<I, O, E> for T
-where
-    T: Parser<I, O, E>,
-{
-    fn map<F, O2>(self, map_fn: F) -> impl Parser<I, O2, E>
-    where
-        F: Fn(O) -> O2,
-    {
-        map(self, map_fn)
-    }
-
-    fn to<O2: Clone>(self, item: O2) -> impl Parser<I, O2, E> {
-        self.map(move |_| item.clone())
-    }
-
-    fn then<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, (O, O2), E> {
-        seq2(self, next)
-    }
-
-    fn then_ignore<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, O, E> {
-        seq2(self, next).map(|(this, _)| this)
-    }
-    fn ignore_then<O2, P: Parser<I, O2, E>>(self, next: P) -> impl Parser<I, O2, E> {
-        seq2(self, next).map(|(_, next)| next)
-    }
-    fn surrounded_by<O2>(self, surrounding: impl Parser<I, O2, E>) -> impl Parser<I, O, E> {
-        surrounded_by(self, surrounding)
-    }
-}

src/parser/boxed_parser.rs

@@ -0,0 +1,38 @@
+use crate::parser::{ParseResult, Parser, ParserInput, Representation};
+
+pub struct BoxedParser<'a, I, O, E>
+where
+    I: ParserInput,
+{
+    inner: Box<dyn Parser<I, O, E> + 'a>,
+}
+
+impl<'a, I, O, E> BoxedParser<'a, I, O, E>
+where
+    I: ParserInput,
+{
+    pub(crate) fn new<P>(inner: P) -> Self
+    where
+        P: Parser<I, O, E> + 'a,
+    {
+        BoxedParser {
+            inner: Box::new(inner),
+        }
+    }
+}
+
+impl<'a, I: ParserInput, O, E> Parser<I, O, E> for BoxedParser<'a, I, O, E> {
+    fn representation(&self) -> Representation {
+        self.inner.representation()
+    }
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        self.inner.parse(input)
+    }
+
+    fn boxed<'b>(self) -> BoxedParser<'b, I, O, E>
+    where
+        Self: Sized + 'b,
+    {
+        self
+    }
+}

src/parser/mod.rs

@@ -0,0 +1,179 @@
+mod boxed_parser;
+mod named_parser;
+mod parser_input;
+mod representation;
+
+use std::rc::Rc;
+
+pub use boxed_parser::BoxedParser;
+pub use named_parser::NamedParser;
+pub use parser_input::ParserInput;
+pub use representation::Representation;
+
+pub type ParseResult<I, O, E> = Result<(O, I), E>;
+
+pub trait Parser<I, O, E>
+where
+    I: ParserInput,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E>;
+
+    fn representation(&self) -> Representation;
+
+    fn boxed<'a>(self) -> BoxedParser<'a, I, O, E>
+    where
+        Self: Sized + 'a,
+    {
+        BoxedParser::new(self)
+    }
+
+    fn map<'a, F, O2>(self, map_fn: F) -> BoxedParser<'a, I, O2, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        E: 'a,
+        O: 'a,
+        O2: 'a,
+        F: Fn(O) -> O2 + 'a,
+    {
+        crate::combinators::map(self, map_fn).boxed()
+    }
+
+    fn to<'a, O2>(self, item: O2) -> BoxedParser<'a, I, O2, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O: 'a,
+        O2: Clone + 'a,
+        E: 'a,
+    {
+        self.map(move |_| item.clone())
+    }
+
+    fn then<'a, P, O2>(self, next_parser: P) -> BoxedParser<'a, I, (O, O2), E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O: 'a,
+        O2: 'a,
+        E: 'a,
+        P: Parser<I, O2, E> + 'a,
+    {
+        crate::sequence::tuple2(self, next_parser).boxed()
+    }
+
+    fn ignore_then<'a, P, O2>(self, next_parser: P) -> BoxedParser<'a, I, O2, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O: 'a,
+        O2: 'a,
+        E: 'a,
+        P: Parser<I, O2, E> + 'a,
+    {
+        crate::sequence::tuple2(self, next_parser).map(|(_, next_output)| next_output)
+    }
+
+    fn then_ignore<'a, P, O2>(self, next_parser: P) -> BoxedParser<'a, I, O, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O: 'a,
+        O2: 'a,
+        E: 'a,
+        P: Parser<I, O2, E> + 'a,
+    {
+        crate::sequence::tuple2(self, next_parser).map(|(this_output, _)| this_output)
+    }
+
+    fn delimited<'a, P1, O1, P2, O2>(self, left: P1, right: P2) -> BoxedParser<'a, I, O, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O1: 'a,
+        O2: 'a,
+        O: 'a,
+        E: 'a,
+        P1: Parser<I, O1, E> + 'a,
+        P2: Parser<I, O2, E> + 'a,
+    {
+        crate::sequence::seq((left, self, right)).map(|(_, output, _)| output)
+    }
+
+    fn surrounded_by<'a, P, O1>(self, surrounding: P) -> BoxedParser<'a, I, O, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+        O1: 'a,
+        O: 'a,
+        E: 'a,
+        P: Parser<I, O1, E> + 'a,
+    {
+        BoxedParser::new(move |input| {
+            let p1 = |i| surrounding.parse(i);
+            let p2 = |i| surrounding.parse(i);
+            let main = |i| self.parse(i);
+            crate::sequence::seq((p1, main, p2))
+                .map(|(_, output, _)| output)
+                .parse(input)
+        })
+    }
+
+    fn optional<'a>(self) -> BoxedParser<'a, I, Option<O>, E>
+    where
+        I: Clone + 'a,
+        O: 'a,
+        E: 'a,
+        Self: Sized + 'a,
+    {
+        crate::combinators::optional(self).boxed()
+    }
+
+    fn named<'a>(self, parser_name: &str) -> NamedParser<'a, I, O, E>
+    where
+        Self: Sized + 'a,
+        I: 'a,
+    {
+        NamedParser::new(self.boxed(), parser_name.to_string())
+    }
+}
+
+impl<I: ParserInput, O, E, F> Parser<I, O, E> for F
+where
+    F: Fn(I) -> ParseResult<I, O, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        self(input)
+    }
+
+    fn representation(&self) -> Representation {
+        Representation::new("NOT IMPL'D")
+    }
+}
+
+impl<I: ParserInput, O, E, F> Parser<I, O, E> for (F, Representation)
+where
+    F: Fn(I) -> ParseResult<I, O, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        self.0(input)
+    }
+
+    fn representation(&self) -> Representation {
+        self.1.clone()
+    }
+}
+
+impl<I, O, E, T> Parser<I, O, E> for Rc<T>
+where
+    I: ParserInput,
+    T: Parser<I, O, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        self.as_ref().parse(input)
+    }
+
+    fn representation(&self) -> Representation {
+        self.as_ref().representation()
+    }
+}

src/parser/named_parser.rs

@@ -0,0 +1,36 @@
+use super::boxed_parser::BoxedParser;
+use crate::parser::{ParseResult, Parser, ParserInput, Representation};
+
+pub struct NamedParser<'a, I, O, E>
+where
+    I: ParserInput,
+{
+    inner_parser: BoxedParser<'a, I, O, E>,
+    name: String,
+}
+
+impl<'a, I, O, E> NamedParser<'a, I, O, E>
+where
+    I: ParserInput,
+{
+    pub(super) fn new(inner_parser: BoxedParser<'a, I, O, E>, name: String) -> Self
+    where
+        I: 'a,
+    {
+        NamedParser { inner_parser, name }
+    }
+
+    pub fn get_name(&'a self) -> &'a str {
+        self.name.as_ref()
+    }
+}
+
+impl<'a, I: ParserInput, O, E> Parser<I, O, E> for NamedParser<'a, I, O, E> {
+    fn representation(&self) -> Representation {
+        self.inner_parser.representation()
+    }
+
+    fn parse(&self, input: I) -> ParseResult<I, O, E> {
+        self.inner_parser.parse(input)
+    }
+}

src/parser/parser_input.rs

@@ -0,0 +1,11 @@
+pub trait ParserInput: std::fmt::Debug {
+    type Output;
+    fn next_token() -> Self::Output;
+}
+
+impl ParserInput for &str {
+    type Output = ();
+    fn next_token() -> Self::Output {
+        ()
+    }
+}

src/parser/representation.rs

@@ -0,0 +1,66 @@
+#[derive(Debug, Clone, PartialEq)]
+pub struct Representation {
+    val: String,
+}
+
+impl Representation {
+    pub fn new(from: &str) -> Self {
+        Self {
+            val: from.to_string(),
+        }
+    }
+
+    pub(crate) fn from_choice(
+        choice_parser_reps: &mut impl Iterator<Item = Representation>,
+    ) -> Self {
+        let mut buf = String::new();
+        let mut iter = choice_parser_reps.peekable();
+        loop {
+            let rep = match iter.next() {
+                Some(r) => r,
+                None => break,
+            };
+            buf.push_str(&rep.val);
+            match iter.peek() {
+                Some(_) => {
+                    buf.push_str(" | ");
+                }
+                None => {
+                    break;
+                }
+            }
+        }
+
+        Representation::new(&buf)
+    }
+
+    pub(crate) fn from_sequence(
+        sequence_representations: &mut impl Iterator<Item = Representation>,
+    ) -> Self {
+        let mut buf = String::new();
+        let mut iter = sequence_representations.peekable();
+        loop {
+            let rep = match iter.next() {
+                Some(r) => r,
+                None => break,
+            };
+            buf.push_str(&rep.val);
+            match iter.peek() {
+                Some(_) => {
+                    buf.push_str(" ");
+                }
+                None => {
+                    break;
+                }
+            }
+        }
+
+        Representation::new(&buf)
+    }
+
+    // TODO use at_least, at_most
+    pub(crate) fn repeated(underlying: Representation, at_least: u16, _at_most: u16) -> Self {
+        let sigil = if at_least == 0 { "*" } else { "+" };
+        Representation::new(&format!("({}){}", underlying.val, sigil))
+    }
+}

src/primitives.rs

@@ -1,110 +0,0 @@
-use crate::{
-    representation::{Representation, EBNF},
-    ParseResult, Parser, ParserExtension,
-};
-
-pub fn literal<'a>(expected: &'static str) -> impl Parser<&'a str, &'a str, ()> {
-    let p = move |input: &'a str| match input.get(0..expected.len()) {
-        Some(next) if next == expected => Ok((next, &input[expected.len()..])),
-        _ => Err(((), input)),
-    };
-    let production = EBNF::StringTerminal(expected.into());
-    p.to_anno()
-        .with_repr(Representation::new().with_production(production))
-}
-
-pub fn literal_char<'a>(expected: char) -> impl Parser<&'a str, char, ()> {
-    (move |input: &'a str| match input.chars().next() {
-        Some(ch) if ch == expected => Ok((expected, &input[ch.len_utf8()..])),
-        _ => Err(((), input)),
-    })
-    .to_anno()
-    .with_repr(Representation::new().with_production(EBNF::CharTerminal(expected)))
-}
-
-pub fn one_of<'a>(items: &'static str) -> impl Parser<&'a str, char, ()> {
-    (move |input: &'a str| {
-        if let Some(ch) = input.chars().next() {
-            if items.contains(ch) {
-                let (_first, rest) = input.split_at(1);
-                return Ok((ch, rest));
-            }
-        }
-        Err(((), input))
-    })
-    .to_anno()
-    .with_repr(
-        Representation::new().with_production(EBNF::Alternation(
-            items
-                .chars()
-                .map(|ch| EBNF::CharTerminal(ch))
-                .collect::<Vec<_>>(),
-        )),
-    )
-}
-
-/// Parses a standard identifier in a programming language
-pub fn identifier(input: &str) -> ParseResult<&str, String, ()> {
-    let mut chars = input.chars();
-    let mut buf = String::new();
-
-    match chars.next() {
-        Some(ch) if ch.is_alphabetic() => buf.push(ch),
-        _ => return Err(((), input)),
-    }
-
-    for next in chars {
-        if next.is_alphanumeric() {
-            buf.push(next);
-        } else {
-            break;
-        }
-    }
-
-    let next_index = buf.len();
-    Ok((buf, &input[next_index..]))
-}
-
-pub struct Whitespace;
-
-impl Parser<&str, char, ()> for Whitespace {
-    fn name(&self) -> Option<String> {
-        Some("whitespace".into())
-    }
-
-    fn representation(&self) -> Representation {
-        Representation::new().with_production(EBNF::LabeledTerminal("whitespace".into()))
-    }
-
-    fn parse<'a>(&self, input: &'a str) -> ParseResult<&'a str, char, ()> {
-        match input.chars().next() {
-            Some(ch) if ch.is_whitespace() => Ok((ch, &input[1..])),
-            _ => Err(((), input)),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn primitive_parsers() {
-        let parser = literal_char('f');
-        assert_eq!(Ok(('f', "unky")), parser.parse("funky"));
-
-        let repr = parser.representation();
-        assert!(matches!(repr.production(), EBNF::CharTerminal('f')));
-
-        let parser = one_of("asdf");
-        let production = parser.representation().production();
-        assert!(
-            matches!(production, EBNF::Alternation(v) if matches!(v.as_slice(), [
-               EBNF::CharTerminal('a'),
-               EBNF::CharTerminal('s'),
-               EBNF::CharTerminal('d'),
-               EBNF::CharTerminal('f'),
-            ]))
-        );
-    }
-}

src/primitives/mod.rs

@@ -0,0 +1,108 @@
+use crate::parser::{ParseResult, Parser, ParserInput, Representation};
+
+pub fn literal_char(expected: char) -> impl Fn(&str) -> ParseResult<&str, char, &str> {
+    move |input| match input.chars().next() {
+        Some(ch) if ch == expected => Ok((expected, &input[ch.len_utf8()..])),
+        _ => Err(input),
+    }
+}
+
+pub fn literal<'a>(expected: &'static str) -> impl Parser<&'a str, &'a str, &'a str> {
+    println!("literal call expected: {}", expected);
+    let rep = Representation::new(expected);
+    let p = move |input: &'a str| match input.get(0..expected.len()) {
+        Some(next) if next == expected => Ok((expected, &input[expected.len()..])),
+        _ => Err(input),
+    };
+    (p, rep)
+}
+
+pub fn any_char(input: &str) -> ParseResult<&str, char, &str> {
+    match input.chars().next() {
+        Some(ch) => Ok((ch, &input[ch.len_utf8()..])),
+        None => Err(input),
+    }
+}
+
+pub fn one_of<'a>(items: &'static str) -> impl Parser<&'a str, &'a str, &'a str> {
+    let p = move |input: &'a str| {
+        if let Some(ch) = input.chars().next() {
+            if items.contains(ch) {
+                let (first, rest) = input.split_at(1);
+                return Ok((first, rest));
+            }
+        }
+        Err(input)
+    };
+
+    let mut s = String::new();
+    for ch in items.chars() {
+        s.push(ch);
+        s.push_str(" | ");
+    }
+    let rep = Representation::new(&s);
+    (p, rep)
+}
+
+pub fn pred<P, F, I, O>(parser: P, pred_fn: F) -> impl Parser<I, O, I>
+where
+    I: ParserInput,
+    P: Parser<I, O, I>,
+    F: Fn(&O) -> bool,
+{
+    let orig_rep = parser.representation();
+    (
+        move |input| {
+            parser.parse(input).and_then(|(result, rest)| {
+                if pred_fn(&result) {
+                    Ok((result, rest))
+                } else {
+                    Err(rest)
+                }
+            })
+        },
+        Representation::new(&format!("{:?} if <PREDICATE>", orig_rep)),
+    )
+}
+
+/// Parses a standard identifier in a programming language
+pub fn identifier(input: &str) -> ParseResult<&str, String, &str> {
+    let mut chars = input.chars();
+    let mut buf = String::new();
+
+    match chars.next() {
+        Some(ch) if ch.is_alphabetic() => buf.push(ch),
+        _ => return Err(input),
+    }
+
+    for next in chars {
+        if next.is_alphanumeric() {
+            buf.push(next);
+        } else {
+            break;
+        }
+    }
+
+    let next_index = buf.len();
+    Ok((buf, &input[next_index..]))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_identifier() {
+        assert_eq!(
+            identifier("bongo1beans").unwrap(),
+            (("bongo1beans".to_string(), ""))
+        );
+        assert_eq!(identifier("2bongo1beans"), Err("2bongo1beans"));
+    }
+
+    #[test]
+    fn test_pred() {
+        let p = pred(any_char, |c| *c == 'f');
+        assert_eq!(p.parse("frog"), Ok(('f', "rog")));
+    }
+}

src/representation.rs

@@ -1,129 +0,0 @@
-use std::fmt;
-
-use crate::util::intersperse_option;
-
-#[derive(Debug, Clone)]
-pub struct Representation {
-    production_output: EBNF,
-}
-
-impl Representation {
-    pub fn show(&self) -> String {
-        self.production_output.to_string()
-    }
-
-    pub fn production(&self) -> EBNF {
-        self.production_output.clone()
-    }
-
-    pub fn new() -> Self {
-        Self {
-            production_output: EBNF::None,
-        }
-    }
-
-    pub fn with_production(self, production_output: EBNF) -> Self {
-        Self {
-            production_output,
-            ..self
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub enum EBNF {
-    None,
-    Nonterminal(String),
-    CharTerminal(char),
-    StringTerminal(String),
-    LabeledTerminal(String),
-    Alternation(Vec<EBNF>),
-    Sequence(Vec<EBNF>),
-    Repeated {
-        inner: Box<EBNF>,
-        more_than_once: bool,
-    },
-}
-
-impl EBNF {
-    fn needs_wrapping(&self) -> bool {
-        match self {
-            EBNF::None => false,
-            EBNF::Nonterminal(_) => false,
-            EBNF::CharTerminal(_) => false,
-            EBNF::StringTerminal(_) => false,
-            EBNF::LabeledTerminal(_) => false,
-            EBNF::Sequence(items) => items.len() > 1,
-            EBNF::Alternation(_) => true,
-            EBNF::Repeated { .. } => false,
-        }
-    }
-}
-
-impl fmt::Display for EBNF {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            //TODO should try to show the name if possible
-            EBNF::None => write!(f, "<no-representation>"),
-            EBNF::CharTerminal(ch) => write!(f, "'{ch}'"),
-            EBNF::Alternation(items) => {
-                for item in intersperse_option(items.iter()) {
-                    match item {
-                        None => write!(f, " | ")?,
-                        Some(item) => write!(f, "{item}")?,
-                    }
-                }
-                write!(f, "")
-            }
-            EBNF::Nonterminal(name) => write!(f, "{name}"),
-            EBNF::StringTerminal(term) => write!(f, r#""{term}""#),
-            EBNF::LabeledTerminal(s) => write!(f, "<{s}>"),
-            EBNF::Repeated {
-                inner,
-                more_than_once,
-            } => {
-                let sigil = if *more_than_once { '+' } else { '*' };
-                if inner.needs_wrapping() {
-                    write!(f, "[{inner}]{sigil}")
-                } else {
-                    write!(f, "{inner}{sigil}")
-                }
-            }
-            EBNF::Sequence(items) => {
-                for item in intersperse_option(items.iter()) {
-                    if let Some(item) = item {
-                        write!(f, "{item}")?;
-                    } else {
-                        write!(f, " ")?;
-                    }
-                }
-                write!(f, "")
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_ebnf_print() {
-        let example = EBNF::Alternation(vec![
-            EBNF::CharTerminal('f'),
-            EBNF::CharTerminal('a'),
-            EBNF::CharTerminal('k'),
-            EBNF::CharTerminal('e'),
-        ]);
-
-        assert_eq!(example.to_string(), "'f' | 'a' | 'k' | 'e'");
-
-        let example = EBNF::Alternation(vec![
-            EBNF::Nonterminal("other-rule".into()),
-            EBNF::CharTerminal('q'),
-            EBNF::CharTerminal('m'),
-            EBNF::StringTerminal("focus".into()),
-        ]);
-        assert_eq!(example.to_string(), "other-rule | 'q' | 'm' | \"focus\"");
-    }
-}

src/sequence.rs

@@ -1,102 +0,0 @@
-use crate::{
-    representation::{Representation, EBNF},
-    ParseResult, Parser, ParserExtension,
-};
-
-pub fn sequence<S, I, O, E>(sequence: S) -> impl Parser<I, O, E>
-where
-    S: Sequence<I, O, E>,
-{
-    let repr = sequence.repr();
-
-    (move |input| -> ParseResult<I, O, E> { sequence.parse(input) })
-        .to_anno()
-        .with_repr(repr)
-}
-
-pub fn surrounded_by<I, O1, O2, E>(
-    main: impl Parser<I, O1, E>,
-    surrounding: impl Parser<I, O2, E>,
-) -> impl Parser<I, O1, E> {
-    let s_prod = surrounding.representation().production();
-    let main_prod = main.representation().production();
-
-    (move |input| {
-        let (_result1, rest1) = surrounding.parse(input)?;
-        let (result2, rest2) = main.parse(rest1)?;
-        let (_result3, rest3) = surrounding.parse(rest2)?;
-        Ok((result2, rest3))
-    })
-    .to_anno()
-    .with_repr(Representation::new().with_production(EBNF::Sequence(vec![
-        s_prod.clone(),
-        main_prod,
-        s_prod,
-    ])))
-}
-
-pub fn seq2<I, O1, O2, E>(
-    first: impl Parser<I, O1, E>,
-    second: impl Parser<I, O2, E>,
-) -> impl Parser<I, (O1, O2), E> {
-    sequence((first, second))
-}
-
-pub trait Sequence<I, O, E> {
-    fn parse(&self, input: I) -> ParseResult<I, O, E>;
-    fn repr(&self) -> Representation;
-}
-
-impl<P1, P2, I, O1, O2, E> Sequence<I, (O1, O2), E> for (P1, P2)
-where
-    P1: Parser<I, O1, E>,
-    P2: Parser<I, O2, E>,
-{
-    fn parse(&self, input: I) -> ParseResult<I, (O1, O2), E> {
-        let p1 = &self.0;
-        let p2 = &self.1;
-        p1.parse(input).and_then(|(result1, rest)| {
-            p2.parse(rest)
-                .map(|(result2, rest2)| ((result1, result2), rest2))
-        })
-    }
-
-    fn repr(&self) -> Representation {
-        let p1 = &self.0;
-        let p2 = &self.1;
-        Representation::new().with_production(EBNF::Sequence(vec![
-            p1.representation().production(),
-            p2.representation().production(),
-        ]))
-    }
-}
-
-impl<P1, P2, P3, I, O1, O2, O3, E> Sequence<I, (O1, O2, O3), E> for (P1, P2, P3)
-where
-    P1: Parser<I, O1, E>,
-    P2: Parser<I, O2, E>,
-    P3: Parser<I, O3, E>,
-{
-    fn parse(&self, input: I) -> ParseResult<I, (O1, O2, O3), E> {
-        let p1 = &self.0;
-        let p2 = &self.1;
-        let p3 = &self.2;
-
-        let (result1, rest1) = p1.parse(input)?;
-        let (result2, rest2) = p2.parse(rest1)?;
-        let (result3, rest3) = p3.parse(rest2)?;
-
-        Ok(((result1, result2, result3), rest3))
-    }
-
-    fn repr(&self) -> Representation {
-        let p1 = &self.0;
-        let p2 = &self.1;
-        let p3 = &self.2;
-        Representation::new().with_production(EBNF::Sequence(vec![
-            p1.representation().production(),
-            p2.representation().production(),
-            p3.representation().production(),
-        ]))
-    }
-}

src/sequence/mod.rs

@@ -0,0 +1,195 @@
+use crate::parser::{ParseResult, Parser, ParserInput, Representation};
+
+pub fn tuple2<P1, P2, I, O1, O2, E>(parser1: P1, parser2: P2) -> impl Parser<I, (O1, O2), E>
+where
+    I: ParserInput,
+    P1: Parser<I, O1, E>,
+    P2: Parser<I, O2, E>,
+{
+    seq((parser1, parser2))
+}
+
+pub fn seq<T, I, O, E>(sequence: T) -> impl Parser<I, O, E>
+where
+    I: ParserInput,
+    T: Sequence<I, O, E>,
+{
+    let rep = sequence.representation();
+    let p = move |input| sequence.parse(input);
+    (p, rep)
+}
+
+/* TODO - eventually rewrite this parser combinator in Schala. Seeing what this
+ * code that makes heavy use of type variables and abstraction over types looks like
+ * in Schala's type system should be educational
+ */
+
+pub trait Sequence<I, O, E> {
+    fn parse(&self, input: I) -> ParseResult<I, O, E>;
+    fn representation(&self) -> Representation;
+}
+
+impl<I, O1, O2, E, P1, P2> Sequence<I, (O1, O2), E> for (P1, P2)
+where
+    I: ParserInput,
+    P1: Parser<I, O1, E>,
+    P2: Parser<I, O2, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, (O1, O2), E> {
+        let parser1 = &self.0;
+        let parser2 = &self.1;
+        parser1.parse(input).and_then(|(result1, rest1)| {
+            parser2
+                .parse(rest1)
+                .map(|(result2, rest2)| ((result1, result2), rest2))
+        })
+    }
+
+    fn representation(&self) -> Representation {
+        let mut iter = [self.0.representation(), self.1.representation()].into_iter();
+        Representation::from_sequence(&mut iter)
+    }
+}
+
+impl<I, O1, O2, O3, E, P1, P2, P3> Sequence<I, (O1, O2, O3), E> for (P1, P2, P3)
+where
+    I: ParserInput,
+    P1: Parser<I, O1, E>,
+    P2: Parser<I, O2, E>,
+    P3: Parser<I, O3, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, (O1, O2, O3), E> {
+        let parser1 = &self.0;
+        let parser2 = &self.1;
+        let parser3 = &self.2;
+
+        let (result1, rest1) = parser1.parse(input)?;
+        let (result2, rest2) = parser2.parse(rest1)?;
+        let (result3, rest3) = parser3.parse(rest2)?;
+
+        Ok(((result1, result2, result3), rest3))
+    }
+
+    fn representation(&self) -> Representation {
+        let mut iter = [
+            self.0.representation(),
+            self.1.representation(),
+            self.2.representation(),
+        ]
+        .into_iter();
+        Representation::from_sequence(&mut iter)
+    }
+}
+
+impl<I, O1, O2, O3, O4, E, P1, P2, P3, P4> Sequence<I, (O1, O2, O3, O4), E> for (P1, P2, P3, P4)
+where
+    I: ParserInput,
+    P1: Parser<I, O1, E>,
+    P2: Parser<I, O2, E>,
+    P3: Parser<I, O3, E>,
+    P4: Parser<I, O4, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, (O1, O2, O3, O4), E> {
+        let parser1 = &self.0;
+        let parser2 = &self.1;
+        let parser3 = &self.2;
+        let parser4 = &self.3;
+
+        let (result1, rest1) = parser1.parse(input)?;
+        let (result2, rest2) = parser2.parse(rest1)?;
+        let (result3, rest3) = parser3.parse(rest2)?;
+        let (result4, rest4) = parser4.parse(rest3)?;
+
+        Ok(((result1, result2, result3, result4), rest4))
+    }
+
+    fn representation(&self) -> Representation {
+        let mut iter = [
+            self.0.representation(),
+            self.1.representation(),
+            self.2.representation(),
+            self.3.representation(),
+        ]
+        .into_iter();
+        Representation::from_sequence(&mut iter)
+    }
+}
+
+impl<I, O1, O2, O3, O4, O5, E, P1, P2, P3, P4, P5> Sequence<I, (O1, O2, O3, O4, O5), E>
+    for (P1, P2, P3, P4, P5)
+where
+    I: ParserInput,
+    P1: Parser<I, O1, E>,
+    P2: Parser<I, O2, E>,
+    P3: Parser<I, O3, E>,
+    P4: Parser<I, O4, E>,
+    P5: Parser<I, O5, E>,
+{
+    fn parse(&self, input: I) -> ParseResult<I, (O1, O2, O3, O4, O5), E> {
+        let parser1 = &self.0;
+        let parser2 = &self.1;
+        let parser3 = &self.2;
+        let parser4 = &self.3;
+        let parser5 = &self.4;
+
+        let (result1, rest1) = parser1.parse(input)?;
+        let (result2, rest2) = parser2.parse(rest1)?;
+        let (result3, rest3) = parser3.parse(rest2)?;
+        let (result4, rest4) = parser4.parse(rest3)?;
+        let (result5, rest5) = parser5.parse(rest4)?;
+
+        Ok(((result1, result2, result3, result4, result5), rest5))
+    }
+
+    fn representation(&self) -> Representation {
+        let mut iter = [
+            self.0.representation(),
+            self.1.representation(),
+            self.2.representation(),
+            self.3.representation(),
+            self.4.representation(),
+        ]
+        .into_iter();
+        Representation::from_sequence(&mut iter)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::combinators::repeated;
+    use crate::primitives::{identifier, literal};
+
+    #[test]
+    fn test_tuple2() {
+        let p = tuple2(identifier, tuple2(literal(" "), literal("ruts")));
+        let (output, _rest) = p.parse("fort1 ruts").unwrap();
+        assert_eq!(output, ("fort1".into(), (" ", "ruts")));
+
+        let p = identifier.then(literal(" ")).then(literal("ruts"));
+        let (output, _rest) = p.parse("fort1 ruts").unwrap();
+        assert_eq!(output, (("fort1".into(), " "), "ruts"));
+    }
+
+    #[test]
+    fn test_seq() {
+        let p = seq((
+            literal("bong").to(10),
+            repeated(literal(" ")).to(()),
+            literal("hits").to(20),
+        ));
+        assert_eq!(p.parse("bong        hits").unwrap(), ((10, (), 20), ""));
+
+        let p = seq((
+            literal("alpha").to(10),
+            repeated(literal(" ")).to(()),
+            repeated(literal("-")).to(()),
+            repeated(literal(" ")),
+            literal("beta"),
+        ));
+        assert_eq!(
+            p.parse("alpha      ------   beta gamma").unwrap(),
+            ((10, (), (), vec![" ", " ", " "], "beta"), " gamma")
+        );
+    }
+}

src/test/mod.rs

@@ -1,98 +0,0 @@
-mod sexp;
-
-use super::*;
-
-#[test]
-fn basic_parsing() {
-    let (parsed, rest) = literal("a").parse("a yolo").unwrap();
-    assert_eq!(parsed, "a");
-    assert_eq!(rest, " yolo");
-
-    fn bare_function_parser(input: &str) -> ParseResult<&str, i32, String> {
-        match input.chars().next() {
-            Some('0') => Ok((0, &input[1..])),
-            Some('1') => Ok((1, &input[1..])),
-            _ => Err(("lol a parse error".to_string(), input)),
-        }
-    }
-
-    assert_eq!(bare_function_parser.parse("0foo"), Ok((0, "foo")));
-    assert_eq!(
-        bare_function_parser.parse("xfoo"),
-        Err(("lol a parse error".to_string(), "xfoo"))
-    );
-}
-
-#[test]
-fn test_sequence() {
-    let parser = seq2(literal("bongo"), seq2(literal(" "), literal("jonzzz")));
-    let output = parser.parse("bongo jonzzz").unwrap();
-    assert_eq!(output.0 .0, "bongo");
-    assert_eq!(output.0 .1, (" ", "jonzzz"));
-    assert_eq!(output.1, "");
-}
-
-#[test]
-fn test_choice() {
-    let parser = choice((literal("bongo"), literal("sucy"), literal("ara")));
-
-    let output = parser.parse("ara hajimete").unwrap();
-    assert_eq!(("ara", " hajimete"), output);
-}
-
-#[test]
-fn test_map() {
-    let parser =
-        seq2(literal("a"), literal("b")).map(|(a, _b): (&str, &str)| (a.to_uppercase(), 59));
-    let output = parser.parse("abcd").unwrap();
-    assert_eq!((("A".to_owned(), 59), "cd"), output);
-
-    let spaces = repeated(literal_char(' ')).at_least(1);
-    let parser = seq2(literal("lute"), spaces).to(500);
-    assert_eq!(parser.parse("lute    "), Ok((500, "")));
-    assert_eq!(parser.parse("lute"), Err(((), "")));
-}
-
-#[test]
-fn test_combinators() {
-    let parser = repeated(literal_char('a')).to(10).then(literal_char('b'));
-    let output = parser.parse("aaaaaaaabcd").unwrap();
-    assert_eq! {((10, 'b'), "cd"), output};
-}
-
-#[test]
-fn test_optional() {
-    let parser = seq2(
-        optional(literal("alpha")),
-        seq2(repeated(literal(" ")), literal("beta")),
-    );
-
-    let output1 = parser.parse("   beta").unwrap();
-    assert_eq!(output1.0 .0, None);
-    let output2 = parser.parse("alpha   beta").unwrap();
-    assert_eq!(output2.0 .0, Some("alpha"));
-}
-
-#[test]
-fn test_repeated() {
-    let spaces = repeated(literal_char(' ')).at_least(1);
-    let bongo = literal("bongo");
-    let parser = repeated(bongo).separated_by(map(spaces, |_| ()));
-    let output = parser.parse("bongo bongo     bongo           bongo");
-    let output = output.unwrap();
-    assert_eq!(output.0, vec!["bongo", "bongo", "bongo", "bongo"]);
-    assert_eq!(parser.representation().show(), r#"["bongo" ' '+]*"#);
-
-    let bongos = repeated(literal("bongo"));
-    let output = bongos.parse("tra la la").unwrap();
-    assert_eq!(output.0.len(), 0);
-    assert_eq!(output.1, "tra la la");
-
-    assert_eq!(bongos.representation().show(), r#""bongo"*"#);
-}
-
-#[test]
-fn test_named_parser() {
-    let parser = literal("yokel").to_named("yokelparser");
-    assert_eq!(parser.name(), Some("yokelparser".to_string()));
-}

src/test/sexp.rs

@@ -1,103 +0,0 @@
-use crate::*;
-
-#[derive(Debug, PartialEq)]
-enum Expr {
-    Atom(Atom),
-    List(Vec<Expr>),
-    Quote(Vec<Expr>),
-}
-
-#[derive(Clone, Debug, PartialEq)]
-enum Atom {
-    Num(i64),
-    Str(String),
-    Bool(bool),
-    Symbol(String),
-}
-
-fn parse_bool(input: &str) -> ParseResult<&str, Atom, ()> {
-    choice((
-        literal("#t").to(Atom::Bool(true)),
-        literal("#f").to(Atom::Bool(false)),
-    ))
-    .parse(input)
-}
-
-fn parse_symbol(input: &str) -> ParseResult<&str, Atom, ()> {
-    identifier.map(Atom::Symbol).parse(input)
-}
-
-fn parse_number(input: &str) -> ParseResult<&str, Atom, ()> {
-    repeated(one_of("1234567890"))
-        .at_least(1)
-        .map(|n| Atom::Num(n.iter().collect::<String>().parse::<i64>().unwrap()))
-        .parse(input)
-}
-
-fn parse_atom(input: &str) -> ParseResult<&str, Atom, ()> {
-    choice((parse_symbol, parse_bool, parse_number)).parse(input)
-}
-
-#[test]
-fn test_parse_atom() {
-    let output = parse_atom.parse("#t").unwrap();
-    assert_eq!(output.0, Atom::Bool(true));
-
-    let output = parse_atom.parse("384").unwrap();
-    assert_eq!(output.0, Atom::Num(384));
-}
-
-fn parse_expr(input: &str) -> ParseResult<&str, Expr, ()> {
-    choice((parse_list, parse_atom.map(Expr::Atom))).parse(input)
-}
-
-fn parse_list(input: &str) -> ParseResult<&str, Expr, ()> {
-    literal_char('(')
-        .ignore_then(
-            repeated(parse_expr)
-                .separated_by(repeated(Whitespace).at_least(1).to(()))
-                .allow_trailing(true),
-        )
-        .then_ignore(literal_char(')'))
-        .map(Expr::List)
-        .parse(input)
-}
-
-#[test]
-fn test_parse_list() {
-    let output = parse_list.parse("(1 2 (1 2) 9999 3)").unwrap();
-    assert_eq!(output.1, "");
-}
-
-fn parse_sexp(input: &str) -> ParseResult<&str, Expr, ()> {
-    parse_list.surrounded_by(repeated(Whitespace)).parse(input)
-}
-
-#[test]
-fn test_parse_sexp() {
-    let output = parse_expr("(add 1 2)").unwrap();
-    assert_eq!(
-        output.0,
-        Expr::List(vec![
-            Expr::Atom(Atom::Symbol("add".to_string())),
-            Expr::Atom(Atom::Num(1)),
-            Expr::Atom(Atom::Num(2))
-        ])
-    );
-    assert_eq!(output.1, "");
-
-    let complex_input = r#"
-(add (mul 28 9)
-    (if (eq a b) (jump #t) (hula 44))
-)"#
-    .trim();
-
-    let output = parse_sexp(complex_input).unwrap();
-    assert_eq!(output.1, "");
-}
-
-#[ignore = "won't work until representations can be passed more easily around"]
-#[test]
-fn test_parse_sexp_repr() {
-    assert_eq!(parse_sexp.representation().show(), r#"["bongo" ' '+]*"#);
-}

src/testutil/mod.rs

@@ -0,0 +1,135 @@
+use crate::choice::choice;
+use crate::combinators::repeated;
+use crate::primitives::{any_char, literal, literal_char, one_of, pred};
+use crate::sequence::seq;
+use crate::Parser;
+
+/*
+    * JSON BNF
+    * <JSON>     ::= <value>
+<value>    ::= <object> | <array> | <boolean> | <string> | <number> | <null>
+<array>    ::= "[" [<value>] {"," <value>}* "]"
+<object>   ::= "{" [<property>] {"," <property>}* "}"
+<property> ::= <string> ":" <value>
+*/
+#[derive(Debug, Clone, PartialEq)]
+pub enum JsonValue {
+    Null,
+    Bool(bool),
+    Str(String),
+    Num(f64),
+    Array(Vec<JsonValue>),
+    Object(Vec<(String, JsonValue)>),
+}
+
+pub trait JsonParser<'a, T>: Parser<&'a str, T, &'a str> {}
+impl<'a, T, P> JsonParser<'a, T> for P where P: Parser<&'a str, T, &'a str> {}
+
+pub fn json_null<'a>() -> impl JsonParser<'a, JsonValue> {
+    literal("null").to(JsonValue::Null)
+}
+
+pub fn json_bool<'a>() -> impl JsonParser<'a, JsonValue> {
+    choice((
+        literal("true").to(JsonValue::Bool(true)),
+        literal("false").to(JsonValue::Bool(false)),
+    ))
+}
+
+pub fn json_number<'a>() -> impl JsonParser<'a, JsonValue> {
+    fn digit<'a>() -> impl JsonParser<'a, &'a str> {
+        one_of("1234567890")
+    }
+
+    fn digits<'a>() -> impl JsonParser<'a, Vec<&'a str>> {
+        repeated(digit()).at_least(1)
+    }
+
+    let json_number_inner = choice((
+        seq((digits(), literal(".").ignore_then(digits()).optional())).map(
+            |(mut digits, maybe_decimal)| {
+                if let Some(decimal_digits) = maybe_decimal {
+                    digits.push(".");
+                    digits.extend(decimal_digits.into_iter());
+                }
+                digits.into_iter().collect::<String>()
+            },
+        ),
+        literal(".").ignore_then(digits()).map(|decimal_digits| {
+            let mut d = vec!["."];
+            d.extend(decimal_digits.into_iter());
+            d.into_iter().collect::<String>()
+        }),
+    ))
+    .map(|digits| digits.parse::<f64>().unwrap());
+
+    literal("-")
+        .optional()
+        .then(json_number_inner)
+        .map(|(maybe_sign, mut val)| {
+            if maybe_sign.is_some() {
+                val *= -1.0;
+            }
+            JsonValue::Num(val)
+        })
+}
+
+pub fn json_string_raw<'a>() -> impl JsonParser<'a, String> {
+    seq((
+        literal_char('"'),
+        repeated(pred(any_char, |ch| *ch != '"')),
+        literal_char('"'),
+    ))
+    .map(|(_, s, _)| s.iter().cloned().collect::<String>())
+}
+
+pub fn json_string<'a>() -> impl JsonParser<'a, JsonValue> {
+    json_string_raw().map(JsonValue::Str)
+}
+
+fn whitespace<'a>() -> impl JsonParser<'a, ()> {
+    repeated(choice((
+        literal_char('\t'),
+        literal_char('\n'),
+        literal_char(' '),
+    )))
+    .to(())
+}
+
+pub fn json_array<'a>() -> impl JsonParser<'a, JsonValue> {
+    move |input| {
+        let val = json_value().surrounded_by(whitespace());
+
+        repeated(val)
+            .separated_by(literal(","), false)
+            .delimited(literal_char('['), literal_char(']'))
+            .map(JsonValue::Array)
+            .parse(input)
+    }
+}
+
+pub fn json_object<'a>() -> impl JsonParser<'a, JsonValue> {
+    move |input| {
+        let kv = json_string_raw()
+            .surrounded_by(whitespace())
+            .then_ignore(literal_char(':'))
+            .then(json_value().surrounded_by(whitespace()));
+
+        repeated(kv)
+            .separated_by(literal_char(','), false)
+            .delimited(literal_char('{'), literal_char('}'))
+            .map(JsonValue::Object)
+            .parse(input)
+    }
+}
+
+pub fn json_value<'a>() -> impl JsonParser<'a, JsonValue> {
+    choice((
+        json_null(),
+        json_bool(),
+        json_number(),
+        json_string(),
+        json_array(),
+        json_object(),
+    ))
+}

src/util.rs

@@ -1,46 +0,0 @@
-use std::iter::Peekable;
-
-pub(crate) fn intersperse_option<I: Iterator>(iterator: I) -> impl Iterator<Item = Option<I::Item>>
-where
-    I::Item: Clone,
-{
-    intersperse(iterator.map(Some), None)
-}
-
-pub(crate) fn intersperse<I: Iterator>(iterator: I, separator: I::Item) -> Intersperse<I>
-where
-    I::Item: Clone,
-{
-    Intersperse {
-        inner: iterator.peekable(),
-        separator,
-        needs_sep: false,
-    }
-}
-
-pub struct Intersperse<I>
-where
-    I: Iterator,
-{
-    inner: Peekable<I>,
-    separator: I::Item,
-    needs_sep: bool,
-}
-
-impl<I> Iterator for Intersperse<I>
-where
-    I: Iterator,
-    I::Item: Clone,
-{
-    type Item = I::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.needs_sep && self.inner.peek().is_some() {
-            self.needs_sep = false;
-            Some(self.separator.clone())
-        } else {
-            self.needs_sep = true;
-            self.inner.next()
-        }
-    }
-}

tests/joplin-cfg.json

@@ -0,0 +1,49 @@
+{
+	"$schema": "https://joplinapp.org/schema/settings.json",
+	"locale": "en_GB",
+	"sync.target": 6,
+	"markdown.plugin.softbreaks": false,
+	"markdown.plugin.typographer": false,
+	"spellChecker.language": "en-US",
+	"ui.layout": {
+	        "key": "root",
+	        "children": [
+		                {
+		                        "key": "sideBar",
+		                        "width": 250,
+		                        "visible": true
+		                },
+		                {
+		                        "key": "noteList",
+		                        "width": 250,
+		                        "visible": true
+		                },
+		                {
+		                        "key": "editor",
+		                        "visible": true,
+		                        "width": 1493
+		                },
+		                {
+		                        "key": "plugin-view-joplin.plugin.note.tabs-note.tabs.panel",
+		                        "context": {
+			                                "pluginId": "joplin.plugin.note.tabs"
+		                        },
+		                        "visible": true
+		                }
+	        ],
+	        "visible": true
+	},
+	"noteVisiblePanes": [
+	        "editor",
+	        "viewer"
+	],
+	"theme": 4,
+	"sync.6.username": "webdav",
+	"net.ignoreTlsErrors": true,
+	"style.editor.contentMaxWidth": 600,
+	"editor.codeView": true,
+	"markdown.plugin.sub": true,
+	"markdown.plugin.sup": true,
+	"markdown.plugin.multitable": true
+}
+

tests/json_parser.rs

@@ -0,0 +1,117 @@
+use parser_combinator::primitives::literal;
+use parser_combinator::testutil::{
+    json_array, json_bool, json_null, json_number, json_object, json_string, JsonValue,
+};
+use parser_combinator::{Parser, Representation};
+
+use proptest::prelude::*;
+
+use rstest::*;
+
+proptest! {
+    #[test]
+    fn doesnt_crash(s in "\\PC*") {
+        let _output = json_object().parse(&s);
+    }
+
+    #[test]
+    fn parse_string(s in r#"[^"]+"#) {
+        let input = format!("\"{}\"", s);
+        let output = json_string().parse(&input).unwrap();
+        match output {
+            (JsonValue::Str(output_s), "") if output_s == s => (),
+            _ => panic!(),
+        }
+    }
+}
+
+#[test]
+fn test_parsing() {
+    let output = literal("a").parse("a yolo");
+    assert_eq!(output.unwrap(), ("a", " yolo"));
+}
+
+#[test]
+fn parse_json_primitives() {
+    assert_eq!(
+        json_string().parse(r#""yolo swagg""#).unwrap(),
+        (JsonValue::Str("yolo swagg".into()), "")
+    );
+
+    assert_eq!(
+        json_number().parse("-383").unwrap().0,
+        JsonValue::Num(-383f64)
+    );
+    assert_eq!(
+        json_number().parse("-.383").unwrap().0,
+        JsonValue::Num(-0.383)
+    );
+    assert_eq!(
+        json_number().parse(".383").unwrap().0,
+        JsonValue::Num(0.383)
+    );
+    assert_eq!(
+        json_number().parse("-1.383").unwrap().0,
+        JsonValue::Num(-1.383)
+    );
+}
+
+#[rstest]
+#[case(r#"[ 4, 9, "ara",]"#)]
+fn parse_json_array_err(#[case] input: &str) {
+    assert!(json_array().parse(input).is_err());
+}
+
+#[rstest]
+#[case("[[],[]]", (JsonValue::Array(vec![JsonValue::Array(vec![]), JsonValue::Array(vec![])]), ""))]
+#[case(r#"[  4,   9,   "foo"   ]"#, (
+            JsonValue::Array(vec![
+                JsonValue::Num(4.),
+                JsonValue::Num(9.0),
+                JsonValue::Str("foo".to_string())
+            ]),
+            ""
+        ))]
+#[case(r#"[8,null,[],5],{}"#,
+        (
+            JsonValue::Array(vec![
+                JsonValue::Num(8.),
+                JsonValue::Null,
+                JsonValue::Array(vec![]),
+                JsonValue::Num(5.),
+            ]),
+            ",{}"
+        ))]
+fn parse_json_array(#[case] input: &str, #[case] expected: (JsonValue, &str)) {
+    assert_eq!(json_array().parse(input).unwrap(), expected);
+}
+
+#[test]
+fn parse_json_object() {
+    assert_eq!(
+        json_object().parse(r#"{ "a": 23}"#).unwrap().0,
+        JsonValue::Object(vec![("a".into(), JsonValue::Num(23.))])
+    );
+    assert_eq!(
+        json_object().parse(r#"{}"#).unwrap().0,
+        JsonValue::Object(vec![])
+    );
+}
+
+#[test]
+fn parse_json_document() {
+    let test_json = include_str!("joplin-cfg.json");
+    let parsed_json = json_object().parse(test_json);
+    assert!(parsed_json.is_ok());
+}
+
+#[rstest]
+#[case(json_null().representation(), Representation::new("null"))]
+#[case(json_bool().representation(), Representation::new("true | false"))]
+#[case(json_number().representation(), Representation::new("- | ε (1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 0 | )+ . (1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 0 | )+ | ε | . (1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 0 | )+"))]
+fn representations_test(
+    #[case] parser_representation: Representation,
+    #[case] expected: Representation,
+) {
+    assert_eq!(parser_representation, expected);
+}