From c176c1c91869394069785a0e641a7348fb2e272c Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sat, 10 Mar 2018 18:20:19 -0800
Subject: [PATCH] Autoparser - token output

---
 src/schala_lang/autoparser.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/schala_lang/autoparser.rs b/src/schala_lang/autoparser.rs
index 9a235f8..8816178 100644
--- a/src/schala_lang/autoparser.rs
+++ b/src/schala_lang/autoparser.rs
@@ -1,4 +1,5 @@
 use schala_lib::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, LanguageOutput};
+use itertools::Itertools;
 
 use schala_lang::{tokenizing, parsing};
 
@@ -20,6 +21,14 @@ impl ProgrammingLanguageInterface for Schala {
 
   fn evaluate_in_repl(&mut self, input: &str, options: &EvalOptions) -> LanguageOutput {
     let mut output = LanguageOutput::default();
+
+    let tokens = tokenizing::tokenize(input);
+    if options.debug_tokens {
+      let token_string = tokens.iter().map(|t| format!("{:?}<L:{},C:{}>", t.token_type, t.offset.0, t.offset.1)).join(", ");
+      output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", token_string)));
+    }
+
+    output.add_output(format!("{:?}", tokens));
     output
   }
 }