Introduce explicit Token type and parse parens

1 files changed, 86 insertions, 24 deletions

diff --git a/rust/src/parser.rs b/rust/src/parser.rs
index 812444d..50cf918 100644
--- a/rust/src/parser.rs
+++ b/rust/src/parser.rs
@@ -1,42 +1,80 @@
 use crate::ast::*;
 
+#[derive(Debug, PartialEq)]
+enum Token {
+    LParen,
+    RParen,
+    Word(String),
+}
+
 pub fn parse(arg: &str) -> Value {
-    if let Some(token) = tokenize(arg).first() {
-        return parse_number(token)
-            .or(parse_bool(token))
-            .or(parse_symbol(token))
-            .unwrap();
+    let tokens = tokenize(arg);
+    parse_value(&tokens[0]).unwrap()
+}
+
+fn parse_value(token: &Token) -> Result<Value, String> {
+    parse_number(token)
+        .or(parse_bool(token))
+        .or(parse_symbol(token))
+}
+
+fn tokenize(arg: &str) -> Vec<Token> {
+    let mut result = Vec::new();
+    let mut word = String::new();
+
+    for c in arg.chars() {
+        match c {
+            '(' => {
+                terminate(&mut result, &mut word);
+                result.push(Token::LParen)
+            }
+            ')' => {
+                terminate(&mut result, &mut word);
+                result.push(Token::RParen)
+            }
+            c if c.is_whitespace() => terminate(&mut result, &mut word),
+            c => word.push(c),
+        }
     }
-    panic!("No value to parse");
+    terminate(&mut result, &mut word);
+    result
 }
 
-fn tokenize(arg: &str) -> Vec<String> {
-    arg.split(|c: char| c.is_whitespace())
-        .filter(|s| !s.is_empty())
-        .map(|s| s.to_string())
-        .collect()
+fn terminate(result: &mut Vec<Token>, word: &mut String) {
+    if !word.is_empty() {
+        let w = word.clone();
+        result.push(Token::Word(w));
+        word.clear();
+    }
 }
 
-fn parse_symbol(token: &str) -> Result<Value, String> {
-    Ok(Value::Sym(token.to_string()))
+fn parse_symbol(token: &Token) -> Result<Value, String> {
+    match token {
+        Token::Word(s) => Ok(Value::Sym(s.clone())),
+        _ => Err("Expected a symbol".to_string()),
+    }
 }
 
-fn parse_bool(token: &str) -> Result<Value, String> {
-    token
-        .parse::<bool>()
-        .map(Value::Bool)
-        .map_err(|e| e.to_string())
+fn parse_bool(token: &Token) -> Result<Value, String> {
+    match token {
+        Token::Word(s) => s
+            .parse::<bool>()
+            .map(Value::Bool)
+            .map_err(|e| e.to_string()),
+        _ => Err("Expected a boolean".to_string()),
+    }
 }
 
-fn parse_number(token: &str) -> Result<Value, String> {
-    token
-        .parse::<i32>()
-        .map(Value::Num)
-        .map_err(|e| e.to_string())
+fn parse_number(token: &Token) -> Result<Value, String> {
+    match token {
+        Token::Word(s) => s.parse::<i32>().map(Value::Num).map_err(|e| e.to_string()),
+        _ => Err("Expected an integer".to_string()),
+    }
 }
 
 #[cfg(test)]
 mod tests {
+    use super::Token::*;
     use super::Value;
     use super::Value::*;
     use super::{parse, tokenize};
@@ -70,7 +108,31 @@ mod tests {
 
     #[test]
     fn tokenize_several_values() {
-        assert_eq!(vec!["42", "foo", "true"], tokenize("42 foo    \ntrue  "));
+        assert_eq!(
+            vec![
+                Word("42".to_string()),
+                Word("foo".to_string()),
+                Word("true".to_string())
+            ],
+            tokenize("42 foo    \ntrue  ")
+        );
+    }
+
+    #[test]
+    fn tokenize_string_with_parens() {
+        assert_eq!(
+            vec![
+                LParen,
+                LParen,
+                RParen,
+                Word("42".to_string()),
+                RParen,
+                Word("true".to_string()),
+                LParen,
+            ],
+            tokenize("(   \r() 42)  \ntrue(  ")
+        );
+    }
     }
 
     impl Arbitrary for Value {