Tokenize a sequence of whitespace separated values

1 files changed, 20 insertions, 6 deletions

diff --git a/rust/src/parser.rs b/rust/src/parser.rs
index 6b18a9d..b3d76fb 100644
--- a/rust/src/parser.rs
+++ b/rust/src/parser.rs
@@ -1,11 +1,20 @@
 use crate::ast::*;
 
 pub fn parse(arg: &str) -> Value {
-    let token = arg.trim();
-    parse_number(token)
-        .or(parse_bool(token))
-        .or(parse_symbol(token))
-        .unwrap()
+    if let Some(token) = tokenize(arg).first() {
+        return parse_number(token)
+            .or(parse_bool(token))
+            .or(parse_symbol(token))
+            .unwrap();
+    }
+    panic!("No value to parse");
+}
+
+fn tokenize(arg: &str) -> Vec<String> {
+    arg.split(|c: char| c.is_whitespace())
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string())
+        .collect()
 }
 
 fn parse_symbol(token: &str) -> Result<Value, String> {
@@ -28,8 +37,8 @@ fn parse_number(token: &str) -> Result<Value, String> {
 
 #[cfg(test)]
 mod tests {
-    use super::parse;
     use super::Value::*;
+    use super::{parse, tokenize};
     use proptest::prelude::*;
 
     proptest! {
@@ -57,4 +66,9 @@ mod tests {
         assert_eq!(Sym("foo".to_string()), parse("  foo  \n\r"));
         assert_eq!(Num(-42), parse("\n-42"));
     }
+
+    #[test]
+    fn tokenize_several_values() {
+        assert_eq!(vec!["42", "foo", "true"], tokenize("42 foo    \ntrue  "));
+    }
 }