jonas/lispers
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tokenizer): allow more ascii chars as symbols

Browse Source
This commit is contained in:
Jonas Röger 2024-11-05 21:12:02 +01:00
parent 53e6ac8132
commit 578e0c2435
Signed by: jonas
GPG Key ID: 4000EB35E1AE0F07
3 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/lisp/expression.rs
View File

@ -277,12 +277,12 @@ fn prelude_gt(env: &Environment, expr: Expression) -> Result<Expression, EvalErr
pub fn eval_prelude(expr: Expression) -> Result<Expression, EvalError> { pub fn eval_prelude(expr: Expression) -> Result<Expression, EvalError> {
let mut prelude = Environment::new(); let mut prelude = Environment::new();
prelude.set("add".to_string(), Expression::Function(prelude_add)); prelude.set("+".to_string(), Expression::Function(prelude_add));
prelude.set("lambda".to_string(), Expression::Function(prelude_lambda)); prelude.set("lambda".to_string(), Expression::Function(prelude_lambda));
prelude.set("if".to_string(), Expression::Function(prelude_if)); prelude.set("if".to_string(), Expression::Function(prelude_if));
prelude.set("eq".to_string(), Expression::Function(prelude_eq)); prelude.set("==".to_string(), Expression::Function(prelude_eq));
prelude.set("lt".to_string(), Expression::Function(prelude_lt)); prelude.set("<".to_string(), Expression::Function(prelude_lt));
prelude.set("gt".to_string(), Expression::Function(prelude_gt)); prelude.set(">".to_string(), Expression::Function(prelude_gt));
prelude.set("let".to_string(), Expression::Function(prelude_let)); prelude.set("let".to_string(), Expression::Function(prelude_let));
eval(&prelude, expr) eval(&prelude, expr)

2
src/main.rs
View File

@ -4,7 +4,7 @@ use lisp::eval_prelude;
use parser::ExpressionStream; use parser::ExpressionStream;
fn main() { fn main() {
let program = "((lambda (x y) (add (if (lt x 10) (add x 10) x) y)) 2 20)"; let program = "((lambda (x y) (+ (if (< x 10) (+ x 10) x) y)) 2 20)";
for r in ExpressionStream::from_char_stream(program.chars()) { for r in ExpressionStream::from_char_stream(program.chars()) {
match r { match r {

21
src/parser/tokenizer.rs
View File

@ -252,12 +252,15 @@ where
{ {
let mut sym = String::new(); let mut sym = String::new();
// Allow some special chars and alphanumeric
while let Some(c) = reader.next() { while let Some(c) = reader.next() {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' { match c {
sym.push(c); '_' | '-' | '<' | '>' | '=' | '*' | '/' | '+' | '%' | '!' | '?' => sym.push(c),
} else { c if c.is_ascii_alphanumeric() => sym.push(c),
reader.step_back(1); _ => {
break; reader.step_back(1);
break;
}
} }
} }
@ -334,11 +337,11 @@ where
#[test] #[test]
fn test_tokenize() { fn test_tokenize() {
let test_str = "(\"abcdefg( )123\" )(\n\t 'nil true \"true\")00987463 123.125 ."; let test_str = "(\"abcdefg( )123\" )(\n\t 'nil true \"true\")00987463 123.125 . 0+-*/go=";
let result: Vec<_> = tokenize(&mut test_str.chars()).collect(); let result: Vec<_> = tokenize(&mut test_str.chars()).collect();
assert_eq!(result.len(), 12); assert_eq!(result.len(), 13);
assert_eq!(result[0].clone().unwrap(), Token::ParOpen); assert_eq!(result[0].clone().unwrap(), Token::ParOpen);
assert_eq!( assert_eq!(
result[1].clone().unwrap(), result[1].clone().unwrap(),
@ -357,4 +360,8 @@ fn test_tokenize() {
assert_eq!(result[9].clone().unwrap(), Token::IntLiteral(987463)); assert_eq!(result[9].clone().unwrap(), Token::IntLiteral(987463));
assert_eq!(result[10].clone().unwrap(), Token::FloatLiteral(123.125)); assert_eq!(result[10].clone().unwrap(), Token::FloatLiteral(123.125));
assert_eq!(result[11].clone().unwrap(), Token::Dot); assert_eq!(result[11].clone().unwrap(), Token::Dot);
assert_eq!(
result[12].clone().unwrap(),
Token::Symbol("0+-*/go=".to_string())
);
} }