feat: add native_lisp_function macro
- refactor project layout to use child crates - lispers-core: parser and evaluator - lispers-macro: proc macros
This commit is contained in:
2
lispers-core/src/lib.rs
Normal file
2
lispers-core/src/lib.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
pub mod lisp;
|
||||
pub mod parser;
|
||||
151
lispers-core/src/lisp/environment.rs
Normal file
151
lispers-core/src/lisp/environment.rs
Normal file
@@ -0,0 +1,151 @@
|
||||
use super::{expression::Expression, prelude::mk_prelude};
|
||||
use std::{cell::RefCell, collections::HashMap, rc::Rc};
|
||||
|
||||
#[derive(PartialEq, Clone, Debug)]
|
||||
/// A Environment is a stack of `EnvironmentLayer`s. Each `EnvironmentLayer` is a mapping from
|
||||
/// variable names to their values.
|
||||
pub struct Environment<'a> {
|
||||
/// The current mapping.
|
||||
layer: EnvironmentLayer,
|
||||
/// The outer _fallback_ mapping.
|
||||
outer: Option<&'a Environment<'a>>,
|
||||
/// A shared layer taking precendence over the outer layer, but not the current layer.
|
||||
shared: Rc<RefCell<EnvironmentLayer>>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Clone, Debug)]
|
||||
/// A concrete EnvironmentLayer, containing a mapping from symbol names to Expressions.
|
||||
pub struct EnvironmentLayer {
|
||||
symbols: HashMap<String, Expression>,
|
||||
}
|
||||
|
||||
impl EnvironmentLayer {
|
||||
/// Construct an empty `EnvironmentLayer`.
|
||||
pub fn new() -> Self {
|
||||
EnvironmentLayer {
|
||||
symbols: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a value in the `EnvironmentLayer`.
|
||||
pub fn set(&mut self, key: String, value: Expression) {
|
||||
self.symbols.insert(key, value);
|
||||
}
|
||||
|
||||
/// Get a value in the `EnvironmentLayer`.
|
||||
pub fn get(&self, key: &str) -> Option<Expression> {
|
||||
self.symbols.get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<HashMap<String, Expression>> for EnvironmentLayer {
|
||||
fn from(map: HashMap<String, Expression>) -> Self {
|
||||
EnvironmentLayer { symbols: map }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Environment<'a> {
|
||||
/// Construct an empty `Environment`.
|
||||
pub fn new() -> Self {
|
||||
Environment {
|
||||
layer: EnvironmentLayer::new(),
|
||||
outer: None,
|
||||
shared: Rc::new(RefCell::new(EnvironmentLayer::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an `Environment` from a `EnvironmentLayer` with no outer `Environment`.
|
||||
pub fn from_layer(layer: EnvironmentLayer) -> Self {
|
||||
Environment {
|
||||
layer,
|
||||
outer: None,
|
||||
shared: Rc::new(RefCell::new(EnvironmentLayer::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a new `Environment` with `self` as the outer `Environment`.
|
||||
pub fn mk_inner(&self) -> Environment {
|
||||
Environment {
|
||||
layer: EnvironmentLayer::new(),
|
||||
outer: Some(self),
|
||||
shared: self.shared.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a new `Environment` with `self` as the outer `Environment` and `layer` as the
|
||||
pub fn overlay(&'a self, layer: EnvironmentLayer) -> Environment<'a> {
|
||||
Environment {
|
||||
layer,
|
||||
outer: Some(&self),
|
||||
shared: self.shared.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a value in the shared layer.
|
||||
///
|
||||
/// Panics:
|
||||
/// - if the shared layer cannot be borrowed mutably.
|
||||
pub fn shared_set(&self, key: String, value: Expression) {
|
||||
match self.shared.try_borrow_mut() {
|
||||
Ok(mut shared) => shared.set(key, value),
|
||||
Err(e) => panic!("Cannot borrow shared layer mutably. ({})", e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a value from the shared layer.
|
||||
pub fn shared_get(&self, key: &str) -> Option<Expression> {
|
||||
self.shared.borrow().get(key)
|
||||
}
|
||||
|
||||
/// Get a value from the `Environment`, without looking at the shared layer.
|
||||
pub fn layer_get(&self, key: &str) -> Option<Expression> {
|
||||
if let Some(e) = self.layer.get(key) {
|
||||
Some(e)
|
||||
} else {
|
||||
self.outer?.layer_get(key).clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a value from the `Environment`.
|
||||
pub fn get(&self, key: &str) -> Option<Expression> {
|
||||
if let Some(e) = self.layer.get(key) {
|
||||
Some(e)
|
||||
} else if let Some(e) = self.shared_get(key) {
|
||||
Some(e)
|
||||
} else {
|
||||
self.outer?.layer_get(key).clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a value in the current `EnvironmentLayer`.
|
||||
pub fn set(&mut self, key: String, value: Expression) {
|
||||
self.layer.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Environment<'_> {
|
||||
/// Get the default prelude layer
|
||||
fn default() -> Self {
|
||||
let mut d = EnvironmentLayer::new();
|
||||
mk_prelude(&mut d);
|
||||
Environment {
|
||||
layer: d,
|
||||
outer: None,
|
||||
shared: Rc::new(RefCell::new(EnvironmentLayer::new())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_environment() {
|
||||
let mut env = Environment::new();
|
||||
env.set("a".to_string(), Expression::Integer(1));
|
||||
env.set("b".to_string(), Expression::Integer(2));
|
||||
let mut inner = env.mk_inner();
|
||||
inner.set("a".to_string(), Expression::Integer(3));
|
||||
assert_eq!(inner.get("a"), Some(Expression::Integer(3)));
|
||||
assert_eq!(inner.get("b"), Some(Expression::Integer(2)));
|
||||
assert_eq!(env.get("a"), Some(Expression::Integer(1)));
|
||||
assert_eq!(env.get("b"), Some(Expression::Integer(2)));
|
||||
assert_eq!(env.get("c"), None);
|
||||
}
|
||||
111
lispers-core/src/lisp/eval.rs
Normal file
111
lispers-core/src/lisp/eval.rs
Normal file
@@ -0,0 +1,111 @@
|
||||
use std::fmt::Display;
|
||||
|
||||
use super::environment::Environment;
|
||||
use super::environment::EnvironmentLayer;
|
||||
use super::expression::Expression;
|
||||
|
||||
#[derive(Debug)]
|
||||
/// All possible evaluation errors
|
||||
pub enum EvalError {
|
||||
SymbolNotBound(String),
|
||||
NotAFunction(Expression),
|
||||
NotANumber(Expression),
|
||||
ArgumentError(String),
|
||||
TypeError(String),
|
||||
NotASymbol(Expression),
|
||||
RuntimeError(String),
|
||||
}
|
||||
|
||||
impl Display for EvalError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
EvalError::SymbolNotBound(s) => write!(f, "Symbol {} is not bound", s),
|
||||
EvalError::NotAFunction(e) => write!(f, "Expression {} is not a function", e),
|
||||
EvalError::NotANumber(e) => write!(f, "Expression {} is not a number", e),
|
||||
EvalError::ArgumentError(s) => write!(f, "Argument error: {}", s),
|
||||
EvalError::TypeError(s) => write!(f, "Type error: {}", s),
|
||||
EvalError::NotASymbol(e) => write!(f, "Expression {} is not a symbol", e),
|
||||
EvalError::RuntimeError(s) => write!(f, "Runtime error: {}", s),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A CellIterator is a convenience struct to iterate a linked cons list.
|
||||
/// The Iterator returns Ok(Expression) as long, as there are elements in the list.
|
||||
/// Err(EvalError) is returned when the right side of a cons cell is not another cons cell or nil.
|
||||
pub struct CellIterator {
|
||||
expr: Option<Expression>,
|
||||
}
|
||||
|
||||
impl CellIterator {
|
||||
pub fn new(expr: Expression) -> CellIterator {
|
||||
CellIterator { expr: Some(expr) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for CellIterator {
|
||||
type Item = Result<Expression, EvalError>;
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(expr) = self.expr.take() {
|
||||
match expr {
|
||||
Expression::Cell(head, tail) => {
|
||||
self.expr = Some(*tail);
|
||||
return Some(Ok(*head));
|
||||
}
|
||||
Expression::Nil => {
|
||||
return None;
|
||||
}
|
||||
_ => {
|
||||
return Some(Err(EvalError::TypeError(
|
||||
"Expected a cell or nil".to_string(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatch an anonymous function call. Evaluates `body` in `env`, binding `args` to `argument_symbols`
|
||||
fn dispatch_anonymous_function(
|
||||
env: &Environment,
|
||||
argument_symbols: Vec<String>,
|
||||
body: Expression,
|
||||
args: Expression,
|
||||
) -> Result<Expression, EvalError> {
|
||||
let mut args: Vec<Expression> = args.try_into()?;
|
||||
|
||||
let mut overlay = EnvironmentLayer::new();
|
||||
|
||||
if args.len() != argument_symbols.len() {
|
||||
return Err(EvalError::ArgumentError(format!(
|
||||
"Exprected {} arguments, got {}",
|
||||
argument_symbols.len(),
|
||||
args.len()
|
||||
)));
|
||||
}
|
||||
|
||||
for (arg, symbol) in args.iter_mut().zip(argument_symbols.iter()) {
|
||||
overlay.set(symbol.to_owned(), eval(env, arg.to_owned())?);
|
||||
}
|
||||
|
||||
eval(&env.overlay(overlay), body)
|
||||
}
|
||||
|
||||
/// Evaluate an expression inside an environment
|
||||
pub fn eval(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
match expr {
|
||||
Expression::Cell(lhs, rhs) => match eval(env, *lhs)? {
|
||||
Expression::Function(f) => f(env, *rhs),
|
||||
Expression::AnonymousFunction {
|
||||
argument_symbols,
|
||||
body,
|
||||
} => dispatch_anonymous_function(env, argument_symbols, *body, *rhs),
|
||||
a => Err(EvalError::NotAFunction(a)),
|
||||
},
|
||||
Expression::Quote(e) => Ok(*e),
|
||||
Expression::Symbol(s) => env.get(&s).ok_or(EvalError::SymbolNotBound(s)),
|
||||
x => Ok(x),
|
||||
}
|
||||
}
|
||||
329
lispers-core/src/lisp/expression.rs
Normal file
329
lispers-core/src/lisp/expression.rs
Normal file
@@ -0,0 +1,329 @@
|
||||
use std::any::Any;
|
||||
use std::fmt::Debug;
|
||||
use std::fmt::Display;
|
||||
use std::ops::Deref;
|
||||
use std::ops::DerefMut;
|
||||
|
||||
use as_any::AsAny;
|
||||
|
||||
use super::environment::Environment;
|
||||
use super::eval::CellIterator;
|
||||
use super::eval::EvalError;
|
||||
|
||||
/// A trait for foreign data types that can be used in lisp expressions.
|
||||
/// Note: This trait requires explicit implementation of:
|
||||
/// - partial_cmp_impl
|
||||
/// - clone_impl
|
||||
/// - eq_impl
|
||||
/// - as_any_box
|
||||
/// to ensure object safety.
|
||||
pub trait ForeignData: Debug + Display + AsAny {
|
||||
fn partial_cmp_impl(&self, other: &dyn ForeignData) -> Option<std::cmp::Ordering>;
|
||||
fn clone_impl(&self) -> Box<dyn ForeignData>;
|
||||
fn eq_impl(&self, other: &dyn ForeignData) -> bool;
|
||||
fn as_any_box(self: Box<Self>) -> Box<dyn Any>;
|
||||
}
|
||||
|
||||
impl<T: Debug + Display + AsAny + PartialOrd + PartialEq + Clone + 'static> ForeignData for T {
|
||||
fn partial_cmp_impl(&self, other: &dyn ForeignData) -> Option<std::cmp::Ordering> {
|
||||
if let Some(other) = other.as_any().downcast_ref::<T>() {
|
||||
self.partial_cmp(other)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn clone_impl(&self) -> Box<dyn ForeignData> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn eq_impl(&self, other: &dyn ForeignData) -> bool {
|
||||
if let Some(other) = other.as_any().downcast_ref::<T>() {
|
||||
self.eq(other)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_box(self: Box<Self>) -> Box<dyn Any> {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper struct around any foreign data type. This struct is used to convert
|
||||
/// any T implementing ForeignData to an Expression and vice versa.
|
||||
#[derive(Debug)]
|
||||
pub struct ForeignDataWrapper<T: ForeignData>(pub Box<T>);
|
||||
impl<T: ForeignData> ForeignDataWrapper<T> {
|
||||
/// Create a new ForeignDataWrapper from an object implementing ForeignData.
|
||||
pub fn new(data: T) -> Self {
|
||||
ForeignDataWrapper(Box::new(data))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ForeignData> Deref for ForeignDataWrapper<T> {
|
||||
type Target = T;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ForeignData> DerefMut for ForeignDataWrapper<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
/// A Store struct for foreign data types injected in expressions.
|
||||
pub struct ForeignDataStore {
|
||||
/// The actual foreign data.
|
||||
data: Box<dyn ForeignData>,
|
||||
}
|
||||
|
||||
/// The ForeignDataStore struct is used to store any foreign data type in an Expression
|
||||
/// and cannot be constructed outside of this scope.
|
||||
impl ForeignDataStore {
|
||||
/// Create a new ForeignDataStore from a ForeignData trait object.
|
||||
fn new(data: Box<dyn ForeignData>) -> Self {
|
||||
ForeignDataStore { data }
|
||||
}
|
||||
|
||||
/// Get the contained box as an Any-Box with type info of the actual data.
|
||||
fn as_any_box(self) -> Box<dyn Any> {
|
||||
self.data.as_any_box()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ForeignDataStore {
|
||||
fn clone(&self) -> Self {
|
||||
ForeignDataStore {
|
||||
data: self.data.clone_impl(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for ForeignDataStore {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.data.eq_impl(other.data.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for ForeignDataStore {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
self.data.partial_cmp_impl(other.data.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for ForeignDataStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.data)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, PartialOrd)]
|
||||
/// A sum type of all possible lisp expressions.
|
||||
pub enum Expression {
|
||||
/// The classic lisp cons cell aka (a . b) used to construct expressions.
|
||||
Cell(Box<Expression>, Box<Expression>),
|
||||
/// A function expression pointing to native code.
|
||||
Function(fn(&Environment, Expression) -> Result<Expression, EvalError>),
|
||||
/// A anonymous function expression consisting of bound symbols and a body expression.
|
||||
AnonymousFunction {
|
||||
argument_symbols: Vec<String>,
|
||||
body: Box<Expression>,
|
||||
},
|
||||
/// A foreign data expression.
|
||||
ForeignExpression(ForeignDataStore),
|
||||
/// A Quoted expression.
|
||||
Quote(Box<Expression>),
|
||||
/// A symbol.
|
||||
Symbol(String),
|
||||
/// Integer values.
|
||||
Integer(i64),
|
||||
/// Float values.
|
||||
Float(f64),
|
||||
/// String values.
|
||||
String(String),
|
||||
/// True
|
||||
True,
|
||||
/// Nil
|
||||
Nil,
|
||||
}
|
||||
|
||||
impl<T: ForeignData> From<ForeignDataWrapper<T>> for Expression {
|
||||
fn from(value: ForeignDataWrapper<T>) -> Expression {
|
||||
Expression::ForeignExpression(ForeignDataStore::new(value.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ForeignData> TryFrom<Expression> for ForeignDataWrapper<T> {
|
||||
type Error = EvalError;
|
||||
fn try_from(value: Expression) -> Result<Self, Self::Error> {
|
||||
match value {
|
||||
Expression::ForeignExpression(f) => match f.as_any_box().downcast::<T>() {
|
||||
Ok(data) => Ok(ForeignDataWrapper(data)),
|
||||
Err(_) => Err(EvalError::TypeError(
|
||||
"Expression is not a ForeignDataWrapper".to_string(),
|
||||
)),
|
||||
},
|
||||
_ => Err(EvalError::TypeError(
|
||||
"Expression is not a ForeignDataWrapper".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<fn(&Environment, Expression) -> Result<Expression, EvalError>> for Expression {
|
||||
fn from(f: fn(&Environment, Expression) -> Result<Expression, EvalError>) -> Self {
|
||||
Expression::Function(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<Expression>> for Expression {
|
||||
fn from(mut value: Vec<Expression>) -> Self {
|
||||
let mut current = Expression::Nil;
|
||||
|
||||
for e in value.iter_mut().rev() {
|
||||
current = Expression::Cell(Box::new(e.to_owned()), Box::new(current));
|
||||
}
|
||||
|
||||
current
|
||||
}
|
||||
}
|
||||
|
||||
impl From<(Expression, Expression)> for Expression {
|
||||
fn from(value: (Expression, Expression)) -> Self {
|
||||
Expression::Cell(Box::new(value.0), Box::new(value.1))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Expression> for i64 {
|
||||
type Error = EvalError;
|
||||
fn try_from(value: Expression) -> Result<i64, Self::Error> {
|
||||
match value {
|
||||
Expression::Integer(i) => Ok(i),
|
||||
_ => Err(EvalError::TypeError(
|
||||
"Expression is not an Integer".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Expression> for f64 {
|
||||
type Error = EvalError;
|
||||
fn try_from(value: Expression) -> Result<f64, Self::Error> {
|
||||
match value {
|
||||
Expression::Integer(i) => Ok(i as f64),
|
||||
Expression::Float(f) => Ok(f),
|
||||
_ => Err(EvalError::TypeError(
|
||||
"Expression is not a Float".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Expression> for String {
|
||||
type Error = EvalError;
|
||||
fn try_from(value: Expression) -> Result<String, Self::Error> {
|
||||
match value {
|
||||
Expression::String(s) => Ok(s),
|
||||
_ => Err(EvalError::TypeError(
|
||||
"Expression is not a String".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Expression> for Vec<Expression> {
|
||||
type Error = EvalError;
|
||||
|
||||
fn try_from(value: Expression) -> Result<Vec<Expression>, Self::Error> {
|
||||
CellIterator::new(value).collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl<ToExpr> TryFrom<Expression> for Vec<ToExpr>
|
||||
where
|
||||
ToExpr: TryFrom<Expression, Error = EvalError>,
|
||||
{
|
||||
type Error = EvalError;
|
||||
|
||||
fn try_from(value: Expression) -> Result<Vec<ToExpr>, Self::Error> {
|
||||
CellIterator::new(value)
|
||||
.map(|x| x?.try_into() as Result<ToExpr, EvalError>)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl<ToExpr, const N: usize> TryFrom<Expression> for [ToExpr; N]
|
||||
where
|
||||
ToExpr: TryFrom<Expression, Error = EvalError>,
|
||||
{
|
||||
type Error = EvalError;
|
||||
|
||||
fn try_from(value: Expression) -> Result<[ToExpr; N], Self::Error> {
|
||||
let buf: Vec<ToExpr> = value.try_into()?;
|
||||
let n = buf.len();
|
||||
|
||||
buf.try_into()
|
||||
.map_err(|_| EvalError::ArgumentError(format!("Expected {} arguments, got {}", N, n)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> TryFrom<Expression> for [Expression; N] {
|
||||
type Error = EvalError;
|
||||
|
||||
fn try_from(value: Expression) -> Result<[Expression; N], Self::Error> {
|
||||
let buf: Vec<Expression> = value.try_into()?;
|
||||
let n = buf.len();
|
||||
|
||||
buf.try_into()
|
||||
.map_err(|_| EvalError::ArgumentError(format!("Expected {} arguments, got {}", N, n)))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Expression> for (Expression, Expression) {
|
||||
type Error = EvalError;
|
||||
fn try_from(value: Expression) -> Result<(Expression, Expression), Self::Error> {
|
||||
match value {
|
||||
Expression::Cell(a, b) => Ok((*a, *b)),
|
||||
_ => Err(EvalError::TypeError(
|
||||
"Expression must be a Cell".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Expression {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Expression::ForeignExpression(e) => write!(f, "{}", e),
|
||||
Expression::Cell(a, b) => {
|
||||
match self.clone().try_into() as Result<Vec<Expression>, EvalError> {
|
||||
Ok(lst) => write!(
|
||||
f,
|
||||
"({})",
|
||||
lst.iter()
|
||||
.map(|e| e.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" ")
|
||||
),
|
||||
Err(_) => write!(f, "({} . {})", a, b),
|
||||
}
|
||||
}
|
||||
Expression::Function(_) => write!(f, "<function>"),
|
||||
Expression::AnonymousFunction {
|
||||
argument_symbols,
|
||||
body,
|
||||
} => write!(f, "(lambda ({}) {})", argument_symbols.join(" "), body),
|
||||
Expression::Quote(e) => write!(f, "'{}", e),
|
||||
Expression::Symbol(s) => write!(f, "{}", s),
|
||||
Expression::Integer(i) => write!(f, "{}", i),
|
||||
Expression::Float(fl) => write!(f, "{}", fl),
|
||||
Expression::String(s) => write!(f, "\"{}\"", s),
|
||||
Expression::True => write!(f, "true"),
|
||||
Expression::Nil => write!(f, "nil"),
|
||||
}
|
||||
}
|
||||
}
|
||||
8
lispers-core/src/lisp/mod.rs
Normal file
8
lispers-core/src/lisp/mod.rs
Normal file
@@ -0,0 +1,8 @@
|
||||
pub mod environment;
|
||||
pub mod eval;
|
||||
pub mod expression;
|
||||
pub mod prelude;
|
||||
|
||||
pub use environment::Environment;
|
||||
pub use eval::eval;
|
||||
pub use expression::Expression;
|
||||
279
lispers-core/src/lisp/prelude.rs
Normal file
279
lispers-core/src/lisp/prelude.rs
Normal file
@@ -0,0 +1,279 @@
|
||||
use super::environment::Environment;
|
||||
use super::environment::EnvironmentLayer;
|
||||
use super::eval::eval;
|
||||
use super::eval::CellIterator;
|
||||
use super::eval::EvalError;
|
||||
use super::expression::Expression;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub fn prelude_add(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
|
||||
match eval(env, a)? {
|
||||
Expression::Integer(a) => match eval(env, b)? {
|
||||
Expression::Integer(b) => Ok(Expression::Integer(a + b)),
|
||||
Expression::Float(b) => Ok(Expression::Float(a as f64 + b)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
Expression::Float(a) => match eval(env, b)? {
|
||||
Expression::Float(b) => Ok(Expression::Float(a + b)),
|
||||
Expression::Integer(b) => Ok(Expression::Float(a + b as f64)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_sub(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
|
||||
match eval(env, a)? {
|
||||
Expression::Integer(a) => match eval(env, b)? {
|
||||
Expression::Integer(b) => Ok(Expression::Integer(a - b)),
|
||||
Expression::Float(b) => Ok(Expression::Float(a as f64 - b)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
Expression::Float(a) => match eval(env, b)? {
|
||||
Expression::Float(b) => Ok(Expression::Float(a - b)),
|
||||
Expression::Integer(b) => Ok(Expression::Float(a - b as f64)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_mul(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
|
||||
match eval(env, a)? {
|
||||
Expression::Integer(a) => match eval(env, b)? {
|
||||
Expression::Integer(b) => Ok(Expression::Integer(a * b)),
|
||||
Expression::Float(b) => Ok(Expression::Float(a as f64 * b)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
Expression::Float(a) => match eval(env, b)? {
|
||||
Expression::Float(b) => Ok(Expression::Float(a * b)),
|
||||
Expression::Integer(b) => Ok(Expression::Float(a * b as f64)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_div(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
|
||||
match eval(env, a)? {
|
||||
Expression::Integer(a) => match eval(env, b)? {
|
||||
Expression::Integer(b) => Ok(Expression::Integer(a / b)),
|
||||
Expression::Float(b) => Ok(Expression::Float(a as f64 / b)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
Expression::Float(a) => match eval(env, b)? {
|
||||
Expression::Float(b) => Ok(Expression::Float(a / b)),
|
||||
Expression::Integer(b) => Ok(Expression::Float(a / b as f64)),
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
},
|
||||
x => Err(EvalError::NotANumber(x)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_lambda(_env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [args, body]: [Expression; 2] = expr.try_into()?;
|
||||
let mut arg_exprs: Vec<Expression> = args.try_into()?;
|
||||
let argument_symbols: Vec<String> = arg_exprs
|
||||
.iter_mut()
|
||||
.map(|a| match a {
|
||||
Expression::Symbol(s) => Ok(s.to_owned()),
|
||||
x => Err(EvalError::NotASymbol(x.to_owned())),
|
||||
})
|
||||
.collect::<Result<Vec<String>, EvalError>>()?;
|
||||
Ok(Expression::AnonymousFunction {
|
||||
argument_symbols,
|
||||
body: Box::new(body),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn prelude_defun(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [name, args, body]: [Expression; 3] = expr.try_into()?;
|
||||
let name = match name {
|
||||
Expression::Symbol(s) => s,
|
||||
x => return Err(EvalError::NotASymbol(x)),
|
||||
};
|
||||
let mut arg_exprs: Vec<Expression> = args.try_into()?;
|
||||
let argument_symbols: Vec<String> = arg_exprs
|
||||
.iter_mut()
|
||||
.map(|a| match a.to_owned() {
|
||||
Expression::Symbol(s) => Ok(s),
|
||||
x => Err(EvalError::NotASymbol(x)),
|
||||
})
|
||||
.collect::<Result<Vec<String>, EvalError>>()?;
|
||||
|
||||
let f = Expression::AnonymousFunction {
|
||||
argument_symbols,
|
||||
body: Box::new(body),
|
||||
};
|
||||
env.shared_set(name, f.clone());
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
pub fn prelude_define(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [name, value] = expr.try_into()?;
|
||||
let name = match name {
|
||||
Expression::Symbol(s) => s,
|
||||
x => return Err(EvalError::NotASymbol(x)),
|
||||
};
|
||||
let value = eval(env, value)?;
|
||||
env.shared_set(name, value.clone());
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub fn prelude_let(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [bindings, body] = expr.try_into()?;
|
||||
|
||||
let bindings = CellIterator::new(eval(env, bindings)?)
|
||||
.map(|e| {
|
||||
let (s, e) = e?.try_into()?;
|
||||
if let Expression::Symbol(s) = s {
|
||||
Ok((s, eval(env, e)?))
|
||||
} else {
|
||||
Err(EvalError::ArgumentError(
|
||||
"Let bindings must be an alist with elements (symbol . expr)".to_string(),
|
||||
))
|
||||
}
|
||||
})
|
||||
.collect::<Result<HashMap<String, Expression>, EvalError>>()?;
|
||||
|
||||
eval(&env.overlay(bindings.into()), body)
|
||||
}
|
||||
|
||||
pub fn prelude_if(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [predicate, e_then, e_else] = expr.try_into()?;
|
||||
|
||||
match eval(env, predicate)? {
|
||||
Expression::Nil => eval(env, e_else),
|
||||
_ => eval(env, e_then),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_eq(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
let a = eval(env, a)?;
|
||||
let b = eval(env, b)?;
|
||||
|
||||
if a == b {
|
||||
Ok(Expression::True)
|
||||
} else {
|
||||
Ok(Expression::Nil)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_lt(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
let a = eval(env, a)?;
|
||||
let b = eval(env, b)?;
|
||||
|
||||
if a < b {
|
||||
Ok(Expression::True)
|
||||
} else {
|
||||
Ok(Expression::Nil)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_gt(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
let a = eval(env, a)?;
|
||||
let b = eval(env, b)?;
|
||||
|
||||
if a > b {
|
||||
Ok(Expression::True)
|
||||
} else {
|
||||
Ok(Expression::Nil)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_not(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a] = expr.try_into()?;
|
||||
match eval(env, a)? {
|
||||
Expression::Nil => Ok(Expression::True),
|
||||
_ => Ok(Expression::Nil),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_set(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [s, e] = expr.try_into()?;
|
||||
|
||||
match eval(env, s)? {
|
||||
Expression::Symbol(s) => {
|
||||
let e = eval(env, e)?;
|
||||
env.shared_set(s, e.clone());
|
||||
Ok(e)
|
||||
}
|
||||
x => Err(EvalError::NotASymbol(x)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prelude_print(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [e] = expr.try_into()?;
|
||||
let e = eval(env, e)?;
|
||||
println!("Prelude: {}", e);
|
||||
Ok(e)
|
||||
}
|
||||
|
||||
pub fn prelude_cons(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [a, b] = expr.try_into()?;
|
||||
Ok(Expression::Cell(
|
||||
Box::new(eval(env, a)?),
|
||||
Box::new(eval(env, b)?),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn prelude_car(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [arg] = expr.try_into()?;
|
||||
let (a, _) = eval(env, arg)?.try_into()?;
|
||||
Ok(a)
|
||||
}
|
||||
|
||||
pub fn prelude_cdr(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [arg] = expr.try_into()?;
|
||||
let (_, b) = eval(env, arg)?.try_into()?;
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
pub fn prelude_eval(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let [e] = expr.try_into()?;
|
||||
eval(env, eval(env, e)?)
|
||||
}
|
||||
|
||||
pub fn prelude_progn(env: &Environment, expr: Expression) -> Result<Expression, EvalError> {
|
||||
let mut result = Expression::Nil;
|
||||
|
||||
for e in CellIterator::new(expr) {
|
||||
result = eval(env, e?)?;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn mk_prelude(layer: &mut EnvironmentLayer) {
|
||||
layer.set("+".to_string(), Expression::Function(prelude_add));
|
||||
layer.set("-".to_string(), Expression::Function(prelude_sub));
|
||||
layer.set("*".to_string(), Expression::Function(prelude_mul));
|
||||
layer.set("/".to_string(), Expression::Function(prelude_div));
|
||||
layer.set("lambda".to_string(), Expression::Function(prelude_lambda));
|
||||
layer.set("defun".to_string(), Expression::Function(prelude_defun));
|
||||
layer.set("define".to_string(), Expression::Function(prelude_define));
|
||||
layer.set("if".to_string(), Expression::Function(prelude_if));
|
||||
layer.set("=".to_string(), Expression::Function(prelude_eq));
|
||||
layer.set("<".to_string(), Expression::Function(prelude_lt));
|
||||
layer.set(">".to_string(), Expression::Function(prelude_gt));
|
||||
layer.set("not".to_string(), Expression::Function(prelude_not));
|
||||
layer.set("let".to_string(), Expression::Function(prelude_let));
|
||||
layer.set("set".to_string(), Expression::Function(prelude_set));
|
||||
layer.set("print".to_string(), Expression::Function(prelude_print));
|
||||
layer.set("cons".to_string(), Expression::Function(prelude_cons));
|
||||
layer.set("car".to_string(), Expression::Function(prelude_car));
|
||||
layer.set("cdr".to_string(), Expression::Function(prelude_cdr));
|
||||
layer.set("eval".to_string(), Expression::Function(prelude_eval));
|
||||
layer.set("progn".to_string(), Expression::Function(prelude_progn));
|
||||
}
|
||||
6
lispers-core/src/parser/mod.rs
Normal file
6
lispers-core/src/parser/mod.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
pub mod parser;
|
||||
pub mod token;
|
||||
pub mod tokenizer;
|
||||
|
||||
pub use parser::ExpressionStream;
|
||||
pub use parser::ParserError;
|
||||
163
lispers-core/src/parser/parser.rs
Normal file
163
lispers-core/src/parser/parser.rs
Normal file
@@ -0,0 +1,163 @@
|
||||
use super::token::Token;
|
||||
use super::tokenizer::tokenize;
|
||||
use super::tokenizer::TokenStream;
|
||||
use super::tokenizer::TokenizerError;
|
||||
use crate::lisp::Expression;
|
||||
use std::iter::Peekable;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ParserError {
|
||||
UnexpectedToken(Token),
|
||||
TokenizerError(TokenizerError),
|
||||
UnexpectedEndOfInput,
|
||||
}
|
||||
|
||||
impl From<TokenizerError> for ParserError {
|
||||
fn from(value: TokenizerError) -> Self {
|
||||
ParserError::TokenizerError(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_list<I>(stream: &mut Peekable<TokenStream<I>>) -> Result<Expression, ParserError>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
let mut list = Vec::new();
|
||||
|
||||
loop {
|
||||
match stream.peek() {
|
||||
// Return current list or nil
|
||||
Some(Ok(Token::ParClose)) => {
|
||||
stream.next();
|
||||
if list.len() == 0 {
|
||||
return Ok(Expression::Nil);
|
||||
} else {
|
||||
return Ok(list.into());
|
||||
}
|
||||
}
|
||||
// Switch to cons-pair parsing
|
||||
Some(Ok(Token::Dot)) => {
|
||||
stream.next();
|
||||
if list.len() > 1 || list.len() == 0 {
|
||||
return Err(ParserError::UnexpectedToken(Token::Dot));
|
||||
} else {
|
||||
let second_expr = parse_expression(stream)?;
|
||||
match stream.next() {
|
||||
Some(Ok(Token::ParClose)) => {
|
||||
return Ok(Expression::Cell(
|
||||
Box::new(list[0].to_owned()),
|
||||
Box::new(second_expr),
|
||||
));
|
||||
}
|
||||
Some(Ok(t)) => {
|
||||
return Err(ParserError::UnexpectedToken(t));
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
return Err(e.into());
|
||||
}
|
||||
None => {
|
||||
return Err(ParserError::UnexpectedEndOfInput);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
list.push(parse_expression(stream)?);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_expression<I>(stream: &mut Peekable<TokenStream<I>>) -> Result<Expression, ParserError>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
match stream.next() {
|
||||
Some(Ok(Token::ParOpen)) => parse_list(stream),
|
||||
Some(Ok(Token::Nil)) => Ok(Expression::Nil),
|
||||
Some(Ok(Token::IntLiteral(n))) => Ok(Expression::Integer(n)),
|
||||
Some(Ok(Token::FloatLiteral(f))) => Ok(Expression::Float(f)),
|
||||
Some(Ok(Token::StringLiteral(s))) => Ok(Expression::String(s)),
|
||||
Some(Ok(Token::True)) => Ok(Expression::True),
|
||||
Some(Ok(Token::Symbol(s))) => Ok(Expression::Symbol(s)),
|
||||
Some(Ok(Token::Quote)) => Ok(Expression::Quote(Box::new(parse_expression(stream)?))),
|
||||
Some(Err(e)) => Err(ParserError::TokenizerError(e)),
|
||||
Some(Ok(x)) => Err(ParserError::UnexpectedToken(x)),
|
||||
None => Err(ParserError::UnexpectedEndOfInput),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ExpressionStream<I: Iterator<Item = char>> {
|
||||
token_stream: Peekable<TokenStream<I>>,
|
||||
}
|
||||
|
||||
impl<I: Iterator<Item = char>> ExpressionStream<I> {
|
||||
pub fn from_token_stream(token_stream: TokenStream<I>) -> Self {
|
||||
ExpressionStream {
|
||||
token_stream: token_stream.peekable(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_char_stream(char_stream: I) -> Self {
|
||||
ExpressionStream {
|
||||
token_stream: tokenize(char_stream).peekable(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: Iterator<Item = char>> Iterator for ExpressionStream<I> {
|
||||
type Item = Result<Expression, ParserError>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.token_stream.peek() == None {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(parse_expression(&mut self.token_stream))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parser() {
|
||||
let input = "(1 2 3) (4 5 6) (1 . 2) (1 . (2 . (3))) \"test\" '(a b c true nil)";
|
||||
let ts = tokenize(input.chars());
|
||||
let es = ExpressionStream::from_token_stream(ts);
|
||||
let exprs = es.collect::<Result<Vec<Expression>, ParserError>>();
|
||||
assert_eq!(
|
||||
exprs,
|
||||
Ok(vec![
|
||||
vec![
|
||||
Expression::Integer(1),
|
||||
Expression::Integer(2),
|
||||
Expression::Integer(3),
|
||||
]
|
||||
.into(),
|
||||
vec![
|
||||
Expression::Integer(4),
|
||||
Expression::Integer(5),
|
||||
Expression::Integer(6),
|
||||
]
|
||||
.into(),
|
||||
Expression::Cell(
|
||||
Box::new(Expression::Integer(1)),
|
||||
Box::new(Expression::Integer(2)),
|
||||
),
|
||||
vec![
|
||||
Expression::Integer(1),
|
||||
Expression::Integer(2),
|
||||
Expression::Integer(3),
|
||||
]
|
||||
.into(),
|
||||
Expression::String("test".to_string()),
|
||||
Expression::Quote(Box::new(
|
||||
vec![
|
||||
Expression::Symbol("a".to_string()),
|
||||
Expression::Symbol("b".to_string()),
|
||||
Expression::Symbol("c".to_string()),
|
||||
Expression::True,
|
||||
Expression::Nil,
|
||||
]
|
||||
.into()
|
||||
)),
|
||||
])
|
||||
);
|
||||
}
|
||||
14
lispers-core/src/parser/token.rs
Normal file
14
lispers-core/src/parser/token.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
#[derive(Debug, PartialEq, Clone)]
|
||||
/// Sum type of different tokens
|
||||
pub enum Token {
|
||||
FloatLiteral(f64),
|
||||
IntLiteral(i64),
|
||||
Dot,
|
||||
Nil,
|
||||
ParClose,
|
||||
ParOpen,
|
||||
Quote,
|
||||
StringLiteral(String),
|
||||
Symbol(String),
|
||||
True,
|
||||
}
|
||||
367
lispers-core/src/parser/tokenizer.rs
Normal file
367
lispers-core/src/parser/tokenizer.rs
Normal file
@@ -0,0 +1,367 @@
|
||||
use super::token::Token;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
/// Errors the tokenizer can yield.
|
||||
pub enum TokenizerError {
|
||||
/// The tokenizer could not read the associated sequence.
|
||||
UnmatchedSequence(String),
|
||||
}
|
||||
|
||||
/// A reader used to wrap the `TokenStream`.
|
||||
/// When reading, it starts with the staging buffer of the stream, once
|
||||
/// it's end is reached, the input stream is copied character wise to
|
||||
/// the staging buffer.
|
||||
struct StagingReader<'a, I> {
|
||||
head: usize,
|
||||
stream: &'a mut TokenStream<I>,
|
||||
}
|
||||
|
||||
impl<'a, I> StagingReader<'a, I>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
/// Create a new StagingReader for a stream.
|
||||
fn new(stream: &'a mut TokenStream<I>) -> StagingReader<'a, I> {
|
||||
StagingReader { head: 0, stream }
|
||||
}
|
||||
/// Step back the reader's head by `n` chars, stopping at 0
|
||||
fn step_back(&mut self, n: usize) {
|
||||
if self.head >= n {
|
||||
self.head -= n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, I> Iterator for StagingReader<'a, I>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
type Item = char;
|
||||
|
||||
/// Get the char at `self.head`. If it is in the staging buffer, return it and increase `self.head` by 1.
|
||||
/// It it is not in the staging buffer, copy one char from the input stream to the staging buffer.
|
||||
/// Returns `None` when the input stream is empty and `self.head` points after the staging buffer.
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(c) = self.stream.staging.get(self.head) {
|
||||
self.head += 1;
|
||||
Some(*c)
|
||||
} else {
|
||||
let next_char = self.stream.input.next()?;
|
||||
self.stream.staging.push(next_char);
|
||||
self.head += 1;
|
||||
Some(next_char)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator yielding tokens scanned from a stream of characters.
|
||||
pub struct TokenStream<InputStream> {
|
||||
staging: Vec<char>,
|
||||
input: InputStream,
|
||||
error: bool,
|
||||
}
|
||||
|
||||
impl<I> TokenStream<I>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
fn new(input: I) -> TokenStream<I> {
|
||||
TokenStream {
|
||||
staging: Vec::new(),
|
||||
input,
|
||||
error: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_whitespace(&mut self) {
|
||||
// Drop whitespace of the staging buffer
|
||||
while let Some(c) = self.staging.first() {
|
||||
if c.is_whitespace() {
|
||||
self.staging.remove(0);
|
||||
} else {
|
||||
return; // Readable character next, keep input untouched
|
||||
}
|
||||
}
|
||||
|
||||
// Staging buffer is empty, drop whitespace from input
|
||||
while let Some(c) = self.input.next() {
|
||||
if !c.is_whitespace() {
|
||||
self.staging.push(c);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_scanners(&mut self) -> Option<(Token, usize)> {
|
||||
let scanners = [
|
||||
scan_symbol,
|
||||
scan_string_literal,
|
||||
scan_integer,
|
||||
scan_float,
|
||||
scan_true,
|
||||
scan_quote,
|
||||
scan_dot,
|
||||
scan_nil,
|
||||
scan_par_close,
|
||||
scan_par_open,
|
||||
];
|
||||
|
||||
scanners
|
||||
.iter()
|
||||
.filter_map(|scanner| {
|
||||
let mut reader = StagingReader::new(self);
|
||||
let token = scanner(&mut reader)?;
|
||||
Some((token, reader.head))
|
||||
})
|
||||
.max_by_key(|pair| pair.1)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I> Iterator for TokenStream<I>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
type Item = Result<Token, TokenizerError>;
|
||||
|
||||
/// Get the next scanned token, consuming as much characters from the
|
||||
/// wrapped input stream as neccessary. If nothing could be scanned and the input
|
||||
/// stream has still elements an error is returned. Each successive call to
|
||||
/// `next` will then return `None`.
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.error {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.skip_whitespace();
|
||||
|
||||
match self.run_scanners() {
|
||||
Some((tkn, n_read)) => {
|
||||
self.staging.drain(0..n_read);
|
||||
Some(Ok(tkn))
|
||||
}
|
||||
None if self.staging.is_empty() => None,
|
||||
None => {
|
||||
let remaining = self.staging.iter().collect();
|
||||
self.staging.clear();
|
||||
self.error = true;
|
||||
Some(Err(TokenizerError::UnmatchedSequence(remaining)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the tokenizer on an iterator of chars and return an
|
||||
/// iterator of tokens as a result.
|
||||
pub fn tokenize<I>(input: I) -> TokenStream<I>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
TokenStream::new(input)
|
||||
}
|
||||
|
||||
// ================== Scanner definitions ================== //
|
||||
|
||||
fn scan_par_open<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
match reader.next()? {
|
||||
'(' => Some(Token::ParOpen),
|
||||
_ => {
|
||||
reader.step_back(1);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_par_close<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
match reader.next()? {
|
||||
')' => Some(Token::ParClose),
|
||||
_ => {
|
||||
reader.step_back(1);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_dot<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
match reader.next()? {
|
||||
'.' => Some(Token::Dot),
|
||||
_ => {
|
||||
reader.step_back(1);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_string_literal<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
let mut lit = String::new();
|
||||
|
||||
if reader.next()? == '"' {
|
||||
for c in reader {
|
||||
match c {
|
||||
'"' => {
|
||||
return Some(Token::StringLiteral(lit));
|
||||
}
|
||||
c => {
|
||||
lit.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return None;
|
||||
}
|
||||
|
||||
fn scan_nil<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
if reader.next()? == 'n' && reader.next()? == 'i' && reader.next()? == 'l' {
|
||||
Some(Token::Nil)
|
||||
} else {
|
||||
reader.step_back(3);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_quote<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
if let Some('\'') = reader.next() {
|
||||
Some(Token::Quote)
|
||||
} else {
|
||||
reader.step_back(1);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_symbol<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
let mut sym = String::new();
|
||||
|
||||
// Allow some special chars and alphanumeric
|
||||
while let Some(c) = reader.next() {
|
||||
match c {
|
||||
'_' | '-' | '<' | '>' | '=' | '*' | '/' | '+' | '%' | '!' | '?' => sym.push(c),
|
||||
c if c.is_ascii_alphanumeric() => sym.push(c),
|
||||
_ => {
|
||||
reader.step_back(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sym.len() > 0 {
|
||||
Some(Token::Symbol(sym))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_true<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
if reader.next()? == 't'
|
||||
&& reader.next()? == 'r'
|
||||
&& reader.next()? == 'u'
|
||||
&& reader.next()? == 'e'
|
||||
{
|
||||
Some(Token::True)
|
||||
} else {
|
||||
reader.step_back(4);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_integer<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
let mut buf = String::new();
|
||||
|
||||
while let Some(c) = reader.next() {
|
||||
if c.is_ascii_digit() {
|
||||
buf.push(c);
|
||||
} else {
|
||||
reader.step_back(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if buf.len() > 0 {
|
||||
buf.parse().map(Token::IntLiteral).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_float<I>(reader: &mut StagingReader<I>) -> Option<Token>
|
||||
where
|
||||
I: Iterator<Item = char>,
|
||||
{
|
||||
let mut buf = String::new();
|
||||
let mut has_dot = false;
|
||||
|
||||
while let Some(c) = reader.next() {
|
||||
if c.is_ascii_digit() {
|
||||
buf.push(c);
|
||||
} else if c == '.' && !has_dot {
|
||||
buf.push(c);
|
||||
has_dot = true;
|
||||
} else {
|
||||
reader.step_back(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if buf.len() > 0 && has_dot {
|
||||
buf.parse().map(Token::FloatLiteral).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize() {
|
||||
let test_str = "(\"abcdefg( )123\" )(\n\t 'nil true \"true\")00987463 123.125 . 0+-*/go=";
|
||||
|
||||
let result: Vec<_> = tokenize(&mut test_str.chars()).collect();
|
||||
|
||||
assert_eq!(result.len(), 13);
|
||||
assert_eq!(result[0].clone().unwrap(), Token::ParOpen);
|
||||
assert_eq!(
|
||||
result[1].clone().unwrap(),
|
||||
Token::StringLiteral(String::from("abcdefg( )123"))
|
||||
);
|
||||
assert_eq!(result[2].clone().unwrap(), Token::ParClose);
|
||||
assert_eq!(result[3].clone().unwrap(), Token::ParOpen);
|
||||
assert_eq!(result[4].clone().unwrap(), Token::Quote);
|
||||
assert_eq!(result[5].clone().unwrap(), Token::Nil);
|
||||
assert_eq!(result[6].clone().unwrap(), Token::True);
|
||||
assert_eq!(
|
||||
result[7].clone().unwrap(),
|
||||
Token::StringLiteral(String::from("true"))
|
||||
);
|
||||
assert_eq!(result[8].clone().unwrap(), Token::ParClose);
|
||||
assert_eq!(result[9].clone().unwrap(), Token::IntLiteral(987463));
|
||||
assert_eq!(result[10].clone().unwrap(), Token::FloatLiteral(123.125));
|
||||
assert_eq!(result[11].clone().unwrap(), Token::Dot);
|
||||
assert_eq!(
|
||||
result[12].clone().unwrap(),
|
||||
Token::Symbol("0+-*/go=".to_string())
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user