jweinst1
12/18/2016 - 2:13 AM

lisptokenizer.js

//general lisp tokenizer

var tokenize = function(input) {
  return input.replace(/\(/g, ' ( ').replace(/\)/g, ' ) ').trim().split(/\s+/);
};

/*   tokenize("(+ 1 2)")
=> [ '(', '+', '1', '2', ')' ]
   tokenize("(+ 'hllo world' 2)")
=> [ '(', '+', '\'hllo', 'world\'', '2', ')' ]
   tokenize("(+ 'hllo world' 2)")
=> [ '(', '+', '\'hllo', 'world\'', '2', ')' ]
   */