From fac76f1b54ed2bdc2e16d8da76f98692e52b6167 Mon Sep 17 00:00:00 2001 From: Kelvin Sherlock Date: Sat, 23 Jul 2016 15:20:56 -0400 Subject: [PATCH] shell tokenizer -- support extended characters. --- mpw-shell-token.rl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mpw-shell-token.rl b/mpw-shell-token.rl index 7de7fc4..6919991 100644 --- a/mpw-shell-token.rl +++ b/mpw-shell-token.rl @@ -68,15 +68,25 @@ '<' %push_token => { tokens.emplace_back("<", '<'); }; - # these should be eval-only too... - '||' %push_token => { tokens.emplace_back("||", '||'); }; - '|' %push_token => { tokens.emplace_back("|", '|'); }; + # macroman ∑, ∑∑ + 0xb7 0xb7 %push_token => { tokens.emplace_back("\xb7\xb7", 0xb7b7); }; + 0xb7 %push_token => { tokens.emplace_back("\xb7", 0xb7); }; - '&&' - %push_token => { tokens.emplace_back("&&", '&&'); }; + # macroman ≥, ≥≥ + 0xb3 0xb3 %push_token => { tokens.emplace_back("\xb3\xb3", 0xb3b3); }; + 0xb3 %push_token => { tokens.emplace_back("\xb3", 0xb3); }; # eval-only. + '||' when eval + %push_token => { tokens.emplace_back("||", '||'); }; + '|' when eval + %push_token => { tokens.emplace_back("|", '|'); }; + + '&&' when eval + %push_token => { tokens.emplace_back("&&", '&&'); }; + + '(' when eval %push_token => { tokens.emplace_back("(", '('); };