Macintosh-SW/mpw-shell
1
0
Fork 0
mirror of https://github.com/ksherlock/mpw-shell.git synced 2024-06-07 19:29:42 +00:00
Code Issues Projects Releases Wiki Activity

fix up tokenizer a little bit

Browse Source
This commit is contained in:
Kelvin Sherlock 2016-07-27 16:28:11 -04:00
parent fe76877693
commit 9b16e98133
1 changed files with 6 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
mpw-shell-token.rl
View File

@ -28,13 +28,13 @@
schar = [^'] $push_back; schar = [^'] $push_back;
sstring = sstring =
['] schar** ['] ['] ${ quoted = true; } schar** [']
${ quoted = true; }
$err{ throw sstring_error(); } $err{ throw sstring_error(); }
; ;
# if eof, should push escape...
escape_seq = escape_seq =
escape escape $err{ scratch.push_back(escape); }
( (
'f' ${scratch.push_back('\f'); } 'f' ${scratch.push_back('\f'); }
| 'n' ${scratch.push_back('\n'); /* \r ? */ } | 'n' ${scratch.push_back('\n'); /* \r ? */ }
@ -46,8 +46,7 @@
# double-quoted string. # double-quoted string.
dchar = escape_seq | (any - escape - ["]) $push_back; dchar = escape_seq | (any - escape - ["]) $push_back;
dstring = dstring =
["] dchar** ["] ["] ${ quoted = true; } dchar** ["]
${ quoted = true; }
$err{ throw dstring_error(); } $err{ throw dstring_error(); }
; ;
@ -57,7 +56,7 @@
# > == start state (single char tokens or common prefix) # > == start state (single char tokens or common prefix)
# % == final state (multi char tokens w/ unique prefix) # % == final state (multi char tokens w/ unique prefix)
# $ == all states # $ == all states
char = any - escape - ['"];
main := |* main := |*
ws+ >push_token; ws+ >push_token;
'>>' %push_token => { tokens.emplace_back(">>", '>>'); }; '>>' %push_token => { tokens.emplace_back(">>", '>>'); };
@ -146,7 +145,7 @@
dstring ; dstring ;
escape_seq; escape_seq;
(any-escape-['"]) => push_back; char => push_back;
*| *|
; ;
}%% }%%
@ -206,10 +205,6 @@ std::vector<token> tokenize(const std::string &s, bool eval)
%%write exec; %%write exec;
if (cs == tokenizer_error) {
throw std::runtime_error("MPW Shell - Lexer error.");
}
if (!scratch.empty() || quoted) { if (!scratch.empty() || quoted) {
tokens.emplace_back(std::move(scratch)); tokens.emplace_back(std::move(scratch));
scratch.clear(); scratch.clear();