diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestAntlrParser.kt index ada57937e..e3c3a5f9e 100644 --- a/compilerAst/test/TestAntlrParser.kt +++ b/compilerAst/test/TestAntlrParser.kt @@ -68,7 +68,30 @@ class TestAntlrParser { val parseTree = parseModule(srcGood) assertEquals(parseTree.block().size, 2) } - + + @Test + fun testWindowsAndMacNewlinesAreAlsoFine() { + val nlWin = "\r\n" + val nlUnix = "\n" + val nlMac = "\r" + + // a good mix of all kinds of newlines: + val srcText = + "foo {" + + nlWin + + "}" + + nlUnix + + nlMac + // both these newlines should be "eaten up" by just one EOL token + "bar {" + + nlMac + + nlWin + + nlUnix + // all three should be "eaten up" by just one EOL token + "}" + + val parseTree = parseModule(srcText) + assertEquals(parseTree.block().size, 2) + } + @Test fun testProg8Ast() { // can create charstreams from many other sources as well; diff --git a/parser/antlr/prog8.g4 b/parser/antlr/prog8.g4 index 4c72bf3e8..4735c71d7 100644 --- a/parser/antlr/prog8.g4 +++ b/parser/antlr/prog8.g4 @@ -14,10 +14,16 @@ grammar prog8; package prog8.parser; } +//TODO: why isn't testWindowsAndMacNewlinesAreAlsoFine *failing* with this old stuff?! +// note: when we build the whole thing with it, then non-Unix newlines actually DO NOT work with it...?! +//LINECOMMENT : '\n' [ \t]* COMMENT -> channel(HIDDEN); +//COMMENT : ';' ~[\n]* -> channel(HIDDEN) ; +//EOL : '\n'+ ; LINECOMMENT : ('\r'? '\n' | '\r') [ \t]* COMMENT -> channel(HIDDEN); COMMENT : ';' ~[\r\n]* -> channel(HIDDEN) ; -WS : [ \t] -> skip ; EOL : ('\r'? '\n' | '\r')+ ; + +WS : [ \t] -> skip ; // WS2 : '\\' EOL -> skip; VOID: 'void'; NAME : [a-zA-Z][a-zA-Z0-9_]* ;