prog8/compilerAst/test/TestAntlrParser.kt

152 lines
5.2 KiB
Kotlin

package prog8tests
import org.antlr.v4.runtime.*
import org.antlr.v4.runtime.misc.ParseCancellationException
import org.junit.jupiter.api.Test
import prog8.ast.IStringEncoding
import prog8.ast.antlr.toAst
import prog8.ast.statements.Block
import prog8.parser.ParsingFailedError
import prog8.parser.prog8Lexer
import prog8.parser.prog8Parser
import java.nio.file.Path
import kotlin.test.*
class TestAntlrParser {
class MyErrorListener: ConsoleErrorListener() {
override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) {
throw ParsingFailedError("line $line:$charPositionInLine $msg")
}
}
class MyErrorStrategy: BailErrorStrategy() {
override fun recover(recognizer: Parser?, e: RecognitionException?) {
try {
// let it fill in e in all the contexts
super.recover(recognizer, e)
} catch (pce: ParseCancellationException) {
reportError(recognizer, e)
}
}
override fun recoverInline(recognizer: Parser?): Token {
throw InputMismatchException(recognizer)
}
}
private fun parseModule(srcText: String): prog8Parser.ModuleContext {
return parseModule(CharStreams.fromString(srcText))
}
private fun parseModule(srcFile: Path): prog8Parser.ModuleContext {
return parseModule(CharStreams.fromPath(srcFile))
}
private fun parseModule(srcStream: CharStream): prog8Parser.ModuleContext {
val errorListener = MyErrorListener()
val lexer = prog8Lexer(srcStream)
lexer.removeErrorListeners()
lexer.addErrorListener(errorListener)
val tokens = CommonTokenStream(lexer)
val parser = prog8Parser(tokens)
parser.errorHandler = MyErrorStrategy()
parser.removeErrorListeners()
parser.addErrorListener(errorListener)
return parser.module()
}
object TestStringEncoding: IStringEncoding {
override fun encodeString(str: String, altEncoding: Boolean): List<Short> {
TODO("Not yet implemented")
}
override fun decodeString(bytes: List<Short>, altEncoding: Boolean): String {
TODO("Not yet implemented")
}
}
@Test
fun testModuleSourceNeedNotEndWithNewline() {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40)
// before the fix, prog8Parser would have reported (thrown) "missing <EOL> at '<EOF>'"
val parseTree = parseModule(srcText)
assertEquals(parseTree.block().size, 1)
}
@Test
fun testModuleSourceMayEndWithNewline() {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40)
val parseTree = parseModule(srcText)
assertEquals(parseTree.block().size, 1)
}
@Test
fun testAllBlocksButLastMustEndWithNewline() {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
// BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end
val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl
// GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed
val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}"
assertFailsWith<ParsingFailedError> { parseModule(srcBad) }
val parseTree = parseModule(srcGood)
assertEquals(parseTree.block().size, 2)
}
@Test
fun testWindowsAndMacNewlinesAreAlsoFine() {
val nlWin = "\r\n"
val nlUnix = "\n"
val nlMac = "\r"
//parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath())
// a good mix of all kinds of newlines:
val srcText =
"foo {" +
nlMac +
nlWin +
"}" +
nlMac + // <-- do test a single \r (!) where an EOL is expected
"bar {" +
nlUnix +
"}" +
nlUnix + nlMac // both should be "eaten up" by just one EOL token
"combi {" +
nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token
"}" +
nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline)
val parseTree = parseModule(srcText)
assertEquals(parseTree.block().size, 2)
}
@Test
fun testProg8Ast() {
// can create charstreams from many other sources as well;
val charstream = CharStreams.fromString("""
main {
sub start() {
return
}
}
""")
val lexer = prog8Lexer(charstream)
val tokens = CommonTokenStream(lexer)
val parser = prog8Parser(tokens)
parser.errorHandler = BailErrorStrategy()
// parser.removeErrorListeners()
// parser.addErrorListener(MyErrorListener())
val ast = parser.module().toAst("test", false, Path.of(""), TestStringEncoding)
assertIs<Block>(ast.statements.first())
assertEquals((ast.statements.first() as Block).name, "main")
}
}