package prog8tests import org.antlr.v4.runtime.* import org.antlr.v4.runtime.misc.ParseCancellationException import org.junit.jupiter.api.Test import prog8.ast.IBuiltinFunctions import prog8.ast.IMemSizer import prog8.ast.IStringEncoding import prog8.ast.antlr.toAst import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.Expression import prog8.ast.expressions.InferredTypes import prog8.ast.expressions.NumericLiteralValue import prog8.ast.statements.Block import prog8.parser.ParsingFailedError import prog8.parser.prog8Lexer import prog8.parser.prog8Parser import java.nio.file.Path import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertIs class TestAntlrParser { class MyErrorListener: ConsoleErrorListener() { override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { throw ParsingFailedError("line $line:$charPositionInLine $msg") } } class MyErrorStrategy: BailErrorStrategy() { override fun recover(recognizer: Parser?, e: RecognitionException?) { try { // let it fill in e in all the contexts super.recover(recognizer, e) } catch (pce: ParseCancellationException) { reportError(recognizer, e) } } override fun recoverInline(recognizer: Parser?): Token { throw InputMismatchException(recognizer) } } private fun parseModule(srcText: String): prog8Parser.ModuleContext { return parseModule(CharStreams.fromString(srcText)) } private fun parseModule(srcFile: Path): prog8Parser.ModuleContext { return parseModule(CharStreams.fromPath(srcFile)) } private fun parseModule(srcStream: CharStream): prog8Parser.ModuleContext { val errorListener = MyErrorListener() val lexer = prog8Lexer(srcStream) lexer.removeErrorListeners() lexer.addErrorListener(errorListener) val tokens = CommonTokenStream(lexer) val parser = prog8Parser(tokens) parser.errorHandler = MyErrorStrategy() parser.removeErrorListeners() parser.addErrorListener(errorListener) return parser.module() } object DummyEncoding: IStringEncoding { override fun encodeString(str: String, altEncoding: Boolean): List { TODO("Not yet implemented") } override fun decodeString(bytes: List, altEncoding: Boolean): String { TODO("Not yet implemented") } } object DummyFunctions: IBuiltinFunctions { override val names: Set = emptySet() override val purefunctionNames: Set = emptySet() override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() } object DummyMemsizer: IMemSizer { override fun memorySize(dt: DataType): Int = 0 } @Test fun testModuleSourceNeedNotEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) // before the fix, prog8Parser would have reported (thrown) "missing at ''" val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @Test fun testModuleSourceMayEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @Test fun testAllBlocksButLastMustEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" assertFailsWith { parseModule(srcBad) } val parseTree = parseModule(srcGood) assertEquals(parseTree.block().size, 2) } @Test fun testWindowsAndMacNewlinesAreAlsoFine() { val nlWin = "\r\n" val nlUnix = "\n" val nlMac = "\r" //parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath()) // a good mix of all kinds of newlines: val srcText = "foo {" + nlMac + nlWin + "}" + nlMac + // <-- do test a single \r (!) where an EOL is expected "bar {" + nlUnix + "}" + nlUnix + nlMac // both should be "eaten up" by just one EOL token "combi {" + nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 2) } @Test fun testInterleavedEolAndCommentBeforeFirstBlock() { // issue: #47 val srcText = """ ; comment ; comment blockA { } """ val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @Test fun testInterleavedEolAndCommentBetweenBlocks() { // issue: #47 val srcText = """ blockA { } ; comment ; comment blockB { } """ val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 2) } @Test fun testInterleavedEolAndCommentAfterLastBlock() { // issue: #47 val srcText = """ blockA { } ; comment ; comment """ val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @Test fun testNewlineBetweenTwoBlocksOrDirectivesStillRequired() { // issue: #47 // block and block assertFailsWith{ parseModule(""" blockA { } blockB { } """) } // block and directive assertFailsWith{ parseModule(""" blockB { } %import textio """) } // The following two are bogus due to directive *args* expected to follow the directive name. // Leaving them in anyways. // dir and block assertFailsWith{ parseModule(""" %import textio blockB { } """) } assertFailsWith{ parseModule(""" %import textio %import syslib """) } } /* @Test fun testImportLibraryModule() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) //assertFailsWith(){ importer.importLibraryModule("import_file_with_syntax_error") } } */ @Test fun testProg8Ast() { // can create charstreams from many other sources as well; val charstream = CharStreams.fromString(""" main { sub start() { return } } """) val lexer = prog8Lexer(charstream) val tokens = CommonTokenStream(lexer) val parser = prog8Parser(tokens) parser.errorHandler = BailErrorStrategy() // parser.removeErrorListeners() // parser.addErrorListener(MyErrorListener()) val ast = parser.module().toAst("test", Path.of(""), DummyEncoding) assertIs(ast.statements.first()) assertEquals((ast.statements.first() as Block).name, "main") } }