diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index d9a220344..4e1dc4f4c 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -16,7 +16,6 @@ import java.nio.file.Path import java.nio.file.Paths -class ParsingFailedError(override var message: String) : Exception(message) fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') @@ -56,7 +55,7 @@ class ModuleImporter(private val program: Program, } private fun importModule(stream: CharStream, modulePath: Path): Module { - val parser = Prog8Parser() + val parser = Prog8Parser val sourceText = stream.toString() val moduleAst = parser.parseModule(sourceText) moduleAst.program = program diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 5a7a4dfc6..d3d10eea4 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -1,47 +1,50 @@ package prog8.parser import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.misc.ParseCancellationException -import prog8.ast.antlr.toAst import prog8.ast.Module +import prog8.ast.antlr.toAst +import prog8.ast.base.Position +import java.nio.file.Path -class Prog8ErrorStrategy: BailErrorStrategy() { - override fun recover(recognizer: Parser?, e: RecognitionException?) { - try { - // let it - super.recover(recognizer, e) // fills in exception e in all the contexts - // ...then throws ParseCancellationException, which is - // *deliberately* not a RecognitionException. However, we don't try any - // error recovery, therefore report an error in this case, too. - } catch (pce: ParseCancellationException) { - reportError(recognizer, e) - } - } +open class ParsingFailedError(override var message: String) : Exception(message) - override fun recoverInline(recognizer: Parser?): Token { - throw InputMismatchException(recognizer) +class ParseError(override var message: String, val position: Position, cause: RuntimeException) + : ParsingFailedError("${position.toClickableStr()}$message") { + init { + initCause(cause) } } -object ThrowErrorListener: BaseErrorListener() { - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - throw ParsingFailedError("$e: $msg") - } +private fun RecognitionException.getPosition(provenance: String) : Position { + val offending = this.offendingToken + val line = offending.line + val beginCol = offending.charPositionInLine + val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? + val pos = Position(provenance, line, beginCol, endCol) + return pos } -class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorListener) { +object Prog8Parser { - fun parseModule(sourceText: String): Module { - val chars = CharStreams.fromString(sourceText) + fun parseModule(srcPath: Path): Module { + return parseModule(CharStreams.fromPath(srcPath), srcPath.fileName.toString()) + } + + fun parseModule(srcText: String): Module { + return parseModule(CharStreams.fromString(srcText), "") + } + + private fun parseModule(chars: CharStream, provenance: String): Module { + val antlrErrorListener = AntlrErrorListener(provenance) val lexer = Prog8ANTLRLexer(chars) lexer.removeErrorListeners() - lexer.addErrorListener(errorListener) + lexer.addErrorListener(antlrErrorListener) val tokens = CommonTokenStream(lexer) val parser = Prog8ANTLRParser(tokens) - parser.errorHandler = Prog8ErrorStrategy() + parser.errorHandler = Prog8ErrorStrategy parser.removeErrorListeners() - parser.addErrorListener(errorListener) + parser.addErrorListener(antlrErrorListener) val parseTree = parser.module() val moduleName = "anonymous" @@ -54,4 +57,42 @@ class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorList } return module } + + private object Prog8ErrorStrategy: BailErrorStrategy() { + private fun fillIn(e: RecognitionException?, ctx: ParserRuleContext?) { + var context = ctx + while (context != null) { + context.exception = e + context = context.getParent() + } + } + + override fun reportInputMismatch(recognizer: Parser?, e: InputMismatchException?) { + super.reportInputMismatch(recognizer, e) + } + + override fun recover(recognizer: Parser?, e: RecognitionException?) { + fillIn(e, recognizer!!.context) + reportError(recognizer, e) + } + + override fun recoverInline(recognizer: Parser?): Token { + val e = InputMismatchException(recognizer) + fillIn(e, recognizer!!.context) + reportError(recognizer, e) + throw e + } + } + + private class AntlrErrorListener(val sourceCodeProvenance: String): BaseErrorListener() { + override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { + if (e == null) { + TODO("no RecognitionException - create your own ParseError") + //throw ParseError() + } else { + throw ParseError(msg, e.getPosition(sourceCodeProvenance), e) + } + } + } + } diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestAntlrParser.kt index 9034685a4..d30e15bdd 100644 --- a/compilerAst/test/TestAntlrParser.kt +++ b/compilerAst/test/TestAntlrParser.kt @@ -1,107 +1,30 @@ package prog8tests -import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.misc.ParseCancellationException import org.junit.jupiter.api.Test -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding -import prog8.ast.Program -import prog8.ast.antlr.toAst -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import prog8.ast.statements.Block -import prog8.parser.* +import prog8.parser.ParseError +import prog8.parser.Prog8Parser.parseModule import java.nio.file.Path import kotlin.test.* -class TestAntlrParser { - - class MyErrorListener: ConsoleErrorListener() { - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - throw ParsingFailedError("line $line:$charPositionInLine $msg") - } - } - - class MyErrorStrategy: BailErrorStrategy() { - override fun recover(recognizer: Parser?, e: RecognitionException?) { - try { - // let it - super.recover(recognizer, e) // fills in exception e in all the contexts - // ...then throws ParseCancellationException, which is - // *deliberately* not a RecognitionException. However, we don't try any - // error recovery, therefore report an error in this case, too. - } catch (pce: ParseCancellationException) { - reportError(recognizer, e) - } - } - - override fun recoverInline(recognizer: Parser?): Token { - throw InputMismatchException(recognizer) - } - } - - private fun parseModule(srcText: String): Prog8ANTLRParser.ModuleContext { - return parseModule(CharStreams.fromString(srcText)) - } - - private fun parseModule(srcFile: Path): Prog8ANTLRParser.ModuleContext { - return parseModule(CharStreams.fromPath(srcFile)) - } - - private fun parseModule(srcStream: CharStream): Prog8ANTLRParser.ModuleContext { - val errorListener = MyErrorListener() - val lexer = Prog8ANTLRLexer(srcStream) - lexer.removeErrorListeners() - lexer.addErrorListener(errorListener) - val tokens = CommonTokenStream(lexer) - val parser = Prog8ANTLRParser(tokens) - parser.errorHandler = MyErrorStrategy() - parser.removeErrorListeners() - parser.addErrorListener(errorListener) - return parser.module() - } - - object DummyEncoding: IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("Not yet implemented") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("Not yet implemented") - } - } - - object DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - object DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } +class TestProg8Parser { @Test fun testModuleSourceNeedNotEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) - // before the fix, Prog8ANTLRParser would have reported (thrown) "missing at ''" - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + // #45: Prog8ANTLRParser would report (throw) "missing at ''" + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test fun testModuleSourceMayEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -114,9 +37,9 @@ class TestAntlrParser { // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" - assertFailsWith { parseModule(srcBad) } - val parseTree = parseModule(srcGood) - assertEquals(parseTree.block().size, 2) + assertFailsWith { parseModule(srcBad) } + val module = parseModule(srcGood) + assertEquals(2, module.statements.size) } @Test @@ -143,8 +66,8 @@ class TestAntlrParser { "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 2) + val module = parseModule(srcText) + assertEquals(2, module.statements.size) } @Test @@ -158,8 +81,8 @@ class TestAntlrParser { blockA { } """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -175,8 +98,8 @@ class TestAntlrParser { blockB { } """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 2) + val module = parseModule(srcText) + assertEquals(2, module.statements.size) } @Test @@ -190,8 +113,8 @@ class TestAntlrParser { ; comment """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -199,14 +122,14 @@ class TestAntlrParser { // issue: #47 // block and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" blockA { } blockB { } """) } // block and directive - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" blockB { } %import textio """) } @@ -215,37 +138,58 @@ class TestAntlrParser { // Leaving them in anyways. // dir and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" %import textio blockB { } """) } - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" %import textio %import syslib """) } } - /* @Test - fun testImportLibraryModule() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + fun testErrorLocationForSourceFromString() { + val srcText = "bad * { }\n" - //assertFailsWith(){ importer.importLibraryModule("import_file_with_syntax_error") } + assertFailsWith { parseModule(srcText) } + try { + parseModule(srcText) + } catch (e: ParseError) { + // Note: assertContains expects *actual* value first + assertContains(e.position.file, Regex("^$")) + assertEquals(1, e.position.line, "line; should be 1-based") + assertEquals(4, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(4, e.position.endCol, "endCol; should be 0-based") + } + } + + @Test + fun testErrorLocationForSourceFromPath() { + val filename = "file_with_syntax_error.p8" + val path = Path.of("test", "fixtures", filename) + + assertFailsWith { parseModule(path) } + try { + parseModule(path) + } catch (e: ParseError) { + assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } } - */ @Test fun testProg8Ast() { - val parseTree = parseModule(""" + val module = parseModule(""" main { sub start() { return } } """) - val ast = parseTree.toAst("test", Path.of(""), DummyEncoding) - assertIs(ast.statements.first()) - assertEquals((ast.statements.first() as Block).name, "main") + assertIs(module.statements.first()) + assertEquals((module.statements.first() as Block).name, "main") } } diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt new file mode 100644 index 000000000..3b18d5cae --- /dev/null +++ b/compilerAst/test/TestModuleImporter.kt @@ -0,0 +1,81 @@ +package prog8tests + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.IStringEncoding +import prog8.ast.Program +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import prog8.parser.ModuleImporter +import prog8.parser.ParseError +import java.nio.file.Path +import org.junit.jupiter.api.Test +import kotlin.test.* + + +class TestModuleImporter { + + object DummyEncoding: IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("Not yet implemented") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("Not yet implemented") + } + } + + object DummyFunctions: IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() + } + + object DummyMemsizer: IMemSizer { + override fun memorySize(dt: DataType): Int = 0 + } + + @Test + fun testImportModuleWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val filename = "file_with_syntax_error.p8" + val act = { importer.importModule(Path.of("test", "fixtures", filename )) } + + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + + @Test + fun testImportLibraryModuleImportingBadModule() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val importing = "import_file_with_syntax_error" + val imported = "file_with_syntax_error" + val act = { importer.importLibraryModule(importing) } + + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals(imported + ".p8", e.position.file, "provenance; should be the importED file's name, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + +} diff --git a/compilerAst/test/fixtures/file_with_syntax_error.p8 b/compilerAst/test/fixtures/file_with_syntax_error.p8 new file mode 100644 index 000000000..6b9930fe0 --- /dev/null +++ b/compilerAst/test/fixtures/file_with_syntax_error.p8 @@ -0,0 +1,2 @@ +; test expects the following 2nd (!) line: +bad { } ; -> missing EOL at '}' (ie: *after* the '{') diff --git a/compilerAst/test/fixtures/import_file_with_syntax_error.p8 b/compilerAst/test/fixtures/import_file_with_syntax_error.p8 new file mode 100644 index 000000000..a6de5368f --- /dev/null +++ b/compilerAst/test/fixtures/import_file_with_syntax_error.p8 @@ -0,0 +1 @@ +%import file_with_syntax_error diff --git a/compilerAst/test/fixtures/import_import_nonexisting.p8 b/compilerAst/test/fixtures/import_import_nonexisting.p8 new file mode 100644 index 000000000..c9c418fa3 --- /dev/null +++ b/compilerAst/test/fixtures/import_import_nonexisting.p8 @@ -0,0 +1 @@ +%import import_nonexisting diff --git a/compilerAst/test/fixtures/import_nonexisting.p8 b/compilerAst/test/fixtures/import_nonexisting.p8 new file mode 100644 index 000000000..3d028ba09 --- /dev/null +++ b/compilerAst/test/fixtures/import_nonexisting.p8 @@ -0,0 +1 @@ +%import i_do_not_exist