diff --git a/.idea/compiler.xml b/.idea/compiler.xml index a39e0f0fb..af8a35b20 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -2,5 +2,6 @@ \ No newline at end of file diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index fc07eece1..0cf6e7eac 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -8,8 +8,7 @@ import prog8.ast.Module import prog8.ast.base.* import prog8.ast.expressions.* import prog8.ast.statements.* -import prog8.parser.CustomLexer -import prog8.parser.prog8Parser +import prog8.parser.Prog8ANTLRParser import java.io.CharConversionException import java.io.File import java.nio.file.Path @@ -19,7 +18,7 @@ import java.nio.file.Path private data class NumericLiteral(val number: Number, val datatype: DataType) -internal fun prog8Parser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { +internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name val directives = this.directive().map { it.toAst() } val blocks = this.block().map { it.toAst(Module.isLibrary(source), encoding) } @@ -27,6 +26,7 @@ internal fun prog8Parser.ModuleContext.toAst(name: String, source: Path, encodin } private fun ParserRuleContext.toPosition() : Position { + /* val customTokensource = this.start.tokenSource as? CustomLexer val filename = when { @@ -34,11 +34,14 @@ private fun ParserRuleContext.toPosition() : Position { start.tokenSource.sourceName == IntStream.UNKNOWN_SOURCE_NAME -> "@internal@" else -> File(start.inputStream.sourceName).name } + */ + val filename = start.inputStream.sourceName + // note: be ware of TAB characters in the source text, they count as 1 column... return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine + stop.text.length) } -private fun prog8Parser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { val blockstatements = block_statement().map { when { it.variabledeclaration()!=null -> it.variabledeclaration().toAst(encoding) @@ -52,10 +55,10 @@ private fun prog8Parser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStri return Block(identifier().text, integerliteral()?.toAst()?.number?.toInt(), blockstatements.toMutableList(), isInLibrary, toPosition()) } -private fun prog8Parser.Statement_blockContext.toAst(encoding: IStringEncoding): MutableList = +private fun Prog8ANTLRParser.Statement_blockContext.toAst(encoding: IStringEncoding): MutableList = statement().asSequence().map { it.toAst(encoding) }.toMutableList() -private fun prog8Parser.VariabledeclarationContext.toAst(encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringEncoding) : Statement { vardecl()?.let { return it.toAst(encoding) } varinitializer()?.let { @@ -111,7 +114,7 @@ private fun prog8Parser.VariabledeclarationContext.toAst(encoding: IStringEncodi throw FatalAstException("weird variable decl $this") } -private fun prog8Parser.SubroutinedeclarationContext.toAst(encoding: IStringEncoding) : Subroutine { +private fun Prog8ANTLRParser.SubroutinedeclarationContext.toAst(encoding: IStringEncoding) : Subroutine { return when { subroutine()!=null -> subroutine().toAst(encoding) asmsubroutine()!=null -> asmsubroutine().toAst(encoding) @@ -120,7 +123,7 @@ private fun prog8Parser.SubroutinedeclarationContext.toAst(encoding: IStringEnco } } -private fun prog8Parser.StatementContext.toAst(encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.StatementContext.toAst(encoding: IStringEncoding) : Statement { val vardecl = variabledeclaration()?.toAst(encoding) if(vardecl!=null) return vardecl @@ -188,7 +191,7 @@ private fun prog8Parser.StatementContext.toAst(encoding: IStringEncoding) : Stat throw FatalAstException("unprocessed source text (are we missing ast conversion rules for parser elements?): $text") } -private fun prog8Parser.AsmsubroutineContext.toAst(encoding: IStringEncoding): Subroutine { +private fun Prog8ANTLRParser.AsmsubroutineContext.toAst(encoding: IStringEncoding): Subroutine { val inline = this.inline()!=null val subdecl = asmsub_decl().toAst() val statements = statement_block()?.toAst(encoding) ?: mutableListOf() @@ -197,7 +200,7 @@ private fun prog8Parser.AsmsubroutineContext.toAst(encoding: IStringEncoding): S subdecl.asmClobbers, null, true, inline, statements, toPosition()) } -private fun prog8Parser.RomsubroutineContext.toAst(): Subroutine { +private fun Prog8ANTLRParser.RomsubroutineContext.toAst(): Subroutine { val subdecl = asmsub_decl().toAst() val address = integerliteral().toAst().number.toInt() return Subroutine(subdecl.name, subdecl.parameters, subdecl.returntypes, @@ -213,7 +216,7 @@ private class AsmsubDecl(val name: String, val asmReturnvaluesRegisters: List, val asmClobbers: Set) -private fun prog8Parser.Asmsub_declContext.toAst(): AsmsubDecl { +private fun Prog8ANTLRParser.Asmsub_declContext.toAst(): AsmsubDecl { val name = identifier().text val params = asmsub_params()?.toAst() ?: emptyList() val returns = asmsub_returns()?.toAst() ?: emptyList() @@ -236,7 +239,7 @@ private class AsmSubroutineReturn(val type: DataType, val statusflag: Statusflag?, val position: Position) -private fun prog8Parser.Asmsub_returnsContext.toAst(): List +private fun Prog8ANTLRParser.Asmsub_returnsContext.toAst(): List = asmsub_return().map { val register = it.register().text var registerorpair: RegisterOrPair? = null @@ -255,7 +258,7 @@ private fun prog8Parser.Asmsub_returnsContext.toAst(): List toPosition()) } -private fun prog8Parser.Asmsub_paramsContext.toAst(): List +private fun Prog8ANTLRParser.Asmsub_paramsContext.toAst(): List = asmsub_param().map { val vardecl = it.vardecl() val datatype = vardecl.datatype()?.toAst() ?: DataType.UNDEFINED @@ -272,7 +275,7 @@ private fun prog8Parser.Asmsub_paramsContext.toAst(): List { +private fun Prog8ANTLRParser.Sub_return_partContext.toAst(): List { val returns = sub_returns() ?: return emptyList() return returns.datatype().map { it.toAst() } } -private fun prog8Parser.Sub_paramsContext.toAst(): List = +private fun Prog8ANTLRParser.Sub_paramsContext.toAst(): List = vardecl().map { val datatype = it.datatype()?.toAst() ?: DataType.UNDEFINED SubroutineParameter(it.varname.text, datatype, it.toPosition()) } -private fun prog8Parser.Assign_targetContext.toAst(encoding: IStringEncoding) : AssignTarget { +private fun Prog8ANTLRParser.Assign_targetContext.toAst(encoding: IStringEncoding) : AssignTarget { val identifier = scoped_identifier() return when { identifier!=null -> AssignTarget(identifier.toAst(), null, null, toPosition()) @@ -338,20 +341,20 @@ private fun prog8Parser.Assign_targetContext.toAst(encoding: IStringEncoding) : } } -private fun prog8Parser.ClobberContext.toAst() : Set { +private fun Prog8ANTLRParser.ClobberContext.toAst() : Set { val names = this.cpuregister().map { it.text } return names.map { CpuRegister.valueOf(it) }.toSet() } -private fun prog8Parser.DatatypeContext.toAst() = DataType.valueOf(text.uppercase()) +private fun Prog8ANTLRParser.DatatypeContext.toAst() = DataType.valueOf(text.uppercase()) -private fun prog8Parser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = +private fun Prog8ANTLRParser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = ArrayIndex(expression().toAst(encoding), toPosition()) -private fun prog8Parser.DirectiveContext.toAst() : Directive = +private fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = Directive(directivename.text, directivearg().map { it.toAst() }, toPosition()) -private fun prog8Parser.DirectiveargContext.toAst() : DirectiveArg { +private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { val str = stringliteral() if(str?.ALT_STRING_ENCODING() != null) throw AstException("${toPosition()} can't use alternate string encodings for directive arguments") @@ -359,7 +362,7 @@ private fun prog8Parser.DirectiveargContext.toAst() : DirectiveArg { return DirectiveArg(stringliteral()?.text, identifier()?.text, integerliteral()?.toAst()?.number?.toInt(), toPosition()) } -private fun prog8Parser.IntegerliteralContext.toAst(): NumericLiteral { +private fun Prog8ANTLRParser.IntegerliteralContext.toAst(): NumericLiteral { fun makeLiteral(text: String, radix: Int): NumericLiteral { val integer: Int var datatype = DataType.UBYTE @@ -403,14 +406,14 @@ private fun prog8Parser.IntegerliteralContext.toAst(): NumericLiteral { val terminal: TerminalNode = children[0] as TerminalNode val integerPart = this.intpart.text return when (terminal.symbol.type) { - prog8Parser.DEC_INTEGER -> makeLiteral(integerPart, 10) - prog8Parser.HEX_INTEGER -> makeLiteral(integerPart.substring(1), 16) - prog8Parser.BIN_INTEGER -> makeLiteral(integerPart.substring(1), 2) + Prog8ANTLRParser.DEC_INTEGER -> makeLiteral(integerPart, 10) + Prog8ANTLRParser.HEX_INTEGER -> makeLiteral(integerPart.substring(1), 16) + Prog8ANTLRParser.BIN_INTEGER -> makeLiteral(integerPart.substring(1), 2) else -> throw FatalAstException(terminal.text) } } -private fun prog8Parser.ExpressionContext.toAst(encoding: IStringEncoding) : Expression { +private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) : Expression { val litval = literalvalue() if(litval!=null) { @@ -487,35 +490,35 @@ private fun prog8Parser.ExpressionContext.toAst(encoding: IStringEncoding) : Exp throw FatalAstException(text) } -private fun prog8Parser.StringliteralContext.toAst(): StringLiteralValue = +private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue = StringLiteralValue(unescape(this.STRING().text, toPosition()), ALT_STRING_ENCODING()!=null, toPosition()) -private fun prog8Parser.ArrayindexedContext.toAst(encoding: IStringEncoding): ArrayIndexedExpression { +private fun Prog8ANTLRParser.ArrayindexedContext.toAst(encoding: IStringEncoding): ArrayIndexedExpression { return ArrayIndexedExpression(scoped_identifier().toAst(), arrayindex().toAst(encoding), toPosition()) } -private fun prog8Parser.Expression_listContext.toAst(encoding: IStringEncoding) = expression().map{ it.toAst(encoding) } +private fun Prog8ANTLRParser.Expression_listContext.toAst(encoding: IStringEncoding) = expression().map{ it.toAst(encoding) } -private fun prog8Parser.IdentifierContext.toAst() : IdentifierReference = +private fun Prog8ANTLRParser.IdentifierContext.toAst() : IdentifierReference = IdentifierReference(listOf(text), toPosition()) -private fun prog8Parser.Scoped_identifierContext.toAst() : IdentifierReference = +private fun Prog8ANTLRParser.Scoped_identifierContext.toAst() : IdentifierReference = IdentifierReference(NAME().map { it.text }, toPosition()) -private fun prog8Parser.FloatliteralContext.toAst() = text.toDouble() +private fun Prog8ANTLRParser.FloatliteralContext.toAst() = text.toDouble() -private fun prog8Parser.BooleanliteralContext.toAst() = when(text) { +private fun Prog8ANTLRParser.BooleanliteralContext.toAst() = when(text) { "true" -> true "false" -> false else -> throw FatalAstException(text) } -private fun prog8Parser.ArrayliteralContext.toAst(encoding: IStringEncoding) : Array = +private fun Prog8ANTLRParser.ArrayliteralContext.toAst(encoding: IStringEncoding) : Array = expression().map { it.toAst(encoding) }.toTypedArray() -private fun prog8Parser.If_stmtContext.toAst(encoding: IStringEncoding): IfStatement { +private fun Prog8ANTLRParser.If_stmtContext.toAst(encoding: IStringEncoding): IfStatement { val condition = expression().toAst(encoding) val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() @@ -525,11 +528,11 @@ private fun prog8Parser.If_stmtContext.toAst(encoding: IStringEncoding): IfState return IfStatement(condition, trueScope, elseScope, toPosition()) } -private fun prog8Parser.Else_partContext.toAst(encoding: IStringEncoding): MutableList { +private fun Prog8ANTLRParser.Else_partContext.toAst(encoding: IStringEncoding): MutableList { return statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) } -private fun prog8Parser.Branch_stmtContext.toAst(encoding: IStringEncoding): BranchStatement { +private fun Prog8ANTLRParser.Branch_stmtContext.toAst(encoding: IStringEncoding): BranchStatement { val branchcondition = branchcondition().toAst() val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() @@ -539,11 +542,11 @@ private fun prog8Parser.Branch_stmtContext.toAst(encoding: IStringEncoding): Bra return BranchStatement(branchcondition, trueScope, elseScope, toPosition()) } -private fun prog8Parser.BranchconditionContext.toAst() = BranchCondition.valueOf( +private fun Prog8ANTLRParser.BranchconditionContext.toAst() = BranchCondition.valueOf( text.substringAfter('_').uppercase() ) -private fun prog8Parser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop { +private fun Prog8ANTLRParser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop { val loopvar = identifier().toAst() val iterable = expression()!!.toAst(encoding) val scope = @@ -554,9 +557,9 @@ private fun prog8Parser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop return ForLoop(loopvar, iterable, scope, toPosition()) } -private fun prog8Parser.BreakstmtContext.toAst() = Break(toPosition()) +private fun Prog8ANTLRParser.BreakstmtContext.toAst() = Break(toPosition()) -private fun prog8Parser.WhileloopContext.toAst(encoding: IStringEncoding): WhileLoop { +private fun Prog8ANTLRParser.WhileloopContext.toAst(encoding: IStringEncoding): WhileLoop { val condition = expression().toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -564,7 +567,7 @@ private fun prog8Parser.WhileloopContext.toAst(encoding: IStringEncoding): While return WhileLoop(condition, scope, toPosition()) } -private fun prog8Parser.RepeatloopContext.toAst(encoding: IStringEncoding): RepeatLoop { +private fun Prog8ANTLRParser.RepeatloopContext.toAst(encoding: IStringEncoding): RepeatLoop { val iterations = expression()?.toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -572,7 +575,7 @@ private fun prog8Parser.RepeatloopContext.toAst(encoding: IStringEncoding): Repe return RepeatLoop(iterations, scope, toPosition()) } -private fun prog8Parser.UntilloopContext.toAst(encoding: IStringEncoding): UntilLoop { +private fun Prog8ANTLRParser.UntilloopContext.toAst(encoding: IStringEncoding): UntilLoop { val untilCondition = expression().toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -580,13 +583,13 @@ private fun prog8Parser.UntilloopContext.toAst(encoding: IStringEncoding): Until return UntilLoop(scope, untilCondition, toPosition()) } -private fun prog8Parser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { +private fun Prog8ANTLRParser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { val condition = expression().toAst(encoding) val choices = this.when_choice()?.map { it.toAst(encoding) }?.toMutableList() ?: mutableListOf() return WhenStatement(condition, choices, toPosition()) } -private fun prog8Parser.When_choiceContext.toAst(encoding: IStringEncoding): WhenChoice { +private fun Prog8ANTLRParser.When_choiceContext.toAst(encoding: IStringEncoding): WhenChoice { val values = expression_list()?.toAst(encoding) val stmt = statement()?.toAst(encoding) val stmtBlock = statement_block()?.toAst(encoding)?.toMutableList() ?: mutableListOf() @@ -596,7 +599,7 @@ private fun prog8Parser.When_choiceContext.toAst(encoding: IStringEncoding): Whe return WhenChoice(values?.toMutableList(), scope, toPosition()) } -private fun prog8Parser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl { +private fun Prog8ANTLRParser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl { return VarDecl( VarDeclType.VAR, datatype()?.toAst() ?: DataType.UNDEFINED, diff --git a/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt b/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt index efb769bf3..dd73a29cc 100644 --- a/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt +++ b/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt @@ -9,7 +9,7 @@ internal class CommentHandlingTokenStream(lexer: Lexer) : CommonTokenStream(lexe fun commentTokens() : List { // extract the comments - val commentTokenChannel = prog8Lexer.channelNames.indexOf("HIDDEN") + val commentTokenChannel = Prog8ANTLRLexer.channelNames.indexOf("HIDDEN") val theLexer = tokenSource as Lexer return get(0, size()) .asSequence() diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 94faa6ee4..d9a220344 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -18,8 +18,6 @@ import java.nio.file.Paths class ParsingFailedError(override var message: String) : Exception(message) -internal class CustomLexer(val modulePath: Path, input: CharStream?) : prog8Lexer(input) - fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') internal fun pathFrom(stringPath: String, vararg rest: String): Path = FileSystems.getDefault().getPath(stringPath, *rest) @@ -44,7 +42,10 @@ class ModuleImporter(private val program: Program, if(!Files.isReadable(filePath)) throw ParsingFailedError("No such file: $filePath") - return importModule(CharStreams.fromPath(filePath), filePath) + val content = filePath.toFile().readText() + val cs = CharStreams.fromString(content) + + return importModule(cs, filePath) } fun importLibraryModule(name: String): Module? { @@ -54,40 +55,10 @@ class ModuleImporter(private val program: Program, return executeImportDirective(import, Paths.get("")) } - private class MyErrorListener: ConsoleErrorListener() { - var numberOfErrors: Int = 0 - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - numberOfErrors++ - when (recognizer) { - is CustomLexer -> System.err.println("${recognizer.modulePath}:$line:$charPositionInLine: $msg") - is prog8Parser -> System.err.println("${recognizer.inputStream.sourceName}:$line:$charPositionInLine: $msg") - else -> System.err.println("$line:$charPositionInLine $msg") - } - if(numberOfErrors>=5) - throw ParsingFailedError("There are too many parse errors. Stopping.") - } - } - private fun importModule(stream: CharStream, modulePath: Path): Module { - val moduleName = moduleName(modulePath.fileName) - val lexer = CustomLexer(modulePath, stream) - lexer.removeErrorListeners() - val lexerErrors = MyErrorListener() - lexer.addErrorListener(lexerErrors) - val tokens = CommentHandlingTokenStream(lexer) - val parser = prog8Parser(tokens) - parser.removeErrorListeners() - parser.addErrorListener(MyErrorListener()) - val parseTree = parser.module() - val numberOfErrors = parser.numberOfSyntaxErrors + lexerErrors.numberOfErrors - if(numberOfErrors > 0) - throw ParsingFailedError("There are $numberOfErrors errors in '$moduleName'.") - - // You can do something with the parsed comments: - // tokens.commentTokens().forEach { println(it) } - - // convert to Ast - val moduleAst = parseTree.toAst(moduleName, modulePath, encoder) + val parser = Prog8Parser() + val sourceText = stream.toString() + val moduleAst = parser.parseModule(sourceText) moduleAst.program = program moduleAst.linkParents(program.namespace) program.modules.add(moduleAst) diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt new file mode 100644 index 000000000..51b60ef44 --- /dev/null +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -0,0 +1,65 @@ +package prog8.parser + +import org.antlr.v4.runtime.* +import org.antlr.v4.runtime.misc.ParseCancellationException +import prog8.ast.antlr.toAst +import prog8.ast.Module +import prog8.ast.base.Position +import prog8.ast.IStringEncoding + + +object DummyEncoding: IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("move StringEncoding out of compilerAst") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("move StringEncoding out of compilerAst") + } +} + +class Prog8ErrorStrategy: BailErrorStrategy() { + override fun recover(recognizer: Parser?, e: RecognitionException?) { + try { + // let it + super.recover(recognizer, e) // fills in exception e in all the contexts + // ...then throws ParseCancellationException, which is + // *deliberately* not a RecognitionException. However, we don't try any + // error recovery, therefore report an error in this case, too. + } catch (pce: ParseCancellationException) { + reportError(recognizer, e) + } + } + + override fun recoverInline(recognizer: Parser?): Token { + throw InputMismatchException(recognizer) + } +} + +object ThrowErrorListener: BaseErrorListener() { + override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { + throw ParsingFailedError("$e: $msg") + } +} + +class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorListener) { + + fun parseModule(sourceText: String): Module { + val chars = CharStreams.fromString(sourceText) + val lexer = Prog8ANTLRLexer(chars) + lexer.removeErrorListeners() + lexer.addErrorListener(errorListener) + val tokens = CommonTokenStream(lexer) + val parser = Prog8ANTLRParser(tokens) + parser.errorHandler = Prog8ErrorStrategy() + parser.removeErrorListeners() + parser.addErrorListener(errorListener) + + val parseTree = parser.module() + + // TODO: use Module ctor directly + val moduleAst = parseTree.toAst("anonymous", pathFrom(""), DummyEncoding) + + return moduleAst + } +} diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestAntlrParser.kt index bb90bfb34..9034685a4 100644 --- a/compilerAst/test/TestAntlrParser.kt +++ b/compilerAst/test/TestAntlrParser.kt @@ -29,8 +29,11 @@ class TestAntlrParser { class MyErrorStrategy: BailErrorStrategy() { override fun recover(recognizer: Parser?, e: RecognitionException?) { try { - // let it fill in e in all the contexts - super.recover(recognizer, e) + // let it + super.recover(recognizer, e) // fills in exception e in all the contexts + // ...then throws ParseCancellationException, which is + // *deliberately* not a RecognitionException. However, we don't try any + // error recovery, therefore report an error in this case, too. } catch (pce: ParseCancellationException) { reportError(recognizer, e) } @@ -41,21 +44,21 @@ class TestAntlrParser { } } - private fun parseModule(srcText: String): prog8Parser.ModuleContext { + private fun parseModule(srcText: String): Prog8ANTLRParser.ModuleContext { return parseModule(CharStreams.fromString(srcText)) } - private fun parseModule(srcFile: Path): prog8Parser.ModuleContext { + private fun parseModule(srcFile: Path): Prog8ANTLRParser.ModuleContext { return parseModule(CharStreams.fromPath(srcFile)) } - private fun parseModule(srcStream: CharStream): prog8Parser.ModuleContext { + private fun parseModule(srcStream: CharStream): Prog8ANTLRParser.ModuleContext { val errorListener = MyErrorListener() - val lexer = prog8Lexer(srcStream) + val lexer = Prog8ANTLRLexer(srcStream) lexer.removeErrorListeners() lexer.addErrorListener(errorListener) val tokens = CommonTokenStream(lexer) - val parser = prog8Parser(tokens) + val parser = Prog8ANTLRParser(tokens) parser.errorHandler = MyErrorStrategy() parser.removeErrorListeners() parser.addErrorListener(errorListener) @@ -88,7 +91,7 @@ class TestAntlrParser { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) - // before the fix, prog8Parser would have reported (thrown) "missing at ''" + // before the fix, Prog8ANTLRParser would have reported (thrown) "missing at ''" val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @@ -234,22 +237,14 @@ class TestAntlrParser { @Test fun testProg8Ast() { - // can create charstreams from many other sources as well; - val charstream = CharStreams.fromString(""" + val parseTree = parseModule(""" main { sub start() { return } } """) - val lexer = prog8Lexer(charstream) - val tokens = CommonTokenStream(lexer) - val parser = prog8Parser(tokens) - parser.errorHandler = BailErrorStrategy() -// parser.removeErrorListeners() -// parser.addErrorListener(MyErrorListener()) - - val ast = parser.module().toAst("test", Path.of(""), DummyEncoding) + val ast = parseTree.toAst("test", Path.of(""), DummyEncoding) assertIs(ast.statements.first()) assertEquals((ast.statements.first() as Block).name, "main") } diff --git a/parser/antlr/prog8.g4 b/parser/antlr/Prog8ANTLR.g4 similarity index 97% rename from parser/antlr/prog8.g4 rename to parser/antlr/Prog8ANTLR.g4 index 9a2cc4fe4..af05c0aec 100644 --- a/parser/antlr/prog8.g4 +++ b/parser/antlr/Prog8ANTLR.g4 @@ -8,7 +8,9 @@ NOTES: */ -grammar prog8; +// -> java classes Prog8ANTLRParser and Prog8ANTLRLexer, +// both NOT to be used from Kotlin code, but ONLY through Kotlin class Prog8Parser +grammar Prog8ANTLR; @header { package prog8.parser;