From 137a89da15b56c408efd47f716a8fa7145ec9e39 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 4 Jul 2021 18:09:16 +0200 Subject: [PATCH] * fix (hack) .name, .source and .position of Modules from the parser (via temp. subclass ParsedModule) The temporary subclass ParsedModule : Module is introduced to concentrate all the workaround stuff in one place *while still not changing any public signature* such as of the Module ctor. The convention used to indicate stuff from resources is still "" not "@embedded@"- *note that this is caught by 3 tests in compiler* --- compilerAst/src/prog8/ast/AstToplevel.kt | 2 +- .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 4 +- compilerAst/src/prog8/parser/Prog8Parser.kt | 87 ++++++++++++------ compilerAst/test/TestProg8Parser.kt | 89 ++++++++++++++++--- 4 files changed, 141 insertions(+), 41 deletions(-) diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 216d0bcb5..1d57f8508 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -339,7 +339,7 @@ class Program(val name: String, } } -class Module(override val name: String, +open class Module(override val name: String, override var statements: MutableList, override val position: Position, val source: Path) : Node, INameScope { diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index 0cf6e7eac..d7abc240d 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -41,7 +41,7 @@ private fun ParserRuleContext.toPosition() : Position { return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine + stop.text.length) } -private fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { +internal fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Block { val blockstatements = block_statement().map { when { it.variabledeclaration()!=null -> it.variabledeclaration().toAst(encoding) @@ -351,7 +351,7 @@ private fun Prog8ANTLRParser.DatatypeContext.toAst() = DataType.valueOf(text.upp private fun Prog8ANTLRParser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = ArrayIndex(expression().toAst(encoding), toPosition()) -private fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = +internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = Directive(directivename.text, directivearg().map { it.toAst() }, toPosition()) private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index bc7182a4a..35901b82e 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -4,6 +4,8 @@ import org.antlr.v4.runtime.* import prog8.ast.Module import prog8.ast.antlr.toAst import prog8.ast.base.Position +import prog8.ast.statements.Block +import prog8.ast.statements.Directive import kotlin.io.path.Path @@ -16,19 +18,10 @@ class ParseError(override var message: String, val position: Position, cause: Ru } } -private fun RecognitionException.getPosition(provenance: String) : Position { - val offending = this.offendingToken - val line = offending.line - val beginCol = offending.charPositionInLine - val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? - val pos = Position(provenance, line, beginCol, endCol) - return pos -} - object Prog8Parser { fun parseModule(src: SourceCode): Module { - val antlrErrorListener = AntlrErrorListener(src.origin) + val antlrErrorListener = AntlrErrorListener(src) val lexer = Prog8ANTLRLexer(src.getCharStream()) lexer.removeErrorListeners() lexer.addErrorListener(antlrErrorListener) @@ -40,24 +33,55 @@ object Prog8Parser { val parseTree = parser.module() - // FIXME: hacking together a name for the module: - var moduleName = src.origin - if (moduleName.startsWith("") + + private class ParsedModule(src: SourceCode) : Module( + // FIXME: hacking together a name for the module: + name = src.pathString() + .substringBeforeLast(".") + .substringAfterLast("/") + .substringAfterLast("\\") + .replace("String@", "anonymous_"), + // FIXME: hacking together a path + source = Path(src.pathString()), + statements = mutableListOf(), + position = Position(src.origin, 1, 0, 0) + ) { + val provenance = Pair(src, Triple(1, 0, 0)) + + /** + * Adds a [Directive] to [statements] and + * sets this Module as its [parent]. + * Note: you can only add [Directive]s or [Block]s to a Module. + */ + fun add(child: Directive) { + child.linkParents(this) + statements.add(child) + } + /** + * Adds a [Block] to [statements] and + * sets this Module as its [parent]. + * Note: you can only add [Directive]s or [Block]s to a Module. + */ + fun add(child: Block) { + child.linkParents(this) + statements.add(child) + } } private object Prog8ErrorStrategy: BailErrorStrategy() { @@ -86,15 +110,24 @@ object Prog8Parser { } } - private class AntlrErrorListener(val sourceCodeProvenance: String): BaseErrorListener() { + private class AntlrErrorListener(val src: SourceCode): BaseErrorListener() { override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { if (e == null) { TODO("no RecognitionException - create your own ParseError") //throw ParseError() } else { - throw ParseError(msg, e.getPosition(sourceCodeProvenance), e) + throw ParseError(msg, e.getPosition(src.origin), e) } } } + private fun RecognitionException.getPosition(file: String) : Position { + val offending = this.offendingToken + val line = offending.line + val beginCol = offending.charPositionInLine + val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? + val pos = Position(file, line, beginCol, endCol) + return pos + } + } diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 381feffa6..a780643cc 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,16 +1,27 @@ package prog8tests import org.junit.jupiter.api.Test +import prog8.ast.Node +import prog8.ast.base.Position import kotlin.test.* import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* import prog8.ast.statements.Block +import prog8.ast.statements.Directive import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode class TestProg8Parser { + val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! + val fixturesDir = workingDir.resolve("test/fixtures") + + @Test + fun testDirectoriesSanityCheck() { + assertEquals("compilerAst", workingDir.fileName.toString()) + assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") + } @Test fun testModuleSourceNeedNotEndWithNewline() { @@ -165,6 +176,51 @@ class TestProg8Parser { assertEquals(1, module.statements.size) } + + @Test + fun testModuleNameForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + + // Note: assertContains has *actual* as first param + assertContains(module.name, Regex("^anonymous_[0-9a-f]+$")) + } + + @Test + fun testModuleNameForSourceFromPath() { + val path = fixturesDir.resolve("simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + + assertEquals(path.nameWithoutExtension, module.name) + } + + + fun assertPosition(actual: Position, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) { + assertEquals(expLine, actual.line, ".position.line (1-based)") + assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + assertEquals(expFile, actual.file, ".position.file") + } + + fun assertPosition(actual: Position, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) { + assertEquals(expLine, actual.line, ".position.line (1-based)") + assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + // Note: assertContains expects *actual* value first + assertContains(actual.file, expFile, ".position.file") + } + + fun assertPositionOf(actual: Node, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + fun assertPositionOf(actual: Node, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + @Test fun testErrorLocationForSourceFromString() { val srcText = "bad * { }\n" @@ -173,30 +229,41 @@ class TestProg8Parser { try { parseModule(SourceCode.of(srcText)) } catch (e: ParseError) { - // Note: assertContains expects *actual* value first - assertContains(e.position.file, Regex("^$")) - assertEquals(1, e.position.line, "line; should be 1-based") - assertEquals(4, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(4, e.position.endCol, "endCol; should be 0-based") + assertPosition(e.position, Regex("^$"), 1, 4, 4) } } @Test fun testErrorLocationForSourceFromPath() { - val filename = "file_with_syntax_error.p8" - val path = Path.of("test", "fixtures", filename) + val path = fixturesDir.resolve("file_with_syntax_error.p8") assertFailsWith { parseModule(SourceCode.fromPath(path)) } try { parseModule(SourceCode.fromPath(path)) } catch (e: ParseError) { - assertEquals(path.absolutePathString(), e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + assertPosition(e.position, path.absolutePathString(), 2, 6, 6) } } + @Test + fun testModulePositionForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + assertPositionOf(module, Regex("^$"), 1, 0, 0) + } + + @Test + fun testModulePositionForSourceFromPath() { + val path = fixturesDir.resolve("simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + assertPositionOf(module, path.absolutePathString(), 1, 0, 0) + } + + @Test fun testProg8Ast() { val module = parseModule(SourceCode.of("""