package prog8tests.ast import io.kotest.assertions.fail import io.kotest.assertions.throwables.shouldThrow import io.kotest.assertions.withClue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.or import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe import io.kotest.matchers.string.shouldContain import io.kotest.matchers.string.shouldStartWith import io.kotest.matchers.types.instanceOf import prog8.ast.IFunctionCall import prog8.ast.Module import prog8.ast.Node import prog8.ast.Program import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode import prog8tests.ast.helpers.* import prog8tests.ast.helpers.DummyFunctions import prog8tests.ast.helpers.DummyMemsizer import kotlin.io.path.Path import kotlin.io.path.isRegularFile import kotlin.io.path.name import kotlin.io.path.nameWithoutExtension class TestProg8Parser: FunSpec( { context("Newline at end") { test("is not required - #40, fixed by #45") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val src = SourceCode.Text("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) // #40: Prog8ANTLRParser would report (throw) "missing at ''" val module = parseModule(src) module.statements.size shouldBe 1 } test("is still accepted - #40, fixed by #45") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } } context("Newline") { test("is required after each block except the last") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" shouldThrow { parseModule(SourceCode.Text(srcBad)) } val module = parseModule(SourceCode.Text(srcGood)) module.statements.size shouldBe 2 } test("is required between two Blocks or Directives - #47") { // block and block shouldThrow{ parseModule(SourceCode.Text(""" blockA { } blockB { } """)) } // block and directive shouldThrow{ parseModule(SourceCode.Text(""" blockB { } %import textio """)) } // The following two are bogus due to directive *args* expected to follow the directive name. // Leaving them in anyways. // dir and block shouldThrow{ parseModule(SourceCode.Text(""" %import textio blockB { } """)) } shouldThrow{ parseModule(SourceCode.Text(""" %import textio %import syslib """)) } } test("can be Win, Unix or mixed, even mixed") { val nlWin = "\r\n" val nlUnix = "\n" val nlMac = "\r" //parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath()) // a good mix of all kinds of newlines: val srcText = "foo {" + nlMac + nlWin + "}" + nlMac + // <-- do test a single \r (!) where an EOL is expected "bar {" + nlUnix + "}" + nlUnix + nlMac // both should be "eaten up" by just one EOL token "combi {" + nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 2 } } context("EOLsInterleavedWithComments") { test("are ok before first block - #47") { // issue: #47 val srcText = """ ; comment ; comment blockA { } """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } test("are ok between blocks - #47") { // issue: #47 val srcText = """ blockA { } ; comment ; comment blockB { } """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 2 } test("are ok after last block - #47") { // issue: #47 val srcText = """ blockA { } ; comment ; comment """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } } context("ImportDirectives") { test("should not be looked into by the parser") { val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") assumeNotExists(fixturesDir, "i_do_not_exist.p8") val text = "%import ${importedNoExt.name}" val module = parseModule(SourceCode.Text(text)) module.statements.size shouldBe 1 } } context("EmptySourcecode") { test("from an empty string should result in empty Module") { val module = parseModule(SourceCode.Text("")) module.statements.size shouldBe 0 } test("from an empty file should result in empty Module") { val path = assumeReadableFile(fixturesDir, "empty.p8") val module = parseModule(SourceCode.File(path)) module.statements.size shouldBe 0 } } context("NameOfModule") { test("parsed from a string") { val srcText = """ main { } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) // Note: assertContains has *actual* as first param module.name shouldContain Regex("^$") } test("parsed from a file") { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.File(path)) module.name shouldBe path.nameWithoutExtension } } context("PositionOfAstNodesAndParseErrors") { fun assertPosition( actual: Position, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) { require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) if (expLine != null) actual.line shouldBe expLine if (expStartCol != null) actual.startCol shouldBe expStartCol if (expEndCol != null) actual.endCol shouldBe expEndCol if (expFile != null) actual.file shouldBe expFile } fun assertPosition( actual: Position, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) { require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) if (expLine != null) actual.line shouldBe expLine if (expStartCol != null) actual.startCol shouldBe expStartCol if (expEndCol != null) actual.endCol shouldBe expEndCol if (expFile != null) actual.file shouldContain expFile } fun assertPositionOf( actual: Node, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) fun assertPositionOf( actual: Node, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) test("in ParseError from bad string source code") { val srcText = "bad * { }\n" val e = shouldThrow { parseModule(SourceCode.Text(srcText)) } assertPosition(e.position, Regex("^$"), 1, 4, 4) } test("in ParseError from bad file source code") { val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") val e = shouldThrow { parseModule(SourceCode.File(path)) } assertPosition(e.position, SourceCode.relative(path).toString(), 2, 6) } test("of Module parsed from a string") { val srcText = """ main { } """ val module = parseModule(SourceCode.Text(srcText)) assertPositionOf(module, Regex("^$"), 1, 0) } test("of Module parsed from a file") { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.File(path)) assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) } test("of non-root Nodes parsed from file") { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.File(path)) val mpf = module.position.file assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) val mainBlock = module.statements.filterIsInstance()[0] assertPositionOf(mainBlock, mpf, 2, 0, 3) val startSub = mainBlock.statements.filterIsInstance()[0] assertPositionOf(startSub, mpf, 3, 4, 6) } test("of non-root Nodes parsed from a string") { val srcText = """ %zeropage basicsafe main { sub start() { ubyte foo = 42 ubyte bar when (foo) { 23 -> bar = 'x' 42 -> bar = 'y' else -> bar = 'z' } } } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) val mpf = module.position.file val targetDirective = module.statements.filterIsInstance()[0] assertPositionOf(targetDirective, mpf, 1, 0, 8) val mainBlock = module.statements.filterIsInstance()[0] assertPositionOf(mainBlock, mpf, 2, 0, 3) val startSub = mainBlock.statements.filterIsInstance()[0] assertPositionOf(startSub, mpf, 3, 4, 6) val declFoo = startSub.statements.filterIsInstance()[0] assertPositionOf(declFoo, mpf, 4, 8, 12) val rhsFoo = declFoo.value!! assertPositionOf(rhsFoo, mpf, 4, 20, 21) val declBar = startSub.statements.filterIsInstance()[1] assertPositionOf(declBar, mpf, 5, 8, 12) val whenStmt = startSub.statements.filterIsInstance()[0] assertPositionOf(whenStmt, mpf, 6, 8, 11) assertPositionOf(whenStmt.choices[0], mpf, 7, 12, 13) assertPositionOf(whenStmt.choices[1], mpf, 8, 12, 13) assertPositionOf(whenStmt.choices[2], mpf, 9, 12, 15) } } context("PositionFile") { fun assertSomethingForAllNodes(module: Module, asserter: (Node) -> Unit) { asserter(module) module.statements.forEach(asserter) module.statements.filterIsInstance().forEach { b -> asserter(b) b.statements.forEach(asserter) b.statements.filterIsInstance().forEach { s -> asserter(s) s.statements.forEach(asserter) } } } test("isn't absolute for filesystem paths") { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.File(path)) assertSomethingForAllNodes(module) { Path(it.position.file).isAbsolute shouldBe false Path(it.position.file).isRegularFile() shouldBe true } } test("is mangled string id for string sources") { val srcText=""" %zeropage basicsafe main { sub start() { ubyte aa=99 aa++ } } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) assertSomethingForAllNodes(module) { it.position.file shouldStartWith SourceCode.stringSourcePrefix } } test("is library prefixed path for resources") { val resource = SourceCode.Resource("prog8lib/math.p8") val module = parseModule(resource) assertSomethingForAllNodes(module) { it.position.file shouldStartWith SourceCode.libraryFilePrefix } } } context("CharLiterals") { test("in argument position, no altEnc") { val src = SourceCode.Text(""" main { sub start() { chrout('\n') } } """) val module = parseModule(src) val startSub = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val funCall = startSub.statements.filterIsInstance().first() funCall.args[0] shouldBe(instanceOf()) val char = funCall.args[0] as CharLiteral char.value shouldBe '\n' } test("on rhs of block-level var decl, no AltEnc") { val src = SourceCode.Text(""" main { ubyte c = 'x' } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.value shouldBe 'x' rhs.altEncoding shouldBe false } test("on rhs of block-level const decl, with AltEnc") { val src = SourceCode.Text(""" main { const ubyte c = @'x' } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.value shouldBe 'x' rhs.altEncoding shouldBe true } test("on rhs of subroutine-level var decl, no AltEnc") { val src = SourceCode.Text(""" main { sub start() { ubyte c = 'x' } } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.value shouldBe 'x' rhs.altEncoding shouldBe false } test("on rhs of subroutine-level const decl, with AltEnc") { val src = SourceCode.Text(""" main { sub start() { const ubyte c = @'x' } } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.value shouldBe 'x' rhs.altEncoding shouldBe true } } context("Ranges") { test("in for-loops") { val module = parseModule(SourceCode.Text(""" main { sub start() { ubyte ub for ub in "start" downto "end" { ; #0 } for ub in "something" { ; #1 } for ub in @'a' to 'f' { ; #2 } for ub in false to true { ; #3 } for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check! } } } """)) val iterables = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance() .map { it.iterable } iterables.size shouldBe 5 val it0 = iterables[0] as RangeExpr it0.from shouldBe instanceOf() it0.to shouldBe instanceOf() val it1 = iterables[1] as StringLiteralValue it1.value shouldBe "something" val it2 = iterables[2] as RangeExpr it2.from shouldBe instanceOf() it2.to shouldBe instanceOf() val it3 = iterables[3] as RangeExpr it3.from shouldBe instanceOf() it3.to shouldBe instanceOf() val it4 = iterables[4] as RangeExpr it4.from shouldBe instanceOf() it4.to shouldBe instanceOf() } } test("testCharLiteralConstValue") { val char1 = CharLiteral('A', false, Position.DUMMY) val char2 = CharLiteral('z', true, Position.DUMMY) val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder) char1.constValue(program).number.toInt() shouldBe 65 char2.constValue(program).number.toInt() shouldBe 122 } test("testLiteralValueComparisons") { val ten = NumericLiteralValue(DataType.UWORD, 10.0, Position.DUMMY) val nine = NumericLiteralValue(DataType.UBYTE, 9.0, Position.DUMMY) ten shouldBe ten nine shouldNotBe ten (ten != ten) shouldBe false (ten != nine) shouldBe true (ten > nine) shouldBe true (ten >= nine) shouldBe true (ten >= ten) shouldBe true (ten > ten) shouldBe false (ten < nine) shouldBe false (ten <= nine) shouldBe false (ten <= ten) shouldBe true (ten < ten) shouldBe false val abc = StringLiteralValue("abc", false, Position.DUMMY) val abd = StringLiteralValue("abd", false, Position.DUMMY) abc shouldBe abc (abc!=abd) shouldBe true (abc!=abc) shouldBe false } test("testAnonScopeStillContainsVarsDirectlyAfterParse") { val src = SourceCode.Text(""" main { sub start() { repeat { ubyte xx = 99 xx++ } } } """) val module = parseModule(src) val mainBlock = module.statements.single() as Block val start = mainBlock.statements.single() as Subroutine val repeatbody = (start.statements.single() as RepeatLoop).body withClue("no vars moved to main block") { mainBlock.statements.any { it is VarDecl } shouldBe false } withClue("no vars moved to start sub") { start.statements.any { it is VarDecl } shouldBe false } withClue("\"var is still in repeat block (anonymousscope") { repeatbody.statements[0] shouldBe instanceOf() } val initvalue = (repeatbody.statements[0] as VarDecl).value as? NumericLiteralValue initvalue?.number?.toInt() shouldBe 99 repeatbody.statements[1] shouldBe instanceOf() // the ast processing steps used in the compiler, will eventually move the var up to the containing scope (subroutine). } test("testLabelsWithAnonScopesParsesFine") { val src = SourceCode.Text(""" main { sub start() { goto mylabeloutside if true { if true { goto labeloutside goto iflabel } iflabel: } repeat { goto labelinside labelinside: } labeloutside: } } """) val module = parseModule(src) val mainBlock = module.statements.single() as Block val start = mainBlock.statements.single() as Subroutine val labels = start.statements.filterIsInstance