package prog8tests.ast import com.github.michaelbull.result.getOrElse import io.kotest.assertions.fail import io.kotest.assertions.throwables.shouldThrow import io.kotest.assertions.withClue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.or import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe import io.kotest.matchers.string.shouldContain import io.kotest.matchers.string.shouldStartWith import io.kotest.matchers.types.instanceOf import prog8.ast.IFunctionCall import prog8.ast.Module import prog8.ast.Node import prog8.ast.Program import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.code.core.* import prog8.code.target.C64Target import prog8.code.target.cbm.PetsciiEncoding import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8tests.helpers.* import kotlin.io.path.Path import kotlin.io.path.isRegularFile import kotlin.io.path.name import kotlin.io.path.nameWithoutExtension class TestProg8Parser: FunSpec( { context("Newline at end") { test("is not required - #40, fixed by #45") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val src = SourceCode.Text("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) // #40: Prog8ANTLRParser would report (throw) "missing at ''" val module = parseModule(src) module.statements.size shouldBe 1 } test("is still accepted - #40, fixed by #45") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } } context("Newline") { test("is required after each block except the last") { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" shouldThrow { parseModule(SourceCode.Text(srcBad)) } val module = parseModule(SourceCode.Text(srcGood)) module.statements.size shouldBe 2 } test("is required between two Blocks or Directives - #47") { // block and block shouldThrow{ parseModule( SourceCode.Text(""" blockA { } blockB { } """)) } // block and directive shouldThrow{ parseModule( SourceCode.Text(""" blockB { } %import textio """)) } // The following two are bogus due to directive *args* expected to follow the directive name. // Leaving them in anyways. // dir and block shouldThrow{ parseModule( SourceCode.Text(""" %import textio blockB { } """)) } shouldThrow{ parseModule( SourceCode.Text(""" %import textio %import syslib """)) } } test("can be Win, Unix or mixed, even mixed") { val nlWin = "\r\n" val nlUnix = "\n" val nlMac = "\r" //parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath()) // a good mix of all kinds of newlines: val srcText = "foo {" + nlMac + nlWin + "}" + nlMac + // <-- do test a single \r (!) where an EOL is expected "bar {" + nlUnix + "}" + nlUnix + nlMac // both should be "eaten up" by just one EOL token "combi {" + nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 2 } } context("EOLsInterleavedWithComments") { test("are ok before first block - #47") { // issue: #47 val srcText = """ ; comment ; comment blockA { } """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } test("are ok between blocks - #47") { // issue: #47 val srcText = """ blockA { } ; comment ; comment blockB { } """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 2 } test("are ok after last block - #47") { // issue: #47 val srcText = """ blockA { } ; comment ; comment """ val module = parseModule(SourceCode.Text(srcText)) module.statements.size shouldBe 1 } } context("ImportDirectives") { test("should not be looked into by the parser") { val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") assumeNotExists(fixturesDir, "i_do_not_exist.p8") val text = "%import ${importedNoExt.name}" val module = parseModule(SourceCode.Text(text)) module.statements.size shouldBe 1 } } context("EmptySourcecode") { test("from an empty string should result in empty Module") { val module = parseModule(SourceCode.Text("")) module.statements.size shouldBe 0 } test("from an empty file should result in empty Module") { val path = assumeReadableFile(fixturesDir, "ast_empty.p8") val module = parseModule(SourceCode.File(path)) module.statements.size shouldBe 0 } } context("NameOfModule") { test("parsed from a string") { val srcText = """ main { } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) // Note: assertContains has *actual* as first param module.name shouldContain Regex("^string:[0-9a-f\\-]+$") } test("parsed from a file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") val module = parseModule(SourceCode.File(path)) module.name shouldBe path.nameWithoutExtension } } context("PositionOfAstNodesAndParseErrors") { fun assertPosition( actual: Position, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) { require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) if (expLine != null) actual.line shouldBe expLine if (expStartCol != null) actual.startCol shouldBe expStartCol if (expEndCol != null) actual.endCol shouldBe expEndCol if (expFile != null) actual.file shouldBe expFile } fun assertPosition( actual: Position, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) { require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) if (expLine != null) actual.line shouldBe expLine if (expStartCol != null) actual.startCol shouldBe expStartCol if (expEndCol != null) actual.endCol shouldBe expEndCol if (expFile != null) actual.file shouldContain expFile } fun assertPositionOf( actual: Node, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) fun assertPositionOf( actual: Node, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null ) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) test("in ParseError from bad string source code") { val srcText = "bad * { }\n" val e = shouldThrow { parseModule(SourceCode.Text(srcText)) } assertPosition(e.position, Regex("^string:[0-9a-f\\-]+$"), 1, 4, 4) } test("in ParseError from bad file source code") { val path = assumeReadableFile(fixturesDir, "ast_file_with_syntax_error.p8") val e = shouldThrow { parseModule(SourceCode.File(path)) } assertPosition(e.position, SourceCode.relative(path).toString(), 2, 6) } test("of Module parsed from a string") { val srcText = """ main { } """ val module = parseModule(SourceCode.Text(srcText)) assertPositionOf(module, Regex("^string:[0-9a-f\\-]+$"), 1, 0) } test("of Module parsed from a file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") val module = parseModule(SourceCode.File(path)) assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) } test("of non-root Nodes parsed from file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") val module = parseModule(SourceCode.File(path)) val mpf = module.position.file assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) val mainBlock = module.statements.filterIsInstance()[0] assertPositionOf(mainBlock, mpf, 2, 1, 4) val startSub = mainBlock.statements.filterIsInstance()[0] assertPositionOf(startSub, mpf, 3, 5, 7) } test("of non-root Nodes parsed from a string") { val srcText = """ %zeropage basicsafe main { sub start() { ubyte foo = 42 ubyte bar when foo { 23 -> bar = 'x' 42 -> bar = 'y' else -> bar = 'z' } } } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) val mpf = module.position.file val targetDirective = module.statements.filterIsInstance()[0] assertPositionOf(targetDirective, mpf, 1, 1, 9) val mainBlock = module.statements.filterIsInstance()[0] assertPositionOf(mainBlock, mpf, 2, 1, 4) val startSub = mainBlock.statements.filterIsInstance()[0] assertPositionOf(startSub, mpf, 3, 5, 7) val declFoo = startSub.statements.filterIsInstance()[0] assertPositionOf(declFoo, mpf, 4, 9, 13) val rhsFoo = declFoo.value!! assertPositionOf(rhsFoo, mpf, 4, 21, 22) val declBar = startSub.statements.filterIsInstance()[1] assertPositionOf(declBar, mpf, 5, 9, 13) val whenStmt = startSub.statements.filterIsInstance()[0] assertPositionOf(whenStmt, mpf, 6, 9, 12) assertPositionOf(whenStmt.choices[0], mpf, 7, 13, 14) assertPositionOf(whenStmt.choices[1], mpf, 8, 13, 14) assertPositionOf(whenStmt.choices[2], mpf, 9, 13, 16) } } context("PositionFile") { fun assertSomethingForAllNodes(module: Module, asserter: (Node) -> Unit) { asserter(module) module.statements.forEach(asserter) module.statements.filterIsInstance().forEach { b -> asserter(b) b.statements.forEach(asserter) b.statements.filterIsInstance().forEach { s -> asserter(s) s.statements.forEach(asserter) } } } test("isn't absolute for filesystem paths") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") val module = parseModule(SourceCode.File(path)) assertSomethingForAllNodes(module) { Path(it.position.file).isAbsolute shouldBe false Path(it.position.file).isRegularFile() shouldBe true } } test("is mangled string id for string sources") { val srcText=""" %zeropage basicsafe main { sub start() { ubyte aa=99 aa++ } } """.trimIndent() val module = parseModule(SourceCode.Text(srcText)) assertSomethingForAllNodes(module) { it.position.file shouldStartWith SourceCode.stringSourcePrefix } } test("is library prefixed path for resources") { val resource = SourceCode.Resource("prog8lib/math.p8") val module = parseModule(resource) assertSomethingForAllNodes(module) { it.position.file shouldStartWith SourceCode.libraryFilePrefix } } } context("CharLiterals") { test("in argument position, no altEnc") { val src = SourceCode.Text(""" main { sub start() { chrout('\n') } } """) val module = parseModule(src) val startSub = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val funCall = startSub.statements.filterIsInstance().first() funCall.args[0] shouldBe(instanceOf()) val char = funCall.args[0] as CharLiteral char.value shouldBe '\n' } test("on rhs of block-level var decl, default encoding") { val src = SourceCode.Text(""" main { ubyte c = 'x' } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.encoding shouldBe Encoding.DEFAULT rhs.value shouldBe 'x' } test("on rhs of block-level const decl, with screencode enc") { val src = SourceCode.Text(""" main { const ubyte c = sc:'x' } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.encoding shouldBe Encoding.SCREENCODES rhs.value shouldBe 'x' } test("on rhs of block-level const decl, with iso encoding") { val src = SourceCode.Text(""" main { const ubyte c = iso:'_' } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.encoding shouldBe Encoding.ISO rhs.value shouldBe '_' } test("on rhs of subroutine-level var decl, default encoding") { val src = SourceCode.Text(""" main { sub start() { ubyte c = 'x' } } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.value shouldBe 'x' rhs.encoding shouldBe Encoding.DEFAULT } test("on rhs of subroutine-level const decl, screencode encoded") { val src = SourceCode.Text(""" main { sub start() { const ubyte c = sc:'x' } } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.encoding shouldBe Encoding.SCREENCODES rhs.value shouldBe 'x' } test("on rhs of subroutine-level const decl, iso encoding") { val src = SourceCode.Text(""" main { sub start() { const ubyte c = iso:'_' } } """) val module = parseModule(src) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as CharLiteral rhs.encoding shouldBe Encoding.ISO rhs.value shouldBe '_' } } context("Strings") { test("default encoding") { val source = """ main { str name = "name" }""" val module = parseModule(SourceCode.Text(source)) val decl = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] val rhs = decl.value as StringLiteral rhs.encoding shouldBe Encoding.DEFAULT rhs.value shouldBe "name" } test("string encodings") { val source = """ main { str name1 = petscii:"Name" str name2 = sc:"Name" str name3 = iso:"Name" }""" val module = parseModule(SourceCode.Text(source)) val (decl1, decl2, decl3) = module .statements.filterIsInstance()[0] .statements.filterIsInstance() val rhs1 = decl1.value as StringLiteral val rhs2 = decl2.value as StringLiteral val rhs3 = decl3.value as StringLiteral rhs1.encoding shouldBe Encoding.PETSCII rhs1.value shouldBe "Name" rhs2.encoding shouldBe Encoding.SCREENCODES rhs2.value shouldBe "Name" rhs3.encoding shouldBe Encoding.ISO rhs3.value shouldBe "Name" } } context("Ranges") { test("in for-loops") { val module = parseModule( SourceCode.Text(""" main { sub start() { ubyte ub for ub in "start" downto "end" { ; #0 } for ub in "something" { ; #1 } for ub in sc:'a' to 'f' { ; #2 } for ub in false to true { ; #3 } for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check! } } } """)) val iterables = module .statements.filterIsInstance()[0] .statements.filterIsInstance()[0] .statements.filterIsInstance() .map { it.iterable } iterables.size shouldBe 5 val it0 = iterables[0] as RangeExpression it0.from shouldBe instanceOf() it0.to shouldBe instanceOf() val it1 = iterables[1] as StringLiteral it1.value shouldBe "something" val it2 = iterables[2] as RangeExpression it2.from shouldBe instanceOf() it2.to shouldBe instanceOf() val it3 = iterables[3] as RangeExpression it3.from shouldBe instanceOf() it3.to shouldBe instanceOf() val it4 = iterables[4] as RangeExpression it4.from shouldBe instanceOf() it4.to shouldBe instanceOf() } } test("testCharLiteralConstValue") { val char1 = CharLiteral('A', Encoding.PETSCII, Position.DUMMY) val char2 = CharLiteral('z', Encoding.SCREENCODES, Position.DUMMY) val char3 = CharLiteral('_', Encoding.ISO, Position.DUMMY) val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder) char1.constValue(program).number.toInt() shouldBe 65 char2.constValue(program).number.toInt() shouldBe 122 char3.constValue(program).number.toInt() shouldBe 95 } test("testLiteralValueComparisons") { val ten = NumericLiteral(DataType.UWORD, 10.0, Position.DUMMY) val nine = NumericLiteral(DataType.UBYTE, 9.0, Position.DUMMY) ten shouldBe ten nine shouldNotBe ten (ten != ten) shouldBe false (ten != nine) shouldBe true (ten > nine) shouldBe true (ten >= nine) shouldBe true (ten >= ten) shouldBe true (ten > ten) shouldBe false (ten < nine) shouldBe false (ten <= nine) shouldBe false (ten <= ten) shouldBe true (ten < ten) shouldBe false val abc = StringLiteral("abc", Encoding.PETSCII, Position.DUMMY) val abd = StringLiteral("abd", Encoding.PETSCII, Position.DUMMY) abc shouldBe abc (abc!=abd) shouldBe true (abc!=abc) shouldBe false } test("testAnonScopeStillContainsVarsDirectlyAfterParse") { val src = SourceCode.Text(""" main { sub start() { repeat { ubyte xx = 99 xx++ } } } """) val module = parseModule(src) val mainBlock = module.statements.single() as Block val start = mainBlock.statements.single() as Subroutine val repeatbody = (start.statements.single() as RepeatLoop).body withClue("no vars moved to main block") { mainBlock.statements.any { it is VarDecl } shouldBe false } withClue("no vars moved to start sub") { start.statements.any { it is VarDecl } shouldBe false } withClue("\"var is still in repeat block (anonymousscope") { repeatbody.statements[0] shouldBe instanceOf() } val initvalue = (repeatbody.statements[0] as VarDecl).value as? NumericLiteral initvalue?.number?.toInt() shouldBe 99 repeatbody.statements[1] shouldBe instanceOf() // the ast processing steps used in the compiler, will eventually move the var up to the containing scope (subroutine). } test("testLabelsWithAnonScopesParsesFine") { val src = SourceCode.Text(""" main { sub start() { goto mylabeloutside if true { if true { goto labeloutside goto iflabel } iflabel: } repeat { goto labelinside labelinside: } labeloutside: } } """) val module = parseModule(src) val mainBlock = module.statements.single() as Block val start = mainBlock.statements.single() as Subroutine val labels = start.statements.filterIsInstance