prog8/compilerAst/test/TestProg8Parser.kt
2021-11-18 01:05:16 +01:00

776 lines
28 KiB
Kotlin

package prog8tests.ast
import io.kotest.assertions.fail
import io.kotest.assertions.throwables.shouldThrow
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.or
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import io.kotest.matchers.string.shouldStartWith
import io.kotest.matchers.types.instanceOf
import prog8.ast.IFunctionCall
import prog8.ast.Module
import prog8.ast.Node
import prog8.ast.Program
import prog8.ast.base.DataType
import prog8.ast.base.Position
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.parser.ParseError
import prog8.parser.Prog8Parser.parseModule
import prog8.parser.SourceCode
import prog8tests.ast.helpers.*
import prog8tests.ast.helpers.DummyFunctions
import prog8tests.ast.helpers.DummyMemsizer
import kotlin.io.path.Path
import kotlin.io.path.isRegularFile
import kotlin.io.path.name
import kotlin.io.path.nameWithoutExtension
class TestProg8Parser: FunSpec( {
context("Newline at end") {
test("is not required - #40, fixed by #45") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val src = SourceCode.Text("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40)
// #40: Prog8ANTLRParser would report (throw) "missing <EOL> at '<EOF>'"
val module = parseModule(src)
module.statements.size shouldBe 1
}
test("is still accepted - #40, fixed by #45") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40)
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
}
context("Newline") {
test("is required after each block except the last") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
// BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end
val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl
// GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed
val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}"
shouldThrow<ParseError> { parseModule(SourceCode.Text(srcBad)) }
val module = parseModule(SourceCode.Text(srcGood))
module.statements.size shouldBe 2
}
test("is required between two Blocks or Directives - #47") {
// block and block
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
blockA {
} blockB {
}
""")) }
// block and directive
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
blockB {
} %import textio
""")) }
// The following two are bogus due to directive *args* expected to follow the directive name.
// Leaving them in anyways.
// dir and block
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
%import textio blockB {
}
""")) }
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
%import textio %import syslib
""")) }
}
test("can be Win, Unix or mixed, even mixed") {
val nlWin = "\r\n"
val nlUnix = "\n"
val nlMac = "\r"
//parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath())
// a good mix of all kinds of newlines:
val srcText =
"foo {" +
nlMac +
nlWin +
"}" +
nlMac + // <-- do test a single \r (!) where an EOL is expected
"bar {" +
nlUnix +
"}" +
nlUnix + nlMac // both should be "eaten up" by just one EOL token
"combi {" +
nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token
"}" +
nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline)
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 2
}
}
context("EOLsInterleavedWithComments") {
test("are ok before first block - #47") {
// issue: #47
val srcText = """
; comment
; comment
blockA {
}
"""
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
test("are ok between blocks - #47") {
// issue: #47
val srcText = """
blockA {
}
; comment
; comment
blockB {
}
"""
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 2
}
test("are ok after last block - #47") {
// issue: #47
val srcText = """
blockA {
}
; comment
; comment
"""
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
}
context("ImportDirectives") {
test("should not be looked into by the parser") {
val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist")
assumeNotExists(fixturesDir, "i_do_not_exist.p8")
val text = "%import ${importedNoExt.name}"
val module = parseModule(SourceCode.Text(text))
module.statements.size shouldBe 1
}
}
context("EmptySourcecode") {
test("from an empty string should result in empty Module") {
val module = parseModule(SourceCode.Text(""))
module.statements.size shouldBe 0
}
test("from an empty file should result in empty Module") {
val path = assumeReadableFile(fixturesDir, "empty.p8")
val module = parseModule(SourceCode.File(path))
module.statements.size shouldBe 0
}
}
context("NameOfModule") {
test("parsed from a string") {
val srcText = """
main {
}
""".trimIndent()
val module = parseModule(SourceCode.Text(srcText))
// Note: assertContains has *actual* as first param
module.name shouldContain Regex("^<String@[0-9a-f\\-]+>$")
}
test("parsed from a file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
val module = parseModule(SourceCode.File(path))
module.name shouldBe path.nameWithoutExtension
}
}
context("PositionOfAstNodesAndParseErrors") {
fun assertPosition(
actual: Position,
expFile: String? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) {
require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
if (expLine != null) actual.line shouldBe expLine
if (expStartCol != null) actual.startCol shouldBe expStartCol
if (expEndCol != null) actual.endCol shouldBe expEndCol
if (expFile != null) actual.file shouldBe expFile
}
fun assertPosition(
actual: Position,
expFile: Regex? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) {
require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
if (expLine != null) actual.line shouldBe expLine
if (expStartCol != null) actual.startCol shouldBe expStartCol
if (expEndCol != null) actual.endCol shouldBe expEndCol
if (expFile != null) actual.file shouldContain expFile
}
fun assertPositionOf(
actual: Node,
expFile: String? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) =
assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
fun assertPositionOf(
actual: Node,
expFile: Regex? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) =
assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
test("in ParseError from bad string source code") {
val srcText = "bad * { }\n"
val e = shouldThrow<ParseError> { parseModule(SourceCode.Text(srcText)) }
assertPosition(e.position, Regex("^<String@[0-9a-f\\-]+>$"), 1, 4, 4)
}
test("in ParseError from bad file source code") {
val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8")
val e = shouldThrow<ParseError> { parseModule(SourceCode.File(path)) }
assertPosition(e.position, SourceCode.relative(path).toString(), 2, 6)
}
test("of Module parsed from a string") {
val srcText = """
main {
}
"""
val module = parseModule(SourceCode.Text(srcText))
assertPositionOf(module, Regex("^<String@[0-9a-f\\-]+>$"), 1, 0)
}
test("of Module parsed from a file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
val module = parseModule(SourceCode.File(path))
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
}
test("of non-root Nodes parsed from file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
val module = parseModule(SourceCode.File(path))
val mpf = module.position.file
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
val mainBlock = module.statements.filterIsInstance<Block>()[0]
assertPositionOf(mainBlock, mpf, 2, 0, 3)
val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
assertPositionOf(startSub, mpf, 3, 4, 6)
}
test("of non-root Nodes parsed from a string") {
val srcText = """
%zeropage basicsafe
main {
sub start() {
ubyte foo = 42
ubyte bar
when (foo) {
23 -> bar = 'x'
42 -> bar = 'y'
else -> bar = 'z'
}
}
}
""".trimIndent()
val module = parseModule(SourceCode.Text(srcText))
val mpf = module.position.file
val targetDirective = module.statements.filterIsInstance<Directive>()[0]
assertPositionOf(targetDirective, mpf, 1, 0, 8)
val mainBlock = module.statements.filterIsInstance<Block>()[0]
assertPositionOf(mainBlock, mpf, 2, 0, 3)
val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
assertPositionOf(startSub, mpf, 3, 4, 6)
val declFoo = startSub.statements.filterIsInstance<VarDecl>()[0]
assertPositionOf(declFoo, mpf, 4, 8, 12)
val rhsFoo = declFoo.value!!
assertPositionOf(rhsFoo, mpf, 4, 20, 21)
val declBar = startSub.statements.filterIsInstance<VarDecl>()[1]
assertPositionOf(declBar, mpf, 5, 8, 12)
val whenStmt = startSub.statements.filterIsInstance<WhenStatement>()[0]
assertPositionOf(whenStmt, mpf, 6, 8, 11)
assertPositionOf(whenStmt.choices[0], mpf, 7, 12, 13)
assertPositionOf(whenStmt.choices[1], mpf, 8, 12, 13)
assertPositionOf(whenStmt.choices[2], mpf, 9, 12, 15)
}
}
context("PositionFile") {
fun assertSomethingForAllNodes(module: Module, asserter: (Node) -> Unit) {
asserter(module)
module.statements.forEach(asserter)
module.statements.filterIsInstance<Block>().forEach { b ->
asserter(b)
b.statements.forEach(asserter)
b.statements.filterIsInstance<Subroutine>().forEach { s ->
asserter(s)
s.statements.forEach(asserter)
}
}
}
test("isn't absolute for filesystem paths") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
val module = parseModule(SourceCode.File(path))
assertSomethingForAllNodes(module) {
Path(it.position.file).isAbsolute shouldBe false
Path(it.position.file).isRegularFile() shouldBe true
}
}
test("is mangled string id for string sources")
{
val srcText="""
%zeropage basicsafe
main {
sub start() {
ubyte aa=99
aa++
}
}
""".trimIndent()
val module = parseModule(SourceCode.Text(srcText))
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.stringSourcePrefix
}
}
test("is library prefixed path for resources")
{
val resource = SourceCode.Resource("prog8lib/math.p8")
val module = parseModule(resource)
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.libraryFilePrefix
}
}
}
context("CharLiterals") {
test("in argument position, no altEnc") {
val src = SourceCode.Text("""
main {
sub start() {
chrout('\n')
}
}
""")
val module = parseModule(src)
val startSub = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
val funCall = startSub.statements.filterIsInstance<IFunctionCall>().first()
funCall.args[0] shouldBe(instanceOf<CharLiteral>())
val char = funCall.args[0] as CharLiteral
char.value shouldBe '\n'
}
test("on rhs of block-level var decl, no AltEnc") {
val src = SourceCode.Text("""
main {
ubyte c = 'x'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
}
test("on rhs of block-level const decl, with AltEnc") {
val src = SourceCode.Text("""
main {
const ubyte c = @'x'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
test("on rhs of subroutine-level var decl, no AltEnc") {
val src = SourceCode.Text("""
main {
sub start() {
ubyte c = 'x'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
}
test("on rhs of subroutine-level const decl, with AltEnc") {
val src = SourceCode.Text("""
main {
sub start() {
const ubyte c = @'x'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
}
context("Ranges") {
test("in for-loops") {
val module = parseModule(SourceCode.Text("""
main {
sub start() {
ubyte ub
for ub in "start" downto "end" { ; #0
}
for ub in "something" { ; #1
}
for ub in @'a' to 'f' { ; #2
}
for ub in false to true { ; #3
}
for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check!
}
}
}
"""))
val iterables = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<ForLoop>()
.map { it.iterable }
iterables.size shouldBe 5
val it0 = iterables[0] as RangeExpr
it0.from shouldBe instanceOf<StringLiteralValue>()
it0.to shouldBe instanceOf<StringLiteralValue>()
val it1 = iterables[1] as StringLiteralValue
it1.value shouldBe "something"
val it2 = iterables[2] as RangeExpr
it2.from shouldBe instanceOf<CharLiteral>()
it2.to shouldBe instanceOf<CharLiteral>()
val it3 = iterables[3] as RangeExpr
it3.from shouldBe instanceOf<NumericLiteralValue>()
it3.to shouldBe instanceOf<NumericLiteralValue>()
val it4 = iterables[4] as RangeExpr
it4.from shouldBe instanceOf<NumericLiteralValue>()
it4.to shouldBe instanceOf<NumericLiteralValue>()
}
}
test("testCharLiteralConstValue") {
val char1 = CharLiteral('A', false, Position.DUMMY)
val char2 = CharLiteral('z', true, Position.DUMMY)
val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder)
char1.constValue(program).number.toInt() shouldBe 65
char2.constValue(program).number.toInt() shouldBe 122
}
test("testLiteralValueComparisons") {
val ten = NumericLiteralValue(DataType.UWORD, 10.0, Position.DUMMY)
val nine = NumericLiteralValue(DataType.UBYTE, 9.0, Position.DUMMY)
ten shouldBe ten
nine shouldNotBe ten
(ten != ten) shouldBe false
(ten != nine) shouldBe true
(ten > nine) shouldBe true
(ten >= nine) shouldBe true
(ten >= ten) shouldBe true
(ten > ten) shouldBe false
(ten < nine) shouldBe false
(ten <= nine) shouldBe false
(ten <= ten) shouldBe true
(ten < ten) shouldBe false
val abc = StringLiteralValue("abc", false, Position.DUMMY)
val abd = StringLiteralValue("abd", false, Position.DUMMY)
abc shouldBe abc
(abc!=abd) shouldBe true
(abc!=abc) shouldBe false
}
test("testAnonScopeStillContainsVarsDirectlyAfterParse") {
val src = SourceCode.Text("""
main {
sub start() {
repeat {
ubyte xx = 99
xx++
}
}
}
""")
val module = parseModule(src)
val mainBlock = module.statements.single() as Block
val start = mainBlock.statements.single() as Subroutine
val repeatbody = (start.statements.single() as RepeatLoop).body
withClue("no vars moved to main block") {
mainBlock.statements.any { it is VarDecl } shouldBe false
}
withClue("no vars moved to start sub") {
start.statements.any { it is VarDecl } shouldBe false
}
withClue("\"var is still in repeat block (anonymousscope") {
repeatbody.statements[0] shouldBe instanceOf<VarDecl>()
}
val initvalue = (repeatbody.statements[0] as VarDecl).value as? NumericLiteralValue
initvalue?.number?.toInt() shouldBe 99
repeatbody.statements[1] shouldBe instanceOf<PostIncrDecr>()
// the ast processing steps used in the compiler, will eventually move the var up to the containing scope (subroutine).
}
test("testLabelsWithAnonScopesParsesFine") {
val src = SourceCode.Text("""
main {
sub start() {
goto mylabeloutside
if true {
if true {
goto labeloutside
goto iflabel
}
iflabel:
}
repeat {
goto labelinside
labelinside:
}
labeloutside:
}
}
""")
val module = parseModule(src)
val mainBlock = module.statements.single() as Block
val start = mainBlock.statements.single() as Subroutine
val labels = start.statements.filterIsInstance<Label>()
labels.size shouldBe 1
}
test("logical operator 'not' priority") {
val src = SourceCode.Text("""
main {
sub start() {
ubyte xx
xx = not 4 and not 5
xx = not 4 or not 5
xx = not 4 xor not 5
}
}
""")
val module = parseModule(src)
val start = (module.statements.single() as Block).statements.single() as Subroutine
val andAssignmentExpr = (start.statements[1] as Assignment).value
val orAssignmentExpr = (start.statements[2] as Assignment).value
val xorAssignmentExpr = (start.statements[3] as Assignment).value
fun correctPrios(expr: Expression, operator: String) {
withClue("not should have higher prio as the other logical operators") {
expr shouldBe instanceOf<BinaryExpression>()
val binExpr = expr as BinaryExpression
binExpr.operator shouldBe operator
(binExpr.left as PrefixExpression).operator shouldBe "not"
(binExpr.right as PrefixExpression).operator shouldBe "not"
}
}
correctPrios(andAssignmentExpr, "and")
correctPrios(orAssignmentExpr, "or")
correctPrios(xorAssignmentExpr, "xor")
}
test("inferred type correct for binaryexpression") {
val src = SourceCode.Text("""
main {
ubyte bb
uword ww
ubyte bb2 = not bb or not ww ; expression combining ubyte and uword
}
""")
val module = parseModule(src)
val program = Program("test", DummyFunctions, DummyMemsizer, DummyStringEncoder)
program.addModule(module)
val bb2 = (module.statements.single() as Block).statements[2] as VarDecl
val expr = bb2.value as BinaryExpression
println(expr)
expr.operator shouldBe "or"
expr.left.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
expr.right.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UWORD
expr.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
}
test("inferred type for typecasted expressions with logical operators") {
val src=SourceCode.Text("""
main {
ubyte bb
uword ww
uword qq = (not bb as uword)
uword zz = not bb or not ww
ubyte bb2 = not bb or not ww
uword zz2 = (not bb as uword) or not ww
}
""")
val module = parseModule(src)
val program = Program("test", DummyFunctions, DummyMemsizer, DummyStringEncoder)
program.addModule(module)
val stmts = (module.statements.single() as Block).statements
stmts.size shouldBe 6
val qq = (stmts[2] as VarDecl).value as TypecastExpression
val zz = (stmts[3] as VarDecl).value as BinaryExpression
val bb2 = (stmts[4] as VarDecl).value as BinaryExpression
val zz2 = (stmts[5] as VarDecl).value as BinaryExpression
qq.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UWORD
zz.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
bb2.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
zz2.operator shouldBe "or"
val left = zz2.left as TypecastExpression
val right = zz2.right as PrefixExpression
left.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UWORD
right.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UWORD
zz2.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE // 'or' causes UBYTE result
}
test("type cast from byte to ubyte as desired target type") {
val src = SourceCode.Text("""
main {
ubyte r
ubyte ub = (cos8(r)/2 + 100) as ubyte
}""")
val module = parseModule(src)
val program = Program("test", DummyFunctions, DummyMemsizer, DummyStringEncoder)
program.addModule(module)
val stmts = (module.statements.single() as Block).statements
stmts.size shouldBe 2
val ubexpr = (stmts[1] as VarDecl).value as TypecastExpression
ubexpr.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
}
test("assignment isAugmented correctness") {
val src = SourceCode.Text("""
main {
sub start() {
ubyte r
ubyte q
r = q*3 ; #1 no
r = r*3 ; #2 yes
r = 3*r ; #3 yes
r = 3*q ; #4 no
r = 5+r ; #5 yes
r = 5-r ; #6 no
r = r-5 ; #7 yes
r = not r ; #8 yes
r = not q ; #9 no
r = (q+r)+5 ; #10 yes
r = q+(r+5) ; #11 yes
r = (q+r)-5 ; #12 yes TODO FIX THIS ONE
r = q+(r-5) ; #13 yes TODO FIX THIS ONE
}
}""")
// TODO fix augmented result for the above mentioned assignments
// TODO certain typecast expressions are also augmentable?? what does this even mean, and does the generated code make a difference???
val module = parseModule(src)
val program = Program("test", DummyFunctions, DummyMemsizer, DummyStringEncoder)
program.addModule(module)
val stmts = program.entrypoint.statements
val expectedResults = listOf(
false, true, true,
false, true, false,
true, true, false,
true, true, true,
true
)
stmts.size shouldBe 15
expectedResults.size shouldBe stmts.size-2
for((idx, pp) in stmts.drop(2).zip(expectedResults).withIndex()) {
val assign = pp.first as Assignment
val expected = pp.second
withClue("#${idx+1}: should be augmentable but isn't : $assign") {
assign.isAugmentable shouldBe expected
assign.value shouldBe (instanceOf<PrefixExpression>() or instanceOf<BinaryExpression>() or instanceOf<TypecastExpression>())
}
}
}
})