prog8/compilerAst/test/TestProg8Parser.kt

658 lines
24 KiB
Kotlin
Raw Normal View History

package prog8tests.ast
import io.kotest.assertions.throwables.shouldThrow
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import io.kotest.matchers.string.shouldStartWith
import io.kotest.matchers.types.instanceOf
2021-10-10 22:22:04 +00:00
import prog8.ast.IFunctionCall
import prog8.ast.Module
2021-10-10 22:22:04 +00:00
import prog8.ast.Node
2021-10-29 22:05:55 +00:00
import prog8.ast.Program
2021-10-20 21:08:40 +00:00
import prog8.ast.base.DataType
2021-10-10 22:22:04 +00:00
import prog8.ast.base.Position
import prog8.ast.expressions.*
2021-10-10 22:22:04 +00:00
import prog8.ast.statements.*
import prog8.parser.ParseError
import prog8.parser.Prog8Parser.parseModule
import prog8.parser.SourceCode
2021-10-29 22:05:55 +00:00
import prog8tests.ast.helpers.*
import prog8tests.ast.helpers.DummyFunctions
import prog8tests.ast.helpers.DummyMemsizer
import kotlin.io.path.Path
import kotlin.io.path.isRegularFile
2021-10-10 22:22:04 +00:00
import kotlin.io.path.name
import kotlin.io.path.nameWithoutExtension
class TestProg8Parser: FunSpec( {
context("Newline at end") {
test("is not required - #40, fixed by #45") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val src = SourceCode.Text("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40)
// #40: Prog8ANTLRParser would report (throw) "missing <EOL> at '<EOF>'"
val module = parseModule(src)
module.statements.size shouldBe 1
}
test("is still accepted - #40, fixed by #45") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40)
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
}
context("Newline") {
test("is required after each block except the last") {
val nl = "\n" // say, Unix-style (different flavours tested elsewhere)
// BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end
val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl
// GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed
val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}"
shouldThrow<ParseError> { parseModule(SourceCode.Text(srcBad)) }
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcGood))
module.statements.size shouldBe 2
}
2021-06-18 19:55:03 +00:00
test("is required between two Blocks or Directives - #47") {
// block and block
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
blockA {
} blockB {
}
""")) }
// block and directive
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
blockB {
} %import textio
""")) }
// The following two are bogus due to directive *args* expected to follow the directive name.
// Leaving them in anyways.
// dir and block
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
%import textio blockB {
}
""")) }
shouldThrow<ParseError>{ parseModule(SourceCode.Text("""
%import textio %import syslib
""")) }
}
2021-06-18 19:55:03 +00:00
test("can be Win, Unix or mixed, even mixed") {
val nlWin = "\r\n"
val nlUnix = "\n"
val nlMac = "\r"
//parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath())
// a good mix of all kinds of newlines:
val srcText =
"foo {" +
nlMac +
nlWin +
"}" +
nlMac + // <-- do test a single \r (!) where an EOL is expected
"bar {" +
nlUnix +
"}" +
nlUnix + nlMac // both should be "eaten up" by just one EOL token
"combi {" +
nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token
"}" +
nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline)
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 2
}
2021-06-18 19:55:03 +00:00
}
context("EOLsInterleavedWithComments") {
test("are ok before first block - #47") {
// issue: #47
val srcText = """
; comment
; comment
blockA {
}
"""
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
test("are ok between blocks - #47") {
// issue: #47
val srcText = """
blockA {
}
; comment
; comment
blockB {
}
"""
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 2
}
test("are ok after last block - #47") {
// issue: #47
val srcText = """
blockA {
}
; comment
; comment
"""
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
module.statements.size shouldBe 1
}
}
context("ImportDirectives") {
test("should not be looked into by the parser") {
val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist")
assumeNotExists(fixturesDir, "i_do_not_exist.p8")
val text = "%import ${importedNoExt.name}"
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(text))
module.statements.size shouldBe 1
}
}
context("EmptySourcecode") {
test("from an empty string should result in empty Module") {
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(""))
module.statements.size shouldBe 0
}
test("from an empty file should result in empty Module") {
val path = assumeReadableFile(fixturesDir, "empty.p8")
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.File(path))
module.statements.size shouldBe 0
}
}
context("NameOfModule") {
test("parsed from a string") {
val srcText = """
main {
}
""".trimIndent()
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
// Note: assertContains has *actual* as first param
module.name shouldContain Regex("^<String@[0-9a-f\\-]+>$")
}
test("parsed from a file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.File(path))
module.name shouldBe path.nameWithoutExtension
}
}
context("PositionOfAstNodesAndParseErrors") {
fun assertPosition(
actual: Position,
expFile: String? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) {
require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
if (expLine != null) actual.line shouldBe expLine
if (expStartCol != null) actual.startCol shouldBe expStartCol
if (expEndCol != null) actual.endCol shouldBe expEndCol
if (expFile != null) actual.file shouldBe expFile
}
fun assertPosition(
actual: Position,
expFile: Regex? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) {
require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
if (expLine != null) actual.line shouldBe expLine
if (expStartCol != null) actual.startCol shouldBe expStartCol
if (expEndCol != null) actual.endCol shouldBe expEndCol
if (expFile != null) actual.file shouldContain expFile
}
fun assertPositionOf(
actual: Node,
expFile: String? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) =
assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
fun assertPositionOf(
actual: Node,
expFile: Regex? = null,
expLine: Int? = null,
expStartCol: Int? = null,
expEndCol: Int? = null
) =
assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
test("in ParseError from bad string source code") {
val srcText = "bad * { }\n"
val e = shouldThrow<ParseError> { parseModule(SourceCode.Text(srcText)) }
assertPosition(e.position, Regex("^<String@[0-9a-f\\-]+>$"), 1, 4, 4)
}
test("in ParseError from bad file source code") {
val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8")
2021-06-18 19:55:03 +00:00
val e = shouldThrow<ParseError> { parseModule(SourceCode.File(path)) }
assertPosition(e.position, SourceCode.relative(path).toString(), 2, 6)
}
test("of Module parsed from a string") {
val srcText = """
main {
}
"""
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
assertPositionOf(module, Regex("^<String@[0-9a-f\\-]+>$"), 1, 0)
}
2021-06-18 19:55:03 +00:00
test("of Module parsed from a file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.File(path))
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
}
test("of non-root Nodes parsed from file") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.File(path))
val mpf = module.position.file
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
val mainBlock = module.statements.filterIsInstance<Block>()[0]
assertPositionOf(mainBlock, mpf, 2, 0, 3)
val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
assertPositionOf(startSub, mpf, 3, 4, 6)
}
test("of non-root Nodes parsed from a string") {
val srcText = """
%zeropage basicsafe
main {
sub start() {
ubyte foo = 42
ubyte bar
when (foo) {
23 -> bar = 'x'
42 -> bar = 'y'
else -> bar = 'z'
}
}
}
""".trimIndent()
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text(srcText))
val mpf = module.position.file
val targetDirective = module.statements.filterIsInstance<Directive>()[0]
assertPositionOf(targetDirective, mpf, 1, 0, 8)
val mainBlock = module.statements.filterIsInstance<Block>()[0]
assertPositionOf(mainBlock, mpf, 2, 0, 3)
val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
assertPositionOf(startSub, mpf, 3, 4, 6)
val declFoo = startSub.statements.filterIsInstance<VarDecl>()[0]
assertPositionOf(declFoo, mpf, 4, 8, 12)
val rhsFoo = declFoo.value!!
assertPositionOf(rhsFoo, mpf, 4, 20, 21)
val declBar = startSub.statements.filterIsInstance<VarDecl>()[1]
assertPositionOf(declBar, mpf, 5, 8, 12)
val whenStmt = startSub.statements.filterIsInstance<WhenStatement>()[0]
assertPositionOf(whenStmt, mpf, 6, 8, 11)
assertPositionOf(whenStmt.choices[0], mpf, 7, 12, 13)
assertPositionOf(whenStmt.choices[1], mpf, 8, 12, 13)
assertPositionOf(whenStmt.choices[2], mpf, 9, 12, 15)
}
}
context("PositionFile") {
fun assertSomethingForAllNodes(module: Module, asserter: (Node) -> Unit) {
asserter(module)
module.statements.forEach(asserter)
module.statements.filterIsInstance<Block>().forEach { b ->
asserter(b)
b.statements.forEach(asserter)
b.statements.filterIsInstance<Subroutine>().forEach { s ->
asserter(s)
s.statements.forEach(asserter)
}
}
}
test("isn't absolute for filesystem paths") {
val path = assumeReadableFile(fixturesDir, "simple_main.p8")
val module = parseModule(SourceCode.File(path))
assertSomethingForAllNodes(module) {
Path(it.position.file).isAbsolute shouldBe false
Path(it.position.file).isRegularFile() shouldBe true
}
}
test("is mangled string id for string sources")
{
val srcText="""
%zeropage basicsafe
main {
sub start() {
ubyte aa=99
aa++
}
}
""".trimIndent()
val module = parseModule(SourceCode.Text(srcText))
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.stringSourcePrefix
}
}
test("is library prefixed path for resources")
{
val resource = SourceCode.Resource("prog8lib/math.p8")
val module = parseModule(resource)
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.libraryFilePrefix
}
}
}
context("CharLiterals") {
test("in argument position, no altEnc") {
2021-10-13 16:55:56 +00:00
val src = SourceCode.Text("""
main {
sub start() {
chrout('\n')
}
}
""")
val module = parseModule(src)
val startSub = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
val funCall = startSub.statements.filterIsInstance<IFunctionCall>().first()
funCall.args[0] shouldBe(instanceOf<CharLiteral>())
val char = funCall.args[0] as CharLiteral
char.value shouldBe '\n'
}
test("on rhs of block-level var decl, no AltEnc") {
2021-10-13 16:55:56 +00:00
val src = SourceCode.Text("""
main {
ubyte c = 'x'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
}
test("on rhs of block-level const decl, with AltEnc") {
2021-10-13 16:55:56 +00:00
val src = SourceCode.Text("""
main {
const ubyte c = @'x'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
test("on rhs of subroutine-level var decl, no AltEnc") {
2021-10-13 16:55:56 +00:00
val src = SourceCode.Text("""
main {
sub start() {
ubyte c = 'x'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
}
test("on rhs of subroutine-level const decl, with AltEnc") {
2021-10-13 16:55:56 +00:00
val src = SourceCode.Text("""
main {
sub start() {
const ubyte c = @'x'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
}
context("Ranges") {
test("in for-loops") {
2021-10-13 16:55:56 +00:00
val module = parseModule(SourceCode.Text("""
main {
sub start() {
ubyte ub
for ub in "start" downto "end" { ; #0
}
for ub in "something" { ; #1
}
for ub in @'a' to 'f' { ; #2
}
for ub in false to true { ; #3
}
for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check!
}
}
}
"""))
val iterables = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<ForLoop>()
.map { it.iterable }
iterables.size shouldBe 5
val it0 = iterables[0] as RangeExpr
it0.from shouldBe instanceOf<StringLiteralValue>()
it0.to shouldBe instanceOf<StringLiteralValue>()
val it1 = iterables[1] as StringLiteralValue
it1.value shouldBe "something"
val it2 = iterables[2] as RangeExpr
it2.from shouldBe instanceOf<CharLiteral>()
it2.to shouldBe instanceOf<CharLiteral>()
val it3 = iterables[3] as RangeExpr
it3.from shouldBe instanceOf<NumericLiteralValue>()
it3.to shouldBe instanceOf<NumericLiteralValue>()
val it4 = iterables[4] as RangeExpr
it4.from shouldBe instanceOf<NumericLiteralValue>()
it4.to shouldBe instanceOf<NumericLiteralValue>()
}
}
test("testCharLiteralConstValue") {
2021-10-29 22:05:55 +00:00
val char1 = CharLiteral('A', false, Position.DUMMY)
val char2 = CharLiteral('z', true, Position.DUMMY)
val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder)
char1.constValue(program).number.toInt() shouldBe 65
char2.constValue(program).number.toInt() shouldBe 122
2021-10-29 22:05:55 +00:00
}
test("testLiteralValueComparisons") {
2021-10-20 21:08:40 +00:00
val ten = NumericLiteralValue(DataType.UWORD, 10, Position.DUMMY)
val nine = NumericLiteralValue(DataType.UBYTE, 9, Position.DUMMY)
ten shouldBe ten
nine shouldNotBe ten
(ten != ten) shouldBe false
(ten != nine) shouldBe true
2021-10-20 21:08:40 +00:00
(ten > nine) shouldBe true
(ten >= nine) shouldBe true
(ten >= ten) shouldBe true
(ten > ten) shouldBe false
2021-10-20 21:08:40 +00:00
(ten < nine) shouldBe false
(ten <= nine) shouldBe false
(ten <= ten) shouldBe true
(ten < ten) shouldBe false
2021-10-20 21:08:40 +00:00
val abc = StringLiteralValue("abc", false, Position.DUMMY)
val abd = StringLiteralValue("abd", false, Position.DUMMY)
abc shouldBe abc
(abc!=abd) shouldBe true
(abc!=abc) shouldBe false
2021-10-20 21:08:40 +00:00
}
test("testAnonScopeStillContainsVarsDirectlyAfterParse") {
val src = SourceCode.Text("""
main {
sub start() {
repeat {
ubyte xx = 99
xx++
}
}
}
""")
val module = parseModule(src)
val mainBlock = module.statements.single() as Block
val start = mainBlock.statements.single() as Subroutine
val repeatbody = (start.statements.single() as RepeatLoop).body
withClue("no vars moved to main block") {
mainBlock.statements.any { it is VarDecl } shouldBe false
}
withClue("no vars moved to start sub") {
start.statements.any { it is VarDecl } shouldBe false
}
withClue("\"var is still in repeat block (anonymousscope") {
repeatbody.statements[0] shouldBe instanceOf<VarDecl>()
}
val initvalue = (repeatbody.statements[0] as VarDecl).value as? NumericLiteralValue
initvalue?.number?.toInt() shouldBe 99
repeatbody.statements[1] shouldBe instanceOf<PostIncrDecr>()
// the ast processing steps used in the compiler, will eventually move the var up to the containing scope (subroutine).
}
test("testLabelsWithAnonScopesParsesFine") {
val src = SourceCode.Text("""
main {
sub start() {
goto mylabeloutside
if true {
if true {
goto labeloutside
goto iflabel
}
iflabel:
}
repeat {
goto labelinside
labelinside:
}
labeloutside:
}
}
""")
val module = parseModule(src)
val mainBlock = module.statements.single() as Block
val start = mainBlock.statements.single() as Subroutine
val labels = start.statements.filterIsInstance<Label>()
labels.size shouldBe 1
}
test("logical operator 'not' priority") {
val src = SourceCode.Text("""
main {
sub start() {
ubyte xx
xx = not 4 and not 5
xx = not 4 or not 5
xx = not 4 xor not 5
}
}
""")
val module = parseModule(src)
val start = (module.statements.single() as Block).statements.single() as Subroutine
val andAssignmentExpr = (start.statements[1] as Assignment).value
val orAssignmentExpr = (start.statements[2] as Assignment).value
val xorAssignmentExpr = (start.statements[3] as Assignment).value
fun correctPrios(expr: Expression, operator: String) {
withClue("not should have higher prio as the other logical operators") {
expr shouldBe instanceOf<BinaryExpression>()
val binExpr = expr as BinaryExpression
binExpr.operator shouldBe operator
(binExpr.left as PrefixExpression).operator shouldBe "not"
(binExpr.right as PrefixExpression).operator shouldBe "not"
}
}
correctPrios(andAssignmentExpr, "and")
correctPrios(orAssignmentExpr, "or")
correctPrios(xorAssignmentExpr, "xor")
}
})