From 264b894909caec60fb61cb7d15e5be10efa35551 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Mon, 19 Oct 2026 21:59:56 +0200 Subject: [PATCH] Add linked-list for-loop iteration with via/until syntax --- .../compiler/astprocessing/AstChecker.kt | 74 ++++ .../compiler/astprocessing/AstExtensions.kt | 13 + .../compiler/astprocessing/CodeDesugarer.kt | 91 +++++ .../astprocessing/ImplicitForIteratorDecls.kt | 89 ++++- compiler/test/TestLinkedForLoops.kt | 346 +++++++++++++++++ .../prog8/ast/antlr/Antlr2KotlinVisitor.kt | 12 +- .../src/prog8/ast/statements/AstStatements.kt | 11 +- compilerAst/src/prog8/ast/walk/AstWalker.kt | 1 + compilerAst/src/prog8/ast/walk/IAstVisitor.kt | 1 + docs/source/libraries.rst | 41 ++ linked-list-iteration-plan.md | 364 ++++++++++++++++++ parser/src/main/antlr/Prog8ANTLR.g4 | 10 +- syntax-files/Jetbrains-IDEs/Prog8.xml | 2 +- syntax-files/Kate/prog8.xml | 1 + syntax-files/NotepadPlusPlus/Prog8.xml | 2 +- syntax-files/NotepadPlusPlus/Prog8dark.xml | 2 +- syntax-files/SublimeText/Prog8.sublime-syntax | 2 +- syntax-files/Vim/prog8.vim | 2 +- syntax-files/nano/prog8.nanorc | 2 +- 19 files changed, 1051 insertions(+), 15 deletions(-) create mode 100644 compiler/test/TestLinkedForLoops.kt create mode 100644 linked-list-iteration-plan.md diff --git a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt index 09614823d..b33dbe5ae 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt @@ -262,6 +262,16 @@ internal class AstChecker(private val program: Program, override fun visit(forLoop: ForLoop) { + if (forLoop.linkedVia != null || forLoop.isLinkedHeaderShorthand(program)) { + checkLinkedForLoop(forLoop) + super.visit(forLoop) + return + } + + if (forLoop.linkedUntil != null) { + errors.err("'until' can only be used in linked-list for loops", forLoop.position) + } + fun checkUnsignedLoopDownto0(range: RangeExpression?) { if(range==null) return @@ -388,6 +398,70 @@ internal class AstChecker(private val program: Program, super.visit(forLoop) } + + private fun checkLinkedForLoop(forLoop: ForLoop) { + val loopVarDecl = forLoop.loopVar.targetVarDecl() + if (loopVarDecl == null || !loopVarDecl.datatype.isPointer) { + errors.err("linked-list for loop requires a pointer loop variable", forLoop.position) + return + } + val nodeType = loopVarDecl.datatype + val iterableType = forLoop.iterable.inferType(program).getOrUndef() + if (!iterableType.isPointer) { + errors.err("linked-list for loop start must be a pointer", forLoop.position) + return + } + + val nodeStruct = nodeType.dereference().subType + if (nodeStruct == null) { + errors.err("linked-list for loop node type must be a struct", forLoop.position) + return + } + + val linkField: String = when { + forLoop.linkedVia != null -> { + if (iterableType != nodeType) { + errors.err("linked-list for loop start must be a pointer to the same node type", forLoop.position) + return + } + forLoop.linkedVia!! + } + else -> { + val headerStruct = iterableType.dereference().subType + val headerStart = listOf("head", "first").firstOrNull { headerStruct?.getFieldType(it) == nodeType } + if (iterableType != nodeType && headerStart == null) { + errors.err("linked-list for loop start must be a pointer to the same node type or a list header with 'head'/'first' field", forLoop.position) + return + } + listOf("next", "succ").firstOrNull { nodeStruct.getFieldType(it)?.isPointer == true } + ?: run { + errors.err("linked-list for loop node type must have a 'next' or 'succ' field", forLoop.position) + return + } + } + } + + val fieldType = nodeStruct.getFieldType(linkField) + if (fieldType == null || !fieldType.isPointer || fieldType != nodeType) { + errors.err("linked-list for loop 'via' field must be a pointer to the same node type", forLoop.position) + return + } + + if (forLoop.step != null) { + errors.err("linked-list for loop cannot use 'step'", forLoop.position) + } + forLoop.linkedUntil?.let { untilExpr -> + val untilType = untilExpr.inferType(program).getOrUndef() + val addressOfTargetType = (untilExpr as? prog8.ast.expressions.AddressOf)?.identifier?.inferType(program)?.getOrUndef() + val isValidUntil = (untilType.isPointer && untilType == nodeType) || + (untilExpr is prog8.ast.expressions.NumericLiteral && untilExpr.number == 0.0) || + addressOfTargetType == nodeType + if (!isValidUntil) { + errors.err("linked-list for loop 'until' must be a compatible node pointer or 0", forLoop.position) + } + } + } + override fun visit(jump: Jump) { val ident = jump.target as? IdentifierReference if(ident!=null) { diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index f59772e94..610fea17f 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -154,6 +154,19 @@ internal fun Expression.flatten(): List { return if (this is ArrayLiteral) this.value.flatMap { it.flatten() } else listOf(this) } +internal fun DataType.isLinkedListType(): Boolean { + if (!isPointer) return false + val struct = dereference().subType ?: return false + val linkFields = listOf("next", "succ", "head", "first") + return linkFields.any { struct.getFieldType(it)?.isPointer == true } +} + +internal fun ForLoop.isLinkedHeaderShorthand(program: Program): Boolean { + if (linkedVia != null) return true + val iterableType = iterable.inferType(program).getOrUndef() + return iterableType.isLinkedListType() +} + internal fun List.flattenArgs(): List { return this.flatMap { it.flatten() } } diff --git a/compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt b/compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt index 15fce00aa..89779a5b3 100644 --- a/compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt +++ b/compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt @@ -1009,6 +1009,12 @@ _after: } override fun after(forLoop: ForLoop, parent: Node): Iterable { + if (forLoop.linkedVia != null || forLoop.isLinkedHeaderShorthand(program)) { + // leave the invalid step combination for AstChecker, desugaring would hide it + if (forLoop.step != null) return noModifications + return desugarLinkedForLoop(forLoop, parent) + } + // Desugar reverse array/string iteration // - for x in array step -1 -> index loop len-1 downto 0 // - for c in string step -1 -> similar via index @@ -1028,6 +1034,91 @@ _after: return noModifications } + private fun desugarLinkedForLoop(forLoop: ForLoop, parent: Node): Iterable { + val pos = forLoop.position + val nodeRef = forLoop.loopVar.copy() + + // Resolve explicit or header-shorthand metadata. + val startExpr: prog8.ast.expressions.Expression + val linkField: String + val sentinelExpr: prog8.ast.expressions.Expression + + if (forLoop.linkedVia != null) { + startExpr = forLoop.iterable + linkField = forLoop.linkedVia!! + sentinelExpr = forLoop.linkedUntil ?: prog8.ast.expressions.NumericLiteral.optimalInteger(0, pos) + } else { + val iterableType = forLoop.iterable.inferType(program).getOrUndef() + val loopVarType = forLoop.loopVar.targetVarDecl()?.datatype + val nodeStruct = loopVarType?.dereference()?.subType + val forwardLinks = listOf("next", "succ") + linkField = forwardLinks.firstOrNull { nodeStruct?.getFieldType(it)?.isPointer == true } ?: "next" + + val headerStruct = iterableType.dereference().subType + val headName = if (iterableType.isPointer && iterableType != loopVarType) { + listOf("head", "first").firstOrNull { headerStruct?.getFieldType(it)?.isPointer == true } + } else null + + if (headName == null) { + startExpr = forLoop.iterable + sentinelExpr = forLoop.linkedUntil ?: prog8.ast.expressions.NumericLiteral.optimalInteger(0, pos) + } else { + startExpr = prog8.ast.expressions.BinaryExpression(forLoop.iterable, ".", prog8.ast.expressions.IdentifierReference(listOf(headName), pos), pos) + val tailName = listOf("tail", "last").firstOrNull { headerStruct?.getFieldType(it)?.isPointer == true } + sentinelExpr = forLoop.linkedUntil ?: if (tailName != null) { + prog8.ast.expressions.BinaryExpression(forLoop.iterable.copy(), ".", prog8.ast.expressions.IdentifierReference(listOf(tailName), pos), pos) + } else { + prog8.ast.expressions.NumericLiteral.optimalInteger(0, pos) + } + } + } + + // Declare the loop variable if it is not already declared. + val statements = mutableListOf() + val names = forLoop.loopVar.nameInSource + val scope = forLoop.definingScope + val existing = if (names.size == 1) scope.lookup(names.single()) else null + if (existing !is prog8.ast.statements.VarDecl) { + val iterableType = forLoop.iterable.inferType(program).getOrUndef() + val effectiveType = forLoop.loopVarType ?: iterableType + val decl = prog8.ast.statements.VarDecl.builder(effectiveType, pos) + .names(names.single()) + .type(prog8.ast.statements.VarDeclType.VAR) + .build() + statements.add(decl) + } + + val initAssign = prog8.ast.statements.Assignment( + prog8.ast.statements.AssignTarget(nodeRef.copy(), null, null, null, false, position = pos), + startExpr, + prog8.ast.statements.AssignmentOrigin.OPTIMIZER, + pos + ) + statements.add(initAssign) + + val condition = prog8.ast.expressions.BinaryExpression(nodeRef.copy(), "!=", sentinelExpr, pos) + val fieldAccess = prog8.ast.expressions.BinaryExpression(nodeRef.copy(), ".", prog8.ast.expressions.IdentifierReference(listOf(linkField), pos), pos) + val tailAssign = prog8.ast.statements.Assignment( + prog8.ast.statements.AssignTarget(nodeRef.copy(), null, null, null, false, position = pos), + fieldAccess, + prog8.ast.statements.AssignmentOrigin.OPTIMIZER, + pos + ) + val whileBody = prog8.ast.statements.AnonymousScope( + mutableListOf().apply { + addAll(forLoop.body.statements) + add(tailAssign) + }, + pos + ) + val whileLoop = prog8.ast.statements.WhileLoop(condition, whileBody, pos) + statements.add(whileLoop) + + val replacement = prog8.ast.statements.AnonymousScope(statements, pos) + replacement.linkParents(parent) + return listOf(prog8.ast.walk.AstReplaceNode(forLoop, replacement, parent)) + } + private fun desugarReverseArrayStringForLoop(forLoop: ForLoop, parent: Node, iterableDt: DataType): Iterable { val pos = forLoop.position // Create index variable - make unique per loop position to avoid hoisting collisions diff --git a/compiler/src/prog8/compiler/astprocessing/ImplicitForIteratorDecls.kt b/compiler/src/prog8/compiler/astprocessing/ImplicitForIteratorDecls.kt index d912d34d7..0d5a69393 100644 --- a/compiler/src/prog8/compiler/astprocessing/ImplicitForIteratorDecls.kt +++ b/compiler/src/prog8/compiler/astprocessing/ImplicitForIteratorDecls.kt @@ -4,6 +4,8 @@ import prog8.ast.INameScope import prog8.ast.IStatementContainer import prog8.ast.Node import prog8.ast.Program +import prog8.ast.expressions.AddressOf +import prog8.ast.expressions.NumericLiteral import prog8.ast.expressions.RangeExpression import prog8.ast.statements.ForLoop import prog8.ast.statements.VarDecl @@ -34,10 +36,23 @@ class ImplicitForIteratorDecls( if (!inferredIterableType.isKnown) return emptyList() val iterableType = inferredIterableType.getOrUndef() - if (!iterableType.isIterable && forLoop.iterable !is RangeExpression) + + val isLinked = forLoop.linkedVia != null || iterableType.isLinkedListType() + if (!iterableType.isIterable && forLoop.iterable !is RangeExpression && !isLinked) return emptyList() - val elementType = iterableType.elementType() + val elementType = if (isLinked) { + linkedNodeType(forLoop, iterableType) ?: return emptyList() + } else { + iterableType.elementType() + } + + if (isLinked) { + validateLinkedForLoop(forLoop, iterableType, elementType) + if (!errors.noErrors()) + return emptyList() + } + val scope = forLoop.definingScope val existing = scope.lookup(names) @@ -111,6 +126,76 @@ class ImplicitForIteratorDecls( return listOf(AstInsert.before(forLoop, declaration, container)) } + + private fun validateLinkedForLoop(forLoop: ForLoop, iterableType: DataType, nodeType: DataType) { + val nodeStruct = nodeType.dereference().subType + if (nodeStruct == null) { + errors.err("linked-list for loop node type must be a struct", forLoop.position) + return + } + + val linkField: String = when { + forLoop.linkedVia != null -> { + if (iterableType != nodeType) { + errors.err("linked-list for loop start must be a pointer to the same node type", forLoop.position) + return + } + forLoop.linkedVia!! + } + else -> { + val headerStruct = iterableType.dereference().subType + val headerStart = listOf("head", "first").firstOrNull { headerStruct?.getFieldType(it) == nodeType } + if (iterableType != nodeType && headerStart == null) { + errors.err("linked-list for loop start must be a pointer to the same node type or a list header with 'head'/'first' field", forLoop.position) + return + } + listOf("next", "succ").firstOrNull { nodeStruct.getFieldType(it)?.isPointer == true } + ?: run { + errors.err("linked-list for loop node type must have a 'next' or 'succ' field", forLoop.position) + return + } + } + } + + val fieldType = nodeStruct.getFieldType(linkField) + if (fieldType == null || !fieldType.isPointer || fieldType != nodeType) { + errors.err("linked-list for loop 'via' field must be a pointer to the same node type", forLoop.position) + return + } + + if (forLoop.step != null) { + errors.err("linked-list for loop cannot use 'step'", forLoop.position) + } + + forLoop.linkedUntil?.let { untilExpr -> + val untilType = untilExpr.inferType(program).getOrUndef() + val addressOfTargetType = (untilExpr as? AddressOf)?.identifier?.inferType(program)?.getOrUndef() + val isValidUntil = (untilType.isPointer && untilType == nodeType) || + (untilExpr is NumericLiteral && untilExpr.number == 0.0) || + addressOfTargetType == nodeType + if (!isValidUntil) { + errors.err("linked-list for loop 'until' must be a compatible node pointer or 0", forLoop.position) + } + } + } + + private fun linkedNodeType(forLoop: ForLoop, iterableType: DataType): DataType? { + if (iterableType.isPointer) { + val subType = iterableType.dereference().subType + val forwardLinks = listOf("next", "succ") + if (subType != null && forwardLinks.any { subType.getFieldType(it)?.isPointer == true }) { + return iterableType + } + } + if (!iterableType.isPointer) return null + val headerStruct = iterableType.dereference().subType + val headName = listOf("head", "first").firstOrNull { headerStruct?.getFieldType(it)?.isPointer == true } + if (headName != null) { + return headerStruct?.getFieldType(headName) + } + return null + } + private fun resolveDt(dt: DataType, scope: INameScope, position: Position): DataType { val antlrName = dt.subTypeFromAntlr ?: return dt val resolved = StructTypeResolver.resolve(scope, program, antlrName, position, errors) ?: return dt diff --git a/compiler/test/TestLinkedForLoops.kt b/compiler/test/TestLinkedForLoops.kt new file mode 100644 index 000000000..a042cca14 --- /dev/null +++ b/compiler/test/TestLinkedForLoops.kt @@ -0,0 +1,346 @@ +package prog8tests.compiler + +import io.kotest.core.spec.style.FunSpec +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.collections.shouldContain +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import prog8.ast.Node +import prog8.ast.ParentSentinel +import prog8.ast.expressions.IdentifierReference +import prog8.ast.statements.AnonymousScope +import prog8.ast.statements.Block +import prog8.ast.statements.ForLoop +import prog8.ast.statements.Subroutine +import prog8.ast.walk.AstModification +import prog8.ast.walk.AstWalker +import prog8.ast.walk.IAstVisitor +import prog8.code.core.Position +import prog8.code.source.SourceCode +import prog8.code.target.VMTarget +import prog8.parser.Prog8Parser.parseModule +import prog8.vm.VmRunner +import prog8tests.helpers.ErrorReporterForTests +import prog8tests.helpers.compileText +import kotlin.io.path.readText + + +class TestLinkedForLoops: FunSpec({ + + val outputDir = tempdir().toPath() + + test("parser keeps step on linked loops with via and until") { + val src = """ + main { + sub start() { + for n in head via .next step 1 { + } + for n2 in head via .next until sentinel step 2 { + } + } + } + """ + + val module = parseModule(SourceCode.Text(src), VMTarget()) + val start = module.statements.filterIsInstance().single { it.name == "main" } + .statements.filterIsInstance().single { it.name == "start" } + val loops = start.statements.filterIsInstance() + + loops.size shouldBe 2 + loops[0].linkedVia shouldBe "next" + loops[0].linkedUntil shouldBe null + loops[0].step shouldNotBe null + loops[1].linkedVia shouldBe "next" + loops[1].linkedUntil shouldNotBe null + loops[1].step shouldNotBe null + } + + test("linked for loop with via cannot use step") { + val errors = ErrorReporterForTests() + compileText(VMTarget(), false, """ + main { + struct Node { + ^^Node next + ubyte value + } + sub start() { + ^^Node head = memory("head", sizeof(Node), 0) + for ^^Node n in head via .next step 1 { + n.value = 1 + } + } + } + """, outputDir, errors, writeAssembly = false) shouldBe null + errors.errors.any { it.contains("linked-list for loop cannot use 'step'") } shouldBe true + } + + test("header shorthand linked for loop cannot use step") { + val errors = ErrorReporterForTests() + compileText(VMTarget(), false, """ + main { + struct Node { + ^^Node next + ubyte value + } + struct List { + ^^Node head + ^^Node last + } + sub start() { + ^^List list = memory("list", sizeof(List), 0) + for ^^Node n in list step 1 { + n.value = 1 + } + } + } + """, outputDir, errors, writeAssembly = false) shouldBe null + errors.errors.any { it.contains("linked-list for loop cannot use 'step'") } shouldBe true + } + + test("linkedUntil is traversed by IAstVisitor and AstWalker") { + val pos = Position.DUMMY + val loop = ForLoop( + IdentifierReference(listOf("node"), pos), + IdentifierReference(listOf("head"), pos), + AnonymousScope.empty(), + null, + null, + pos, + linkedVia = "next", + linkedUntil = IdentifierReference(listOf("sentinel"), pos) + ) + + val seenByVisitor = mutableListOf() + val visitor = object : IAstVisitor { + override fun visit(identifier: IdentifierReference) { + seenByVisitor += identifier.nameInSource.single() + } + } + loop.accept(visitor) + seenByVisitor shouldContain "sentinel" + + val seenByWalker = mutableListOf() + val walker = object : AstWalker() { + override fun before(identifier: IdentifierReference, parent: Node): Iterable { + seenByWalker += identifier.nameInSource.single() + return noModifications + } + } + loop.linkParents(ParentSentinel) + loop.accept(walker, ParentSentinel) + seenByWalker shouldContain "sentinel" + } + + test("explicit custom sentinel identifier terminates traversal") { + val src = $$""" + %zeropage basicsafe + main { + struct Node { + ^^Node next + ubyte value + } + sub start() { + ^^Node n1 = memory("n1", sizeof(Node), 0) + ^^Node n2 = memory("n2", sizeof(Node), 0) + ^^Node sentinel = memory("sentinel", sizeof(Node), 0) + n1.value = 11 + n2.value = 22 + sentinel.value = 99 + n1.next = n2 + n2.next = sentinel + ubyte @shared sum = 0 + for ^^Node n in n1 via .next until sentinel { + sum += n.value + } + if sum != 33 { sys.exit(7) } + pokew($4400, $c0ba) + sys.exit(0) + } + } + """ + val result = compileText(VMTarget(), false, src, outputDir)!! + val virtfile = result.compilationOptions.outputDir.resolve(result.compilerAst.name + ".p8ir") + VmRunner().runAndTestProgram(virtfile.readText(), true) { vm -> + vm.memory.getUW(0x4400u).toInt() shouldBe 0xc0ba + } + } + + test("header shorthand uses tail sentinel") { + val src = $$""" + %zeropage basicsafe + main { + struct Node { + ^^Node next + ubyte value + } + struct List { + ^^Node head + ^^Node last + } + sub start() { + ^^Node n1 = memory("n1", sizeof(Node), 0) + ^^Node n2 = memory("n2", sizeof(Node), 0) + ^^Node sentinel = memory("sentinel", sizeof(Node), 0) + ^^List list = memory("list", sizeof(List), 0) + n1.value = 11 + n2.value = 22 + sentinel.value = 99 + n1.next = n2 + n2.next = sentinel + list.head = n1 + list.last = sentinel + ubyte @shared sum = 0 + for ^^Node n in list { + sum += n.value + } + if sum != 33 { sys.exit(7) } + pokew($4400, $c0ba) + sys.exit(0) + } + } + """ + val result = compileText(VMTarget(), false, src, outputDir)!! + val virtfile = result.compilationOptions.outputDir.resolve(result.compilerAst.name + ".p8ir") + VmRunner().runAndTestProgram(virtfile.readText(), true) { vm -> + vm.memory.getUW(0x4400u).toInt() shouldBe 0xc0ba + } + } + + test("header shorthand with node pointer terminates at zero") { + val src = $$""" + %zeropage basicsafe + main { + struct Node { + ^^Node next + ubyte value + } + sub start() { + ^^Node n1 = memory("n1", sizeof(Node), 0) + ^^Node n2 = memory("n2", sizeof(Node), 0) + n1.value = 11 + n2.value = 22 + n1.next = n2 + n2.next = 0 + ^^Node @shared head = n1 + ubyte @shared sum = 0 + for ^^Node n in head { + sum += n.value + } + if sum != 33 { sys.exit(7) } + pokew($4400, $c0ba) + sys.exit(0) + } + } + """ + val result = compileText(VMTarget(), false, src, outputDir)!! + val virtfile = result.compilationOptions.outputDir.resolve(result.compilerAst.name + ".p8ir") + VmRunner().runAndTestProgram(virtfile.readText(), true) { vm -> + vm.memory.getUW(0x4400u).toInt() shouldBe 0xc0ba + } + } + + test("ordinary array, string and range for loops still compile") { + val result = compileText(VMTarget(), false, """ + main { + sub start() { + ubyte[] arr = [1,2,3,4] + ubyte x + for x in arr { + x++ + } + for x in arr step -1 { + x++ + } + str s = "abcd" + ubyte c + for c in s { + c++ + } + for c in s step -1 { + c++ + } + for x in 0 to 3 { + x++ + } + } + } + """, outputDir, writeAssembly = false) + result shouldNotBe null + } + + test("until in ordinary for loop is rejected") { + val errors = ErrorReporterForTests() + compileText(VMTarget(), false, """ + main { + sub start() { + ubyte[] arr = [1,2,3] + ubyte x + for x in arr until 2 { + x++ + } + } + } + """, outputDir, errors, writeAssembly = false) shouldBe null + errors.errors.any { it.contains("'until' can only be used in linked-list for loops") } shouldBe true + } + + test("header shorthand requires node link field") { + val errors = ErrorReporterForTests() + compileText(VMTarget(), false, """ + main { + struct Node { + ubyte value + } + struct List { + ^^Node head + } + sub start() { + ^^List list = memory("list", sizeof(List), 0) + for n in list { + n.value = 1 + } + } + } + """, outputDir, errors, writeAssembly = false) shouldBe null + errors.errors.any { it.contains("linked-list for loop node type must have a 'next' or 'succ' field") } shouldBe true + } + + test("explicit until accepts address-of header field") { + val src = $$""" + %zeropage basicsafe + main { + struct Node { + ^^Node next + ubyte value + } + struct List { + ^^Node head + ^^Node tail + } + sub start() { + ^^Node n1 = memory("n1", sizeof(Node), 0) + ^^Node n2 = memory("n2", sizeof(Node), 0) + ^^List list = memory("list", sizeof(List), 0) + n1.value = 11 + n2.value = 22 + n1.next = n2 + n2.next = &list.tail + list.head = n1 + list.tail = n2 + ubyte @shared sum = 0 + for ^^Node n in list.head via .next until &list.tail { + sum += n.value + } + if sum != 33 { sys.exit(7) } + pokew($4400, $c0ba) + sys.exit(0) + } + } + """ + val result = compileText(VMTarget(), false, src, outputDir)!! + val virtfile = result.compilationOptions.outputDir.resolve(result.compilerAst.name + ".p8ir") + VmRunner().runAndTestProgram(virtfile.readText(), true) { vm -> + vm.memory.getUW(0x4400u).toInt() shouldBe 0xc0ba + } + } +}) diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt b/compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt index fb372cb26..2b6263cc1 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt @@ -737,11 +737,17 @@ class Antlr2KotlinVisitor(val source: SourceCode, private val target: ICompilati override fun visitForloop(ctx: ForloopContext): ForLoop { val loopvar = ctx.scoped_identifier().accept(this) as IdentifierReference - val iterable = ctx.expression(0).accept(this) as Expression + val iterable = ctx.iterable.accept(this) as Expression val scope = stmtBlockOrSingle(ctx.statement_block(), ctx.statement()) val loopVarType = dataTypeFor(ctx.datatype()) - val step = if(ctx.STEP()!=null && ctx.expression().size > 1) ctx.expression(1).accept(this) as Expression else null - return ForLoop(loopvar, iterable, scope, loopVarType, step, ctx.toPosition()) + + val viaField = ctx.VIA()?.let { ctx.viaField.text } + val untilExpr = ctx.UNTIL()?.let { ctx.untilExpr.accept(this) as Expression } + val step = ctx.STEP()?.let { ctx.stepExpr.accept(this) as Expression } + + return ForLoop(loopvar, iterable, scope, loopVarType, step, ctx.toPosition(), + linkedVia = viaField, + linkedUntil = untilExpr) } override fun visitWhileloop(ctx: WhileloopContext): WhileLoop { diff --git a/compilerAst/src/prog8/ast/statements/AstStatements.kt b/compilerAst/src/prog8/ast/statements/AstStatements.kt index f35864eb2..23c5b312e 100644 --- a/compilerAst/src/prog8/ast/statements/AstStatements.kt +++ b/compilerAst/src/prog8/ast/statements/AstStatements.kt @@ -1413,7 +1413,9 @@ class ForLoop(var loopVar: IdentifierReference, var body: AnonymousScope, var loopVarType: DataType?, var step: Expression?, - override val position: Position) : Statement() { + override val position: Position, + var linkedVia: String? = null, + var linkedUntil: Expression? = null) : Statement() { override lateinit var parent: Node override fun linkParents(parent: Node) { @@ -1421,6 +1423,7 @@ class ForLoop(var loopVar: IdentifierReference, loopVar.linkParents(this) iterable.linkParents(this) step?.linkParents(this) + linkedUntil?.linkParents(this) body.linkParents(this) } @@ -1431,6 +1434,7 @@ class ForLoop(var loopVar: IdentifierReference, node===loopVar -> loopVar = replacement as IdentifierReference node===iterable -> iterable = replacement as Expression node===step -> step = replacement as Expression + node===linkedUntil -> linkedUntil = replacement as Expression node===body -> body = replacement as AnonymousScope else -> throw FatalAstException("invalid replace in $this at ${node.position}: $node -> $replacement") } @@ -1441,13 +1445,16 @@ class ForLoop(var loopVar: IdentifierReference, override fun accept(visitor: AstWalker, parent: Node) = visitor.visit(this, parent) override fun toString(): String { val typeStr = if (loopVarType != null) "type: $loopVarType, " else "" + val viaStr = if (linkedVia != null) ", via: $linkedVia" else "" + val untilStr = if (linkedUntil != null) ", until: $linkedUntil" else "" val stepStr = if (step != null) ", step: $step" else "" - return "ForLoop(${typeStr}loopVar: $loopVar, iterable: $iterable$stepStr, pos=$position)" + return "ForLoop(${typeStr}loopVar: $loopVar, iterable: $iterable$viaStr$untilStr$stepStr, pos=$position)" } override fun referencesIdentifier(nameInSource: List): Boolean = loopVar.referencesIdentifier(nameInSource) || iterable.referencesIdentifier(nameInSource) || (step?.referencesIdentifier(nameInSource)==true) || + (linkedUntil?.referencesIdentifier(nameInSource)==true) || body.referencesIdentifier(nameInSource) fun loopVarDt(program: Program): InferredTypes.InferredType { diff --git a/compilerAst/src/prog8/ast/walk/AstWalker.kt b/compilerAst/src/prog8/ast/walk/AstWalker.kt index 701f7dcaa..d94993ad9 100644 --- a/compilerAst/src/prog8/ast/walk/AstWalker.kt +++ b/compilerAst/src/prog8/ast/walk/AstWalker.kt @@ -586,6 +586,7 @@ abstract class AstWalker { forLoop.loopVar.accept(this, forLoop) forLoop.iterable.accept(this, forLoop) forLoop.step?.accept(this, forLoop) + forLoop.linkedUntil?.accept(this, forLoop) forLoop.body.accept(this, forLoop) track(after(forLoop, parent), forLoop, parent) } diff --git a/compilerAst/src/prog8/ast/walk/IAstVisitor.kt b/compilerAst/src/prog8/ast/walk/IAstVisitor.kt index 31c26a59c..48a323d21 100644 --- a/compilerAst/src/prog8/ast/walk/IAstVisitor.kt +++ b/compilerAst/src/prog8/ast/walk/IAstVisitor.kt @@ -154,6 +154,7 @@ interface IAstVisitor { forLoop.loopVar.accept(this) forLoop.iterable.accept(this) forLoop.step?.accept(this) + forLoop.linkedUntil?.accept(this) forLoop.body.accept(this) } diff --git a/docs/source/libraries.rst b/docs/source/libraries.rst index d918d606c..a88b1b254 100644 --- a/docs/source/libraries.rst +++ b/docs/source/libraries.rst @@ -1175,6 +1175,47 @@ to the header: } } +Linked-list iteration +~~~~~~~~~~~~~~~~~~~~~ + +Prog8 can desugar linked-list ``for`` loops into ordinary ``while`` loops. +The explicit form works for arbitrary node layouts and link field names: + + main { + ^^MyList mylist = [] ; zero-initialized header + ^^MyNode n1 = [0, 0, 42] + ^^MyNode n2 = [0, 0, 99] + lists.init(mylist) + lists.add_tail(mylist, n1) + lists.add_tail(mylist, n2) + + ; forward iteration over an Exec/MinList + for node in mylist.Head via .Succ until &mylist.Tail { + txt.print_ub(node.value) + txt.nl() + } + + ; backward iteration + for node in mylist.TailPred via .Pred until &mylist.Head { + txt.print_ub(node.value) + txt.nl() + } + } + +For simple null-terminated lists, ``until`` may be omitted and defaults to ``0``: + + for node in head via .next { + ; use node + } + +Header shorthand is available when the conventional lowercase field names +``next`` or ``succ`` are used: + + for node in head { } ; forward, until 0 + +Exec-style lists use uppercase field names (``Succ``, ``Pred``, ``Head``, +``Tail``), so use the explicit ``via`` form for them. + Custom nodes work the same way as long as the link pointers are the first fields at offset 0; add ``Type``/``Pri``/``Name`` after them if you need ``enqueue`` or ``find_name`` (the ``exec.Node``/``FullNode`` layout). diff --git a/linked-list-iteration-plan.md b/linked-list-iteration-plan.md new file mode 100644 index 000000000..1f0ded9c0 --- /dev/null +++ b/linked-list-iteration-plan.md @@ -0,0 +1,364 @@ +# Linked List Iteration Plan + +## Goal + +Add general linked-list iteration to Prog8 without hardcoding Amiga Exec conventions. + +Supported forms: + +```prog8 +for node in start via .next { } ; until 0 by default +for node in start via .next until sentinel { } ; explicit terminator +for node in list { } ; header shorthand, forward +``` + +All forms desugar to ordinary `while` loops in `CodeDesugarer`, so no backend changes are needed. + +> **Implementation status:** the forms above and conventional-name header shorthand +> are implemented. Struct attributes (`@head`, `@succ`, etc.) described later in this +> document are not yet implemented. + +## Syntax + +### Grammar + +File: `parser/src/main/antlr/Prog8ANTLR.g4` + +Current rule: + +```antlr +forloop : 'for' datatype? scoped_identifier 'in' expression (STEP expression)? EOL? (statement | statement_block) ; +``` + +Proposed rule: + +```antlr +forloop : 'for' datatype? scoped_identifier 'in' iterable=expression + (VIA '.' viaField=identifier)? + (UNTIL untilExpr=expression)? + (STEP stepExpr=expression)? + EOL? (statement | statement_block) ; +``` + +Add named lexer tokens for `VIA` and `UNTIL` rather than using anonymous parser +literals, because the visitor needs stable token accessors. +The link field must use the existing `identifier` rule; there is no generic +`IDENTIFIER` token in this grammar. + +For example, add the tokens alongside the existing keyword tokens: + +```antlr +VIA: 'via'; +UNTIL: 'until'; +``` + +Use the existing zero expression (`0`) as the default and explicit null +sentinel. Do not add a first-class `null` literal for this feature. + +Notes: + +- `via` and `until` are only meaningful for linked-list iteration. +- `step` remains valid only for ordinary array/string/range loops. +- The parser should accept the combined forms; semantic validation rejects invalid combinations. + +### Parser visitor + +File: `compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt` + +Method: `visitForloop(ctx: ForloopContext)` + +Current implementation: + +```kotlin +val loopvar = ctx.scoped_identifier().accept(this) as IdentifierReference +val iterable = ctx.expression(0).accept(this) as Expression +val scope = stmtBlockOrSingle(ctx.statement_block(), ctx.statement()) +val loopVarType = dataTypeFor(ctx.datatype()) +val step = if(ctx.STEP()!=null && ctx.expression().size > 1) ctx.expression(1).accept(this) as Expression else null +return ForLoop(loopvar, iterable, scope, loopVarType, step, ctx.toPosition()) +``` + +Proposed implementation shape: + +```kotlin +val loopvar = ctx.scoped_identifier().accept(this) as IdentifierReference +val iterable = ctx.expression(0).accept(this) as Expression +val scope = stmtBlockOrSingle(ctx.statement_block(), ctx.statement()) +val loopVarType = dataTypeFor(ctx.datatype()) + +val viaField = ctx.viaField()?.text // only the identifier under VIA +val untilExpr = ctx.untilExpression()?.accept(this) as Expression? +val step = ctx.stepExpression()?.accept(this) as Expression? + +return ForLoop( + loopVar = loopvar, + iterable = iterable, + body = scope, + loopVarType = loopVarType, + step = step, + position = ctx.toPosition(), + linkedVia = viaField, + linkedUntil = untilExpr +) +``` + +The grammar should expose labeled contexts such as `untilExpression=expression` +and `stepExpression=expression`, and the visitor must use those labels. Do not +infer clause meaning from `ctx.expression().size`: for example, `via .next +step -1` must remain an invalid linked-loop combination, not reinterpret `-1` +as the sentinel. + +## AST Changes + +File: `compilerAst/src/prog8/ast/statements/AstStatements.kt` + +Extend `ForLoop`: + +```kotlin +class ForLoop(var loopVar: IdentifierReference, + var iterable: Expression, + var body: AnonymousScope, + var loopVarType: DataType?, + var step: Expression?, + override val position: Position, + var linkedVia: String? = null, + var linkedUntil: Expression? = null, + var linkedReverse: Boolean = false, + ) : Statement() +``` + +Keep `position` before the new defaulted parameters so existing positional +construction remains source-compatible, and prefer named arguments for new +construction sites. + +Update: + +- `linkParents()` must link `linkedUntil`. +- `replaceChildNode()` must handle `linkedUntil`. +- `toString()` should include the new fields for debugging. +- `referencesIdentifier()` must include `linkedUntil`. +- `IAstVisitor.visit(ForLoop)` and `AstWalker.visit(ForLoop)` must visit + `linkedUntil`, otherwise identifier replacement, constant folding, and type + checking will skip the sentinel expression. + +Example: + +```kotlin +override fun linkParents(parent: Node) { + this.parent = parent + loopVar.linkParents(this) + iterable.linkParents(this) + step?.linkParents(this) + linkedUntil?.linkParents(this) + body.linkParents(this) +} +``` + +```kotlin +override fun replaceChildNode(node: Node, replacement: Node) { + when { + node === loopVar -> loopVar = replacement as IdentifierReference + node === iterable -> iterable = replacement as Expression + node === step -> step = replacement as Expression + node === linkedUntil -> linkedUntil = replacement as Expression + node === body -> body = replacement as AnonymousScope + else -> throw FatalAstException("invalid replace in $this at ${node.position}: $node -> $replacement") + } + replacement.parent = this +} +``` + +## Type Checking + +File: `compiler/src/prog8/compiler/astprocessing/AstChecker.kt` + +Method: `visit(forLoop: ForLoop)` + +Add an early branch before the normal iterable checks: + +```kotlin +if (forLoop.linkedVia != null || forLoop.linkedReverse || isLinkedHeaderShorthand(forLoop)) { + checkLinkedForLoop(forLoop) + super.visit(forLoop) + return +} +``` + +Implement `checkLinkedForLoop(forLoop)`: + +1. `loopVar.targetVarDecl()` must exist and have a pointer base type. +2. `iterable.inferType(program)` must be a pointer to the same node type, or a list header type with resolvable metadata. +3. `linkedVia` field must exist on the node type and be a pointer to the same node type. +4. `linkedUntil`, if present, must be comparable to the node pointer type. +5. If `linkedReverse` is true, a backward link must be available. +6. Reject `step` when linked-list options are present. + +For header shorthand (`for node in list` with no `via`), require the iterable +type to have resolvable list metadata. `isLinkedHeaderShorthand` must only +return true for a pointer/list-header type with linked-list metadata; ordinary +array, string, range, and other existing `for` loops must continue through the +normal checks. If the expression is intended as a header but metadata cannot +be resolved, report: + +```text +linked-list iteration requires 'via .field', or a list type with @head/@succ/@tail metadata +``` + +## Desugaring + +File: `compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt` + +Method: `after(forLoop: ForLoop, parent: Node)` + +Add a new branch before the existing array/string reverse handling: + +```kotlin +if (forLoop.linkedVia != null || forLoop.linkedReverse || isLinkedHeaderShorthand(forLoop)) { + return desugarLinkedForLoop(forLoop, parent) +} +``` + +### Explicit form + +Input: + +```prog8 +for node in start via .next until sentinel { body } +``` + +Output AST equivalent to: + +```prog8 +node = start +while node != sentinel { + body + node = node.next +} +``` + +Implementation steps: + +1. Resolve `startExpr = forLoop.iterable`. +2. Resolve `linkField = forLoop.linkedVia ?: resolvedDefaultLink`. +3. Resolve `sentinelExpr = forLoop.linkedUntil ?: NumericLiteral.optimalInteger(0, pos)`. +4. Create assignment `node = startExpr`. +5. Create condition `node != sentinelExpr`. +6. Create body: + - original `forLoop.body` + - assignment `node = node.linkField` +7. Replace `ForLoop` with a `VarDecl`? No. The loop variable must already be declared or declared by the for loop. For a typed for loop, create the variable declaration first if needed, matching existing for-loop behavior. For an untyped for loop, reuse the existing variable. +8. Return a list of `AstInsertBefore`, `AstReplaceNode`, or `AstInsertAfter` modifications, depending on how other desugarers in this file operate. + +Update `ImplicitForIteratorDecls` as part of this work. It currently only +creates implicit loop variables for recognized iterable types, so pointer +starts and list headers would otherwise reach desugaring without a declaration. +The linked-loop path must infer the node pointer type, validate an existing +declaration or insert a scoped declaration for an untyped iterator, and keep +symbol-table/parent links valid before constructing assignments. + +The implementation should mirror `desugarReverseArrayStringForLoop`, which already builds synthetic `VarDecl`, `Assignment`, `WhileLoop`, and `AnonymousScope` nodes. + +### Header shorthand + +For `for node in list`, resolve metadata first: + +```kotlin +data class LinkedMetadata( + val start: Expression, + val link: String, + val sentinel: Expression +) +``` + +Resolution order: + +1. Struct attributes: + - `@head`, `@succ`/`@next`, `@pred`/`@prev`, `@tail` +2. Conventional field names: + - head: `head`, `first` + - succ: `succ`, `next` + - pred: `pred`, `prev`, `prevNode` + - tail: `tail`, `last` +3. If the iterable is already a node pointer, use it directly and default to `next`/`succ` with null terminator. + +If resolution fails, leave the node unchanged and let `AstChecker` report the error. + +### Attribute support + +> **Not implemented yet.** The current implementation uses only conventional field +> names; attributes are future work. + +Attributes can be added to struct fields using the existing attribute mechanism. If Prog8 does not already support field attributes, add a minimal `@head`, `@succ`, `@pred`, and `@tail` marker that is stored on `VarDecl` or struct field metadata. + +Files likely involved: + +- `compilerAst` struct field representation +- parser grammar for struct fields +- `AstChecker` for attribute validation + +If attributes are too large for the first implementation, ship conventional names first and add attributes later. + +## Simplified AST + +No changes should be needed in `simpleAst` if desugaring happens in `CodeDesugarer`, because the resulting `WhileLoop` and `Assignment` nodes are already supported by `SimplifiedAstMaker`. + +## Backend Impact + +No backend changes: + +- `codeGenCpu6502` +- `codeGenIntermediate` +- `codeGenVirtual` +- `codeGenM68k` + +All traversal logic is removed during desugaring. + +## Tests + +Files to update or add: + +- `compiler/test/ast/TestProg8Parser.kt` + - parse `via` and `until` +- `compiler/test/ast/TestVariousCompilerAst.kt` + - verify AST fields +- `compiler/test/TestCompilerOnRanges.kt` or a new `TestLinkedForLoops.kt` + - desugaring output +- `compiler/test/codegeneration/TestConstStepForLoops6502.kt` or a new linked-list codegen test + - generated traversal +- `compiler/test/vm/TestVariableStepForLoops.kt` or a new VM test + - execution behavior + +Test cases: + +1. Singly linked list, default null terminator. +2. Explicit `until 0`. +3. Custom sentinel. +4. Exec-style forward traversal with `&list.tail`. +5. Backward traversal with `pred`. +6. Custom field names using `via`. +7. Header shorthand with conventional names. +8. Header shorthand with attributes. +9. Invalid `step` combination. +10. Missing link field. +11. Wrong link field type. +12. Non-pointer loop variable. +13. `-noopt` and optimized builds match. + +## Implementation Order + +1. Add grammar and parser visitor. +2. Add AST fields. +3. Add type checking. +4. Add explicit-form desugaring. +5. Add custom sentinel support. +7. Add conventional-name header shorthand. +8. Add attributes. +9. Add tests. + +## Risks + +- Expression indexing in the parser visitor is fragile; capture contexts explicitly. +- Header shorthand resolution can become implicit and surprising; attributes should be preferred. +- Exec list termination uses `&list.tail`, not `list.tail`. +- List mutation during traversal is allowed but can invalidate terminators; document this. +- Conventional names may collide with unrelated fields; attributes resolve ambiguity. diff --git a/parser/src/main/antlr/Prog8ANTLR.g4 b/parser/src/main/antlr/Prog8ANTLR.g4 index 45501cae4..d82eedce7 100644 --- a/parser/src/main/antlr/Prog8ANTLR.g4 +++ b/parser/src/main/antlr/Prog8ANTLR.g4 @@ -33,6 +33,8 @@ INLINE: 'inline'; PRIVATE: 'private'; PUBLIC: 'public'; STEP: 'step'; +VIA: 'via'; +UNTIL: 'until'; ELSE: 'else'; THEN: 'then'; ENUM: 'enum'; @@ -425,11 +427,15 @@ branch_stmt : branchcondition EOL? (statement | statement_block) EOL? else_part? branchcondition: 'if_cs' | 'if_cc' | 'if_eq' | 'if_z' | 'if_ne' | 'if_nz' | 'if_pl' | 'if_pos' | 'if_mi' | 'if_neg' | 'if_vs' | 'if_vc' ; -forloop : 'for' datatype? scoped_identifier 'in' expression (STEP expression)? EOL? (statement | statement_block) ; +forloop : 'for' datatype? scoped_identifier 'in' iterable=expression + (VIA '.' viaField=identifier)? + (UNTIL untilExpr=expression)? + (STEP stepExpr=expression)? + EOL? (statement | statement_block) ; whileloop: 'while' expression EOL? (statement | statement_block) ; -untilloop: 'do' (statement | statement_block) EOL? 'until' expression ; +untilloop: 'do' (statement | statement_block) EOL? UNTIL expression ; repeatloop: 'repeat' expression? EOL? (statement | statement_block) ; diff --git a/syntax-files/Jetbrains-IDEs/Prog8.xml b/syntax-files/Jetbrains-IDEs/Prog8.xml index 3674da96b..2fb9b6ffc 100644 --- a/syntax-files/Jetbrains-IDEs/Prog8.xml +++ b/syntax-files/Jetbrains-IDEs/Prog8.xml @@ -11,7 +11,7 @@