building syntax support for ptr[x].field

attempting to do this by making '.' an expression operator

compiler/src/prog8/compiler/astprocessing/AstChecker.kt

@@ -62,6 +62,12 @@ internal class AstChecker(private val program: Program,
     }
 
     override fun visit(identifier: IdentifierReference) {
+
+        val parentExpr = identifier.parent as? BinaryExpression
+        if(parentExpr?.operator=="^^") {
+            return  // identifiers will be checked over at the BinaryExpression itself
+        }
+
         if(identifier.nameInSource.any { it.startsWith('_') }) {
             errors.err("identifiers cannot start with an underscore", identifier.position)
         }
@@ -1228,6 +1234,39 @@ internal class AstChecker(private val program: Program,
 
     override fun visit(expr: BinaryExpression) {
         super.visit(expr)
+
+        if(expr.operator==".")
+            throw FatalAstException("temporary operator '.' should have been replaced by '^^' for 'walking the chain' at ${expr.position}")
+
+        if(expr.operator=="^^") {
+            val leftIdentfier = expr.left as? IdentifierReference
+            val leftIndexer = expr.left as? ArrayIndexedExpression
+            val rightIdentifier = expr.right as? IdentifierReference
+            val rightIndexer = expr.right as? ArrayIndexedExpression
+            if(rightIdentifier!=null) {
+                val struct: StructDecl? =
+                    if (leftIdentfier != null) {
+                        // PTR.FIELD
+                        leftIdentfier.targetVarDecl()?.datatype?.subType as? StructDecl
+                    } else if(leftIndexer!=null) {
+                        // ARRAY[x].NAME --> maybe it's a pointer dereference
+                        leftIndexer.arrayvar.targetVarDecl()?.datatype?.subType as? StructDecl
+                    }
+                    else null
+                if (struct != null) {
+                    val field = struct.getFieldType(rightIdentifier.nameInSource.joinToString("."))
+                    if (field == null)
+                        errors.err("no such field '${rightIdentifier.nameInSource.joinToString(".")}' in struct '${struct.name}'", expr.position)
+                } else
+                    errors.err("cannot find struct type", expr.position)
+            } else if(rightIndexer!=null) {
+                TODO("something.field[y]   at ${expr.position}")
+                // TODO I don't think we can evaluate this because it could end up in as a struct instance, which we don't support yet... rewrite or just give an error?
+            } else
+                throw FatalAstException("expected identifier or arrayindexer after ^^ operator at ${expr.position})")
+            return
+        }
+
         checkLongType(expr)
 
         val leftIDt = expr.left.inferType(program)

compiler/src/prog8/compiler/astprocessing/AstExtensions.kt

@@ -132,7 +132,7 @@ internal fun Program.checkIdentifiers(errors: IErrorReporter, options: Compilati
     checker2.visit(this)
 
     if(errors.noErrors()) {
-        val lit2decl = LiteralsToAutoVars(this, errors)
+        val lit2decl = LiteralsToAutoVarsAndRecombineIdentifiers(this, errors)
         lit2decl.visit(this)
         while(errors.noErrors() && lit2decl.applyModifications()>0)
             lit2decl.visit(this)

compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt

@@ -25,6 +25,7 @@ internal class CodeDesugarer(val program: Program, private val errors: IErrorRep
     // - flatten chained assignments
     // - remove alias nodes
     // - replace implicit pointer dereference chains (a.b.c.d) with explicit ones (a^^.b^^.c^^.d)
+    // - replace binary expression with "." operator with one that has a "^^" operator, to signify pointer scope traversal
 
     override fun after(alias: Alias, parent: Node): Iterable<IAstModification> {
         return listOf(IAstModification.Remove(alias, parent as IStatementContainer))
@@ -267,6 +268,10 @@ _after:
             return listOf(IAstModification.ReplaceNode(expr, squareCall, parent))
         }
 
+        if(expr.operator==".") {
+            expr.operator = "^^"
+        }
+
         return noModifications
     }
 

compiler/src/prog8/compiler/astprocessing/LiteralsToAutoVarsAndRecombineIdentifiers.kt

@@ -12,7 +12,7 @@ import prog8.code.ast.PtContainmentCheck
 import prog8.code.core.IErrorReporter
 
 
-internal class LiteralsToAutoVars(private val program: Program, private val errors: IErrorReporter) : AstWalker() {
+internal class LiteralsToAutoVarsAndRecombineIdentifiers(private val program: Program, private val errors: IErrorReporter) : AstWalker() {
 
     override fun after(string: StringLiteral, parent: Node): Iterable<IAstModification> {
         if(string.parent !is VarDecl && string.parent !is WhenChoice) {
@@ -160,4 +160,22 @@ internal class LiteralsToAutoVars(private val program: Program, private val erro
 //        }
         return noModifications
     }
+
+    override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".") {
+            val leftIdent = expr.left as? IdentifierReference
+            val rightIndex = expr.right as? ArrayIndexedExpression
+            if (leftIdent != null && rightIndex != null) {
+                // maybe recombine   IDENTIFIER . ARRAY[IDX]  -->  COMBINEDIDENTIFIER[IDX]
+                val leftTarget = leftIdent.targetStatement(null)
+                if(leftTarget==null || leftTarget !is StructDecl) {
+                    val combinedName = leftIdent.nameInSource + rightIndex.arrayvar.nameInSource
+                    val combined = IdentifierReference(combinedName, leftIdent.position)
+                    val indexer = ArrayIndexedExpression(combined, rightIndex.indexer, leftIdent.position)
+                    return listOf(IAstModification.ReplaceNode(expr, indexer, parent))
+                }
+            }
+        }
+        return noModifications
+    }
 }

compiler/src/prog8/compiler/astprocessing/SimplifiedAstMaker.kt

@@ -665,10 +665,17 @@ class SimplifiedAstMaker(private val program: Program, private val errors: IErro
 
     private fun transform(srcExpr: BinaryExpression): PtBinaryExpression {
         val type = srcExpr.inferType(program).getOrElse { throw FatalAstException("unknown dt") }
-        val expr = PtBinaryExpression(srcExpr.operator, type, srcExpr.position)
+        if(srcExpr.operator=="^^") {
-        expr.add(transformExpression(srcExpr.left))
+            if(srcExpr.left is ArrayIndexedExpression) {
-        expr.add(transformExpression(srcExpr.right))
+                TODO("ptr[x].field dereference, field type=$type  at ${srcExpr.position}")
-        return expr
+            } else
+                TODO("??? deref something ??? at ${srcExpr.position}")
+        } else {
+            val expr = PtBinaryExpression(srcExpr.operator, type, srcExpr.position)
+            expr.add(transformExpression(srcExpr.left))
+            expr.add(transformExpression(srcExpr.right))
+            return expr
+        }
     }
 
     private fun transform(srcCheck: ContainmentCheck): PtExpression {

compilerAst/src/prog8/ast/AstToSourceTextConverter.kt

@@ -92,14 +92,17 @@ class AstToSourceTextConverter(val output: (text: String) -> Unit, val program:
     }
 
     override fun visit(expr: BinaryExpression) {
-        output("(")
+        val isValue = expr.parent is Assignment
+        if(!isValue) output("(")
         expr.left.accept(this)
         if(expr.operator.any { it.isLetter() })
             output(" ${expr.operator} ")
+        else if(expr.operator=="^^")
+            output(".")
         else
             output(expr.operator)
         expr.right.accept(this)
-        output(")")
+        if(!isValue) output(")")
     }
 
     override fun visit(directive: Directive) {

compilerAst/src/prog8/ast/expressions/AstExpressions.kt

@@ -231,6 +231,36 @@ class BinaryExpression(
             "<=", ">=",
             "==", "!=" -> InferredTypes.knownFor(BaseDataType.BOOL)
             "<<", ">>" -> leftDt
+            "." -> InferredTypes.unknown()      // intermediate operator, will be replaced with '^^' after recombining scoped identifiers
+            "^^" -> {
+                val leftIdentfier = left as? IdentifierReference
+                val leftIndexer = left as? ArrayIndexedExpression
+                val rightIdentifier = right as? IdentifierReference
+                val rightIndexer = right as? ArrayIndexedExpression
+                if(rightIdentifier!=null) {
+                    val struct: StructDecl? =
+                        if (leftIdentfier != null) {
+                            // PTR . FIELD
+                            leftIdentfier.targetVarDecl()?.datatype?.subType as? StructDecl
+                        } else if(leftIndexer!=null) {
+                            // ARRAY[x].NAME --> maybe it's a pointer dereference
+                            leftIndexer.arrayvar.targetVarDecl()?.datatype?.subType as? StructDecl
+                        }
+                        else null
+                    if (struct != null) {
+                        val field = struct.getFieldType(rightIdentifier.nameInSource.joinToString("."))
+                        if (field != null)
+                            InferredTypes.knownFor(field)
+                        else
+                            InferredTypes.unknown()
+                    } else
+                        InferredTypes.unknown()
+                } else if(rightIndexer!=null) {
+                    TODO("something.field[x]  at ${right.position}")
+                    // TODO I don't think we can evaluate this type because it could end up in as a struct instance, which we don't support yet... rewrite or just give an error?
+                } else
+                    InferredTypes.unknown()
+            }
             else -> throw FatalAstException("resulting datatype check for invalid operator $operator")
         }
     }
@@ -357,8 +387,10 @@ class ArrayIndexedExpression(var arrayvar: IdentifierReference,
                 target.datatype.isString || target.datatype.isUnsignedWord -> InferredTypes.knownFor(BaseDataType.UBYTE)
                 target.datatype.isArray -> InferredTypes.knownFor(target.datatype.elementType())
                 target.datatype.isPointer -> {
-                    if(target.datatype.subType!=null)
+                    if(target.datatype.subTypeFromAntlr!=null)
-                        TODO("indexing on pointer to struct would yield the struct type itself, this is not yet supported (only pointers) at $position")
+                        InferredTypes.unknown()
+                    else if(target.datatype.subType!=null)
+                        InferredTypes.unknown()    //  TODO("indexing on pointer to struct would yield the struct type itself, this is not yet supported (only pointers) at $position")
                     else
                         InferredTypes.knownFor(target.datatype.sub!!)
                 }

docs/source/todo.rst

@@ -28,6 +28,7 @@ STRUCTS and TYPED POINTERS
 - DONE (for basic types only): allow array syntax on pointers too: ptr[2]  means ptr+sizeof()*2,   ptr[0]  just means  ptr^^  .
 - allow array syntax on pointers to structs too, but what type will ptr[2] have? And it will require  ptr[2].field  to work as well now. Actually that will be the only thing to work for now.
 - pointer arithmetic should follow C:  ptr=ptr+10 adds 10*sizeof() instead of just 10.
+- can we get rid of the '.' -> '^^' operator rewrite?
 - add unit tests for all changes
 - arrays of structs? No -> Just an array of uword pointers to said structs. Can even be @split as the only representation form because that's the default for word arrays.
 - static initialization of structs may be allowed only at block scope and then behaves like arrays; it won't reset to the original value when program is restarted, so beware.  Syntax = TBD

examples/test.p8

@@ -1,22 +1,25 @@
-%import textio
-%import floats
-%zeropage basicsafe
-%option no_sysinit
 
 main {
     sub start() {
+;        rotatedx[i] = A*shipdata.x[i] + B*shipdata.y[i] + C*shipdata.z[i]
+;        cx16.VERA_DATA0 = cx16.VERA_DATA0 & gfx_hires.plot.mask4c[cx16.r2L] | cx16.r12L
+        if cx16.r0[0]!='0' {
+            cx16.r0++
+        }
+
         struct Node {
             bool flag
             ^^Node next
         }
 
-        ^^Node ptr = 2000
+        ^^Node ptr
 
-        txt.print_uw(ptr)
+        bool zz1, zz2
-        txt.nl()
+        ptr = ptr[10].next
-
+        zz2 = ptr[10].next.flag         ;; TODO fix
-        bool derp = ptr[2].flag
+        zz1 = ptr[10].next.next.flag    ;; TODO fix
-        bool derp2 = ptr.next.next[2].flag
+        zz2 = ptr[10].next[2].flag      ;; TODO fix
+        zz1 = ptr[10].next[2].flagz     ;; TODO should complain about flagz only
     }
 }
 

parser/src/main/antlr/Prog8ANTLR.g4

@@ -189,6 +189,7 @@ postincrdecr :  assign_target  operator = ('++' | '--') ;
 expression :
     '(' expression ')'
     | functioncall
+    | left = expression EOL? bop = '.' EOL? right = expression          // "scope traversal operator"
     | <assoc=right> prefix = ('+'|'-'|'~') expression
     | left = expression EOL? bop = ('*' | '/' | '%' ) EOL? right = expression
     | left = expression EOL? bop = ('+' | '-' ) EOL? right = expression

simpleAst/src/prog8/code/ast/AstExpressions.kt

@@ -215,7 +215,7 @@ class PtBinaryExpression(val operator: String, type: DataType, position: Positio
     init {
         if(operator in ComparisonOperators + LogicalOperators)
             require(type.isBool)
-        else
+        else if(operator!="^^")
             require(!type.isBool) { "no bool allowed for this operator $operator"}
     }
 }