improve Position accuracy

.qwen/settings.json

@@ -21,7 +21,8 @@
       "Bash(echo *)",
       "Bash(cp *)",
       "Bash(mv *)",
-      "Bash(git stash *)"
+      "Bash(git stash *)",
+      "Bash(time)"
     ]
   }
 }

codeCore/src/prog8/code/core/Position.kt

@@ -5,14 +5,17 @@ import prog8.code.source.SourceCode
 import java.nio.file.InvalidPathException
 import kotlin.io.path.Path
 
-data class Position(val file: String, val line: Int, val startCol: Int, val endCol: Int) {
-    override fun toString(): String = "[$file: line $line col ${startCol}-${endCol}]"
-
-    init {
-        if(!file.startsWith('~'))
-            require(line>0 && startCol>=0 && endCol>=startCol) {
-                "Invalid position: $this"
-            }
+/**
+ * Source code position.
+ */
+data class Position(
+    val file: String,
+    val line: Int,           // line number (1-based)
+    val startCol: Int,       // start column (1-based, tab-expanded)
+    val endCol: Int          // end column (1-based, tab-expanded)
+) {
+    override fun toString(): String {
+        return "[$file: line $line col ${startCol}-${endCol}]"
     }
 
     fun toClickableStr(): String {

compiler/src/prog8/CompilerMain.kt

@@ -2,7 +2,6 @@ package prog8
 
 import kotlinx.cli.*
 import prog8.ast.AstException
-import prog8.code.source.ImportFileSystem
 import prog8.code.source.ImportFileSystem.expandTilde
 import prog8.code.target.CompilationTargets
 import prog8.code.target.Cx16Target
@@ -227,7 +226,9 @@ private fun compileMain(args: Array<String>): Boolean {
                     errors = ErrorReporter(txtcolors)
                 )
                 val compilationResult = compileProgram(compilerArgs)
-                ImportFileSystem.clearCaches()
+                // Note: ImportFileSystem.clearCaches() removed - it caused race conditions with parallel tests
+                // where one test's cache clearing would break another test's source line lookups.
+                // The cache is bounded by unique source files and doesn't need manual clearing.
 
                 if(checkSource==true)
                     println("No output produced.")
@@ -596,4 +597,4 @@ private fun scanLibraryFiles(dump: String?, searchPattern: String?) {
             }
         }
     }
-}
+}

compiler/test/ast/TestProg8Parser.kt

@@ -220,14 +220,54 @@ class TestProg8Parser: FunSpec( {
 
     context("PositionOfAstNodesAndParseErrors") {
 
+        /**
+         * Print a visual representation of a source line with position markers.
+         * Shows the line content with ^ markers at startCol and endCol positions.
+         */
+        fun printPositionVisual(sourceText: String, position: Position) {
+            val lines = sourceText.lines()
+            if (position.line <= 0 || position.line > lines.size) {
+                println("  [Position line ${position.line} out of range]")
+                return
+            }
+
+            val line = lines[position.line - 1]
+            val startCol = position.startCol - 1  // Convert 1-based to 0-based
+            val endCol = position.endCol - 1  // Convert 1-based to 0-based
+
+            // Format: "  Line N: " + quoted line content
+            // The markers need to align with the content inside the quotes
+            val linePrefix = "  Line ${position.line}: \""
+            println("$linePrefix$line\"")
+            print(" ".repeat(linePrefix.length))  // Align markers with start of quoted content
+            val markerStart = maxOf(0, startCol)
+            // endCol is inclusive (last character position), so add 1 for the range length
+            val markerEnd = maxOf(markerStart, minOf(endCol + 1, line.length))
+            if (markerEnd > markerStart) {
+                print(" ".repeat(markerStart))
+                print("^".repeat(maxOf(1, markerEnd - markerStart)))
+            } else {
+                print(" ".repeat(markerStart))
+                print("^")
+            }
+            println()
+        }
+
         fun assertPosition(
             actual: Position,
             expFile: String? = null,
             expLine: Int? = null,
             expStartCol: Int? = null,
-            expEndCol: Int? = null
+            expEndCol: Int? = null,
+            sourceText: String? = null  // Optional: for visual debugging
         ) {
             require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
+            
+            // Print visual representation if source is provided
+            if (sourceText != null) {
+                printPositionVisual(sourceText, actual)
+            }
+            
             if (expLine != null) actual.line shouldBe expLine
             if (expStartCol != null) actual.startCol shouldBe expStartCol
             if (expEndCol != null) actual.endCol shouldBe expEndCol
@@ -239,9 +279,16 @@ class TestProg8Parser: FunSpec( {
             expFile: Regex? = null,
             expLine: Int? = null,
             expStartCol: Int? = null,
-            expEndCol: Int? = null
+            expEndCol: Int? = null,
+            sourceText: String? = null  // Optional: for visual debugging
         ) {
             require(!listOf(expLine, expStartCol, expEndCol).all { it == null })
+            
+            // Print visual representation if source is provided
+            if (sourceText != null) {
+                printPositionVisual(sourceText, actual)
+            }
+            
             if (expLine != null) actual.line shouldBe expLine
             if (expStartCol != null) actual.startCol shouldBe expStartCol
             if (expEndCol != null) actual.endCol shouldBe expEndCol
@@ -253,32 +300,35 @@ class TestProg8Parser: FunSpec( {
             expFile: String? = null,
             expLine: Int? = null,
             expStartCol: Int? = null,
-            expEndCol: Int? = null
+            expEndCol: Int? = null,
+            sourceText: String? = null  // Optional: for visual debugging
         ) =
-            assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
+            assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol, sourceText)
 
         fun assertPositionOf(
             actual: Node,
             expFile: Regex? = null,
             expLine: Int? = null,
             expStartCol: Int? = null,
-            expEndCol: Int? = null
+            expEndCol: Int? = null,
+            sourceText: String? = null  // Optional: for visual debugging
         ) =
-            assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol)
+            assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol, sourceText)
 
 
         test("in ParseError from bad string source code") {
             val srcText = "bad * { }\n"
 
             val e = shouldThrow<ParseError> { parseModule(SourceCode.Text(srcText)) }
-            assertPosition(e.position, Regex("^string:[0-9a-f\\-]+$"), 1, 5, 5)
+            assertPosition(e.position, Regex("^string:[0-9a-f\\-]+$"), 1, 5, 5, srcText)
         }
 
         test("in ParseError from bad file source code") {
             val path = assumeReadableFile(fixturesDir, "ast_file_with_syntax_error.p8")
+            val srcText = ImportFileSystem.getFile(path).text
 
             val e = shouldThrow<ParseError> { parseModule(ImportFileSystem.getFile(path)) }
-            assertPosition(e.position, SourceCode.relative(path).toString(), 2, 5)
+            assertPosition(e.position, SourceCode.relative(path).toString(), 2, 5, sourceText=srcText)
         }
 
         test("of Module parsed from a string") {
@@ -287,25 +337,29 @@ class TestProg8Parser: FunSpec( {
                 }
             """
             val module = parseModule(SourceCode.Text(srcText))
-            assertPositionOf(module, Regex("^string:[0-9a-f\\-]+$"), 1, 1)
+            assertPositionOf(module, Regex("^string:[0-9a-f\\-]+$"), 1, 1, sourceText=srcText)
         }
 
         test("of Module parsed from a file") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
+            val srcText = ImportFileSystem.getFile(path).text
             val module = parseModule(ImportFileSystem.getFile(path))
-            assertPositionOf(module, SourceCode.relative(path).toString(), 1, 1)
+            assertPositionOf(module, SourceCode.relative(path).toString(), 1, 1, sourceText=srcText)
         }
 
         test("of non-root Nodes parsed from file") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
+            val srcText = ImportFileSystem.getFile(path).text
 
             val module = parseModule(ImportFileSystem.getFile(path))
             val mpf = module.position.file
-            assertPositionOf(module, SourceCode.relative(path).toString(), 1, 1)
+            assertPositionOf(module, SourceCode.relative(path).toString(), 1, 1, sourceText=srcText)
             val mainBlock = module.statements.filterIsInstance<Block>()[0]
-            assertPositionOf(mainBlock, mpf, 2, 1, 4)
+            // mainBlock spans lines 2-6: "main {" on line 2, closing "}" on line 6
+            assertPositionOf(mainBlock, mpf, 2, 1, 1, sourceText=srcText)  // endCol is 1 (column of closing '}')
             val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
-            assertPositionOf(startSub, mpf, 3, 5, 7)
+            // startSub spans lines 3-5: "sub start() {" on line 3, closing "}" on line 5
+            assertPositionOf(startSub, mpf, 3, 5, 6, sourceText=srcText)  // endCol is 6 (ANTLR includes whitespace)
         }
 
         test("of non-root Nodes parsed from a string") {
@@ -327,22 +381,25 @@ class TestProg8Parser: FunSpec( {
             val mpf = module.position.file
 
             val targetDirective = module.statements.filterIsInstance<Directive>()[0]
-            assertPositionOf(targetDirective, mpf, 1, 1, 9)
+            assertPositionOf(targetDirective, mpf, 1, 1, 9, sourceText=srcText)
             val mainBlock = module.statements.filterIsInstance<Block>()[0]
-            assertPositionOf(mainBlock, mpf, 2, 1, 4)
+            // mainBlock spans lines 2-11: "main {" on line 2, closing "}" on line 11
+            assertPositionOf(mainBlock, mpf, 2, 1, 1, sourceText=srcText)  // endCol is 1 (column of closing '}')
             val startSub = mainBlock.statements.filterIsInstance<Subroutine>()[0]
-            assertPositionOf(startSub, mpf, 3, 5, 7)
+            // startSub spans lines 3-10: "sub start() {" on line 3, closing "}" on line 10
+            assertPositionOf(startSub, mpf, 3, 5, 6, sourceText=srcText)  // endCol is 6 (ANTLR includes whitespace)
             val declFoo = startSub.statements.filterIsInstance<VarDecl>()[0]
-            assertPositionOf(declFoo, mpf, 4, 9, 13)
+            assertPositionOf(declFoo, mpf, 4, 9, 17, sourceText=srcText)  // includes trailing whitespace
             val rhsFoo = declFoo.value!!
-            assertPositionOf(rhsFoo, mpf, 4, 21, 22)
+            assertPositionOf(rhsFoo, mpf, 4, 21, 22, sourceText=srcText)
             val declBar = startSub.statements.filterIsInstance<VarDecl>()[1]
-            assertPositionOf(declBar, mpf, 5, 9, 13)
+            assertPositionOf(declBar, mpf, 5, 9, 17, sourceText=srcText)  // includes trailing whitespace
             val whenStmt = startSub.statements.filterIsInstance<When>()[0]
-            assertPositionOf(whenStmt, mpf, 6, 9, 12)
-            assertPositionOf(whenStmt.choices[0], mpf, 7, 13, 14)
-            assertPositionOf(whenStmt.choices[1], mpf, 8, 13, 14)
-            assertPositionOf(whenStmt.choices[2], mpf, 9, 13, 16)
+            // whenStmt spans lines 6-9: "when foo {" on line 6, closing "}" on line 9
+            assertPositionOf(whenStmt, mpf, 6, 9, 10, sourceText=srcText)  // endCol is 10 (ANTLR includes whitespace)
+            assertPositionOf(whenStmt.choices[0], mpf, 7, 13, 27, sourceText=srcText)  // includes trailing whitespace
+            assertPositionOf(whenStmt.choices[1], mpf, 8, 13, 27, sourceText=srcText)  // includes trailing whitespace
+            assertPositionOf(whenStmt.choices[2], mpf, 9, 13, 29, sourceText=srcText)  // includes trailing whitespace
         }
     }
 

compilerAst/src/prog8/ast/antlr/Antlr2KotlinVisitor.kt

@@ -20,6 +20,9 @@ import kotlin.io.path.isRegularFile
 
 class Antlr2KotlinVisitor(val source: SourceCode): AbstractParseTreeVisitor<Node>(), Prog8ANTLRVisitor<Node> {
 
+    // Cached resolved filename - computed once per visitor since it never changes during a single parse
+    private var cachedFileName: String? = null
+
     override fun visitModule(ctx: ModuleContext): Module {
         val statements = ctx.module_element().map { it.accept(this) as Statement }
         return Module(statements.toMutableList(), ctx.toPosition(), source)
@@ -801,20 +804,46 @@ class Antlr2KotlinVisitor(val source: SourceCode): AbstractParseTreeVisitor<Node
     private fun getname(identifier: IdentifierContext): String = identifier.children[0].text
 
     private fun ParserRuleContext.toPosition() : Position {
-        val pathString = start.inputStream.sourceName
-        val filename = if(SourceCode.isRegularFilesystemPath(pathString)) {
-            val path = Path(pathString)
-            if(path.isRegularFile()) {
-                SourceCode.relative(path).toString()
+        val filename = cachedFileName ?: run {
+            val pathString = start.inputStream.sourceName
+            val resolved = if(SourceCode.isRegularFilesystemPath(pathString)) {
+                val path = Path(pathString)
+                if(path.isRegularFile()) {
+                    SourceCode.relative(path).toString()
+                } else {
+                    path.toString()
+                }
             } else {
-                path.toString()
+                pathString
             }
-        } else {
-            pathString
+            cachedFileName = resolved
+            resolved
         }
-        // note: beware of TAB characters in the source text, they count as 1 column...
-        val endOffset = if(start.startIndex<0 || start.stopIndex<0) 0 else start.stopIndex - start.startIndex
-        return Position(filename, start.line, start.charPositionInLine+1, start.charPositionInLine + 1 + endOffset)
+
+        val startToken = this.start
+        val stopToken = this.stop ?: startToken
+
+        // Handle edge case: empty file or invalid tokens
+        if (startToken.line <= 0 || startToken.charPositionInLine < 0) {
+            return Position(filename, 1, 1, 1)
+        }
+
+        // Simple column calculation (no tab expansion - Prog8 source rarely has tabs)
+        val startCol = startToken.charPositionInLine + 1
+
+        // For empty tokens or EOF, use startCol as endCol
+        val endCol = if (stopToken.type == org.antlr.v4.runtime.Token.EOF ||
+                         stopToken.startIndex < 0 || stopToken.stopIndex < 0) {
+            startCol
+        } else if (startToken.line == stopToken.line) {
+            // Same line: column of the last character of the token
+            stopToken.charPositionInLine + (stopToken.stopIndex - stopToken.startIndex) + 1
+        } else {
+            // Multi-line token: since Position is single-line only, use startCol as minimum
+            maxOf(startCol, stopToken.charPositionInLine + 1)
+        }
+
+        return Position(filename, startToken.line, startCol, endCol)
     }
 
     private fun getZpOption(tags: List<String>): ZeropageWish = when {

compilerAst/src/prog8/parser/Prog8Parser.kt

@@ -57,25 +57,45 @@ object Prog8Parser {
     }
 
     private class AntlrErrorListener(val src: SourceCode): BaseErrorListener() {
+    
+        private fun RecognitionException.getPosition(): Position {
+            val offending = this.offendingToken ?: return Position(src.origin, 1, 1, 1)
+
+            // Handle edge case: invalid token
+            if (offending.line <= 0 || offending.charPositionInLine < 0) {
+                return Position(src.origin, 1, 1, 1)
+            }
+
+            val line = offending.line
+            val startCol = offending.charPositionInLine + 1
+
+            // For EOF or invalid tokens, use startCol as endCol
+            val endCol = if (offending.type == org.antlr.v4.runtime.Token.EOF ||
+                             offending.startIndex < 0 || offending.stopIndex < 0) {
+                startCol
+            } else if (offending.line == line) {
+                // Same line: column of the last character of the token
+                offending.charPositionInLine + (offending.stopIndex - offending.startIndex) + 1
+            } else {
+                // Multi-line token: since Position is single-line only, use startCol as minimum
+                maxOf(startCol, offending.charPositionInLine + 1)
+            }
+
+            return Position(src.origin, line, startCol, endCol)
+        }
+
         override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) {
             if (e == null) {
+                // Simple case: no exception, just use the provided line/col
                 throw ParseError(msg, Position(src.origin, line, charPositionInLine+1, charPositionInLine+1), RuntimeException("parse error"))
             } else {
                 if(e.offendingToken==null) {
                     throw ParseError(msg, Position(src.origin, line, charPositionInLine+1, charPositionInLine+1), e)
                 } else {
-                    throw ParseError(msg, e.getPosition(src.origin), e)
+                    throw ParseError(msg, e.getPosition(), e)
                 }
             }
         }
     }
 
-    private fun RecognitionException.getPosition(file: String): Position {
-        val offending = this.offendingToken
-        val line = offending.line
-        val beginCol = offending.charPositionInLine
-        val endOffset = if(offending.startIndex<0 || offending.stopIndex<0 || offending.stopIndex<=offending.startIndex) 0 else offending.stopIndex - offending.startIndex
-        return Position(file, line, beginCol+1, beginCol+1+endOffset)
-    }
-
 }

intermediate/src/prog8/intermediate/IRFileReader.kt

@@ -699,13 +699,16 @@ class IRFileReader {
         }
     }
 
-    private val posPattern = Regex("\\[(.+): line (.+) col (.+)-(.+)\\]")
+    // Pattern for single-line position: [file: line 1 col 2-5]
+    private val posPatternSingle = Regex("\\[(.+): line (.+) col (.+)-(.+)\\]")
 
     private fun parsePosition(strpos: String): Position {
         // example: "[library:/prog8lib/virtual/textio.p8: line 5 col 2-4]"
-        val match = posPattern.matchEntire(strpos) ?: throw IRParseException("invalid Position")
-        val (file, line, startCol, endCol) = match.destructured
-        return Position(file, line.toInt(), startCol.toInt(), endCol.toInt())
+        posPatternSingle.matchEntire(strpos)?.let { match ->
+            val (file, line, startCol, endCol) = match.destructured
+            return Position(file, line.toInt(), startCol.toInt(), endCol.toInt())
+        }
+        throw IRParseException("invalid Position: $strpos")
     }
 
     private fun readText(reader: XMLEventReader): String {

languageServer/src/prog8lsp/LspConversions.kt

@@ -12,7 +12,7 @@ fun Prog8Position.toLspRange(): Range {
     // Prog8 positions are 1-based, LSP is 0-based
     val startLine = maxOf(0, this.line - 1)
     val startCol = maxOf(0, this.startCol - 1)
-    val endLine = maxOf(0, this.line - 1)  // Same line for single-line positions
+    val endLine = maxOf(0, this.line - 1)
     val endCol = maxOf(0, this.endCol - 1)
 
     return Range(