avoid re-reading all source files when sourcelines are requested in the asm

codeCore/src/prog8/code/source/ImportFileSystem.kt

@@ -0,0 +1,60 @@
+package prog8.code.source
+
+import prog8.code.core.Position
+import java.nio.file.Path
+import kotlin.io.path.Path
+
+
+// Resource caching "filesystem".
+// Note that it leaves the decision to load a resource or an actual disk file to the caller.
+
+object ImportFileSystem {
+    fun getFile(path: Path): SourceCode {
+        val cached = cache[path.toString()]
+        if (cached != null) return cached
+        val file = SourceCode.File(path)
+        cache[path.toString()] = file
+        return file
+    }
+
+    fun getResource(name: String): SourceCode {
+        val cached = cache[name]
+        if (cached != null) return cached
+        val resource = SourceCode.Resource(name)
+        cache[name] = resource
+        return resource
+    }
+
+    fun retrieveSourceLine(position: Position): String {
+        if(SourceCode.isLibraryResource(position.file)) {
+            val cached = cache[SourceCode.withoutPrefix(position.file)]
+            if(cached != null)
+                return getLine(cached, position.line)
+        }
+        val cached = cache[position.file]
+        if(cached != null)
+            return getLine(cached, position.line)
+        val path = Path(position.file).toAbsolutePath().normalize()
+        val cached2 = cache[path.toString()]
+        if(cached2 != null)
+            return getLine(cached2, position.line)
+        throw NoSuchElementException("cannot get source line $position")
+    }
+
+    private fun getLine(code: SourceCode, lineIndex: Int): String {
+        var spans = lineSpanCache[code]
+        if(spans==null) {
+            val lineSpans = Regex("^", RegexOption.MULTILINE).findAll(code.text).map { it.range.first }
+            val ends = lineSpans.drop(1) + code.text.length
+            spans = lineSpans.zip(ends).map { (start, end) -> LineSpan(start, end) }.toList().toTypedArray()
+            lineSpanCache[code] = spans
+        }
+        val span = spans[lineIndex - 1]
+        return code.text.substring(span.start, span.end).trim()
+    }
+
+    private class LineSpan(val start: Int, val end: Int)
+
+    private val cache = mutableMapOf<String, SourceCode>()
+    private val lineSpanCache = mutableMapOf<SourceCode, Array<LineSpan>>()
+}

codeCore/src/prog8/code/source/SourceCode.kt

@@ -82,12 +82,13 @@ sealed class SourceCode {
     /**
      * Get [SourceCode] from the file represented by the specified Path.
      * This immediately reads the file fully into memory.
+     * You can only get an instance of this via the ImportFileSystem object.
      *
      * [origin] will be the given path in absolute and normalized form.
      * @throws NoSuchFileException if the file does not exist
      * @throws FileSystemException if the file cannot be read
      */
-    class File(path: Path): SourceCode() {
+    internal class File(path: Path): SourceCode() {
         override val text: String
         override val origin: String
         override val name: String
@@ -111,8 +112,9 @@ sealed class SourceCode {
 
     /**
      * [origin]: `library:/x/y/z.p8` for a given `pathString` of "x/y/z.p8"
+     * You can only get an instance of this via the ImportFileSystem object.
      */
-    class Resource(pathString: String): SourceCode() {
+    internal class Resource(pathString: String): SourceCode() {
         private val normalized = "/" + Path(pathString).normalize().toMutableList().joinToString("/")
 
         override val isFromResources = true

codeCore/src/prog8/code/source/SourceLineCache.kt

@@ -1,33 +0,0 @@
-package prog8.code.source
-
-import prog8.code.core.Position
-import kotlin.io.path.Path
-
-object SourceLineCache {
-    private val cache = mutableMapOf<String, List<String>>()
-
-    private fun getCachedFile(file: String): List<String> {
-        val existing = cache[file]
-        if(existing!=null)
-            return existing
-        if (SourceCode.isRegularFilesystemPath(file)) {
-            val source = SourceCode.File(Path(file))
-            cache[file] = source.text.split('\n', '\r').map { it.trim() }
-            return cache.getValue(file)
-        } else if(SourceCode.isLibraryResource(file)) {
-            val source = SourceCode.Resource(SourceCode.withoutPrefix(file))
-            cache[file] = source.text.split('\n', '\r').map { it.trim()}
-            return cache.getValue(file)
-        }
-        return emptyList()
-    }
-
-    fun retrieveLine(position: Position): String? {
-        if (position.line>0) {
-            val lines = getCachedFile(position.file)
-            if(lines.isNotEmpty())
-                return lines[position.line-1]
-        }
-        return null
-    }
-}

codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt

@@ -7,8 +7,8 @@ import prog8.code.SymbolTable
 import prog8.code.SymbolTableMaker
 import prog8.code.ast.*
 import prog8.code.core.*
+import prog8.code.source.ImportFileSystem
 import prog8.code.source.SourceCode
-import prog8.code.source.SourceLineCache
 import prog8.code.target.Cx16Target
 import prog8.codegen.cpu6502.assignment.*
 import kotlin.io.path.Path
@@ -337,11 +337,8 @@ class AsmGen6502Internal (
 
         lastSourceLineNumber = node.position.line
         val srcComment = "\t; source: ${node.position.file}:${node.position.line}"
-        val line = SourceLineCache.retrieveLine(node.position)
+        val line = ImportFileSystem.retrieveSourceLine(node.position)
-        if(line==null)
+        out("$srcComment   $line", false)
-            out(srcComment, false)
-        else
-            out("$srcComment   $line", false)
     }
 
     internal fun out(str: String, splitlines: Boolean = true) {

compiler/src/prog8/compiler/ModuleImporter.kt

@@ -8,6 +8,7 @@ import prog8.ast.statements.Directive
 import prog8.ast.statements.DirectiveArg
 import prog8.code.core.IErrorReporter
 import prog8.code.core.Position
+import prog8.code.source.ImportFileSystem
 import prog8.code.source.SourceCode
 import prog8.parser.Prog8Parser
 import java.io.File
@@ -32,7 +33,7 @@ class ModuleImporter(private val program: Program,
             if(programPath.exists()) {
                 println("Compiling program ${Path("").absolute().relativize(programPath)}")
                 println("Compiler target: $compilationTargetName")
-                val source = SourceCode.File(programPath)
+                val source = ImportFileSystem.getFile(programPath)
                 return Ok(importModule(source))
             }
         }
@@ -122,8 +123,8 @@ class ModuleImporter(private val program: Program,
 
     private fun getModuleFromResource(name: String, compilationTargetName: String): Result<SourceCode, NoSuchFileException> {
         val result =
-            runCatching { SourceCode.Resource("/prog8lib/$compilationTargetName/$name") }
+            runCatching { ImportFileSystem.getResource("/prog8lib/$compilationTargetName/$name") }
-            .orElse { runCatching { SourceCode.Resource("/prog8lib/$name") }  }
+            .orElse { runCatching { ImportFileSystem.getResource("/prog8lib/$name") }  }
 
         return result.mapError { NoSuchFileException(File(name)) }
     }
@@ -140,7 +141,7 @@ class ModuleImporter(private val program: Program,
 
         locations.forEach {
             try {
-                return Ok(SourceCode.File(it.resolve(fileName)))
+                return Ok(ImportFileSystem.getFile(it.resolve(fileName)))
             } catch (_: NoSuchFileException) {
             }
         }

compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt

@@ -10,6 +10,7 @@ import prog8.ast.expressions.*
 import prog8.ast.statements.*
 import prog8.code.ast.*
 import prog8.code.core.*
+import prog8.code.source.ImportFileSystem
 import prog8.code.source.SourceCode
 import prog8.compiler.builtinFunctionReturnType
 import java.io.File
@@ -769,14 +770,14 @@ class IntermediateAstMaker(private val program: Program, private val errors: IEr
         return if (SourceCode.isLibraryResource(filename)) {
             return com.github.michaelbull.result.runCatching {
                 val physFilename = SourceCode.withoutPrefix(filename)
-                SourceCode.Resource("/prog8lib/$physFilename").text
+                ImportFileSystem.getResource("/prog8lib/$physFilename").text
             }.mapError { NoSuchFileException(File(filename)) }
         } else {
             val sib = Path(source.origin).resolveSibling(filename)
             if (sib.isRegularFile())
-                Ok(SourceCode.File(sib).text)
+                Ok(ImportFileSystem.getFile(sib).text)
             else
-                Ok(SourceCode.File(Path(filename)).text)
+                Ok(ImportFileSystem.getFile(Path(filename)).text)
         }
     }
 

compiler/test/ast/TestProg8Parser.kt

@@ -17,6 +17,7 @@ import prog8.ast.Program
 import prog8.ast.expressions.*
 import prog8.ast.statements.*
 import prog8.code.core.*
+import prog8.code.source.ImportFileSystem
 import prog8.code.source.SourceCode
 import prog8.code.target.C64Target
 import prog8.code.target.VMTarget
@@ -191,7 +192,7 @@ class TestProg8Parser: FunSpec( {
 
         test("from an empty file should result in empty Module") {
             val path = assumeReadableFile(fixturesDir, "ast_empty.p8")
-            val module = parseModule(SourceCode.File(path))
+            val module = parseModule(ImportFileSystem.getFile(path))
             module.statements.size shouldBe 0
         }
     }
@@ -210,7 +211,7 @@ class TestProg8Parser: FunSpec( {
 
         test("parsed from a file") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
-            val module = parseModule(SourceCode.File(path))
+            val module = parseModule(ImportFileSystem.getFile(path))
             module.name shouldBe path.nameWithoutExtension
         }
     }
@@ -274,7 +275,7 @@ class TestProg8Parser: FunSpec( {
         test("in ParseError from bad file source code") {
             val path = assumeReadableFile(fixturesDir, "ast_file_with_syntax_error.p8")
 
-            val e = shouldThrow<ParseError> { parseModule(SourceCode.File(path)) }
+            val e = shouldThrow<ParseError> { parseModule(ImportFileSystem.getFile(path)) }
             assertPosition(e.position, SourceCode.relative(path).toString(), 2, 4)
         }
 
@@ -289,14 +290,14 @@ class TestProg8Parser: FunSpec( {
 
         test("of Module parsed from a file") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
-            val module = parseModule(SourceCode.File(path))
+            val module = parseModule(ImportFileSystem.getFile(path))
             assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
         }
 
         test("of non-root Nodes parsed from file") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
 
-            val module = parseModule(SourceCode.File(path))
+            val module = parseModule(ImportFileSystem.getFile(path))
             val mpf = module.position.file
             assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
             val mainBlock = module.statements.filterIsInstance<Block>()[0]
@@ -359,7 +360,7 @@ class TestProg8Parser: FunSpec( {
 
         test("isn't absolute for filesystem paths") {
             val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
-            val module = parseModule(SourceCode.File(path))
+            val module = parseModule(ImportFileSystem.getFile(path))
             assertSomethingForAllNodes(module) {
                 Path(it.position.file).isAbsolute shouldBe false
                 Path(it.position.file).isRegularFile() shouldBe true
@@ -385,7 +386,7 @@ class TestProg8Parser: FunSpec( {
 
         test("is library prefixed path for resources")
         {
-            val resource = SourceCode.Resource("prog8lib/math.p8")
+            val resource = ImportFileSystem.getResource("prog8lib/math.p8")
             val module = parseModule(resource)
             assertSomethingForAllNodes(module) {
                 SourceCode.isLibraryResource(it.position.file) shouldBe true

compiler/test/ast/TestSourceCode.kt

@@ -5,6 +5,7 @@ import io.kotest.core.spec.style.AnnotationSpec
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNotBe
 import io.kotest.matchers.string.shouldContain
+import prog8.code.source.ImportFileSystem
 import prog8.code.source.SourceCode
 import prog8tests.helpers.*
 import kotlin.io.path.Path
@@ -23,7 +24,7 @@ class TestSourceCode: AnnotationSpec() {
         src.text shouldBe text
         src.isFromResources shouldBe false
         src.isFromFilesystem shouldBe false
-        src.toString().startsWith("prog8.code.core.SourceCode") shouldBe true
+        src.toString().startsWith(SourceCode::class.qualifiedName!!) shouldBe true
     }
 
     @Test
@@ -38,19 +39,19 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromPathWithNonExistingPath() {
         val filename = "i_do_not_exist.p8"
         val path = assumeNotExists(fixturesDir, filename)
-        shouldThrow<NoSuchFileException> { SourceCode.File(path) }
+        shouldThrow<NoSuchFileException> { ImportFileSystem.getFile(path) }
     }
 
     @Test
     fun testFromPathWithMissingExtension_p8() {
         val pathWithoutExt = assumeNotExists(fixturesDir,"simple_main")
         assumeReadableFile(fixturesDir,"ast_simple_main.p8")
-        shouldThrow<NoSuchFileException> { SourceCode.File(pathWithoutExt) }
+        shouldThrow<NoSuchFileException> { ImportFileSystem.getFile(pathWithoutExt) }
     }
 
     @Test
     fun testFromPathWithDirectory() {
-        shouldThrow<FileSystemException> { SourceCode.File(fixturesDir) }
+        shouldThrow<FileSystemException> { ImportFileSystem.getFile(fixturesDir) }
     }
 
 
@@ -62,7 +63,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromPathWithExistingPath() {
         val filename = "ast_simple_main.p8"
         val path = assumeReadableFile(fixturesDir, filename)
-        val src = SourceCode.File(path)
+        val src = ImportFileSystem.getFile(path)
         val expectedOrigin = SourceCode.relative(path).toString()
         src.origin shouldBe expectedOrigin
         src.text shouldBe normalizeLineEndings(path.toFile().readText())
@@ -76,7 +77,7 @@ class TestSourceCode: AnnotationSpec() {
         val filePath = outputDir.resolve("on_the_fly_test_" + text.hashCode().toUInt().toString(16) + ".p8")
         filePath.toFile().writeText(text)
         val path = assumeReadableFile(fixturesDir, filePath)
-        val src = SourceCode.File(path)
+        val src = ImportFileSystem.getFile(path)
         src.text shouldNotBe  path.toFile().readText()      // should be normalized!
         src.text.split('\r', '\n').size shouldBe 5
     }
@@ -86,7 +87,7 @@ class TestSourceCode: AnnotationSpec() {
         val filename = "ast_simple_main.p8"
         val path = Path(".", "test", "..", "test", "fixtures", filename)
         val srcFile = assumeReadableFile(path).toFile()
-        val src = SourceCode.File(path)
+        val src = ImportFileSystem.getFile(path)
         val expectedOrigin = SourceCode.relative(path).toString()
         src.origin shouldBe expectedOrigin
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -96,7 +97,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromResourcesWithExistingP8File_withoutLeadingSlash() {
         val pathString = "prog8lib/math.p8"
         val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
-        val src = SourceCode.Resource(pathString)
+        val src = ImportFileSystem.getResource(pathString)
 
         src.origin shouldBe "library:/$pathString"
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -108,7 +109,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromResourcesWithExistingP8File_withLeadingSlash() {
         val pathString = "/prog8lib/math.p8"
         val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
-        val src = SourceCode.Resource(pathString)
+        val src = ImportFileSystem.getResource(pathString)
 
         src.origin shouldBe "library:$pathString"
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -118,7 +119,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromResourcesWithExistingAsmFile_withoutLeadingSlash() {
         val pathString = "prog8lib/math.asm"
         val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
-        val src = SourceCode.Resource(pathString)
+        val src = ImportFileSystem.getResource(pathString)
 
         src.origin shouldBe "library:/$pathString"
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -129,7 +130,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromResourcesWithExistingAsmFile_withLeadingSlash() {
         val pathString = "/prog8lib/math.asm"
         val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
-        val src = SourceCode.Resource(pathString)
+        val src = ImportFileSystem.getResource(pathString)
 
         src.origin shouldBe "library:$pathString"
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -139,7 +140,7 @@ class TestSourceCode: AnnotationSpec() {
     fun testFromResourcesWithNonNormalizedPath() {
         val pathString = "/prog8lib/../prog8lib/math.p8"
         val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
-        val src = SourceCode.Resource(pathString)
+        val src = ImportFileSystem.getResource(pathString)
 
         src.origin shouldBe "library:/prog8lib/math.p8"
         src.text shouldBe normalizeLineEndings(srcFile.readText())
@@ -152,13 +153,13 @@ class TestSourceCode: AnnotationSpec() {
         val pathString = "/prog8lib/i_do_not_exist"
         assumeNotExists(resourcesDir, pathString.substring(1))
 
-        shouldThrow<NoSuchFileException> { SourceCode.Resource(pathString) }
+        shouldThrow<NoSuchFileException> { ImportFileSystem.getResource(pathString) }
     }
     @Test
     fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() {
         val pathString = "prog8lib/i_do_not_exist"
         assumeNotExists(resourcesDir, pathString)
 
-        shouldThrow<NoSuchFileException> { SourceCode.Resource(pathString) }
+        shouldThrow<NoSuchFileException> { ImportFileSystem.getResource(pathString) }
     }
 }

intermediate/src/prog8/intermediate/IRFileWriter.kt

@@ -3,7 +3,7 @@ package prog8.intermediate
 import prog8.code.core.InternalCompilerException
 import prog8.code.core.Position
 import prog8.code.core.toHex
-import prog8.code.source.SourceLineCache
+import prog8.code.source.ImportFileSystem
 import java.nio.file.Path
 import javax.xml.stream.XMLOutputFactory
 import javax.xml.stream.XMLStreamWriter
@@ -149,10 +149,8 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) {
                 xml.writeStartElement("P8SRC")
                 val sourceTxt = StringBuilder("\n")
                 code.sourceLinesPositions.forEach { pos ->
-                    val line = SourceLineCache.retrieveLine(pos)
+                    val line = ImportFileSystem.retrieveSourceLine(pos)
-                    if(line!=null) {
+                    sourceTxt.append("$pos  $line\n")
-                        sourceTxt.append("$pos  $line\n")
-                    }
                 }
                 xml.writeCData(sourceTxt.toString())
                 xml.writeEndElement()