From 131d5ceb4f97728dff87827a1d8aa9b4265f8ccf Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 21 Dec 2024 00:06:18 +0100 Subject: [PATCH] avoid re-reading all source files when sourcelines are requested in the asm --- .../src/prog8/code/source/ImportFileSystem.kt | 60 +++++++++++++++++++ codeCore/src/prog8/code/source/SourceCode.kt | 6 +- .../src/prog8/code/source/SourceLineCache.kt | 33 ---------- .../src/prog8/codegen/cpu6502/AsmGen.kt | 9 +-- compiler/src/prog8/compiler/ModuleImporter.kt | 9 +-- .../astprocessing/IntermediateAstMaker.kt | 7 ++- compiler/test/ast/TestProg8Parser.kt | 15 ++--- compiler/test/ast/TestSourceCode.kt | 29 ++++----- .../src/prog8/intermediate/IRFileWriter.kt | 8 +-- 9 files changed, 102 insertions(+), 74 deletions(-) create mode 100644 codeCore/src/prog8/code/source/ImportFileSystem.kt delete mode 100644 codeCore/src/prog8/code/source/SourceLineCache.kt diff --git a/codeCore/src/prog8/code/source/ImportFileSystem.kt b/codeCore/src/prog8/code/source/ImportFileSystem.kt new file mode 100644 index 000000000..e9d1887c4 --- /dev/null +++ b/codeCore/src/prog8/code/source/ImportFileSystem.kt @@ -0,0 +1,60 @@ +package prog8.code.source + +import prog8.code.core.Position +import java.nio.file.Path +import kotlin.io.path.Path + + +// Resource caching "filesystem". +// Note that it leaves the decision to load a resource or an actual disk file to the caller. + +object ImportFileSystem { + fun getFile(path: Path): SourceCode { + val cached = cache[path.toString()] + if (cached != null) return cached + val file = SourceCode.File(path) + cache[path.toString()] = file + return file + } + + fun getResource(name: String): SourceCode { + val cached = cache[name] + if (cached != null) return cached + val resource = SourceCode.Resource(name) + cache[name] = resource + return resource + } + + fun retrieveSourceLine(position: Position): String { + if(SourceCode.isLibraryResource(position.file)) { + val cached = cache[SourceCode.withoutPrefix(position.file)] + if(cached != null) + return getLine(cached, position.line) + } + val cached = cache[position.file] + if(cached != null) + return getLine(cached, position.line) + val path = Path(position.file).toAbsolutePath().normalize() + val cached2 = cache[path.toString()] + if(cached2 != null) + return getLine(cached2, position.line) + throw NoSuchElementException("cannot get source line $position") + } + + private fun getLine(code: SourceCode, lineIndex: Int): String { + var spans = lineSpanCache[code] + if(spans==null) { + val lineSpans = Regex("^", RegexOption.MULTILINE).findAll(code.text).map { it.range.first } + val ends = lineSpans.drop(1) + code.text.length + spans = lineSpans.zip(ends).map { (start, end) -> LineSpan(start, end) }.toList().toTypedArray() + lineSpanCache[code] = spans + } + val span = spans[lineIndex - 1] + return code.text.substring(span.start, span.end).trim() + } + + private class LineSpan(val start: Int, val end: Int) + + private val cache = mutableMapOf() + private val lineSpanCache = mutableMapOf>() +} \ No newline at end of file diff --git a/codeCore/src/prog8/code/source/SourceCode.kt b/codeCore/src/prog8/code/source/SourceCode.kt index d8d2de72c..e7eb86028 100644 --- a/codeCore/src/prog8/code/source/SourceCode.kt +++ b/codeCore/src/prog8/code/source/SourceCode.kt @@ -82,12 +82,13 @@ sealed class SourceCode { /** * Get [SourceCode] from the file represented by the specified Path. * This immediately reads the file fully into memory. + * You can only get an instance of this via the ImportFileSystem object. * * [origin] will be the given path in absolute and normalized form. * @throws NoSuchFileException if the file does not exist * @throws FileSystemException if the file cannot be read */ - class File(path: Path): SourceCode() { + internal class File(path: Path): SourceCode() { override val text: String override val origin: String override val name: String @@ -111,8 +112,9 @@ sealed class SourceCode { /** * [origin]: `library:/x/y/z.p8` for a given `pathString` of "x/y/z.p8" + * You can only get an instance of this via the ImportFileSystem object. */ - class Resource(pathString: String): SourceCode() { + internal class Resource(pathString: String): SourceCode() { private val normalized = "/" + Path(pathString).normalize().toMutableList().joinToString("/") override val isFromResources = true diff --git a/codeCore/src/prog8/code/source/SourceLineCache.kt b/codeCore/src/prog8/code/source/SourceLineCache.kt deleted file mode 100644 index 8c5dd513e..000000000 --- a/codeCore/src/prog8/code/source/SourceLineCache.kt +++ /dev/null @@ -1,33 +0,0 @@ -package prog8.code.source - -import prog8.code.core.Position -import kotlin.io.path.Path - -object SourceLineCache { - private val cache = mutableMapOf>() - - private fun getCachedFile(file: String): List { - val existing = cache[file] - if(existing!=null) - return existing - if (SourceCode.isRegularFilesystemPath(file)) { - val source = SourceCode.File(Path(file)) - cache[file] = source.text.split('\n', '\r').map { it.trim() } - return cache.getValue(file) - } else if(SourceCode.isLibraryResource(file)) { - val source = SourceCode.Resource(SourceCode.withoutPrefix(file)) - cache[file] = source.text.split('\n', '\r').map { it.trim()} - return cache.getValue(file) - } - return emptyList() - } - - fun retrieveLine(position: Position): String? { - if (position.line>0) { - val lines = getCachedFile(position.file) - if(lines.isNotEmpty()) - return lines[position.line-1] - } - return null - } -} \ No newline at end of file diff --git a/codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt b/codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt index 8292e1b3f..8cb552138 100644 --- a/codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt +++ b/codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt @@ -7,8 +7,8 @@ import prog8.code.SymbolTable import prog8.code.SymbolTableMaker import prog8.code.ast.* import prog8.code.core.* +import prog8.code.source.ImportFileSystem import prog8.code.source.SourceCode -import prog8.code.source.SourceLineCache import prog8.code.target.Cx16Target import prog8.codegen.cpu6502.assignment.* import kotlin.io.path.Path @@ -337,11 +337,8 @@ class AsmGen6502Internal ( lastSourceLineNumber = node.position.line val srcComment = "\t; source: ${node.position.file}:${node.position.line}" - val line = SourceLineCache.retrieveLine(node.position) - if(line==null) - out(srcComment, false) - else - out("$srcComment $line", false) + val line = ImportFileSystem.retrieveSourceLine(node.position) + out("$srcComment $line", false) } internal fun out(str: String, splitlines: Boolean = true) { diff --git a/compiler/src/prog8/compiler/ModuleImporter.kt b/compiler/src/prog8/compiler/ModuleImporter.kt index aef9d28f1..953780188 100644 --- a/compiler/src/prog8/compiler/ModuleImporter.kt +++ b/compiler/src/prog8/compiler/ModuleImporter.kt @@ -8,6 +8,7 @@ import prog8.ast.statements.Directive import prog8.ast.statements.DirectiveArg import prog8.code.core.IErrorReporter import prog8.code.core.Position +import prog8.code.source.ImportFileSystem import prog8.code.source.SourceCode import prog8.parser.Prog8Parser import java.io.File @@ -32,7 +33,7 @@ class ModuleImporter(private val program: Program, if(programPath.exists()) { println("Compiling program ${Path("").absolute().relativize(programPath)}") println("Compiler target: $compilationTargetName") - val source = SourceCode.File(programPath) + val source = ImportFileSystem.getFile(programPath) return Ok(importModule(source)) } } @@ -122,8 +123,8 @@ class ModuleImporter(private val program: Program, private fun getModuleFromResource(name: String, compilationTargetName: String): Result { val result = - runCatching { SourceCode.Resource("/prog8lib/$compilationTargetName/$name") } - .orElse { runCatching { SourceCode.Resource("/prog8lib/$name") } } + runCatching { ImportFileSystem.getResource("/prog8lib/$compilationTargetName/$name") } + .orElse { runCatching { ImportFileSystem.getResource("/prog8lib/$name") } } return result.mapError { NoSuchFileException(File(name)) } } @@ -140,7 +141,7 @@ class ModuleImporter(private val program: Program, locations.forEach { try { - return Ok(SourceCode.File(it.resolve(fileName))) + return Ok(ImportFileSystem.getFile(it.resolve(fileName))) } catch (_: NoSuchFileException) { } } diff --git a/compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt b/compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt index 0dd99891b..531f1bd3e 100644 --- a/compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt +++ b/compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt @@ -10,6 +10,7 @@ import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.code.ast.* import prog8.code.core.* +import prog8.code.source.ImportFileSystem import prog8.code.source.SourceCode import prog8.compiler.builtinFunctionReturnType import java.io.File @@ -769,14 +770,14 @@ class IntermediateAstMaker(private val program: Program, private val errors: IEr return if (SourceCode.isLibraryResource(filename)) { return com.github.michaelbull.result.runCatching { val physFilename = SourceCode.withoutPrefix(filename) - SourceCode.Resource("/prog8lib/$physFilename").text + ImportFileSystem.getResource("/prog8lib/$physFilename").text }.mapError { NoSuchFileException(File(filename)) } } else { val sib = Path(source.origin).resolveSibling(filename) if (sib.isRegularFile()) - Ok(SourceCode.File(sib).text) + Ok(ImportFileSystem.getFile(sib).text) else - Ok(SourceCode.File(Path(filename)).text) + Ok(ImportFileSystem.getFile(Path(filename)).text) } } diff --git a/compiler/test/ast/TestProg8Parser.kt b/compiler/test/ast/TestProg8Parser.kt index 3706a5192..6182a2e15 100644 --- a/compiler/test/ast/TestProg8Parser.kt +++ b/compiler/test/ast/TestProg8Parser.kt @@ -17,6 +17,7 @@ import prog8.ast.Program import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.code.core.* +import prog8.code.source.ImportFileSystem import prog8.code.source.SourceCode import prog8.code.target.C64Target import prog8.code.target.VMTarget @@ -191,7 +192,7 @@ class TestProg8Parser: FunSpec( { test("from an empty file should result in empty Module") { val path = assumeReadableFile(fixturesDir, "ast_empty.p8") - val module = parseModule(SourceCode.File(path)) + val module = parseModule(ImportFileSystem.getFile(path)) module.statements.size shouldBe 0 } } @@ -210,7 +211,7 @@ class TestProg8Parser: FunSpec( { test("parsed from a file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") - val module = parseModule(SourceCode.File(path)) + val module = parseModule(ImportFileSystem.getFile(path)) module.name shouldBe path.nameWithoutExtension } } @@ -274,7 +275,7 @@ class TestProg8Parser: FunSpec( { test("in ParseError from bad file source code") { val path = assumeReadableFile(fixturesDir, "ast_file_with_syntax_error.p8") - val e = shouldThrow { parseModule(SourceCode.File(path)) } + val e = shouldThrow { parseModule(ImportFileSystem.getFile(path)) } assertPosition(e.position, SourceCode.relative(path).toString(), 2, 4) } @@ -289,14 +290,14 @@ class TestProg8Parser: FunSpec( { test("of Module parsed from a file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") - val module = parseModule(SourceCode.File(path)) + val module = parseModule(ImportFileSystem.getFile(path)) assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) } test("of non-root Nodes parsed from file") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") - val module = parseModule(SourceCode.File(path)) + val module = parseModule(ImportFileSystem.getFile(path)) val mpf = module.position.file assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0) val mainBlock = module.statements.filterIsInstance()[0] @@ -359,7 +360,7 @@ class TestProg8Parser: FunSpec( { test("isn't absolute for filesystem paths") { val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8") - val module = parseModule(SourceCode.File(path)) + val module = parseModule(ImportFileSystem.getFile(path)) assertSomethingForAllNodes(module) { Path(it.position.file).isAbsolute shouldBe false Path(it.position.file).isRegularFile() shouldBe true @@ -385,7 +386,7 @@ class TestProg8Parser: FunSpec( { test("is library prefixed path for resources") { - val resource = SourceCode.Resource("prog8lib/math.p8") + val resource = ImportFileSystem.getResource("prog8lib/math.p8") val module = parseModule(resource) assertSomethingForAllNodes(module) { SourceCode.isLibraryResource(it.position.file) shouldBe true diff --git a/compiler/test/ast/TestSourceCode.kt b/compiler/test/ast/TestSourceCode.kt index 4d73d226d..46d26a3c5 100644 --- a/compiler/test/ast/TestSourceCode.kt +++ b/compiler/test/ast/TestSourceCode.kt @@ -5,6 +5,7 @@ import io.kotest.core.spec.style.AnnotationSpec import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe import io.kotest.matchers.string.shouldContain +import prog8.code.source.ImportFileSystem import prog8.code.source.SourceCode import prog8tests.helpers.* import kotlin.io.path.Path @@ -23,7 +24,7 @@ class TestSourceCode: AnnotationSpec() { src.text shouldBe text src.isFromResources shouldBe false src.isFromFilesystem shouldBe false - src.toString().startsWith("prog8.code.core.SourceCode") shouldBe true + src.toString().startsWith(SourceCode::class.qualifiedName!!) shouldBe true } @Test @@ -38,19 +39,19 @@ class TestSourceCode: AnnotationSpec() { fun testFromPathWithNonExistingPath() { val filename = "i_do_not_exist.p8" val path = assumeNotExists(fixturesDir, filename) - shouldThrow { SourceCode.File(path) } + shouldThrow { ImportFileSystem.getFile(path) } } @Test fun testFromPathWithMissingExtension_p8() { val pathWithoutExt = assumeNotExists(fixturesDir,"simple_main") assumeReadableFile(fixturesDir,"ast_simple_main.p8") - shouldThrow { SourceCode.File(pathWithoutExt) } + shouldThrow { ImportFileSystem.getFile(pathWithoutExt) } } @Test fun testFromPathWithDirectory() { - shouldThrow { SourceCode.File(fixturesDir) } + shouldThrow { ImportFileSystem.getFile(fixturesDir) } } @@ -62,7 +63,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromPathWithExistingPath() { val filename = "ast_simple_main.p8" val path = assumeReadableFile(fixturesDir, filename) - val src = SourceCode.File(path) + val src = ImportFileSystem.getFile(path) val expectedOrigin = SourceCode.relative(path).toString() src.origin shouldBe expectedOrigin src.text shouldBe normalizeLineEndings(path.toFile().readText()) @@ -76,7 +77,7 @@ class TestSourceCode: AnnotationSpec() { val filePath = outputDir.resolve("on_the_fly_test_" + text.hashCode().toUInt().toString(16) + ".p8") filePath.toFile().writeText(text) val path = assumeReadableFile(fixturesDir, filePath) - val src = SourceCode.File(path) + val src = ImportFileSystem.getFile(path) src.text shouldNotBe path.toFile().readText() // should be normalized! src.text.split('\r', '\n').size shouldBe 5 } @@ -86,7 +87,7 @@ class TestSourceCode: AnnotationSpec() { val filename = "ast_simple_main.p8" val path = Path(".", "test", "..", "test", "fixtures", filename) val srcFile = assumeReadableFile(path).toFile() - val src = SourceCode.File(path) + val src = ImportFileSystem.getFile(path) val expectedOrigin = SourceCode.relative(path).toString() src.origin shouldBe expectedOrigin src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -96,7 +97,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromResourcesWithExistingP8File_withoutLeadingSlash() { val pathString = "prog8lib/math.p8" val srcFile = assumeReadableFile(resourcesDir, pathString).toFile() - val src = SourceCode.Resource(pathString) + val src = ImportFileSystem.getResource(pathString) src.origin shouldBe "library:/$pathString" src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -108,7 +109,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromResourcesWithExistingP8File_withLeadingSlash() { val pathString = "/prog8lib/math.p8" val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() - val src = SourceCode.Resource(pathString) + val src = ImportFileSystem.getResource(pathString) src.origin shouldBe "library:$pathString" src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -118,7 +119,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromResourcesWithExistingAsmFile_withoutLeadingSlash() { val pathString = "prog8lib/math.asm" val srcFile = assumeReadableFile(resourcesDir, pathString).toFile() - val src = SourceCode.Resource(pathString) + val src = ImportFileSystem.getResource(pathString) src.origin shouldBe "library:/$pathString" src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -129,7 +130,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromResourcesWithExistingAsmFile_withLeadingSlash() { val pathString = "/prog8lib/math.asm" val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() - val src = SourceCode.Resource(pathString) + val src = ImportFileSystem.getResource(pathString) src.origin shouldBe "library:$pathString" src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -139,7 +140,7 @@ class TestSourceCode: AnnotationSpec() { fun testFromResourcesWithNonNormalizedPath() { val pathString = "/prog8lib/../prog8lib/math.p8" val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() - val src = SourceCode.Resource(pathString) + val src = ImportFileSystem.getResource(pathString) src.origin shouldBe "library:/prog8lib/math.p8" src.text shouldBe normalizeLineEndings(srcFile.readText()) @@ -152,13 +153,13 @@ class TestSourceCode: AnnotationSpec() { val pathString = "/prog8lib/i_do_not_exist" assumeNotExists(resourcesDir, pathString.substring(1)) - shouldThrow { SourceCode.Resource(pathString) } + shouldThrow { ImportFileSystem.getResource(pathString) } } @Test fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() { val pathString = "prog8lib/i_do_not_exist" assumeNotExists(resourcesDir, pathString) - shouldThrow { SourceCode.Resource(pathString) } + shouldThrow { ImportFileSystem.getResource(pathString) } } } diff --git a/intermediate/src/prog8/intermediate/IRFileWriter.kt b/intermediate/src/prog8/intermediate/IRFileWriter.kt index 6ecef4b8f..d383824cb 100644 --- a/intermediate/src/prog8/intermediate/IRFileWriter.kt +++ b/intermediate/src/prog8/intermediate/IRFileWriter.kt @@ -3,7 +3,7 @@ package prog8.intermediate import prog8.code.core.InternalCompilerException import prog8.code.core.Position import prog8.code.core.toHex -import prog8.code.source.SourceLineCache +import prog8.code.source.ImportFileSystem import java.nio.file.Path import javax.xml.stream.XMLOutputFactory import javax.xml.stream.XMLStreamWriter @@ -149,10 +149,8 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) { xml.writeStartElement("P8SRC") val sourceTxt = StringBuilder("\n") code.sourceLinesPositions.forEach { pos -> - val line = SourceLineCache.retrieveLine(pos) - if(line!=null) { - sourceTxt.append("$pos $line\n") - } + val line = ImportFileSystem.retrieveSourceLine(pos) + sourceTxt.append("$pos $line\n") } xml.writeCData(sourceTxt.toString()) xml.writeEndElement()