Apple-2-SW/prog8
1
0
Fork 0
mirror of https://github.com/irmen/prog8.git synced 2024-12-22 18:30:01 +00:00
Code Issues Projects Releases Wiki Activity

avoid re-reading all source files when sourcelines are requested in the asm

Browse Source
This commit is contained in:
Irmen de Jong 2024-12-21 00:06:18 +01:00
parent 512ddd1694
commit 131d5ceb4f
9 changed files with 102 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
codeCore/src/prog8/code/source/ImportFileSystem.kt Normal file
View File

@ -0,0 +1,60 @@
package prog8.code.source
import prog8.code.core.Position
import java.nio.file.Path
import kotlin.io.path.Path
// Resource caching "filesystem".
// Note that it leaves the decision to load a resource or an actual disk file to the caller.
object ImportFileSystem {
fun getFile(path: Path): SourceCode {
val cached = cache[path.toString()]
if (cached != null) return cached
val file = SourceCode.File(path)
cache[path.toString()] = file
return file
}
fun getResource(name: String): SourceCode {
val cached = cache[name]
if (cached != null) return cached
val resource = SourceCode.Resource(name)
cache[name] = resource
return resource
}
fun retrieveSourceLine(position: Position): String {
if(SourceCode.isLibraryResource(position.file)) {
val cached = cache[SourceCode.withoutPrefix(position.file)]
if(cached != null)
return getLine(cached, position.line)
}
val cached = cache[position.file]
if(cached != null)
return getLine(cached, position.line)
val path = Path(position.file).toAbsolutePath().normalize()
val cached2 = cache[path.toString()]
if(cached2 != null)
return getLine(cached2, position.line)
throw NoSuchElementException("cannot get source line $position")
}
private fun getLine(code: SourceCode, lineIndex: Int): String {
var spans = lineSpanCache[code]
if(spans==null) {
val lineSpans = Regex("^", RegexOption.MULTILINE).findAll(code.text).map { it.range.first }
val ends = lineSpans.drop(1) + code.text.length
spans = lineSpans.zip(ends).map { (start, end) -> LineSpan(start, end) }.toList().toTypedArray()
lineSpanCache[code] = spans
}
val span = spans[lineIndex - 1]
return code.text.substring(span.start, span.end).trim()
}
private class LineSpan(val start: Int, val end: Int)
private val cache = mutableMapOf<String, SourceCode>()
private val lineSpanCache = mutableMapOf<SourceCode, Array<LineSpan>>()
}

6
codeCore/src/prog8/code/source/SourceCode.kt
View File

@ -82,12 +82,13 @@ sealed class SourceCode {
/**
* Get [SourceCode] from the file represented by the specified Path.
* This immediately reads the file fully into memory.
* You can only get an instance of this via the ImportFileSystem object.
*
* [origin] will be the given path in absolute and normalized form.
* @throws NoSuchFileException if the file does not exist
* @throws FileSystemException if the file cannot be read
*/
class File(path: Path): SourceCode() {
internal class File(path: Path): SourceCode() {
override val text: String
override val origin: String
override val name: String
@ -111,8 +112,9 @@ sealed class SourceCode {
/**
* [origin]: `library:/x/y/z.p8` for a given `pathString` of "x/y/z.p8"
* You can only get an instance of this via the ImportFileSystem object.
*/
class Resource(pathString: String): SourceCode() {
internal class Resource(pathString: String): SourceCode() {
private val normalized = "/" + Path(pathString).normalize().toMutableList().joinToString("/")
override val isFromResources = true

33
codeCore/src/prog8/code/source/SourceLineCache.kt
View File

@ -1,33 +0,0 @@
package prog8.code.source
import prog8.code.core.Position
import kotlin.io.path.Path
object SourceLineCache {
private val cache = mutableMapOf<String, List<String>>()
private fun getCachedFile(file: String): List<String> {
val existing = cache[file]
if(existing!=null)
return existing
if (SourceCode.isRegularFilesystemPath(file)) {
val source = SourceCode.File(Path(file))
cache[file] = source.text.split('\n', '\r').map { it.trim() }
return cache.getValue(file)
} else if(SourceCode.isLibraryResource(file)) {
val source = SourceCode.Resource(SourceCode.withoutPrefix(file))
cache[file] = source.text.split('\n', '\r').map { it.trim()}
return cache.getValue(file)
}
return emptyList()
}
fun retrieveLine(position: Position): String? {
if (position.line>0) {
val lines = getCachedFile(position.file)
if(lines.isNotEmpty())
return lines[position.line-1]
}
return null
}
}

9
codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt
View File

@ -7,8 +7,8 @@ import prog8.code.SymbolTable
import prog8.code.SymbolTableMaker
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.ImportFileSystem
import prog8.code.source.SourceCode
import prog8.code.source.SourceLineCache
import prog8.code.target.Cx16Target
import prog8.codegen.cpu6502.assignment.*
import kotlin.io.path.Path
@ -337,11 +337,8 @@ class AsmGen6502Internal (
lastSourceLineNumber = node.position.line
val srcComment = "\t; source: ${node.position.file}:${node.position.line}"
val line = SourceLineCache.retrieveLine(node.position)
if(line==null)
out(srcComment, false)
else
out("$srcComment $line", false)
val line = ImportFileSystem.retrieveSourceLine(node.position)
out("$srcComment $line", false)
}
internal fun out(str: String, splitlines: Boolean = true) {

9
compiler/src/prog8/compiler/ModuleImporter.kt
View File

@ -8,6 +8,7 @@ import prog8.ast.statements.Directive
import prog8.ast.statements.DirectiveArg
import prog8.code.core.IErrorReporter
import prog8.code.core.Position
import prog8.code.source.ImportFileSystem
import prog8.code.source.SourceCode
import prog8.parser.Prog8Parser
import java.io.File
@ -32,7 +33,7 @@ class ModuleImporter(private val program: Program,
if(programPath.exists()) {
println("Compiling program ${Path("").absolute().relativize(programPath)}")
println("Compiler target: $compilationTargetName")
val source = SourceCode.File(programPath)
val source = ImportFileSystem.getFile(programPath)
return Ok(importModule(source))
}
}
@ -122,8 +123,8 @@ class ModuleImporter(private val program: Program,
private fun getModuleFromResource(name: String, compilationTargetName: String): Result<SourceCode, NoSuchFileException> {
val result =
runCatching { SourceCode.Resource("/prog8lib/$compilationTargetName/$name") }
.orElse { runCatching { SourceCode.Resource("/prog8lib/$name") } }
runCatching { ImportFileSystem.getResource("/prog8lib/$compilationTargetName/$name") }
.orElse { runCatching { ImportFileSystem.getResource("/prog8lib/$name") } }
return result.mapError { NoSuchFileException(File(name)) }
}
@ -140,7 +141,7 @@ class ModuleImporter(private val program: Program,
locations.forEach {
try {
return Ok(SourceCode.File(it.resolve(fileName)))
return Ok(ImportFileSystem.getFile(it.resolve(fileName)))
} catch (_: NoSuchFileException) {
}
}

7
compiler/src/prog8/compiler/astprocessing/IntermediateAstMaker.kt
View File

@ -10,6 +10,7 @@ import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.ImportFileSystem
import prog8.code.source.SourceCode
import prog8.compiler.builtinFunctionReturnType
import java.io.File
@ -769,14 +770,14 @@ class IntermediateAstMaker(private val program: Program, private val errors: IEr
return if (SourceCode.isLibraryResource(filename)) {
return com.github.michaelbull.result.runCatching {
val physFilename = SourceCode.withoutPrefix(filename)
SourceCode.Resource("/prog8lib/$physFilename").text
ImportFileSystem.getResource("/prog8lib/$physFilename").text
}.mapError { NoSuchFileException(File(filename)) }
} else {
val sib = Path(source.origin).resolveSibling(filename)
if (sib.isRegularFile())
Ok(SourceCode.File(sib).text)
Ok(ImportFileSystem.getFile(sib).text)
else
Ok(SourceCode.File(Path(filename)).text)
Ok(ImportFileSystem.getFile(Path(filename)).text)
}
}

15
compiler/test/ast/TestProg8Parser.kt
View File

@ -17,6 +17,7 @@ import prog8.ast.Program
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.code.core.*
import prog8.code.source.ImportFileSystem
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.code.target.VMTarget
@ -191,7 +192,7 @@ class TestProg8Parser: FunSpec( {
test("from an empty file should result in empty Module") {
val path = assumeReadableFile(fixturesDir, "ast_empty.p8")
val module = parseModule(SourceCode.File(path))
val module = parseModule(ImportFileSystem.getFile(path))
module.statements.size shouldBe 0
}
}
@ -210,7 +211,7 @@ class TestProg8Parser: FunSpec( {
test("parsed from a file") {
val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
val module = parseModule(SourceCode.File(path))
val module = parseModule(ImportFileSystem.getFile(path))
module.name shouldBe path.nameWithoutExtension
}
}
@ -274,7 +275,7 @@ class TestProg8Parser: FunSpec( {
test("in ParseError from bad file source code") {
val path = assumeReadableFile(fixturesDir, "ast_file_with_syntax_error.p8")
val e = shouldThrow<ParseError> { parseModule(SourceCode.File(path)) }
val e = shouldThrow<ParseError> { parseModule(ImportFileSystem.getFile(path)) }
assertPosition(e.position, SourceCode.relative(path).toString(), 2, 4)
}
@ -289,14 +290,14 @@ class TestProg8Parser: FunSpec( {
test("of Module parsed from a file") {
val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
val module = parseModule(SourceCode.File(path))
val module = parseModule(ImportFileSystem.getFile(path))
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
}
test("of non-root Nodes parsed from file") {
val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
val module = parseModule(SourceCode.File(path))
val module = parseModule(ImportFileSystem.getFile(path))
val mpf = module.position.file
assertPositionOf(module, SourceCode.relative(path).toString(), 1, 0)
val mainBlock = module.statements.filterIsInstance<Block>()[0]
@ -359,7 +360,7 @@ class TestProg8Parser: FunSpec( {
test("isn't absolute for filesystem paths") {
val path = assumeReadableFile(fixturesDir, "ast_simple_main.p8")
val module = parseModule(SourceCode.File(path))
val module = parseModule(ImportFileSystem.getFile(path))
assertSomethingForAllNodes(module) {
Path(it.position.file).isAbsolute shouldBe false
Path(it.position.file).isRegularFile() shouldBe true
@ -385,7 +386,7 @@ class TestProg8Parser: FunSpec( {
test("is library prefixed path for resources")
{
val resource = SourceCode.Resource("prog8lib/math.p8")
val resource = ImportFileSystem.getResource("prog8lib/math.p8")
val module = parseModule(resource)
assertSomethingForAllNodes(module) {
SourceCode.isLibraryResource(it.position.file) shouldBe true

29
compiler/test/ast/TestSourceCode.kt
View File

@ -5,6 +5,7 @@ import io.kotest.core.spec.style.AnnotationSpec
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import prog8.code.source.ImportFileSystem
import prog8.code.source.SourceCode
import prog8tests.helpers.*
import kotlin.io.path.Path
@ -23,7 +24,7 @@ class TestSourceCode: AnnotationSpec() {
src.text shouldBe text
src.isFromResources shouldBe false
src.isFromFilesystem shouldBe false
src.toString().startsWith("prog8.code.core.SourceCode") shouldBe true
src.toString().startsWith(SourceCode::class.qualifiedName!!) shouldBe true
}
@Test
@ -38,19 +39,19 @@ class TestSourceCode: AnnotationSpec() {
fun testFromPathWithNonExistingPath() {
val filename = "i_do_not_exist.p8"
val path = assumeNotExists(fixturesDir, filename)
shouldThrow<NoSuchFileException> { SourceCode.File(path) }
shouldThrow<NoSuchFileException> { ImportFileSystem.getFile(path) }
}
@Test
fun testFromPathWithMissingExtension_p8() {
val pathWithoutExt = assumeNotExists(fixturesDir,"simple_main")
assumeReadableFile(fixturesDir,"ast_simple_main.p8")
shouldThrow<NoSuchFileException> { SourceCode.File(pathWithoutExt) }
shouldThrow<NoSuchFileException> { ImportFileSystem.getFile(pathWithoutExt) }
}
@Test
fun testFromPathWithDirectory() {
shouldThrow<FileSystemException> { SourceCode.File(fixturesDir) }
shouldThrow<FileSystemException> { ImportFileSystem.getFile(fixturesDir) }
}
@ -62,7 +63,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromPathWithExistingPath() {
val filename = "ast_simple_main.p8"
val path = assumeReadableFile(fixturesDir, filename)
val src = SourceCode.File(path)
val src = ImportFileSystem.getFile(path)
val expectedOrigin = SourceCode.relative(path).toString()
src.origin shouldBe expectedOrigin
src.text shouldBe normalizeLineEndings(path.toFile().readText())
@ -76,7 +77,7 @@ class TestSourceCode: AnnotationSpec() {
val filePath = outputDir.resolve("on_the_fly_test_" + text.hashCode().toUInt().toString(16) + ".p8")
filePath.toFile().writeText(text)
val path = assumeReadableFile(fixturesDir, filePath)
val src = SourceCode.File(path)
val src = ImportFileSystem.getFile(path)
src.text shouldNotBe path.toFile().readText() // should be normalized!
src.text.split('\r', '\n').size shouldBe 5
}
@ -86,7 +87,7 @@ class TestSourceCode: AnnotationSpec() {
val filename = "ast_simple_main.p8"
val path = Path(".", "test", "..", "test", "fixtures", filename)
val srcFile = assumeReadableFile(path).toFile()
val src = SourceCode.File(path)
val src = ImportFileSystem.getFile(path)
val expectedOrigin = SourceCode.relative(path).toString()
src.origin shouldBe expectedOrigin
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -96,7 +97,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromResourcesWithExistingP8File_withoutLeadingSlash() {
val pathString = "prog8lib/math.p8"
val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
val src = SourceCode.Resource(pathString)
val src = ImportFileSystem.getResource(pathString)
src.origin shouldBe "library:/$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -108,7 +109,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromResourcesWithExistingP8File_withLeadingSlash() {
val pathString = "/prog8lib/math.p8"
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
val src = ImportFileSystem.getResource(pathString)
src.origin shouldBe "library:$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -118,7 +119,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromResourcesWithExistingAsmFile_withoutLeadingSlash() {
val pathString = "prog8lib/math.asm"
val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
val src = SourceCode.Resource(pathString)
val src = ImportFileSystem.getResource(pathString)
src.origin shouldBe "library:/$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -129,7 +130,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromResourcesWithExistingAsmFile_withLeadingSlash() {
val pathString = "/prog8lib/math.asm"
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
val src = ImportFileSystem.getResource(pathString)
src.origin shouldBe "library:$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -139,7 +140,7 @@ class TestSourceCode: AnnotationSpec() {
fun testFromResourcesWithNonNormalizedPath() {
val pathString = "/prog8lib/../prog8lib/math.p8"
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
val src = ImportFileSystem.getResource(pathString)
src.origin shouldBe "library:/prog8lib/math.p8"
src.text shouldBe normalizeLineEndings(srcFile.readText())
@ -152,13 +153,13 @@ class TestSourceCode: AnnotationSpec() {
val pathString = "/prog8lib/i_do_not_exist"
assumeNotExists(resourcesDir, pathString.substring(1))
shouldThrow<NoSuchFileException> { SourceCode.Resource(pathString) }
shouldThrow<NoSuchFileException> { ImportFileSystem.getResource(pathString) }
}
@Test
fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() {
val pathString = "prog8lib/i_do_not_exist"
assumeNotExists(resourcesDir, pathString)
shouldThrow<NoSuchFileException> { SourceCode.Resource(pathString) }
shouldThrow<NoSuchFileException> { ImportFileSystem.getResource(pathString) }
}
}

8
intermediate/src/prog8/intermediate/IRFileWriter.kt
View File

@ -3,7 +3,7 @@ package prog8.intermediate
import prog8.code.core.InternalCompilerException
import prog8.code.core.Position
import prog8.code.core.toHex
import prog8.code.source.SourceLineCache
import prog8.code.source.ImportFileSystem
import java.nio.file.Path
import javax.xml.stream.XMLOutputFactory
import javax.xml.stream.XMLStreamWriter
@ -149,10 +149,8 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) {
xml.writeStartElement("P8SRC")
val sourceTxt = StringBuilder("\n")
code.sourceLinesPositions.forEach { pos ->
val line = SourceLineCache.retrieveLine(pos)
if(line!=null) {
sourceTxt.append("$pos $line\n")
}
val line = ImportFileSystem.retrieveSourceLine(pos)
sourceTxt.append("$pos $line\n")
}
xml.writeCData(sourceTxt.toString())
xml.writeEndElement()