This commit is contained in:
Irmen de Jong 2024-12-20 22:47:05 +01:00
parent 14a213bff9
commit 512ddd1694
37 changed files with 112 additions and 100 deletions

View File

@ -6,6 +6,9 @@ import prog8.code.ast.PtProgram
import prog8.code.core.*
const val internedStringsModuleName = "prog8_interned_strings"
/**
* Tree structure containing all symbol definitions in the program
* (blocks, subroutines, variables (all types), memoryslabs, and labels).

View File

@ -3,7 +3,7 @@ package prog8.code.ast
import prog8.code.core.IMemSizer
import prog8.code.core.IStringEncoding
import prog8.code.core.Position
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
import java.nio.file.Path
// New simplified AST for the code generator.

View File

@ -1,6 +1,6 @@
package prog8.code.core
import prog8.code.core.SourceCode.Companion.LIBRARYFILEPREFIX
import prog8.code.source.SourceCode
import java.nio.file.InvalidPathException
import kotlin.io.path.Path
import kotlin.io.path.absolute
@ -10,7 +10,7 @@ data class Position(val file: String, val line: Int, val startCol: Int, val endC
fun toClickableStr(): String {
if(this===DUMMY)
return ""
if(file.startsWith(LIBRARYFILEPREFIX))
if(SourceCode.isLibraryResource(file))
return "$file:$line:$startCol:"
return try {
val path = Path(file).absolute().normalize().toString()

View File

@ -1,16 +1,11 @@
package prog8.code.core
package prog8.code.source
import java.io.File
import java.io.IOException
import java.nio.file.Path
import java.text.Normalizer
import kotlin.io.path.Path
import kotlin.io.path.readText
const val internedStringsModuleName = "prog8_interned_strings"
/**
* Encapsulates - and ties together - actual source code (=text) and its [origin].
*/
@ -55,14 +50,21 @@ sealed class SourceCode {
/**
* filename prefix to designate library files that will be retreived from internal resources rather than disk
*/
const val LIBRARYFILEPREFIX = "library:"
const val STRINGSOURCEPREFIX = "string:"
private const val LIBRARYFILEPREFIX = "library:"
private const val STRINGSOURCEPREFIX = "string:"
val curdir: Path = Path(".").toAbsolutePath()
fun relative(path: Path): Path = curdir.relativize(path.toAbsolutePath())
fun isRegularFilesystemPath(pathString: String) =
!(pathString.startsWith(LIBRARYFILEPREFIX) || pathString.startsWith(STRINGSOURCEPREFIX))
fun isRegularFilesystemPath(pathString: String) = !isLibraryResource(pathString) && !isStringResource(pathString)
fun isLibraryResource(path: String) = path.startsWith(LIBRARYFILEPREFIX)
fun isStringResource(path: String) = path.startsWith(STRINGSOURCEPREFIX)
fun withoutPrefix(path: String): String {
return if(isLibraryResource(path))
path.removePrefix(LIBRARYFILEPREFIX)
else if(isStringResource(path))
path.removePrefix(STRINGSOURCEPREFIX)
else
path
}
}
/**
@ -124,7 +126,7 @@ sealed class SourceCode {
if (rscURL == null) {
val rscRoot = object {}.javaClass.getResource("/")
throw NoSuchFileException(
File(normalized),
java.io.File(normalized),
reason = "looked in resources rooted at $rscRoot"
)
}
@ -144,34 +146,4 @@ sealed class SourceCode {
override val origin: String = name
override val text: String = "<generated code node, no text representation>"
}
}
object SourceLineCache {
private val cache = mutableMapOf<String, List<String>>()
private fun getCachedFile(file: String): List<String> {
val existing = cache[file]
if(existing!=null)
return existing
if (SourceCode.isRegularFilesystemPath(file)) {
val source = SourceCode.File(Path(file))
cache[file] = source.text.split('\n', '\r').map { it.trim() }
return cache.getValue(file)
} else if(file.startsWith(SourceCode.LIBRARYFILEPREFIX)) {
val source = SourceCode.Resource(file.drop(SourceCode.LIBRARYFILEPREFIX.length))
cache[file] = source.text.split('\n', '\r').map { it.trim()}
return cache.getValue(file)
}
return emptyList()
}
fun retrieveLine(position: Position): String? {
if (position.line>0) {
val lines = getCachedFile(position.file)
if(lines.isNotEmpty())
return lines[position.line-1]
}
return null
}
}

View File

@ -0,0 +1,33 @@
package prog8.code.source
import prog8.code.core.Position
import kotlin.io.path.Path
object SourceLineCache {
private val cache = mutableMapOf<String, List<String>>()
private fun getCachedFile(file: String): List<String> {
val existing = cache[file]
if(existing!=null)
return existing
if (SourceCode.isRegularFilesystemPath(file)) {
val source = SourceCode.File(Path(file))
cache[file] = source.text.split('\n', '\r').map { it.trim() }
return cache.getValue(file)
} else if(SourceCode.isLibraryResource(file)) {
val source = SourceCode.Resource(SourceCode.withoutPrefix(file))
cache[file] = source.text.split('\n', '\r').map { it.trim()}
return cache.getValue(file)
}
return emptyList()
}
fun retrieveLine(position: Position): String? {
if (position.line>0) {
val lines = getCachedFile(position.file)
if(lines.isNotEmpty())
return lines[position.line-1]
}
return null
}
}

View File

@ -7,6 +7,8 @@ import prog8.code.SymbolTable
import prog8.code.SymbolTableMaker
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.source.SourceLineCache
import prog8.code.target.Cx16Target
import prog8.codegen.cpu6502.assignment.*
import kotlin.io.path.Path

View File

@ -8,6 +8,7 @@ import io.kotest.matchers.shouldBe
import prog8.code.SymbolTableMaker
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.codegen.cpu6502.AsmGen6502
import java.nio.file.Files

View File

@ -4,6 +4,7 @@ import io.kotest.matchers.shouldBe
import prog8.code.SymbolTableMaker
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.target.VMTarget
import prog8.codegen.vm.VmAssemblyProgram
import prog8.codegen.vm.VmCodeGen

View File

@ -10,7 +10,7 @@ import prog8.ast.walk.AstWalker
import prog8.ast.walk.IAstModification
import prog8.code.core.ICompilationTarget
import prog8.code.core.IErrorReporter
import prog8.code.core.internedStringsModuleName
import prog8.code.internedStringsModuleName
import prog8.compiler.CallGraph

View File

@ -8,7 +8,7 @@ import prog8.ast.statements.Directive
import prog8.ast.statements.DirectiveArg
import prog8.code.core.IErrorReporter
import prog8.code.core.Position
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
import prog8.parser.Prog8Parser
import java.io.File
import java.nio.file.Path

View File

@ -10,6 +10,7 @@ import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.compiler.builtinFunctionReturnType
import java.io.File
import kotlin.io.path.Path
@ -765,9 +766,10 @@ class IntermediateAstMaker(private val program: Program, private val errors: IEr
private fun loadAsmIncludeFile(filename: String, source: SourceCode): Result<String, NoSuchFileException> {
return if (filename.startsWith(SourceCode.LIBRARYFILEPREFIX)) {
return if (SourceCode.isLibraryResource(filename)) {
return com.github.michaelbull.result.runCatching {
SourceCode.Resource("/prog8lib/${filename.substring(SourceCode.LIBRARYFILEPREFIX.length)}").text
val physFilename = SourceCode.withoutPrefix(filename)
SourceCode.Resource("/prog8lib/$physFilename").text
}.mapError { NoSuchFileException(File(filename)) }
} else {
val sib = Path(source.origin).resolveSibling(filename)

View File

@ -6,6 +6,7 @@ import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.ast.walk.IAstVisitor
import prog8.code.core.*
import prog8.code.internedStringsModuleName
internal class VerifyFunctionArgTypes(val program: Program, val options: CompilationOptions, val errors: IErrorReporter) : IAstVisitor {
@ -21,7 +22,7 @@ internal class VerifyFunctionArgTypes(val program: Program, val options: Compila
}
// remove unused strings from interned strings block
val internedBlock = program.allBlocks.singleOrNull { it.name== internedStringsModuleName }
val internedBlock = program.allBlocks.singleOrNull { it.name == internedStringsModuleName }
internedBlock?.statements?.withIndex()?.reversed()?.forEach { (index, st) ->
if(st is VarDecl && st.scopedName !in allStringRefs) {
internedBlock.statements.removeAt(index)

View File

@ -11,8 +11,8 @@ import io.kotest.matchers.shouldBe
import io.kotest.matchers.string.shouldContain
import prog8.ast.Program
import prog8.code.core.IErrorReporter
import prog8.code.core.SourceCode
import prog8.code.core.internedStringsModuleName
import prog8.code.source.SourceCode
import prog8.code.internedStringsModuleName
import prog8.compiler.ModuleImporter
import prog8.parser.ParseError
import prog8tests.helpers.*

View File

@ -10,7 +10,7 @@ import io.kotest.matchers.string.shouldContain
import prog8.ast.Program
import prog8.ast.statements.Block
import prog8.ast.statements.Subroutine
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.code.target.VMTarget
import prog8.compiler.CallGraph

View File

@ -7,7 +7,7 @@ import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import io.kotest.matchers.string.shouldStartWith
import prog8.code.core.ZeropageType
import prog8.code.core.internedStringsModuleName
import prog8.code.internedStringsModuleName
import prog8.code.target.C64Target
import prog8.code.target.Cx16Target
import prog8.code.target.VMTarget

View File

@ -13,6 +13,7 @@ import prog8.ast.expressions.NumericLiteral
import prog8.ast.expressions.PrefixExpression
import prog8.ast.statements.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.target.*
import prog8tests.helpers.DummyFunctions
import prog8tests.helpers.DummyMemsizer

View File

@ -8,6 +8,7 @@ import io.kotest.matchers.types.shouldBeSameInstanceAs
import prog8.code.*
import prog8.code.ast.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8tests.helpers.DummyMemsizer
import prog8tests.helpers.DummyStringEncoder

View File

@ -6,8 +6,8 @@ import io.kotest.matchers.string.shouldContain
import prog8.ast.AstToSourceTextConverter
import prog8.ast.Module
import prog8.ast.Program
import prog8.code.core.SourceCode
import prog8.code.core.internedStringsModuleName
import prog8.code.source.SourceCode
import prog8.code.internedStringsModuleName
import prog8.parser.ParseError
import prog8.parser.Prog8Parser.parseModule
import prog8tests.helpers.DummyFunctions

View File

@ -13,7 +13,7 @@ import prog8.ast.statements.Block
import prog8.ast.statements.Subroutine
import prog8.ast.statements.VarDecl
import prog8.code.core.Position
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
import prog8.parser.Prog8Parser
import prog8tests.helpers.DummyFunctions
import prog8tests.helpers.DummyMemsizer

View File

@ -9,7 +9,6 @@ import io.kotest.matchers.or
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import io.kotest.matchers.string.shouldStartWith
import io.kotest.matchers.types.instanceOf
import prog8.ast.IFunctionCall
import prog8.ast.Module
@ -18,6 +17,7 @@ import prog8.ast.Program
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.code.target.VMTarget
import prog8.code.target.encodings.PetsciiEncoding
@ -379,7 +379,7 @@ class TestProg8Parser: FunSpec( {
""".trimIndent()
val module = parseModule(SourceCode.Text(srcText))
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.STRINGSOURCEPREFIX
SourceCode.isStringResource(it.position.file) shouldBe true
}
}
@ -388,7 +388,7 @@ class TestProg8Parser: FunSpec( {
val resource = SourceCode.Resource("prog8lib/math.p8")
val module = parseModule(resource)
assertSomethingForAllNodes(module) {
it.position.file shouldStartWith SourceCode.LIBRARYFILEPREFIX
SourceCode.isLibraryResource(it.position.file) shouldBe true
}
}
}

View File

@ -14,8 +14,8 @@ import prog8.ast.Program
import prog8.ast.statements.Block
import prog8.code.ast.PtBlock
import prog8.code.core.Position
import prog8.code.core.SourceCode
import prog8.code.core.internedStringsModuleName
import prog8.code.source.SourceCode
import prog8.code.internedStringsModuleName
import prog8.code.target.C64Target
import prog8tests.helpers.DummyFunctions
import prog8tests.helpers.DummyMemsizer

View File

@ -5,8 +5,7 @@ import io.kotest.core.spec.style.AnnotationSpec
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldContain
import prog8.code.core.SourceCode
import prog8.code.core.SourceCode.Companion.LIBRARYFILEPREFIX
import prog8.code.source.SourceCode
import prog8tests.helpers.*
import kotlin.io.path.Path
@ -99,7 +98,7 @@ class TestSourceCode: AnnotationSpec() {
val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
val src = SourceCode.Resource(pathString)
src.origin shouldBe "$LIBRARYFILEPREFIX/$pathString"
src.origin shouldBe "library:/$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
src.isFromResources shouldBe true
src.isFromFilesystem shouldBe false
@ -111,7 +110,7 @@ class TestSourceCode: AnnotationSpec() {
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
src.origin shouldBe "$LIBRARYFILEPREFIX$pathString"
src.origin shouldBe "library:$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
}
@ -121,7 +120,7 @@ class TestSourceCode: AnnotationSpec() {
val srcFile = assumeReadableFile(resourcesDir, pathString).toFile()
val src = SourceCode.Resource(pathString)
src.origin shouldBe "$LIBRARYFILEPREFIX/$pathString"
src.origin shouldBe "library:/$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
src.isFromResources shouldBe true
}
@ -132,7 +131,7 @@ class TestSourceCode: AnnotationSpec() {
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
src.origin shouldBe "$LIBRARYFILEPREFIX$pathString"
src.origin shouldBe "library:$pathString"
src.text shouldBe normalizeLineEndings(srcFile.readText())
}
@ -142,7 +141,7 @@ class TestSourceCode: AnnotationSpec() {
val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile()
val src = SourceCode.Resource(pathString)
src.origin shouldBe "$LIBRARYFILEPREFIX/prog8lib/math.p8"
src.origin shouldBe "library:/prog8lib/math.p8"
src.text shouldBe normalizeLineEndings(srcFile.readText())
src.isFromResources shouldBe true
}

View File

@ -10,7 +10,7 @@ import prog8.code.ast.PtAssignTarget
import prog8.code.ast.PtAssignment
import prog8.code.core.BaseDataType
import prog8.code.core.DataType
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.code.target.Cx16Target
import prog8.parser.Prog8Parser.parseModule

View File

@ -15,6 +15,7 @@ import prog8.code.ast.PtAssignment
import prog8.code.ast.PtIdentifier
import prog8.code.ast.PtProgram
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.code.target.C64Target
import prog8.code.target.VMTarget
import prog8.codegen.cpu6502.AsmGen6502Internal

View File

@ -9,6 +9,7 @@ import prog8.ast.statements.*
import prog8.ast.walk.AstWalker
import prog8.ast.walk.IAstVisitor
import prog8.code.core.*
import prog8.code.source.SourceCode
object ParentSentinel : Node {

View File

@ -7,6 +7,8 @@ import prog8.ast.statements.*
import prog8.ast.walk.IAstVisitor
import prog8.code.ast.PtLabel
import prog8.code.core.*
import prog8.code.internedStringsModuleName
import prog8.code.source.SourceCode
/*********** Everything starts from here, the Program; zero or more modules *************/

View File

@ -4,6 +4,7 @@ import prog8.ast.expressions.NumericLiteral
import prog8.ast.statements.*
import prog8.ast.walk.IAstVisitor
import prog8.code.core.*
import prog8.code.source.SourceCode
import java.io.PrintStream

View File

@ -8,6 +8,7 @@ import prog8.ast.base.SyntaxError
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.code.core.*
import prog8.code.source.SourceCode
import prog8.parser.Prog8ANTLRParser.*
import kotlin.io.path.Path
import kotlin.io.path.isRegularFile

View File

@ -9,6 +9,7 @@ import prog8.ast.statements.*
import prog8.ast.walk.AstWalker
import prog8.ast.walk.IAstVisitor
import prog8.code.core.*
import prog8.code.internedStringsModuleName
import prog8.code.target.encodings.JapaneseCharacterConverter
import java.io.CharConversionException
import java.util.*

View File

@ -6,7 +6,7 @@ import prog8.ast.antlr.toAst
import prog8.ast.statements.Block
import prog8.ast.statements.Directive
import prog8.code.core.Position
import prog8.code.core.SourceCode
import prog8.code.source.SourceCode
class ParseError(override var message: String, val position: Position, cause: RuntimeException): Exception(message, cause)

View File

@ -20,6 +20,7 @@ TODO
Future Things and Ideas
^^^^^^^^^^^^^^^^^^^^^^^
- SourceLineCache should be removed as a separate thing, it reloads all source files again and splits them by line. It should re-use the already loaded Sources. Wrap it all in a ImporterFileSystem?
- Compiling Libraries: improve ability to create library files in prog8; for instance there's still stuff injected into the start of the start() routine AND there is separate setup logic going on before calling it.
Make up our mind! Maybe all setup does need to be put into start() ? because the program cannot function correctly when the variables aren't initialized properly bss is not cleared etc. etc.
Add a -library $xxxx command line option (and/or some directive) to preselect every setting that is required to make a library at $xxxx rather than a normal loadable and runnable program?

View File

@ -372,9 +372,10 @@ but they have some special properties because they are considered to be *text*.
Strings (without encoding prefix) will be encoded (translated from ASCII/UTF-8) into bytes via the
*default encoding* for the target platform. On the CBM machines, this is CBM PETSCII.
Strings in the default encoding are stored in the machine's default character encoding (which is PETSCII on the CBM machines).
You can choose to store it in another encodings such as ``sc`` (screencodes) or ``iso`` (iso-8859-15).
Here are examples of the possible encodings:
Strings without an encoding prefix are stored in the machine's default character encoding (which is PETSCII on the CBM machines,
but can be something else on other targets).
There are ways to change the encoding: prefix the string with an encoding name, or use the ``%encoding`` directive to
change it for the whole file at once. Here are examples of the possible encodings:
- ``"hello"`` a string translated into the default character encoding (PETSCII on the CBM machines)
- ``petscii:"hello"`` string in CBM PETSCII encoding
@ -386,9 +387,9 @@ Here are examples of the possible encodings:
- ``cp437:"≈ IBM Pc ≈ ♂♀♪☺¶"`` string in "cp437" encoding (IBM PC codepage 437)
- ``kata:"アノ ニホンジン ワ ガイコクジン。 # が # ガ"`` string in "kata" encoding (Katakana)
So the following is a string literal that will be encoded into memory bytes using the iso encoding.
So what follows below is a string literal that will be encoded into memory bytes using the iso encoding.
It can be correctly displayed on the screen only if a iso-8859-15 charset has been activated first
(the Commander X16 has this feature built in)::
(the Commander X16 has this capability)::
iso:"Käse, Straße"

View File

@ -1,22 +1,9 @@
%import textio
%zeropage basicsafe
%option no_sysinit
%import compression
main {
sub start() {
ubyte[] array = [
11,
22,
33,
44,
]
for cx16.r0L in [1,2,3,4,] {
txt.print_ub(cx16.r0L)
txt.nl()
}
for cx16.r0L in array {
txt.print_ub(cx16.r0L)
txt.nl()
}
; compression.decode_rle(0,0,0)
; compression.decode_zx0(0,0)
compression.decode_tscrunch(0,0)
}
}

View File

@ -2,8 +2,8 @@ package prog8.intermediate
import prog8.code.core.InternalCompilerException
import prog8.code.core.Position
import prog8.code.core.SourceLineCache
import prog8.code.core.toHex
import prog8.code.source.SourceLineCache
import java.nio.file.Path
import javax.xml.stream.XMLOutputFactory
import javax.xml.stream.XMLStreamWriter
@ -102,12 +102,12 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) {
is IRSubroutine -> {
xml.writeStartElement("SUB")
xml.writeAttribute("NAME", child.label)
xml.writeAttribute("RETURNTYPE", child.returnType?.typeString(null)?.lowercase() ?: "")
xml.writeAttribute("RETURNTYPE", child.returnType?.irTypeString(null)?.lowercase() ?: "")
xml.writeAttribute("POS", child.position.toString())
xml.writeCharacters("\n")
xml.writeStartElement("PARAMS")
xml.writeCharacters("\n")
child.parameters.forEach { param -> xml.writeCharacters("${param.dt.typeString(null)} ${param.name}\n") }
child.parameters.forEach { param -> xml.writeCharacters("${param.dt.irTypeString(null)} ${param.name}\n") }
xml.writeEndElement()
xml.writeCharacters("\n")
child.chunks.forEach { chunk ->

View File

@ -169,7 +169,7 @@ class IRStMemVar(name: String,
require(!dt.isString)
}
val typeString: String = dt.typeString(length)
val typeString: String = dt.irTypeString(length)
}
class IRStMemorySlab(
@ -219,7 +219,7 @@ class IRStStaticVariable(name: String,
val uninitialized = onetimeInitializationArrayValue==null && onetimeInitializationStringValue==null && onetimeInitializationNumericValue==null
val typeString: String = dt.typeString(length)
val typeString: String = dt.irTypeString(length)
}
class IRStArrayElement(val bool: Boolean?, val number: Double?, val addressOfSymbol: String?) {

View File

@ -6,7 +6,7 @@ import prog8.code.left
import prog8.code.right
fun DataType.typeString(length: Int?): String {
fun DataType.irTypeString(length: Int?): String {
val lengthStr = if(length==0) "" else length.toString()
return when (this.base) {
BaseDataType.BOOL -> "bool"

View File

@ -78,7 +78,7 @@ class Memory {
return Double.fromBits(bits)
}
// for now, no LONG 32-bits and no FLOAT support.
// for now, no LONG 32-bits support
// fun getL(address: Int): UInt {
// return mem[address+3] + 256u*mem[address+2] + 65536u*mem[address+1] + 16777216u*mem[address]
// }