reducing ast dependencies - separate Ast compilation module

This commit is contained in:
Irmen de Jong 2021-02-09 01:06:11 +01:00
parent c97d76dbf2
commit d9244f22c2
26 changed files with 222 additions and 114 deletions

1
.idea/modules.xml generated
View File

@ -3,6 +3,7 @@
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/compiler/compiler.iml" filepath="$PROJECT_DIR$/compiler/compiler.iml" />
<module fileurl="file://$PROJECT_DIR$/compilerAst/compilerAst.iml" filepath="$PROJECT_DIR$/compilerAst/compilerAst.iml" />
<module fileurl="file://$PROJECT_DIR$/dbusCompilerService/dbusCompilerService.iml" filepath="$PROJECT_DIR$/dbusCompilerService/dbusCompilerService.iml" />
<module fileurl="file://$PROJECT_DIR$/docs/docs.iml" filepath="$PROJECT_DIR$/docs/docs.iml" />
<module fileurl="file://$PROJECT_DIR$/examples/examples.iml" filepath="$PROJECT_DIR$/examples/examples.iml" />

View File

@ -1,7 +1,7 @@
plugins {
id 'java'
id 'application'
id "org.jetbrains.kotlin.jvm" version "1.4.21"
id "org.jetbrains.kotlin.jvm" version "1.4.30"
id 'org.jetbrains.dokka' version "0.9.18"
id 'com.github.johnrengelman.shadow' version '6.1.0'
}
@ -18,14 +18,12 @@ repositories {
def prog8version = rootProject.file('compiler/res/version.txt').text.trim()
dependencies {
implementation project(':parser')
implementation project(':compilerAst')
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
// implementation "org.jetbrains.kotlin:kotlin-reflect"
implementation 'org.antlr:antlr4-runtime:4.8'
implementation 'org.jetbrains.kotlinx:kotlinx-cli:0.3.1'
// implementation 'net.razorvine:ksim65:1.8'
// implementation "com.github.hypfvieh:dbus-java:3.2.4"
implementation project(':parser')
testImplementation "org.jetbrains.kotlin:kotlin-test-junit5"
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.2'

View File

@ -11,9 +11,8 @@
<orderEntry type="jdk" jdkName="11" jdkType="JavaSDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinJavaRuntime" level="project" />
<orderEntry type="module" module-name="parser" />
<orderEntry type="library" name="unittest-libs" level="project" />
<orderEntry type="library" name="kotlinx-cli-jvm" level="project" />
<orderEntry type="library" name="antlr-runtime-4.9" level="project" />
<orderEntry type="module" module-name="compilerAst" />
</component>
</module>

View File

@ -151,21 +151,24 @@ internal class AstChecker(private val program: Program,
}
override fun visit(jump: Jump) {
if(jump.identifier!=null) {
val targetStatement = checkFunctionOrLabelExists(jump.identifier, jump)
val ident = jump.identifier
if(ident!=null) {
val targetStatement = checkFunctionOrLabelExists(ident, jump)
if(targetStatement!=null) {
if(targetStatement is BuiltinFunctionStatementPlaceholder)
errors.err("can't jump to a builtin function", jump.position)
}
}
if(jump.address!=null && (jump.address < 0 || jump.address > 65535))
val addr = jump.address
if(addr!=null && (addr < 0 || addr > 65535))
errors.err("jump address must be valid integer 0..\$ffff", jump.position)
super.visit(jump)
}
override fun visit(block: Block) {
if(block.address!=null && (block.address<0 || block.address>65535)) {
val addr = block.address
if(addr!=null && (addr<0 || addr>65535)) {
errors.err("block memory address must be valid integer 0..\$ffff", block.position)
}
@ -316,9 +319,11 @@ internal class AstChecker(private val program: Program,
RegisterOrPair.R13,
RegisterOrPair.R14,
RegisterOrPair.R15 -> { /* no sensible way to count this */ }
null ->
if(p.statusflag!=null)
statusflagCounts[p.statusflag] = statusflagCounts.getValue(p.statusflag) + 1
null -> {
val statusf = p.statusflag
if (statusf != null)
statusflagCounts[statusf] = statusflagCounts.getValue(statusf) + 1
}
}
}
}

View File

@ -28,14 +28,17 @@ internal interface CompilationTarget: IStringEncoding {
}
fun isInRegularRAM(target: AssignTarget, program: Program): Boolean {
val memAddr = target.memoryAddress
val arrayIdx = target.arrayindexed
val ident = target.identifier
when {
target.memoryAddress != null -> {
return when (target.memoryAddress.addressExpression) {
memAddr != null -> {
return when (memAddr.addressExpression) {
is NumericLiteralValue -> {
machine.isRegularRAMaddress((target.memoryAddress.addressExpression as NumericLiteralValue).number.toInt())
machine.isRegularRAMaddress((memAddr.addressExpression as NumericLiteralValue).number.toInt())
}
is IdentifierReference -> {
val decl = (target.memoryAddress.addressExpression as IdentifierReference).targetVarDecl(program)
val decl = (memAddr.addressExpression as IdentifierReference).targetVarDecl(program)
if ((decl?.type == VarDeclType.VAR || decl?.type == VarDeclType.CONST) && decl.value is NumericLiteralValue)
machine.isRegularRAMaddress((decl.value as NumericLiteralValue).number.toInt())
else
@ -44,8 +47,8 @@ internal interface CompilationTarget: IStringEncoding {
else -> false
}
}
target.arrayindexed != null -> {
val targetStmt = target.arrayindexed!!.arrayvar.targetVarDecl(program)
arrayIdx != null -> {
val targetStmt = arrayIdx.arrayvar.targetVarDecl(program)
return if (targetStmt?.type == VarDeclType.MEMORY) {
val addr = targetStmt.value as? NumericLiteralValue
if (addr != null)
@ -54,8 +57,8 @@ internal interface CompilationTarget: IStringEncoding {
false
} else true
}
target.identifier != null -> {
val decl = target.identifier!!.targetVarDecl(program)!!
ident != null -> {
val decl = ident.targetVarDecl(program)!!
return if (decl.type == VarDeclType.MEMORY && decl.value is NumericLiteralValue)
machine.isRegularRAMaddress((decl.value as NumericLiteralValue).number.toInt())
else

View File

@ -177,9 +177,10 @@ internal class AsmGen(private val program: Program,
out("\n\n; ---- block: '${block.name}' ----")
out("${block.name}\t" + (if("force_output" in block.options()) ".block\n" else ".proc\n"))
if(block.address!=null) {
out(".cerror * > ${block.address.toHex()}, 'block address overlaps by ', *-${block.address.toHex()},' bytes'")
out("* = ${block.address.toHex()}")
val addr = block.address
if(addr!=null) {
out(".cerror * > ${addr.toHex()}, 'block address overlaps by ', *-${addr.toHex()},' bytes'")
out("* = ${addr.toHex()}")
}
outputSourceLine(block)
@ -356,10 +357,11 @@ internal class AsmGen(private val program: Program,
}
val asmSubs = statements.filterIsInstance<Subroutine>().filter { it.isAsmSubroutine }
for(sub in asmSubs) {
if(sub.asmAddress!=null) {
val addr = sub.asmAddress
if(addr!=null) {
if(sub.statements.isNotEmpty())
throw AssemblyError("kernel subroutine cannot have statements")
out(" ${sub.name} = ${sub.asmAddress.toHex()}")
out(" ${sub.name} = ${addr.toHex()}")
}
}
}
@ -1268,17 +1270,20 @@ $label nop""")
}
private fun getJumpTarget(jmp: Jump): String {
val ident = jmp.identifier
val label = jmp.generatedLabel
val addr = jmp.address
return when {
jmp.identifier!=null -> {
val target = jmp.identifier.targetStatement(program)
val asmName = asmSymbolName(jmp.identifier)
ident!=null -> {
val target = ident.targetStatement(program)
val asmName = asmSymbolName(ident)
if(target is Label)
"_$asmName" // prefix with underscore to jump to local label
else
asmName
}
jmp.generatedLabel!=null -> jmp.generatedLabel
jmp.address!=null -> jmp.address.toHex()
label!=null -> label
addr!=null -> addr.toHex()
else -> "????"
}
}
@ -1293,12 +1298,12 @@ $label nop""")
when (returnType) {
in NumericDatatypes -> {
assignExpressionToRegister(returnvalue, returnReg.registerOrPair)
assignExpressionToRegister(returnvalue, returnReg.registerOrPair!!)
}
else -> {
// all else take its address and assign that also to AY register pair
val addrofValue = AddressOf(returnvalue as IdentifierReference, returnvalue.position)
assignmentAsmGen.assignExpressionToRegister(addrofValue, returnReg.registerOrPair)
assignmentAsmGen.assignExpressionToRegister(addrofValue, returnReg.registerOrPair!!)
}
}
}

65
compilerAst/build.gradle Normal file
View File

@ -0,0 +1,65 @@
plugins {
id 'antlr'
id 'java'
id "org.jetbrains.kotlin.jvm" version "1.4.30"
}
targetCompatibility = 11
sourceCompatibility = 11
repositories {
mavenCentral()
}
configurations {
// strange antlr plugin issue, see https://github.com/gradle/gradle/issues/820
// this avoids linking in the complete antlr binary jar
compile {
extendsFrom = extendsFrom.findAll { it != configurations.antlr }
}
}
dependencies {
antlr 'org.antlr:antlr4:4.9'
implementation 'org.antlr:antlr4-runtime:4.9'
implementation project(':parser')
// antlr('org.antlr:antlr4:4.9') {
// exclude group: 'com.ibm.icu', module: 'icu4j'
// }
}
compileKotlin {
kotlinOptions {
jvmTarget = "11"
// verbose = true
// freeCompilerArgs += "-XXLanguage:+NewInference"
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = "11"
}
}
sourceSets {
main {
java {
srcDirs = ["${project.projectDir}/src"]
}
resources {
srcDirs = ["${project.projectDir}/res"]
}
}
test {
java {
srcDirs = ["${project.projectDir}/test"]
}
}
}
task wrapper(type: Wrapper) {
gradleVersion = '6.7'
}

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="openjdk-11" jdkType="JavaSDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="KotlinJavaRuntime" level="project" />
<orderEntry type="module" module-name="parser" />
<orderEntry type="library" name="antlr-runtime-4.9" level="project" />
</component>
</module>

View File

@ -392,7 +392,7 @@ object BuiltinFunctionScopePlaceholder : INameScope {
// prefix for struct member variables
internal fun mangledStructMemberName(varName: String, memberName: String) = "prog8struct_${varName}_$memberName"
fun mangledStructMemberName(varName: String, memberName: String) = "prog8struct_${varName}_$memberName"
fun Number.toHex(): String {

View File

@ -644,48 +644,3 @@ private fun prog8Parser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl
toPosition()
)
}
internal fun escape(str: String): String {
val es = str.map {
when(it) {
'\t' -> "\\t"
'\n' -> "\\n"
'\r' -> "\\r"
'"' -> "\\\""
in '\u8000'..'\u80ff' -> "\\x" + (it.toInt() - 0x8000).toString(16).padStart(2, '0')
in '\u0000'..'\u00ff' -> it.toString()
else -> "\\u" + it.toInt().toString(16).padStart(4, '0')
}
}
return es.joinToString("")
}
internal fun unescape(str: String, position: Position): String {
val result = mutableListOf<Char>()
val iter = str.iterator()
while(iter.hasNext()) {
val c = iter.nextChar()
if(c=='\\') {
val ec = iter.nextChar()
result.add(when(ec) {
'\\' -> '\\'
'n' -> '\n'
'r' -> '\r'
'"' -> '"'
'\'' -> '\''
'u' -> {
"${iter.nextChar()}${iter.nextChar()}${iter.nextChar()}${iter.nextChar()}".toInt(16).toChar()
}
'x' -> {
// special hack 0x8000..0x80ff will be outputted verbatim without encoding
val hex = ("" + iter.nextChar() + iter.nextChar()).toInt(16)
(0x8000 + hex).toChar()
}
else -> throw SyntaxError("invalid escape char in string: \\$ec", position)
})
} else {
result.add(c)
}
}
return result.joinToString("")
}

View File

@ -0,0 +1,49 @@
package prog8.ast.antlr
import prog8.ast.base.Position
import prog8.ast.base.SyntaxError
fun escape(str: String): String {
val es = str.map {
when(it) {
'\t' -> "\\t"
'\n' -> "\\n"
'\r' -> "\\r"
'"' -> "\\\""
in '\u8000'..'\u80ff' -> "\\x" + (it.toInt() - 0x8000).toString(16).padStart(2, '0')
in '\u0000'..'\u00ff' -> it.toString()
else -> "\\u" + it.toInt().toString(16).padStart(4, '0')
}
}
return es.joinToString("")
}
fun unescape(str: String, position: Position): String {
val result = mutableListOf<Char>()
val iter = str.iterator()
while(iter.hasNext()) {
val c = iter.nextChar()
if(c=='\\') {
val ec = iter.nextChar()
result.add(when(ec) {
'\\' -> '\\'
'n' -> '\n'
'r' -> '\r'
'"' -> '"'
'\'' -> '\''
'u' -> {
"${iter.nextChar()}${iter.nextChar()}${iter.nextChar()}${iter.nextChar()}".toInt(16).toChar()
}
'x' -> {
// special hack 0x8000..0x80ff will be outputted verbatim without encoding
val hex = ("" + iter.nextChar() + iter.nextChar()).toInt(16)
(0x8000 + hex).toChar()
}
else -> throw SyntaxError("invalid escape char in string: \\$ec", position)
})
} else {
result.add(c)
}
}
return result.joinToString("")
}

View File

@ -43,7 +43,7 @@ enum class DataType {
this == other -> false
this in ByteDatatypes -> false
this in WordDatatypes -> other in ByteDatatypes
this==STR && other==UWORD || this==UWORD && other==STR -> false
this== STR && other== UWORD || this== UWORD && other== STR -> false
else -> true
}
@ -52,7 +52,7 @@ enum class DataType {
this == other -> true
this in ByteDatatypes -> other in ByteDatatypes
this in WordDatatypes -> other in WordDatatypes
this==STR && other==UWORD || this==UWORD && other==STR -> true
this== STR && other== UWORD || this== UWORD && other== STR -> true
else -> false
}
}
@ -128,10 +128,11 @@ val NumericDatatypes = setOf(DataType.UBYTE, DataType.BYTE, DataType.UWORD, Data
val ArrayDatatypes = setOf(DataType.ARRAY_UB, DataType.ARRAY_B, DataType.ARRAY_UW, DataType.ARRAY_W, DataType.ARRAY_F)
val StringlyDatatypes = setOf(DataType.STR, DataType.ARRAY_UB, DataType.ARRAY_B, DataType.UWORD)
val IterableDatatypes = setOf(
DataType.STR,
DataType.ARRAY_UB, DataType.ARRAY_B,
DataType.ARRAY_UW, DataType.ARRAY_W,
DataType.ARRAY_F)
DataType.STR,
DataType.ARRAY_UB, DataType.ARRAY_B,
DataType.ARRAY_UW, DataType.ARRAY_W,
DataType.ARRAY_F
)
val PassByValueDatatypes = NumericDatatypes
val PassByReferenceDatatypes = IterableDatatypes.plus(DataType.STRUCT)
val ArrayElementTypes = mapOf(
@ -140,7 +141,8 @@ val ArrayElementTypes = mapOf(
DataType.ARRAY_UB to DataType.UBYTE,
DataType.ARRAY_W to DataType.WORD,
DataType.ARRAY_UW to DataType.UWORD,
DataType.ARRAY_F to DataType.FLOAT)
DataType.ARRAY_F to DataType.FLOAT
)
val ElementArrayTypes = mapOf(
DataType.BYTE to DataType.ARRAY_B,
DataType.UBYTE to DataType.ARRAY_UB,
@ -148,10 +150,12 @@ val ElementArrayTypes = mapOf(
DataType.UWORD to DataType.ARRAY_UW,
DataType.FLOAT to DataType.ARRAY_F
)
val Cx16VirtualRegisters = listOf(RegisterOrPair.R0, RegisterOrPair.R1, RegisterOrPair.R2, RegisterOrPair.R3,
val Cx16VirtualRegisters = listOf(
RegisterOrPair.R0, RegisterOrPair.R1, RegisterOrPair.R2, RegisterOrPair.R3,
RegisterOrPair.R4, RegisterOrPair.R5, RegisterOrPair.R6, RegisterOrPair.R7,
RegisterOrPair.R8, RegisterOrPair.R9, RegisterOrPair.R10, RegisterOrPair.R11,
RegisterOrPair.R12, RegisterOrPair.R13, RegisterOrPair.R14, RegisterOrPair.R15)
RegisterOrPair.R12, RegisterOrPair.R13, RegisterOrPair.R14, RegisterOrPair.R15
)
// find the parent node of a specific type or interface

View File

@ -2,7 +2,7 @@ package prog8.ast.base
import prog8.parser.ParsingFailedError
// TODO can move to compiler?????? **************
class ErrorReporter {
private enum class MessageSeverity {
WARNING,

View File

@ -147,10 +147,13 @@ class BinaryExpression(var left: Expression, var operator: String, var right: Ex
InferredTypes.unknown()
else {
try {
InferredTypes.knownFor(commonDatatype(
InferredTypes.knownFor(
commonDatatype(
leftDt.typeOrElse(DataType.BYTE),
rightDt.typeOrElse(DataType.BYTE),
null, null).first)
null, null
).first
)
} catch (x: FatalAstException) {
InferredTypes.unknown()
}
@ -704,7 +707,8 @@ internal fun makeRange(fromVal: Int, toVal: Int, stepVal: Int): IntProgression {
}
}
data class IdentifierReference(val nameInSource: List<String>, override val position: Position) : Expression(), IAssignable {
data class IdentifierReference(val nameInSource: List<String>, override val position: Position) : Expression(),
IAssignable {
override lateinit var parent: Node
fun targetStatement(program: Program) =

View File

@ -155,15 +155,15 @@ enum class ZeropageWish {
open class VarDecl(val type: VarDeclType,
private val declaredDatatype: DataType,
val zeropage: ZeropageWish,
var arraysize: ArrayIndex?,
val name: String,
private val structName: String?,
var value: Expression?,
val isArray: Boolean,
val autogeneratedDontRemove: Boolean,
override val position: Position) : Statement() {
private val declaredDatatype: DataType,
val zeropage: ZeropageWish,
var arraysize: ArrayIndex?,
val name: String,
private val structName: String?,
var value: Expression?,
val isArray: Boolean,
val autogeneratedDontRemove: Boolean,
override val position: Position) : Statement() {
override lateinit var parent: Node
var struct: StructDecl? = null // set later (because at parse time, we only know the name)
private set

View File

@ -16,7 +16,8 @@ interface IAstModification {
}
}
class SetExpression(private val setter: (newExpr: Expression) -> Unit, private val newExpr: Expression, private val parent: Node) : IAstModification {
class SetExpression(private val setter: (newExpr: Expression) -> Unit, private val newExpr: Expression, private val parent: Node) :
IAstModification {
override fun perform() {
setter(newExpr)
newExpr.linkParents(parent)
@ -37,7 +38,8 @@ interface IAstModification {
}
}
class InsertAfter(private val after: Statement, private val stmt: Statement, private val parent: INameScope) : IAstModification {
class InsertAfter(private val after: Statement, private val stmt: Statement, private val parent: INameScope) :
IAstModification {
override fun perform() {
val idx = parent.statements.indexOfFirst { it===after } + 1
parent.statements.add(idx, stmt)
@ -45,7 +47,8 @@ interface IAstModification {
}
}
class InsertBefore(private val before: Statement, private val stmt: Statement, private val parent: INameScope) : IAstModification {
class InsertBefore(private val before: Statement, private val stmt: Statement, private val parent: INameScope) :
IAstModification {
override fun perform() {
val idx = parent.statements.indexOfFirst { it===before }
parent.statements.add(idx, stmt)
@ -53,7 +56,8 @@ interface IAstModification {
}
}
class ReplaceNode(private val node: Node, private val replacement: Node, private val parent: Node) : IAstModification {
class ReplaceNode(private val node: Node, private val replacement: Node, private val parent: Node) :
IAstModification {
override fun perform() {
parent.replaceChildNode(node, replacement)
replacement.linkParents(parent)

View File

@ -9,25 +9,25 @@ import prog8.ast.base.Position
import prog8.ast.base.SyntaxError
import prog8.ast.statements.Directive
import prog8.ast.statements.DirectiveArg
import prog8.pathFrom
import java.io.InputStream
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
internal class ParsingFailedError(override var message: String) : Exception(message)
class ParsingFailedError(override var message: String) : Exception(message)
internal class CustomLexer(val modulePath: Path, input: CharStream?) : prog8Lexer(input)
fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.')
internal fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.')
internal fun pathFrom(stringPath: String, vararg rest: String): Path = FileSystems.getDefault().getPath(stringPath, *rest)
internal class ModuleImporter {
class ModuleImporter {
internal fun importModule(program: Program, filePath: Path, encoder: IStringEncoding, compilationTargetName: String): Module {
fun importModule(program: Program, filePath: Path, encoder: IStringEncoding, compilationTargetName: String): Module {
print("importing '${moduleName(filePath.fileName)}'")
if(filePath.parent!=null) {
var importloc = filePath.toString()
@ -45,7 +45,7 @@ internal class ModuleImporter {
return importModule(program, input, filePath, false, encoder, compilationTargetName)
}
internal fun importLibraryModule(program: Program, name: String,
fun importLibraryModule(program: Program, name: String,
encoder: IStringEncoding, compilationTargetName: String): Module? {
val import = Directive("%import", listOf(
DirectiveArg("", name, 42, position = Position("<<<implicit-import>>>", 0, 0, 0))

View File

@ -2,7 +2,7 @@
plugins {
id 'java'
id 'application'
id "org.jetbrains.kotlin.jvm" version "1.4.21"
id "org.jetbrains.kotlin.jvm" version "1.4.30"
id 'com.github.johnrengelman.shadow' version '6.1.0'
}

View File

@ -2,7 +2,7 @@
plugins {
id 'java'
id 'application'
id "org.jetbrains.kotlin.jvm" version "1.4.21"
id "org.jetbrains.kotlin.jvm" version "1.4.30"
id 'com.github.johnrengelman.shadow' version '6.1.0'
}

View File

@ -2,4 +2,5 @@
rm -f *.jar *.asm *.prg *.vm.txt *.vice-mon-list a.out
rm -rf build out
rm -rf compiler/build compilerAst/build dbusCompilerService/build httpCompilerService/build parser/build

View File

@ -1,4 +1,5 @@
include ':parser'
include ':compilerAst'
include ':compiler'
include ':dbusCompilerService'
include ':httpCompilerService'