module import mechanism and ast checking

il65/antlr/examples/imported.ill

@@ -1,6 +0,0 @@
-~ extra  {
-	; this is imported
-
-	X = 42
-	return 44
-}

il65/antlr/examples/test.ill

@@ -1,21 +0,0 @@
-~ main $c003  {
-  memory byte derp = $ffdd1
-  const byte hopla=55-33
-  ; const byte hopla2=55-hopla
-  const float zwop = -1.7014118345e+38
-  const float zwop = -1.7014118346e+38
-
-  A = "derp" * %000100
-
-  %import maghierniet
-  %import maghierniet
-  %import maghierniet
-  %import maghierniet
-
-  return 1+999
-}
-
-%import imported1, 33, aaa
-%import imported2
-%import imported3
-

il65/examples/imported.ill

@@ -0,0 +1,25 @@
+%zp full
+%address 33
+
+~ extra  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+
+~ extra2  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+
+
+~ main  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+

il65/examples/imported2.ill

@@ -0,0 +1,25 @@
+%zp full
+%address 33
+
+~ extra  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+
+~ extra2  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+
+
+~ main  {
+	; this is imported
+
+	X = 42
+	return 44
+}
+

il65/examples/test.ill

@@ -0,0 +1,28 @@
+~ main $c003  {
+  memory byte derp = $ffdd
+  const byte hopla=55-33
+  ; const byte hopla2=55-hopla
+  const float zwop = -1.7014118345e+38
+  const float zwop = -1.7014118345e+38
+
+  A = "derp" * %000100
+
+  return 1+999
+	%breakpoint
+	%asminclude "derp", hopsa
+	%asmbinary "derp", 0, 200
+}
+
+
+%import imported
+%import imported
+
+
+%import imported
+%import imported
+
+%import imported
+%import imported2
+
+%import imported
+

il65/src/il65/Main.kt

@@ -1,15 +1,15 @@
 package il65
 
-import il65.ast.Directive
-import il65.ast.Module
-import il65.ast.SyntaxError
-import il65.ast.toAst
+import il65.ast.*
 import il65.parser.il65Lexer
 import il65.parser.il65Parser
 import org.antlr.v4.runtime.CharStreams
 import org.antlr.v4.runtime.CommonTokenStream
 import org.antlr.v4.runtime.Lexer
+import java.nio.file.Files
+import java.nio.file.Path
 import java.nio.file.Paths
+import kotlin.collections.HashMap
 
 
 class MyTokenStream(lexer: Lexer) : CommonTokenStream(lexer) {
@@ -32,42 +32,49 @@ class MyTokenStream(lexer: Lexer) : CommonTokenStream(lexer) {
 class ParsingFailedError(override var message: String) : Exception(message)
 
 
-fun loadModule(filename: String) : Module {
+private val importedModules : HashMap<String, Module> = hashMapOf()
 
-    val filePath = Paths.get(filename).normalize()
-    val fileLocation = filePath.parent
-    val fileName = filePath.fileName
 
-    println("importing '$fileName'  (from $fileLocation)...")
+
+fun loadModule(filePath: Path) : Module {
+    println("importing '${filePath.fileName}'  (from ${filePath.parent})...")
+    if(!Files.isReadable(filePath))
+        throw ParsingFailedError("No such file: $filePath")
+
+    val moduleName = fileNameWithoutSuffix(filePath)
     val input = CharStreams.fromPath(filePath)
     val lexer = il65Lexer(input)
     val tokens = MyTokenStream(lexer)
     val parser = il65Parser(tokens)
     val parseTree = parser.module()
     if(parser.numberOfSyntaxErrors > 0)
-        throw ParsingFailedError("There are ${parser.numberOfSyntaxErrors} syntax errors in '$fileName'.")
+        throw ParsingFailedError("There are ${parser.numberOfSyntaxErrors} syntax errors in '${filePath.fileName}'.")
 
     // TODO the comments:
     // tokens.commentTokens().forEach { println(it) }
 
     // convert to Ast and optimize
-    var moduleAst = parseTree.toAst(fileName.toString(),true).optimized()
+    val moduleAst = parseTree.toAst(moduleName,true)
+    moduleAst.optimize()
     moduleAst.linkParents()
-
-    val checkResult = moduleAst.checkValid()
-    checkResult.forEach { it.printError() }
-    if(checkResult.isNotEmpty())
-        throw ParsingFailedError("There are ${checkResult.size} errors in '$fileName'.")
+    moduleAst.checkValid()
+    importedModules[moduleAst.name] = moduleAst
 
     // process imports
     val lines = moduleAst.lines.toMutableList()
     val imports = lines
             .mapIndexed { i, it -> Pair(i, it) }
             .filter { (it.second as? Directive)?.directive == "%import" }
-            .map { Pair(it.first, executeImportDirective(it.second as Directive)) }
+            .map { Pair(it.first, executeImportDirective(it.second as Directive, filePath)) }
 
     imports.reversed().forEach {
-            println("IMPORT [in ${moduleAst.name}]: $it")   // TODO
+        if(it.second==null) {
+            // this import was already satisfied. just remove this line.
+            lines.removeAt(it.first)
+        } else {
+            // merge imported lines at this spot
+            lines.addAll(it.first, it.second!!.lines)
+        }
     }
 
     moduleAst.lines = lines
@@ -75,21 +82,44 @@ fun loadModule(filename: String) : Module {
 }
 
 
-fun executeImportDirective(import: Directive): Module {
+fun fileNameWithoutSuffix(filePath: Path) =
+        filePath.fileName.toString().substringBeforeLast('.')
+
+
+fun discoverImportedModule(name: String, importedFrom: Path): Path {
+    val tryName = Paths.get(importedFrom.parent.toString(), name + ".ill")
+    if(Files.exists(tryName))
+        return tryName
+    else
+        throw ParsingFailedError("No such module source file: $tryName")
+}
+
+
+fun executeImportDirective(import: Directive, importedFrom: Path): Module? {
     if(import.directive!="%import" || import.args.size!=1 || import.args[0].name==null)
         throw SyntaxError("invalid import directive", import)
+    val moduleName = import.args[0].name!!
+    if(importedModules.containsKey(moduleName))
+        return null
 
-    return Module("???", emptyList())   // TODO
+    val modulePath = discoverImportedModule(moduleName, importedFrom)
+    val importedModule = loadModule(modulePath)
+    importedModule.checkImportValid()
+
+    return importedModule
 }
 
 
 fun main(args: Array<String>) {
-    println("Reading source file: ${args[0]}")
     try {
-        val moduleAst = loadModule(args[0]).optimized()     // one final global optimization
-        moduleAst.linkParents()  // re-link parents in final configuration
+        val filepath = Paths.get(args[0]).normalize()
+        val moduleAst = loadModule(filepath)
+        moduleAst.optimize()        // one final global optimization
+        moduleAst.linkParents()     // re-link parents in final configuration
+        moduleAst.checkValid()      // check if final tree is valid
 
-        moduleAst.lines.map {
+        // todo compile to asm...
+        moduleAst.lines.forEach {
             println(it)
         }
     } catch(sx: SyntaxError) {

il65/src/il65/ast/AST.kt

@@ -3,6 +3,7 @@ package il65.ast
 import il65.parser.il65Parser
 import org.antlr.v4.runtime.ParserRuleContext
 import org.antlr.v4.runtime.tree.TerminalNode
+import java.nio.file.Paths
 
 
 /**************************** AST Data classes ****************************/
@@ -35,19 +36,19 @@ class ExpressionException(override var message: String) : AstException(message)
 
 class SyntaxError(override var message: String, val node: Node?) : AstException(message) {
     fun printError() {
-        val location = if(node?.position == null)
-            ""
-        else
-            "[line ${node.position!!.line} col ${node.position!!.startCol}-${node.position!!.endCol}] "
-        System.err.println("$location$message")
+        val location = if(node?.position == null) "" else node.position.toString()
+        System.err.println("$location $message")
     }
 }
 
 
-data class Position(val line: Int, val startCol:Int, val endCol: Int)
+data class Position(val file: String, val line: Int, val startCol: Int, val endCol: Int) {
+    override fun toString(): String = "[$file: line $line col $startCol-$endCol]"
+}
 
 
 interface IAstProcessor {
+    fun process(module: Module)
     fun process(expr: PrefixExpression): IExpression
     fun process(expr: BinaryExpression): IExpression
     fun process(directive: Directive): IStatement
@@ -81,9 +82,8 @@ data class Module(val name: String,
         lines.forEach {it.linkParents(this)}
     }
 
-    fun process(processor: IAstProcessor): Module {
-        lines = lines.map { it.process(processor) }
-        return this
+    fun process(processor: IAstProcessor) {
+        processor.process(this)
     }
 }
 
@@ -440,9 +440,10 @@ data class InlineAssembly(val assembly: String) : IStatement {
 /***************** Antlr Extension methods to create AST ****************/
 
 fun ParserRuleContext.toPosition(withPosition: Boolean) : Position? {
+    val file = Paths.get(this.start.inputStream.sourceName).fileName.toString()
     return if (withPosition)
         // note: be ware of TAB characters in the source text, they count as 1 column...
-        Position(start.line, start.charPositionInLine, stop.charPositionInLine+stop.text.length)
+        Position(file, start.line, start.charPositionInLine, stop.charPositionInLine+stop.text.length)
     else
         null
 }

il65/src/il65/ast/AstChecker.kt

@@ -1,22 +1,32 @@
-package il65
+package il65.ast
 
-import il65.ast.*
+import il65.ParsingFailedError
 
 
-fun Module.checkValid() : List<SyntaxError> {
+fun Module.checkValid() {
     val checker = AstChecker()
     this.process(checker)
-    return checker.result()
+    val checkResult = checker.result()
+    checkResult.forEach {
+        it.printError()
+    }
+    if(checkResult.isNotEmpty())
+        throw ParsingFailedError("There are ${checkResult.size} errors in module '$name'.")
 }
 
 
 class AstChecker : IAstProcessor {
     private val checkResult: MutableList<SyntaxError> = mutableListOf()
+    private val blockNames: HashMap<String, Position?> = hashMapOf()
 
     fun result(): List<SyntaxError> {
         return checkResult
     }
 
+    override fun process(module: Module) {
+        module.lines.forEach { it.process(this) }
+    }
+
     override fun process(expr: PrefixExpression): IExpression {
         return expr
     }
@@ -29,6 +39,12 @@ class AstChecker : IAstProcessor {
         if(block.address!=null && (block.address<0 || block.address>65535)) {
             checkResult.add(SyntaxError("block memory address must be valid 0..\$ffff", block))
         }
+        val existing = blockNames[block.name]
+        if(existing!=null) {
+            checkResult.add(SyntaxError("block name conflict, first defined in ${existing.file} line ${existing.line}", block))
+        } else {
+            blockNames[block.name] = block.position
+        }
         block.statements.forEach { it.process(this) }
         return block
     }
@@ -38,8 +54,6 @@ class AstChecker : IAstProcessor {
      */
     override fun process(decl: VarDecl): IStatement {
         fun err(msg: String) {
-            if(decl.value?.position == null)
-                throw AstException("declvalue no pos!")
             checkResult.add(SyntaxError(msg, decl))
         }
         when(decl.type) {
@@ -127,16 +141,21 @@ class AstChecker : IAstProcessor {
                 if(directive.parent !is Module) err("this directive may only occur at module level")
                 if(directive.args.size!=1 || directive.args[0].name==null)
                     err("invalid import directive, expected module name argument")
+                if(directive.args[0].name == (directive.parent as Module).name)
+                    err("invalid import directive, cannot import itself")
             }
             "%breakpoint" -> {
+                if(directive.parent !is Block) err("this directive may only occur in a block")
                 if(directive.args.isNotEmpty())
                     err("invalid breakpoint directive, expected no arguments")
             }
             "%asminclude" -> {
+                if(directive.parent !is Block) err("this directive may only occur in a block")
                 if(directive.args.size!=2 || directive.args[0].str==null || directive.args[1].name==null)
                     err("invalid asminclude directive, expected arguments: \"filename\", scopelabel")
             }
             "%asmbinary" -> {
+                if(directive.parent !is Block) err("this directive may only occur in a block")
                 val errormsg = "invalid asmbinary directive, expected arguments: \"filename\" [, offset [, length ] ]"
                 if(directive.args.isEmpty()) err(errormsg)
                 if(directive.args.isNotEmpty() && directive.args[0].str==null) err(errormsg)

il65/src/il65/ast/AstOptimizer.kt

@@ -1,24 +1,23 @@
-package il65
+package il65.ast
 
-import il65.ast.*
 import kotlin.math.pow
 
 
-fun Module.optimized() : Module {
+fun Module.optimize() {
     val optimizer = AstOptimizer()
-    var result = this.process(optimizer)
+    this.process(optimizer)
+    if(optimizer.optimizationsDone==0)
+        println("[${this.name}] 0 optimizations performed")
+
     while(optimizer.optimizationsDone>0) {
-        println("Optimizations done: ${optimizer.optimizationsDone}")
+        println("[${this.name}] ${optimizer.optimizationsDone} optimizations performed")
         optimizer.reset()
-        result = result.process(optimizer)
+        this.process(optimizer)
     }
-    println("nothing left to process!")
-    return result
 }
 
 
 class AstOptimizer : IAstProcessor {
-
     var optimizationsDone: Int = 0
         private set
 
@@ -26,6 +25,10 @@ class AstOptimizer : IAstProcessor {
         optimizationsDone = 0
     }
 
+    override fun process(module: Module) {
+        module.lines = module.lines.map { it.process(this) }
+    }
+
     override fun process(block: Block): IStatement {
         block.statements = block.statements.map { it.process(this) }
         return block
@@ -102,7 +105,6 @@ class AstOptimizer : IAstProcessor {
         val rightconst = expr.right.constValue()
         return when {
             leftconst != null && rightconst != null -> {
-                println("optimizing $expr")
                 optimizationsDone++
                 evaluator.evaluate(leftconst, expr.operator, rightconst)
             }

il65/src/il65/ast/ImportedAstChecker.kt

@@ -0,0 +1,64 @@
+package il65.ast
+
+import il65.ParsingFailedError
+
+
+fun Module.checkImportValid() {
+    val checker = ImportedAstChecker()
+    this.process(checker)
+    val result = checker.result()
+    result.forEach {
+        it.printError()
+    }
+    if(result.isNotEmpty())
+        throw ParsingFailedError("There are ${result.size} errors in module '$name'.")
+}
+
+
+class ImportedAstChecker : IAstProcessor {
+    private val checkResult: MutableList<SyntaxError> = mutableListOf()
+
+    fun result(): List<SyntaxError> {
+        return checkResult
+    }
+
+    override fun process(module: Module) {
+        val newLines : MutableList<IStatement> = mutableListOf()
+        module.lines.forEach {
+            val stmt = it.process(this)
+            if(stmt is Directive) {
+                if(stmt.parent is Module) {
+                    when(stmt.directive) {
+                        "%output", "%launcher", "%zp", "%address" ->
+                            println("${stmt.position} Warning: ignoring module directive because it was imported: ${stmt.directive}")
+                        else ->
+                            newLines.add(stmt)
+                    }
+                }
+            }
+            else newLines.add(stmt)
+        }
+        module.lines = newLines
+    }
+
+    override fun process(expr: PrefixExpression): IExpression {
+        return expr
+    }
+
+    override fun process(expr: BinaryExpression): IExpression {
+        return expr
+    }
+
+    override fun process(block: Block): IStatement {
+        return block
+    }
+
+    override fun process(decl: VarDecl): IStatement {
+        return decl
+    }
+
+    override fun process(directive: Directive): IStatement {
+        return directive
+    }
+
+}