restructure

il65/examples/test.ill

@@ -9,15 +9,10 @@
   memory byte derpA = abs(-2.5-0.5)
   memory byte derpB = max(1, 2.2, 4.4, 100)
   memory byte cderp = min($ffdd)+ (1/1)
-  memory byte cderp1 = foobar
-  memory byte cderp2 = boo.bar.booz
-  memory byte cderp3 = main.doesnt_exist
   memory byte cderpA = min($ffdd, 10, 20, 30)
   memory byte cderpB = min(1, 2.2, 4.4, 100)
   memory byte derp2 = 2+$ffdd+round(10*sin(3))
-  memory byte derp3 = round2(100*sin(3))
   const byte hopla=55-33
-  const byte hopla2=100+(-main.hopla2)
   const byte hopla3=100+(-hopla)
   const byte hopla4 = 100-hopla
   const byte hopla1=main.hopla
@@ -33,6 +28,8 @@
   byte equalQQ = 4==4
   const byte equalQQ2 = (4+hopla)>0
 
+  equalQQ = foo(33)
+  equalQQ = main.foo(33)
   XY = hopla*2+hopla1
   A = "derp" * %000100
 
@@ -79,6 +76,7 @@ cool:
 
   }
 
+
 some_label_def:   A=44
   return 1+999
 	%breakpoint

il65/src/il65/Main.kt

@@ -1,130 +1,19 @@
 package il65
 
+import java.nio.file.Paths
 import il65.ast.*
+import il65.parser.*
 import il65.optimizing.optimizeExpressions
 import il65.optimizing.optimizeStatements
-import il65.parser.il65Lexer
-import il65.parser.il65Parser
-import org.antlr.v4.runtime.CharStreams
-import org.antlr.v4.runtime.CommonTokenStream
-import org.antlr.v4.runtime.Lexer
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-import kotlin.collections.HashMap
-
-
-class MyTokenStream(lexer: Lexer) : CommonTokenStream(lexer) {
-
-    data class Comment(val type: String, val line: Int, val comment: String)
-
-    fun commentTokens() : List<Comment> {
-        // extract the comments
-        val commentTokenChannel = il65Lexer.channelNames.indexOf("HIDDEN")
-        val theLexer = tokenSource as Lexer
-        return  get(0, size())
-                .filter { it.channel == commentTokenChannel }
-                .map {
-                    Comment(theLexer.vocabulary.getSymbolicName(it.type),
-                            it.line, it.text.substringAfter(';').trim())
-                }
-    }
-}
-
-class ParsingFailedError(override var message: String) : Exception(message)
-
-
-private val importedModules : HashMap<String, Module> = hashMapOf()
-
-
-
-fun loadModule(filePath: Path) : Module {
-    println("importing '${filePath.fileName}'  (from ${filePath.parent})...")
-    if(!Files.isReadable(filePath))
-        throw ParsingFailedError("No such file: $filePath")
-
-    val moduleName = fileNameWithoutSuffix(filePath)
-    val input = CharStreams.fromPath(filePath)
-    val lexer = il65Lexer(input)
-    val tokens = MyTokenStream(lexer)
-    val parser = il65Parser(tokens)
-    val parseTree = parser.module()
-    if(parser.numberOfSyntaxErrors > 0)
-        throw ParsingFailedError("There are ${parser.numberOfSyntaxErrors} syntax errors in '${filePath.fileName}'.")
-
-    // TODO the comments:
-    // tokens.commentTokens().forEach { println(it) }
-
-    // convert to Ast
-    val moduleAst = parseTree.toAst(moduleName,true)
-    importedModules[moduleAst.name] = moduleAst
-
-    // process imports
-    val lines = moduleAst.statements.toMutableList()
-    val imports = lines
-            .mapIndexed { i, it -> Pair(i, it) }
-            .filter { (it.second as? Directive)?.directive == "%import" }
-            .map { Pair(it.first, executeImportDirective(it.second as Directive, filePath)) }
-
-    imports.reversed().forEach {
-        if(it.second==null) {
-            // this import was already satisfied. just remove this line.
-            lines.removeAt(it.first)
-        } else {
-            // merge imported lines at this spot
-            lines.addAll(it.first, it.second!!.statements)
-        }
-    }
-
-    moduleAst.statements = lines
-    return moduleAst
-}
-
-
-fun fileNameWithoutSuffix(filePath: Path) =
-        filePath.fileName.toString().substringBeforeLast('.')
-
-
-fun discoverImportedModule(name: String, importedFrom: Path, position: Position?): Path {
-    val fileName = name + ".ill"
-    val locations = mutableListOf(Paths.get(importedFrom.parent.toString()))
-
-    val propPath = System.getProperty("il65.libdir")
-    if(propPath!=null)
-        locations.add(Paths.get(propPath))
-    val envPath = System.getenv("IL65_LIBDIR")
-    if(envPath!=null)
-        locations.add(Paths.get(envPath))
-    locations.add(Paths.get(Paths.get("").toAbsolutePath().toString(), "lib65"))
-
-    locations.forEach {
-        val file = Paths.get(it.toString(), fileName)
-        if (Files.isReadable(file)) return file
-    }
-
-    throw ParsingFailedError("$position Import: no module source file '$fileName' found  (I've looked in: $locations)")
-}
-
-
-fun executeImportDirective(import: Directive, importedFrom: Path): Module? {
-    if(import.directive!="%import" || import.args.size!=1 || import.args[0].name==null)
-        throw SyntaxError("invalid import directive", import.position)
-    val moduleName = import.args[0].name!!
-    if(importedModules.containsKey(moduleName))
-        return null
-
-    val modulePath = discoverImportedModule(moduleName, importedFrom, import.position)
-    val importedModule = loadModule(modulePath)
-    importedModule.checkImportedValid()
-
-    return importedModule
-}
 
 
 fun main(args: Array<String>) {
     try {
+        println("\nIL65 compiler by Irmen de Jong (irmen@razorvine.net)")
+        println("This software is licensed under the GNU GPL 3.0, see https://www.gnu.org/licenses/gpl.html\n")
+
         val filepath = Paths.get(args[0]).normalize()
-        val moduleAst = loadModule(filepath)
+        val moduleAst = importModule(filepath)
         moduleAst.linkParents()
         val globalNamespace = moduleAst.namespace()
         // globalNamespace.debugPrint()
@@ -137,10 +26,6 @@ fun main(args: Array<String>) {
         moduleAst.statements.forEach {
             println(it)
         }
-//    } catch(sx: SyntaxError) {
-//        System.err.println(sx)
-//    } catch(ex: ExpressionException) {
-//        System.err.println(ex)
     } catch (px: ParsingFailedError) {
         System.err.println(px.message)
     }

il65/src/il65/ast/AST.kt

@@ -130,7 +130,7 @@ interface IStatement : Node {
 
 
 interface IFunctionCall {
-    var location: Identifier
+    var target: Identifier
     var arglist: List<IExpression>
 }
 
@@ -565,20 +565,20 @@ data class Jump(val address: Int?, val identifier: Identifier?) : IStatement {
 }
 
 
-data class FunctionCall(override var location: Identifier, override var arglist: List<IExpression>) : IExpression, IFunctionCall {
+data class FunctionCall(override var target: Identifier, override var arglist: List<IExpression>) : IExpression, IFunctionCall {
     override var position: Position? = null
     override var parent: Node? = null
 
     override fun linkParents(parent: Node) {
         this.parent = parent
-        location.linkParents(this)
+        target.linkParents(this)
         arglist.forEach { it.linkParents(this) }
     }
 
     override fun constValue(namespace: INameScope): LiteralValue? {
         // if the function is a built-in function and the args are consts, should evaluate!
-        if(location.scopedName.size>1) return null
+        if(target.scopedName.size>1) return null
-        return when(location.scopedName[0]){
+        return when(target.scopedName[0]){
             "sin" -> builtin_sin(arglist, position, namespace)
             "cos" -> builtin_cos(arglist, position, namespace)
             "abs" -> builtin_abs(arglist, position, namespace)
@@ -599,17 +599,17 @@ data class FunctionCall(override var location: Identifier, override var arglist:
     }
 
     override fun process(processor: IAstProcessor) = processor.process(this)
-    override fun referencesIdentifier(name: String): Boolean = location.referencesIdentifier(name) || arglist.any{it.referencesIdentifier(name)}
+    override fun referencesIdentifier(name: String): Boolean = target.referencesIdentifier(name) || arglist.any{it.referencesIdentifier(name)}
 }
 
 
-data class FunctionCallStatement(override var location: Identifier, override var arglist: List<IExpression>) : IStatement, IFunctionCall {
+data class FunctionCallStatement(override var target: Identifier, override var arglist: List<IExpression>) : IStatement, IFunctionCall {
     override var position: Position? = null
     override var parent: Node? = null
 
     override fun linkParents(parent: Node) {
         this.parent = parent
-        location.linkParents(this)
+        target.linkParents(this)
         arglist.forEach { it.linkParents(this) }
     }
 

il65/src/il65/ast/AstChecker.kt

@@ -1,6 +1,7 @@
 package il65.ast
 
-import il65.ParsingFailedError
+import il65.functions.BuiltIns
+import il65.parser.ParsingFailedError
 
 /**
  * General checks on the Ast
@@ -97,6 +98,9 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
         if(subroutine.parent !is Block)
             err("subroutines can only be defined in a block (not in other scopes)")
 
+        if(BuiltIns.contains(subroutine.name))
+            err("cannot override a built-in function")
+
         val uniqueNames = subroutine.parameters.map { it.name }.toSet()
         if(uniqueNames.size!=subroutine.parameters.size)
             err("parameter names should be unique")
@@ -154,8 +158,8 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
      * Check the variable declarations (values within range etc)
      */
     override fun process(decl: VarDecl): IStatement {
-        fun err(msg: String) {
+        fun err(msg: String, position: Position?=null) {
-            checkResult.add(SyntaxError(msg, decl.position))
+            checkResult.add(SyntaxError(msg, position ?: decl.position))
         }
 
         // the initializer value can't refer to the variable itself (recursive definition)
@@ -182,13 +186,13 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
                 }
             }
             VarDeclType.MEMORY -> {
-                if(decl.value !is LiteralValue)
+                if(decl.value !is LiteralValue) {
-                    // @todo normal error reporting
+                    err("value of memory var decl is not a literal (it is a ${decl.value!!::class.simpleName}).", decl.value?.position)
-                    throw SyntaxError("value of memory var decl is not a literal (it is a ${decl.value!!::class.simpleName}).", decl.value?.position)
+                } else {
-
+                    val value = decl.value as LiteralValue
-                val value = decl.value as LiteralValue
+                    if (value.intvalue == null || value.intvalue < 0 || value.intvalue > 65535) {
-                if(value.intvalue==null || value.intvalue<0 || value.intvalue>65535) {
+                        err("memory address must be valid integer 0..\$ffff")
-                    err("memory address must be valid integer 0..\$ffff")
+                    }
                 }
             }
         }
@@ -340,6 +344,27 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
         return range
     }
 
+    override fun process(functionCall: FunctionCall): IExpression {
+        // this function call is (part of) an expression, which should be in a statement somewhere.
+        var statementNode: Node? = functionCall
+        while(statementNode !is IStatement && statementNode?.parent != null) statementNode = statementNode.parent
+        if(statementNode==null)
+            throw FatalAstException("cannot determine statement scope of function call expression at ${functionCall.position}")
+
+        checkFunctionExists(functionCall.target, statementNode as IStatement)
+        return super.process(functionCall)
+    }
+
+    override fun process(functionCall: FunctionCallStatement): IStatement {
+        checkFunctionExists(functionCall.target, functionCall)
+        return super.process(functionCall)
+    }
+
+    private fun checkFunctionExists(target: Identifier, statement: IStatement) {
+        if(globalNamespace.lookup(target.scopedName, statement)==null)
+            checkResult.add(SyntaxError("undefined function or subroutine: ${target.scopedName.joinToString(".")}", statement.position))
+    }
+
     private fun checkValueRange(datatype: DataType, value: LiteralValue, position: Position?) : Boolean {
         fun err(msg: String) : Boolean {
             checkResult.add(SyntaxError(msg, position))

il65/src/il65/ast/ImportedAstChecker.kt

@@ -1,6 +1,7 @@
 package il65.ast
 
-import il65.ParsingFailedError
+import il65.parser.ParsingFailedError
+
 
 /**
  * Checks that are specific for imported modules.

il65/src/il65/functions/BuiltinFunctions.kt

@@ -2,6 +2,10 @@ package il65.functions
 
 import il65.ast.*
 
+
+val BuiltIns = listOf("sin", "cos", "abs", "acos", "asin", "tan", "atan", "log", "log10", "sqrt", "max", "min", "round", "rad", "deg")
+
+
 private fun oneDoubleArg(args: List<IExpression>, position: Position?, namespace:INameScope, function: (arg: Double)->Double): LiteralValue {
     if(args.size!=1)
         throw SyntaxError("built-in function requires one floating point argument", position)

il65/src/il65/optimizing/ExpressionOptimizer.kt

@@ -1,6 +1,6 @@
 package il65.optimizing
 
-import il65.ParsingFailedError
+import il65.parser.ParsingFailedError
 import il65.ast.*
 import kotlin.math.pow
 

il65/src/il65/parser/CommentHandlingTokenStream.kt

@@ -0,0 +1,22 @@
+package il65.parser
+
+import org.antlr.v4.runtime.CommonTokenStream
+import org.antlr.v4.runtime.Lexer
+
+
+class CommentHandlingTokenStream(lexer: Lexer) : CommonTokenStream(lexer) {
+
+    data class Comment(val type: String, val line: Int, val comment: String)
+
+    fun commentTokens() : List<Comment> {
+        // extract the comments
+        val commentTokenChannel = il65Lexer.channelNames.indexOf("HIDDEN")
+        val theLexer = tokenSource as Lexer
+        return  get(0, size())
+                .filter { it.channel == commentTokenChannel }
+                .map {
+                    Comment(theLexer.vocabulary.getSymbolicName(it.type),
+                            it.line, it.text.substringAfter(';').trim())
+                }
+    }
+}

il65/src/il65/parser/ModuleParsing.kt

@@ -0,0 +1,93 @@
+package il65.parser
+
+import org.antlr.v4.runtime.CharStreams
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import il65.ast.*
+
+
+class ParsingFailedError(override var message: String) : Exception(message)
+
+
+private val importedModules : HashMap<String, Module> = hashMapOf()
+
+
+
+fun importModule(filePath: Path) : Module {
+    println("importing '${filePath.fileName}'  (from ${filePath.parent})...")
+    if(!Files.isReadable(filePath))
+        throw ParsingFailedError("No such file: $filePath")
+
+    val moduleName = filePath.fileName.toString().substringBeforeLast('.')
+    val input = CharStreams.fromPath(filePath)
+    val lexer = il65Lexer(input)
+    val tokens = CommentHandlingTokenStream(lexer)
+    val parser = il65Parser(tokens)
+    val parseTree = parser.module()
+    if(parser.numberOfSyntaxErrors > 0)
+        throw ParsingFailedError("There are ${parser.numberOfSyntaxErrors} syntax errors in '${filePath.fileName}'.")
+
+    // TODO the comments:
+    // tokens.commentTokens().forEach { println(it) }
+
+    // convert to Ast
+    val moduleAst = parseTree.toAst(moduleName,true)
+    importedModules[moduleAst.name] = moduleAst
+
+    // process imports
+    val lines = moduleAst.statements.toMutableList()
+    val imports = lines
+            .mapIndexed { i, it -> Pair(i, it) }
+            .filter { (it.second as? Directive)?.directive == "%import" }
+            .map { Pair(it.first, executeImportDirective(it.second as Directive, filePath)) }
+
+    imports.reversed().forEach {
+        if(it.second==null) {
+            // this import was already satisfied. just remove this line.
+            lines.removeAt(it.first)
+        } else {
+            // merge imported lines at this spot
+            lines.addAll(it.first, it.second!!.statements)
+        }
+    }
+
+    moduleAst.statements = lines
+    return moduleAst
+}
+
+
+fun discoverImportedModule(name: String, importedFrom: Path, position: Position?): Path {
+    val fileName = name + ".ill"
+    val locations = mutableListOf(Paths.get(importedFrom.parent.toString()))
+
+    val propPath = System.getProperty("il65.libdir")
+    if(propPath!=null)
+        locations.add(Paths.get(propPath))
+    val envPath = System.getenv("IL65_LIBDIR")
+    if(envPath!=null)
+        locations.add(Paths.get(envPath))
+    locations.add(Paths.get(Paths.get("").toAbsolutePath().toString(), "lib65"))
+
+    locations.forEach {
+        val file = Paths.get(it.toString(), fileName)
+        if (Files.isReadable(file)) return file
+    }
+
+    throw ParsingFailedError("$position Import: no module source file '$fileName' found  (I've looked in: $locations)")
+}
+
+
+fun executeImportDirective(import: Directive, importedFrom: Path): Module? {
+    if(import.directive!="%import" || import.args.size!=1 || import.args[0].name==null)
+        throw SyntaxError("invalid import directive", import.position)
+    val moduleName = import.args[0].name!!
+    if(importedModules.containsKey(moduleName))
+        return null
+
+    val modulePath = discoverImportedModule(moduleName, importedFrom, import.position)
+    val importedModule = importModule(modulePath)
+    importedModule.checkImportedValid()
+
+    return importedModule
+}