improved tracking of module file locations, fixed relative importing of submodules from same directory as source module

compiler/src/prog8/ast/AST.kt

@@ -9,6 +9,7 @@ import prog8.compiler.target.c64.Petscii
 import prog8.functions.BuiltinFunctions
 import prog8.functions.NotConstArgumentException
 import prog8.functions.builtinFunctionReturnType
+import prog8.parser.CustomLexer
 import prog8.parser.prog8Parser
 import java.io.File
 import java.nio.file.Path
@@ -1787,18 +1788,22 @@ class RepeatLoop(var body: AnonymousScope,
 
 /***************** Antlr Extension methods to create AST ****************/
 
-fun prog8Parser.ModuleContext.toAst(name: String, isLibrary: Boolean, importedFrom: Path) : Module =
+fun prog8Parser.ModuleContext.toAst(name: String, isLibrary: Boolean, importedFrom: Path) : Module {
-        Module(name, modulestatement().asSequence().map { it.toAst(isLibrary) }.toMutableList(), toPosition(), isLibrary, importedFrom)
+    val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name
+    return Module(nameWithoutSuffix, modulestatement().asSequence().map { it.toAst(isLibrary) }.toMutableList(), toPosition(), isLibrary, importedFrom)
+}
 
 
 private fun ParserRuleContext.toPosition() : Position {
-    val file =
+    val customTokensource = this.start.tokenSource as? CustomLexer
-            if(start.inputStream.sourceName == IntStream.UNKNOWN_SOURCE_NAME)
+    val filename =
-                "@internal@"
+            when {
-            else
+                customTokensource!=null -> customTokensource.modulePath.fileName.toString()
-                File(start.inputStream.sourceName).name
+                start.tokenSource.sourceName == IntStream.UNKNOWN_SOURCE_NAME -> "@internal@"
+                else -> File(start.inputStream.sourceName).name
+            }
     // note: be ware of TAB characters in the source text, they count as 1 column...
-    return Position(file, start.line, start.charPositionInLine, stop.charPositionInLine+stop.text.length)
+    return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine+stop.text.length)
 }
 
 

compiler/src/prog8/parser/ModuleParsing.kt

@@ -5,6 +5,7 @@ import prog8.ast.*
 import prog8.compiler.LauncherType
 import prog8.compiler.OutputType
 import prog8.determineCompilationOptions
+import java.io.File
 import java.io.InputStream
 import java.nio.file.Files
 import java.nio.file.Path
@@ -26,8 +27,12 @@ private class LexerErrorListener: BaseErrorListener() {
 }
 
 
-fun importModule(stream: CharStream, moduleName: String, isLibrary: Boolean): Module {
+internal class CustomLexer(val modulePath: Path, input: CharStream?) : prog8Lexer(input)
-    val lexer = prog8Lexer(stream)
+
+
+fun importModule(stream: CharStream, modulePath: Path, isLibrary: Boolean): Module {
+    val moduleName = modulePath.fileName
+    val lexer = CustomLexer(modulePath, stream)
     val lexerErrors = LexerErrorListener()
     lexer.addErrorListener(lexerErrors)
     val tokens = CommentHandlingTokenStream(lexer)
@@ -35,13 +40,13 @@ fun importModule(stream: CharStream, moduleName: String, isLibrary: Boolean): Mo
     val parseTree = parser.module()
     val numberOfErrors = parser.numberOfSyntaxErrors + lexerErrors.numberOfErrors
     if(numberOfErrors > 0)
-        throw ParsingFailedError("There are $numberOfErrors errors in '$moduleName.p8'.")
+        throw ParsingFailedError("There are $numberOfErrors errors in '$moduleName'.")
 
     // You can do something with the parsed comments:
     // tokens.commentTokens().forEach { println(it) }
 
     // convert to Ast
-    val moduleAst = parseTree.toAst(moduleName, isLibrary, Paths.get(stream.sourceName))
+    val moduleAst = parseTree.toAst(moduleName.toString(), isLibrary, modulePath)
     importedModules[moduleAst.name] = moduleAst
 
     // process imports
@@ -63,7 +68,7 @@ fun importModule(stream: CharStream, moduleName: String, isLibrary: Boolean): Mo
             .asSequence()
             .mapIndexed { i, it -> Pair(i, it) }
             .filter { (it.second as? Directive)?.directive == "%import" }
-            .map { Pair(it.first, executeImportDirective(it.second as Directive, Paths.get("$moduleName.p8"))) }
+            .map { Pair(it.first, executeImportDirective(it.second as Directive, modulePath)) }
             .toList()
 
     imports.reversed().forEach {
@@ -95,9 +100,8 @@ fun importModule(filePath: Path) : Module {
     if(!Files.isReadable(filePath))
         throw ParsingFailedError("No such file: $filePath")
 
-    val moduleName = filePath.fileName.toString().substringBeforeLast('.')
     val input = CharStreams.fromPath(filePath)
-    return importModule(input, moduleName, filePath.parent==null)
+    return importModule(input, filePath, filePath.parent==null)
 }
 
 
@@ -136,7 +140,7 @@ private fun executeImportDirective(import: Directive, importedFrom: Path): Modul
             // load the module from the embedded resource
             resource.use {
                 println("importing '$moduleName' (embedded library)")
-                importModule(CharStreams.fromStream(it), moduleName, true)
+                importModule(CharStreams.fromStream(it), Paths.get("@embedded@/$moduleName"), true)
             }
         } else {
             val modulePath = discoverImportedModuleFile(moduleName, importedFrom, import.position)