added -sourcelines cli option to include src lines in generated assembly (which is now off by default)

codeCore/src/prog8/code/core/CompilationOptions.kt

@@ -16,6 +16,7 @@ class CompilationOptions(val output: OutputType,
                          var optimize: Boolean = false,
                          var asmQuiet: Boolean = false,
                          var asmListfile: Boolean = false,
+                         var includeSourcelines: Boolean = false,
                          var experimentalCodegen: Boolean = false,
                          var varsHighBank: Int? = null,
                          var splitWordArrays: Boolean = false,

codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt

@@ -239,7 +239,7 @@ class AsmGen6502Internal (
     internal fun isTargetCpu(cpu: CpuType) = options.compTarget.machine.cpu == cpu
 
     internal fun outputSourceLine(node: PtNode) {
-        if(node.position===Position.DUMMY)
+        if(!options.includeSourcelines || node.position===Position.DUMMY)
             return
         val srcComment = "\t; source: ${node.position.file}:${node.position.line}"
         val line = SourceLineCache.retrieveLine(node.position)

codeGenIntermediate/src/prog8/codegen/intermediate/AssignmentGen.kt

@@ -13,7 +13,9 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
         if(assignment.target.children.single() is PtMachineRegister)
             throw AssemblyError("assigning to a register should be done by just evaluating the expression into resultregister")
 
-        return translateRegularAssign(assignment)
+        val chunks = translateRegularAssign(assignment)
+        chunks.filterIsInstance<IRCodeChunk>().firstOrNull()?.appendSrcPosition(assignment.position)
+        return chunks
     }
 
     internal fun translate(augAssign: PtAugmentedAssign): IRCodeChunks {
@@ -24,7 +26,7 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
         val memory = augAssign.target.memory
         val array = augAssign.target.array
 
-        return if(ident!=null) {
+        val chunks = if(ident!=null) {
             assignVarAugmented(ident.name, augAssign)
         } else if(memory != null) {
             if(memory.address is PtNumber)
@@ -39,6 +41,8 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
         } else {
             fallbackAssign(augAssign)
         }
+        chunks.filterIsInstance<IRCodeChunk>().firstOrNull()?.appendSrcPosition(augAssign.position)
+        return chunks
     }
 
     private fun assignMemoryAugmented(

codeGenIntermediate/src/prog8/codegen/intermediate/IRCodeGen.kt

@@ -289,6 +289,8 @@ class IRCodeGen(
             }
         }
 
+        chunks.filterIsInstance<IRCodeChunk>().firstOrNull()?.appendSrcPosition(node.position)
+
         return chunks
     }
 

codeGenIntermediate/src/prog8/codegen/intermediate/IRPeepholeOptimizer.kt

@@ -105,7 +105,10 @@ class IRPeepholeOptimizer(private val irprog: IRProgram) {
 
         relabelChunks.forEach { (index, label) ->
             val chunk = IRCodeChunk(label, null)
-            chunk.instructions += sub.chunks[index].instructions
+            val subChunk = sub.chunks[index]
+            chunk.instructions += subChunk.instructions
+            if(subChunk is IRCodeChunk)
+                chunk.appendSrcPositions(subChunk.sourceLinesPositions)
             sub.chunks[index] = chunk
         }
         removeChunks.reversed().forEach { index -> sub.chunks.removeAt(index) }
@@ -140,6 +143,8 @@ class IRPeepholeOptimizer(private val irprog: IRProgram) {
                     if(mayJoinCodeChunks(lastChunk, candidate)) {
                         lastChunk.instructions += candidate.instructions
                         lastChunk.next = candidate.next
+                        if(lastChunk is IRCodeChunk)
+                            lastChunk.appendSrcPositions(candidate.sourceLinesPositions)
                     }
                     else
                         chunks += candidate

compiler/src/prog8/CompilerMain.kt

@@ -46,11 +46,12 @@ private fun compileMain(args: Array<String>): Boolean {
     val outputDir by cli.option(ArgType.String, fullName = "out", description = "directory for output files instead of current directory").default(".")
     val quietAssembler by cli.option(ArgType.Boolean, fullName = "quietasm", description = "don't print assembler output results")
     val sourceDirs by cli.option(ArgType.String, fullName="srcdirs", description = "list of extra paths, separated with ${File.pathSeparator}, to search in for imported modules").multiple().delimiter(File.pathSeparator)
+    val includeSourcelines by cli.option(ArgType.Boolean, fullName = "sourcelines", description = "include original Prog8 source lines in generated asm code")
+    val splitWordArrays by cli.option(ArgType.Boolean, fullName = "splitarrays", description = "treat all word arrays as tagged with @split to make them lsb/msb split in memory")
     val compilationTarget by cli.option(ArgType.String, fullName = "target", description = "target output of the compiler (one of '${C64Target.NAME}', '${C128Target.NAME}', '${Cx16Target.NAME}', '${AtariTarget.NAME}', '${VMTarget.NAME}') (required)")
     val startVm by cli.option(ArgType.Boolean, fullName = "vm", description = "load and run a .p8ir IR source file in the VM")
     val watchMode by cli.option(ArgType.Boolean, fullName = "watch", description = "continuous compilation mode (watch for file changes)")
     val varsHighBank by cli.option(ArgType.Int, fullName = "varshigh", description = "put uninitialized variables in high memory area instead of at the end of the program. On the cx16 target the value specifies the HiRAM bank (0=keep active), on other systems it is ignored.")
-    val splitWordArrays by cli.option(ArgType.Boolean, fullName = "splitarrays", description = "treat all word arrays as tagged with @split to make them lsb/msb split in memory")
     val moduleFiles by cli.argument(ArgType.String, fullName = "modules", description = "main module file(s) to compile").multiple(999)
 
     try {
@@ -114,6 +115,7 @@ private fun compileMain(args: Array<String>): Boolean {
                     dontWriteAssembly != true,
                     quietAssembler == true,
                     asmListfile == true,
+                    includeSourcelines == true,
                     experimentalCodegen == true,
                     varsHighBank,
                     compilationTarget!!,
@@ -180,6 +182,7 @@ private fun compileMain(args: Array<String>): Boolean {
                     dontWriteAssembly != true,
                     quietAssembler == true,
                     asmListfile == true,
+                    includeSourcelines == true,
                     experimentalCodegen == true,
                     varsHighBank,
                     compilationTarget!!,

compiler/src/prog8/compiler/Compiler.kt

@@ -32,6 +32,7 @@ class CompilerArguments(val filepath: Path,
                         val writeAssembly: Boolean,
                         val quietAssembler: Boolean,
                         val asmListfile: Boolean,
+                        val includeSourcelines: Boolean,
                         val experimentalCodegen: Boolean,
                         val varsHighBank: Int?,
                         val compilationTarget: String,
@@ -68,6 +69,7 @@ fun compileProgram(args: CompilerArguments): CompilationResult? {
                 optimize = args.optimize
                 asmQuiet = args.quietAssembler
                 asmListfile = args.asmListfile
+                includeSourcelines = args.includeSourcelines
                 experimentalCodegen = args.experimentalCodegen
                 varsHighBank = args.varsHighBank
                 splitWordArrays = args.splitWordArrays

compiler/test/TestCompilerOnExamples.kt

@@ -29,6 +29,7 @@ private fun compileTheThing(filepath: Path, optimize: Boolean, target: ICompilat
         writeAssembly = true,
         quietAssembler = true,
         asmListfile = false,
+        includeSourcelines = false,
         experimentalCodegen = false,
         varsHighBank = null,
         compilationTarget = target.name,

compiler/test/TestCompilerOptionLibdirs.kt

@@ -46,6 +46,7 @@ class TestCompilerOptionSourcedirs: FunSpec({
             writeAssembly = true,
             quietAssembler = true,
             asmListfile = false,
+            includeSourcelines = false,
             experimentalCodegen = false,
             varsHighBank = null,
             compilationTarget = Cx16Target.NAME,

compiler/test/helpers/compileXyz.kt

@@ -26,6 +26,7 @@ internal fun compileFile(
         writeAssembly = writeAssembly,
         quietAssembler = true,
         asmListfile = false,
+        includeSourcelines = false,
         experimentalCodegen = false,
         varsHighBank = null,
         platform.name,

docs/source/compiling.rst

@@ -160,6 +160,11 @@ One or more .p8 module files
 ``-expericodegen``
     Use experimental code generation backend (*incomplete*).
 
+``-sourcelines``
+    Also include the original prog8 source code lines as comments in the generated assembly code file,
+    mixed in between the actual generated assembly code.
+    This can be useful for debugging purposes to see what assembly was generated for what prog8 source code.
+
 ``-splitarrays``
     Treat all word arrays as tagged with @split so they are all lsb/msb split into memory.
     This removes the need to add @split yourself but some programs may fail to compile with

docs/source/todo.rst

@@ -1,8 +1,6 @@
 TODO
 ====
 
-- add command line option to enable/disable the inclusion of p8 source lines into the generated assembly / p8ir  see outputSourceLine()
-- add a mechanism to pass the original p8 source lines into the p8ir file as comments. Remove the position xml tags.
 - try to reduce the number of uses of temp variables for example in array[idx] -= amount   /
 - investigate McCarthy evaluation again? this may also reduce code size perhaps for things like if a>4 or a<2 .... / - investigate McCarthy evaluation again? this may also reduce code size perhaps for things like if a>4 or a<2 ....
 

httpCompilerService/src/prog8/http/TestHttp.kt

@@ -37,6 +37,7 @@ class RequestParser : Take {
             compilationTarget = "c64",
             symbolDefs = emptyMap(),
             quietAssembler = false,
+            includeSourcelines = false,
             asmListfile = false,
             experimentalCodegen = false,
             splitWordArrays = false,

intermediate/src/prog8/intermediate/IRFileReader.kt

@@ -294,6 +294,11 @@ class IRFileReader {
         skipText(reader)
         val start = reader.nextEvent().asStartElement()
         require(start.name.localPart=="CODE") { "missing CODE" }
+        val next = reader.peek()
+        if(next.isStartElement && next.asStartElement().name.localPart=="P8SRC") {
+            reader.nextEvent()  // skip the P8SRC node
+            while(!reader.nextEvent().isEndElement) { /* skip until end of P8SRC node */ }
+        }
         val label = start.attributes.asSequence().singleOrNull { it.name.localPart == "LABEL" }?.value?.ifBlank { null }
         val text = readText(reader).trim()
         val chunk = IRCodeChunk(label, null)

intermediate/src/prog8/intermediate/IRFileWriter.kt

@@ -3,6 +3,7 @@ package prog8.intermediate
 import prog8.code.core.*
 import java.nio.file.Path
 import javax.xml.stream.XMLOutputFactory
+import javax.xml.stream.XMLStreamWriter
 import kotlin.io.path.bufferedWriter
 import kotlin.io.path.div
 
@@ -126,6 +127,7 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) {
         xml.writeStartElement("CODE")
         chunk.label?.let { xml.writeAttribute("LABEL", chunk.label) }
         // xml.writeAttribute("used-registers", chunk.usedRegisters().toString())
+        writeSourcelines(xml, chunk)
         xml.writeCharacters("\n")
         chunk.instructions.forEach { instr ->
             numInstr++
@@ -136,6 +138,23 @@ class IRFileWriter(private val irProgram: IRProgram, outfileOverride: Path?) {
         xml.writeCharacters("\n")
     }
 
+    private fun writeSourcelines(xml: XMLStreamWriter, code: IRCodeChunk) {
+        if(irProgram.options.includeSourcelines) {
+            if(code.sourceLinesPositions.any {it !== Position.DUMMY}) {
+                xml.writeStartElement("P8SRC")
+                var sourceTxt = StringBuilder("\n")
+                code.sourceLinesPositions.forEach { pos ->
+                    val line = SourceLineCache.retrieveLine(pos)
+                    if(line!=null) {
+                        sourceTxt.append("$pos  $line\n")
+                    }
+                }
+                xml.writeCData(sourceTxt.toString())
+                xml.writeEndElement()
+            }
+        }
+    }
+
     private fun writeInlineBytes(chunk: IRInlineBinaryChunk) {
         xml.writeStartElement("BYTES")
         chunk.label?.let { xml.writeAttribute("LABEL", chunk.label) }

intermediate/src/prog8/intermediate/IRProgram.kt

@@ -447,6 +447,17 @@ class IRCodeChunk(label: String?, next: IRCodeChunkBase?): IRCodeChunkBase(label
     operator fun plusAssign(chunk: IRCodeChunkBase) {
         instructions.addAll(chunk.instructions)
     }
+
+    fun appendSrcPosition(position: Position) {
+        if(sourceLinesPositions.lastOrNull()!=position)
+            sourceLinesPositions.add(position)
+    }
+
+    fun appendSrcPositions(positions: Collection<Position>) {
+        positions.forEach { appendSrcPosition(it) }
+    }
+
+    val sourceLinesPositions = mutableListOf<Position>()
 }
 
 class IRInlineAsmChunk(label: String?,