normalize DOS line endings to just \n, fixes source lines in asm for \r\n source files

codeCore/src/prog8/code/core/SourceCode.kt

@@ -69,7 +69,8 @@ sealed class SourceCode {
      * Turn a plain String into a [SourceCode] object.
      * [origin] will be something like `string:44c56085`.
      */
-    class Text(override val text: String): SourceCode() {
+    class Text(origText: String): SourceCode() {
+        override val text = origText.replace("\\R".toRegex(), "\n")      // normalize line endings
         override val isFromResources = false
         override val isFromFilesystem = false
         override val origin = "$STRINGSOURCEPREFIX${System.identityHashCode(text).toString(16)}"
@@ -95,7 +96,8 @@ sealed class SourceCode {
             val normalized = path.normalize()
             origin = relative(normalized).toString()
             try {
-                text = Normalizer.normalize(normalized.readText(), Normalizer.Form.NFC)
+                val contents = Normalizer.normalize(normalized.readText(), Normalizer.Form.NFC)
+                text = contents.replace("\\R".toRegex(), "\n")      // normalize line endings
                 name = normalized.toFile().nameWithoutExtension
             } catch (nfx: java.nio.file.NoSuchFileException) {
                 throw NoSuchFileException(normalized.toFile()).also { it.initCause(nfx) }
@@ -127,7 +129,8 @@ sealed class SourceCode {
                 )
             }
             val stream = object {}.javaClass.getResourceAsStream(normalized)
-            text = stream!!.reader().use { Normalizer.normalize(it.readText(), Normalizer.Form.NFC) }
+            val contents = stream!!.reader().use { Normalizer.normalize(it.readText(), Normalizer.Form.NFC) }
+            text = contents.replace("\\R".toRegex(), "\n")      // normalize line endings
             name = Path(pathString).toFile().nameWithoutExtension
         }
     }

compiler/test/ast/TestSourceCode.kt

@@ -3,6 +3,7 @@ package prog8tests.ast
 import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.core.spec.style.AnnotationSpec
 import io.kotest.matchers.shouldBe
+import io.kotest.matchers.shouldNotBe
 import io.kotest.matchers.string.shouldContain
 import prog8.code.core.SourceCode
 import prog8.code.core.SourceCode.Companion.LIBRARYFILEPREFIX
@@ -29,6 +30,14 @@ class TestSourceCode: AnnotationSpec() {
         src.toString().startsWith("prog8.code.core.SourceCode") shouldBe true
     }
 
+    @Test
+    fun testFromStringDOSLineEndings() {
+        val text = "main {\r\nline2\r\nline3\r\n}\r\n"
+        val src = SourceCode.Text(text)
+        src.text shouldNotBe text     // because normalized line endings!
+        src.text.split('\r', '\n').size shouldBe 5
+    }
+
     @Test
     fun testFromPathWithNonExistingPath() {
         val filename = "i_do_not_exist.p8"
@@ -60,6 +69,15 @@ class TestSourceCode: AnnotationSpec() {
         src.isFromFilesystem shouldBe true
     }
 
+    @Test
+    fun testFromPathWithExistingPathDOSLineEndings() {
+        val filename = "dos_line_endings.p8"
+        val path = assumeReadableFile(fixturesDir, filename)
+        val src = SourceCode.File(path)
+        src.text shouldNotBe  path.toFile().readText()      // should be normalized!
+        src.text.split('\r', '\n').size shouldBe 7
+    }
+
     @Test
     fun testFromPathWithExistingNonNormalizedPath() {
         val filename = "ast_simple_main.p8"

compiler/test/fixtures/dos_line_endings.p8

@@ -0,0 +1,6 @@
+main {
+	sub start() {
+	}
+}
+
+; CR LF line endings