From 680f5d21eebce879326b449920cf611fd3b5ab67 Mon Sep 17 00:00:00 2001
From: Irmen de Jong <irmen@razorvine.net>
Date: Sat, 3 Feb 2024 12:53:21 +0100
Subject: [PATCH] normalize DOS line endings to just \n, fixes source lines in
 asm for \r\n source files

---
 codeCore/src/prog8/code/core/SourceCode.kt |  9 ++++++---
 compiler/test/ast/TestSourceCode.kt        | 18 ++++++++++++++++++
 compiler/test/fixtures/dos_line_endings.p8 |  6 ++++++
 3 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 compiler/test/fixtures/dos_line_endings.p8

diff --git a/codeCore/src/prog8/code/core/SourceCode.kt b/codeCore/src/prog8/code/core/SourceCode.kt
index 915039fa7..3dc52b9f6 100644
--- a/codeCore/src/prog8/code/core/SourceCode.kt
+++ b/codeCore/src/prog8/code/core/SourceCode.kt
@@ -69,7 +69,8 @@ sealed class SourceCode {
      * Turn a plain String into a [SourceCode] object.
      * [origin] will be something like `string:44c56085`.
      */
-    class Text(override val text: String): SourceCode() {
+    class Text(origText: String): SourceCode() {
+        override val text = origText.replace("\\R".toRegex(), "\n")      // normalize line endings
         override val isFromResources = false
         override val isFromFilesystem = false
         override val origin = "$STRINGSOURCEPREFIX${System.identityHashCode(text).toString(16)}"
@@ -95,7 +96,8 @@ sealed class SourceCode {
             val normalized = path.normalize()
             origin = relative(normalized).toString()
             try {
-                text = Normalizer.normalize(normalized.readText(), Normalizer.Form.NFC)
+                val contents = Normalizer.normalize(normalized.readText(), Normalizer.Form.NFC)
+                text = contents.replace("\\R".toRegex(), "\n")      // normalize line endings
                 name = normalized.toFile().nameWithoutExtension
             } catch (nfx: java.nio.file.NoSuchFileException) {
                 throw NoSuchFileException(normalized.toFile()).also { it.initCause(nfx) }
@@ -127,7 +129,8 @@ sealed class SourceCode {
                 )
             }
             val stream = object {}.javaClass.getResourceAsStream(normalized)
-            text = stream!!.reader().use { Normalizer.normalize(it.readText(), Normalizer.Form.NFC) }
+            val contents = stream!!.reader().use { Normalizer.normalize(it.readText(), Normalizer.Form.NFC) }
+            text = contents.replace("\\R".toRegex(), "\n")      // normalize line endings
             name = Path(pathString).toFile().nameWithoutExtension
         }
     }
diff --git a/compiler/test/ast/TestSourceCode.kt b/compiler/test/ast/TestSourceCode.kt
index 5bdfe0f5c..9997647f3 100644
--- a/compiler/test/ast/TestSourceCode.kt
+++ b/compiler/test/ast/TestSourceCode.kt
@@ -3,6 +3,7 @@ package prog8tests.ast
 import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.core.spec.style.AnnotationSpec
 import io.kotest.matchers.shouldBe
+import io.kotest.matchers.shouldNotBe
 import io.kotest.matchers.string.shouldContain
 import prog8.code.core.SourceCode
 import prog8.code.core.SourceCode.Companion.LIBRARYFILEPREFIX
@@ -29,6 +30,14 @@ class TestSourceCode: AnnotationSpec() {
         src.toString().startsWith("prog8.code.core.SourceCode") shouldBe true
     }
 
+    @Test
+    fun testFromStringDOSLineEndings() {
+        val text = "main {\r\nline2\r\nline3\r\n}\r\n"
+        val src = SourceCode.Text(text)
+        src.text shouldNotBe text     // because normalized line endings!
+        src.text.split('\r', '\n').size shouldBe 5
+    }
+
     @Test
     fun testFromPathWithNonExistingPath() {
         val filename = "i_do_not_exist.p8"
@@ -60,6 +69,15 @@ class TestSourceCode: AnnotationSpec() {
         src.isFromFilesystem shouldBe true
     }
 
+    @Test
+    fun testFromPathWithExistingPathDOSLineEndings() {
+        val filename = "dos_line_endings.p8"
+        val path = assumeReadableFile(fixturesDir, filename)
+        val src = SourceCode.File(path)
+        src.text shouldNotBe  path.toFile().readText()      // should be normalized!
+        src.text.split('\r', '\n').size shouldBe 7
+    }
+
     @Test
     fun testFromPathWithExistingNonNormalizedPath() {
         val filename = "ast_simple_main.p8"
diff --git a/compiler/test/fixtures/dos_line_endings.p8 b/compiler/test/fixtures/dos_line_endings.p8
new file mode 100644
index 000000000..42c62e319
--- /dev/null
+++ b/compiler/test/fixtures/dos_line_endings.p8
@@ -0,0 +1,6 @@
+main {
+	sub start() {
+	}
+}
+
+; CR LF line endings