Added fragment system handling of CR / CRLF. #490

src/main/fragment/cache/fragment-cache-csg65ce02.asm

@@ -1,4 +1,4 @@
-//KICKC FRAGMENT CACHE 16e7ed5268
+//KICKC FRAGMENT CACHE 16e7ed5268 16e7ed6a89
 //FRAGMENT vbuz1=vbuc1
 lda #{c1}
 sta {z1}

src/main/fragment/cache/fragment-cache-mega45gs02.asm

@@ -1,4 +1,4 @@
-//KICKC FRAGMENT CACHE 16e7ed5268
+//KICKC FRAGMENT CACHE 16e7ed5268 16e7ed6a89
 //FRAGMENT vduz1=vduc1
 lda #<{c1}
 sta {z1}

src/main/fragment/cache/fragment-cache-mos6502.asm

@@ -1,4 +1,4 @@
-//KICKC FRAGMENT CACHE 16e7ed5268
+//KICKC FRAGMENT CACHE 16e7ed5268 16e7ed6a89
 //FRAGMENT vbuz1=vbuc1
 lda #{c1}
 sta {z1}

src/main/fragment/cache/fragment-cache-wdc65c02.asm

@@ -1,4 +1,4 @@
-//KICKC FRAGMENT CACHE 16e7ed5268
+//KICKC FRAGMENT CACHE 16e7ed5268 16e7ed6a89
 //FRAGMENT vbuz1=_deref_pbuc1
 lda {c1}
 sta {z1}

src/main/java/dk/camelot64/kickc/fragment/AsmFragmentSystemHash.java

@@ -1,8 +1,11 @@
 package dk.camelot64.kickc.fragment;
 
+import dk.camelot64.kickc.model.InternalError;
 import dk.camelot64.kickc.model.TargetCpu;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 
 /**
@@ -12,59 +15,144 @@ import java.nio.file.Path;
  */
 public class AsmFragmentSystemHash {
 
-   /** Hash of the fragment source files. */
+   /** Hash of the fragment source files on any system where a newline is LF. */
-   private long hash;
+   private Long hashLF;
+
+   /** Hash of the fragment source files on any system where a newline is CRLF. */
+   private Long hashCRLF;
 
    /** Last modified date for the fragment source files. */
    private long lastModified;
 
-
+   public AsmFragmentSystemHash(Long hashLF, Long hashCRLF, long lastModified) {
-   AsmFragmentSystemHash(long hash, long lastModified) {
+      this.hashCRLF = hashCRLF;
+      this.hashLF = hashLF;
       this.lastModified = lastModified;
-      this.hash = hash;
    }
 
    /**
     * Run through the fragment folder and calculate the hash/modify time
+    *
     * @param baseFragmentFolder The fragment folder
+    * @param allSystems Should hash code be calculated for all systems (both LF and CRLF newlines).
+    * If false hash is only calculated for the current system, which is fast.
+    * If true calculate for all systems, which requires reading through all files to count newlines - so it is a lot slower.
     * @return The fragment system hash
     */
-   public static AsmFragmentSystemHash calculate(Path baseFragmentFolder) {
+   public static AsmFragmentSystemHash calculate(Path baseFragmentFolder, boolean allSystems) {
-      long hash = 0;
+      Long hashLF = 0L;
+      Long hashCRLF = 0L;
       long lastModified = 0;
       final TargetCpu.Feature[] cpuFeatures = TargetCpu.Feature.values();
       for(TargetCpu.Feature cpuFeature : cpuFeatures) {
-         hash += cpuFeature.getName().hashCode();
+         hashCRLF += cpuFeature.getName().hashCode();
+         hashLF += cpuFeature.getName().hashCode();
          final Path cpuFeatureFolder = baseFragmentFolder.resolve(cpuFeature.getName());
          final File cpuFeatureFolderFile = cpuFeatureFolder.toFile();
          if(cpuFeatureFolderFile.exists() && cpuFeatureFolderFile.isDirectory()) {
             final File[] files = cpuFeatureFolderFile.listFiles((dir, name) -> name.endsWith(".asm"));
             if(files != null)
                for(File file : files) {
-                  hash += file.length();
+                  long lengthLF;
-                  hash += file.getName().hashCode();
+                  long lengthCRLF;
+                  if(allSystems) {
+                     // Calculate length for all systems by reading the file
+                     int fileNewlines = getFileNewlineCount(file);
+                     if(isSystemLF()) {
+                        lengthLF = file.length();
+                        lengthCRLF = file.length() + fileNewlines;
+                     } else {
+                        lengthLF = file.length() - fileNewlines;
+                        lengthCRLF = file.length();
+                     }
+                  }  else {
+                     // Only find length for current system - set other to zero
+                     if(isSystemLF()) {
+                        lengthLF = file.length();
+                        lengthCRLF = 0;
+                     } else {
+                        lengthLF = 0;
+                        lengthCRLF = file.length();
+                     }
+                  }
+                  hashCRLF += lengthCRLF;
+                  hashLF += lengthLF;
+                  hashCRLF += file.getName().hashCode();
+                  hashLF += file.getName().hashCode();
                   if(file.lastModified() > lastModified)
                      lastModified = file.lastModified();
-
                }
          }
       }
       // Also hash in all synthesis rules
       for(AsmFragmentTemplateSynthesisRule synthesisRule : AsmFragmentTemplateSynthesisRule.getSynthesisRules()) {
-         hash += synthesisRule.hashCode();
+         hashCRLF += synthesisRule.hashCode();
+         hashLF += synthesisRule.hashCode();
       }
-      return new AsmFragmentSystemHash(hash, lastModified);
+      if(!allSystems) {
+         // If not all systems - null out other system hash since it is wrong
+         if(isSystemLF())
+            hashCRLF = null;
+         else
+            hashLF = null;
+      }
+      return new AsmFragmentSystemHash(hashLF, hashCRLF, lastModified);
    }
 
-   public long getHash() {
+   /**
-      return hash;
+    * Get the number of newlines in a file
+    * @param file The file to examine
+    * @return The number of newline characters
+    */
+   private static int getFileNewlineCount(File file) {
+      int fileNewlines = 0;
+      final byte[] fileBytes;
+      try {
+         fileBytes = Files.readAllBytes(file.toPath());
+      } catch(IOException e) {
+         throw new InternalError("Error reading ASM fragment file "+file.getAbsolutePath(),e);
+      }
+      for(int i = 0; i < fileBytes.length; i++) {
+         byte fileByte = fileBytes[i];
+         if(fileByte=='\n') fileNewlines++;
+      }
+      return fileNewlines;
    }
 
-   public String getHashString() {
+   /**
-      return Long.toHexString(hash);
+    * Does the current system use LF as newline
+    *
+    * @return true if the line separator is LF
+    */
+   static boolean isSystemLF() {
+      return System.lineSeparator().length() == 1;
+   }
+
+   public String getHashStringLF() {
+      return hashLF==null?null:Long.toHexString(hashLF);
+   }
+
+   public String getHashStringCRLF() {
+      return hashCRLF==null?null:Long.toHexString(hashCRLF);
+   }
+
+   /** Determines if this hash mathes the passed hash values.
+    * Only examines the value relevant for the current system
+    * @param hashLF Hash of the fragment source files on any system where a newline is LF
+    * @param hashCRLF Hash of the fragment source files on any system where a newline is CRLF
+    * @return true if the hash matches
+    */
+   public boolean matches(String hashLF, String hashCRLF) {
+      if(isSystemLF()) {
+         return getHashStringLF().equals(hashLF);
+      } else {
+         return getHashStringCRLF().equals(hashCRLF);
+      }
    }
 
    public long getLastModified() {
       return lastModified;
    }
+
+
 }

src/main/java/dk/camelot64/kickc/fragment/AsmFragmentTemplateCache.java

@@ -1,6 +1,7 @@
 package dk.camelot64.kickc.fragment;
 
 import dk.camelot64.kickc.CompileLog;
+import dk.camelot64.kickc.model.InternalError;
 import dk.camelot64.kickc.model.TargetCpu;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
@@ -32,7 +33,12 @@ public class AsmFragmentTemplateCache {
    /** Detects any modification of the cache. */
    private boolean modified;
 
-   AsmFragmentTemplateCache(Path cacheFolder, TargetCpu cpu, AsmFragmentSystemHash fragmentSystemHash, Map<String, AsmFragmentTemplate> cache) {
+   /**
+    * Creates a cache
+    *
+    * @param cpu The target CPU
+    */
+   private AsmFragmentTemplateCache(Path cacheFolder, TargetCpu cpu, AsmFragmentSystemHash fragmentSystemHash, Map<String, AsmFragmentTemplate> cache) {
       this.cacheFolder = cacheFolder;
       this.cpu = cpu;
       this.fragmentSystemHash = fragmentSystemHash;
@@ -84,8 +90,18 @@ public class AsmFragmentTemplateCache {
     * @param cpu The CPU to make a cache for
     * @return The new empty cache
     */
-   public static AsmFragmentTemplateCache memory(TargetCpu cpu) {
+   public static AsmFragmentTemplateCache memoryCache(TargetCpu cpu) {
-      return new AsmFragmentTemplateCache(null, cpu, new AsmFragmentSystemHash(0, 0), new LinkedHashMap<>());
+      return new AsmFragmentTemplateCache(null, cpu, new AsmFragmentSystemHash(0L, 0L, 0), new LinkedHashMap<>());
+   }
+
+   /**
+    * Creates a disk fragment cache
+    *
+    * @param cpu The CPU to make a cache for
+    * @return The new empty cache
+    */
+   public static AsmFragmentTemplateCache diskCache(Path cacheFolder, TargetCpu cpu, AsmFragmentSystemHash fragmentSystemHash, Map<String, AsmFragmentTemplate> cache) {
+      return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, cache);
    }
 
    /**
@@ -96,37 +112,41 @@ public class AsmFragmentTemplateCache {
     * @return The map with all best fragments from the cache file. null if the cache file is not found.
     */
    public static AsmFragmentTemplateCache load(TargetCpu cpu, Path baseFragmentFolder, CompileLog log) {
-      final AsmFragmentSystemHash fragmentSystemHash = AsmFragmentSystemHash.calculate(baseFragmentFolder);
+      final AsmFragmentSystemHash fragmentSystemHash = AsmFragmentSystemHash.calculate(baseFragmentFolder, false);
       Path cacheFolder = baseFragmentFolder.resolve(CACHE_FOLDER_NAME);
       final Date before = new Date();
       if(!cacheFolder.toFile().exists()) {
          if(log.isVerboseFragmentLog())
             log.append("Creating fragment cache folder " + cacheFolder.toAbsolutePath().toString());
          cacheFolder.toFile().mkdir();
-         return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
+         return diskCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
       }
       try {
          File cacheFile = cacheFolder.resolve(getCacheFileName(cpu)).toFile();
          if(!cacheFile.exists()) {
-            return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
+            return diskCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
          }
          if(cacheFile.lastModified() < fragmentSystemHash.getLastModified()) {
             if(log.isVerboseFragmentLog())
                log.append("Deleting outdated fragment cache file " + cacheFile.getAbsolutePath());
             cacheFile.delete();
-            return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
+            return diskCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
          }
          LinkedHashMap<String, AsmFragmentTemplate> cache = new LinkedHashMap<>();
          BufferedReader fragmentCacheReader = new BufferedReader(new FileReader(cacheFile));
          String hashLine = fragmentCacheReader.readLine();
-         final String hash = hashLine.substring(HASH_HEADER.length());
+         final String hashPayload = hashLine.substring(HASH_HEADER.length());
+         final String[] hashes = hashPayload.split(" ");
+         final String hashLF = hashes[0];
+         final String hashCRLF = (hashes.length > 1) ? hashes[1] : null;
          // Compare cache file hash with fragment system hash
-         if(!hash.equals(fragmentSystemHash.getHashString())) {
+         if(!fragmentSystemHash.matches(hashLF, hashCRLF)) {
             // Cache file hash does not match fragment system hash
             if(log.isVerboseFragmentLog())
                log.append("Deleting hash mismatch fragment cache file " + cacheFile.getAbsolutePath());
             cacheFile.delete();
-            return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
+            // And make a new cache to hold the fragments
+            return diskCache(cacheFolder, cpu, fragmentSystemHash, new LinkedHashMap<>());
          }
          // Read the first "real" line
          String cacheLine = fragmentCacheReader.readLine();
@@ -154,7 +174,7 @@ public class AsmFragmentTemplateCache {
          final long millis = after.getTime() - before.getTime();
          if(log.isVerboseFragmentLog())
             log.append("Loaded cached fragments " + cache.size() + " from " + cacheFile.getPath() + " in " + millis + "ms");
-         return new AsmFragmentTemplateCache(cacheFolder, cpu, fragmentSystemHash, cache);
+         return diskCache(cacheFolder, cpu, fragmentSystemHash, cache);
       } catch(IOException e) {
          throw new RuntimeException("Error loading fragment cache file " + cacheFolder, e);
       } catch(StringIndexOutOfBoundsException e) {
@@ -178,16 +198,20 @@ public class AsmFragmentTemplateCache {
     *
     * @param log The compile log
     */
-   public void save(CompileLog log) {
+   public void save(Path baseFragmentFolder, CompileLog log) {
       if(!modified)
          return;
       if(this.cacheFolder == null)
          return;
       Date before = new Date();
+      // Calculate a new hash which handles all systems
+      final AsmFragmentSystemHash newFragmentSystemHash = AsmFragmentSystemHash.calculate(baseFragmentFolder, true);
       File cacheFile = this.cacheFolder.resolve(getCacheFileName(cpu)).toFile();
       try {
          PrintStream fragmentFilePrint = new PrintStream(cacheFile);
-         fragmentFilePrint.println(HASH_HEADER + fragmentSystemHash.getHashString());
+         if(newFragmentSystemHash.getHashStringLF() == null || newFragmentSystemHash.getHashStringCRLF() == null)
+            throw new InternalError("Error saving ASM fragment cache file. Not calculated for all systems!");
+         fragmentFilePrint.println(HASH_HEADER + newFragmentSystemHash.getHashStringLF() + " " + newFragmentSystemHash.getHashStringCRLF());
          for(String signature : this.cache.keySet()) {
             AsmFragmentTemplate fragmentTemplate = this.cache.get(signature);
             fragmentFilePrint.println(FRAGMENT_HEADER + signature);

src/main/java/dk/camelot64/kickc/fragment/AsmFragmentTemplateSynthesizer.java

@@ -34,7 +34,7 @@ public class AsmFragmentTemplateSynthesizer {
       if(useFragmentCache)
          this.fragmentCache = AsmFragmentTemplateCache.load(targetCpu, baseFragmentFolder, log);
       else
-         this.fragmentCache = AsmFragmentTemplateCache.memory(targetCpu);
+         this.fragmentCache = AsmFragmentTemplateCache.memoryCache(targetCpu);
    }
 
    /** The folder containing fragment files. */
@@ -58,7 +58,7 @@ public class AsmFragmentTemplateSynthesizer {
    /** Finalize the fragment template synthesizer. */
    void finalize(CompileLog log) {
       if(fragmentCache != null)
-         fragmentCache.save(log);
+         fragmentCache.save(baseFragmentFolder, log);
    }
 
    /**