Used usage module to identify & remove all unneeded fragments. Many unused ones moved into a new "unused" folder to be removed later.

src/main/java/dk/camelot64/kickc/fragment/AsmFragmentManager.java

@@ -5,6 +5,7 @@ import dk.camelot64.kickc.asm.AsmProgram;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
@@ -24,12 +25,13 @@ public class AsmFragmentManager {
     /** Caches all asm fragment templates for all encountered signatures. */
     private static Map<String, List<AsmFragmentTemplate>> fragmentTemplateCache = new LinkedHashMap<>();
 
-    /** Usage Statistics for fragment templates. */
-    private static Map<AsmFragmentTemplate, Integer> fragmentTemplateUsage = new HashMap<>();
-
     /** Special singleton representing that the fragment can not be synthesized or loaded. */
     private static AsmFragmentTemplate UNKNOWN = new AsmFragmentTemplate("UNKNOWN", null);
 
+    static Map<String, List<AsmFragmentTemplate>> getFragmentTemplateCache() {
+        return fragmentTemplateCache;
+    }
+
     public static AsmFragment getFragment(AsmFragmentSignature signature, CompileLog log) {
         AsmFragmentTemplate bestTemplate = bestFragmentCache.get(signature.getSignature());
         if (bestTemplate == UNKNOWN) {
@@ -39,7 +41,7 @@ public class AsmFragmentManager {
             throw new UnknownFragmentException(signature.toString());
         }
         if (bestTemplate == null) {
-            AsmFragmentTemplateSynthesizer synthesizer = new AsmFragmentTemplateSynthesizer(signature, log);
+            AsmFragmentTemplateSynthesizer synthesizer = new AsmFragmentTemplateSynthesizer(signature.getSignature(), log);
             List<AsmFragmentTemplate> candidates = synthesizer.loadOrSynthesizeFragment(signature.getSignature());
             if (candidates.size() == 0) {
                 if (log.isVerboseFragmentLog()) {
@@ -71,7 +73,7 @@ public class AsmFragmentManager {
             bestFragmentCache.put(signature.getSignature(), bestTemplate);
         }
         // Count usages
-        incUsage(bestTemplate);
+        AsmFragmentUsages.incUsage(bestTemplate);
         // Return the resulting fragment instance
         return new AsmFragment(
                 signature.getProgram(),
@@ -81,72 +83,6 @@ public class AsmFragmentManager {
                 signature.getBindings());
     }
 
-    /**
-     * Count one usage of ASM fragment templates - directly or through synthesis
-     * @param fragmentTemplate The template to increment usage of
-     */
-    private static void incUsage(AsmFragmentTemplate fragmentTemplate) {
-        Integer usage = fragmentTemplateUsage.get(fragmentTemplate);
-        if (usage == null) {
-            usage = 0;
-        }
-        fragmentTemplateUsage.put(fragmentTemplate, usage + 1);
-        AsmFragmentTemplate subFragment = fragmentTemplate.getSubFragment();
-        if (subFragment != null) {
-            incUsage(subFragment);
-        }
-    }
-
-    /**
-     * Log the usage of all template fragemnts (both loaded and synthesized).
-     * @param log The compile log to add the output to
-     */
-    public static void logUsages(CompileLog log) {
-
-        ArrayList<String> signatures = new ArrayList<>(fragmentTemplateCache.keySet());
-        Collections.sort(signatures);
-
-        /*
-        log.append("ASM FRAGMENT USAGES");
-        for (String signature : signatures) {
-            List<AsmFragmentTemplate> templates = fragmentTemplateCache.get(signature);
-            for (AsmFragmentTemplate template : templates) {
-                Integer usage = fragmentTemplateUsage.get(template);
-                if(usage==null) usage = 0;
-                log.append(String.format("%8d", usage)+"  "+template.getName());
-            }
-        }
-        */
-
-        // Find all file fragments that were bested by a synthesized fragment
-        log.append("\nREDUNDANT ASM FRAGMENT FILES - REMOVE FROM DISK");
-        for (String signature : signatures) {
-            List<AsmFragmentTemplate> templates = fragmentTemplateCache.get(signature);
-            AsmFragmentTemplate fileTemplate = null;
-            int fileUsage = 0;
-            AsmFragmentTemplate maxTemplate = null;
-            int maxUsage = 0;
-            for (AsmFragmentTemplate template : templates) {
-                Integer usage = fragmentTemplateUsage.get(template);
-                if(usage==null) usage = 0;
-                if(template.isFile()) {
-                    fileTemplate = template;
-                    fileUsage = usage;
-                }
-                if(usage>maxUsage) {
-                    maxUsage = usage;
-                    maxTemplate = template;
-                }
-            }
-            if(fileTemplate!=null && fileUsage==0 && maxUsage>0) {
-                log.append("rm "+fileTemplate.getName()+".asm #synthesized by "+maxTemplate.getName()+" - usages: "+maxUsage);
-            }
-        }
-
-
-
-    }
-
     /**
      * Capable of creating fragments from signatures by loading them or synthesizing them from other smaller fragments.
      * <p>
@@ -155,13 +91,13 @@ public class AsmFragmentManager {
     static class AsmFragmentTemplateSynthesizer {
 
         /** Signature of the fragment being synthesized. */
-        private AsmFragmentSignature signature;
+        private String creating;
 
         /** The log. */
         private CompileLog log;
 
-        AsmFragmentTemplateSynthesizer(AsmFragmentSignature signature, CompileLog log) {
-            this.signature = signature;
+        AsmFragmentTemplateSynthesizer(String creating, CompileLog log) {
+            this.creating = creating;
             this.log = log;
         }
 
@@ -176,7 +112,7 @@ public class AsmFragmentManager {
                 List<AsmFragmentTemplate> synthesized = synth.synthesize(signature, this);
                 if (synthesized != null) {
                     if (log.isVerboseFragmentLog() && synthesized.size() > 0) {
-                        log.append("Finding fragment " + this.signature.getSignature() + " - Successfully synthesized " + synthesized.size() + " fragments " + signature + " (from " + synth.getSubSignature() + ")");
+                        log.append("Finding fragment " + this.creating + " - Successfully synthesized " + synthesized.size() + " fragments " + signature + " (from " + synth.getSubSignature() + ")");
                     }
                     candidates.addAll(synthesized);
                 }
@@ -184,9 +120,14 @@ public class AsmFragmentManager {
             // Load the fragment from disk
             CharStream fragmentCharStream = loadFragment(signature);
             if (fragmentCharStream != null) {
-                candidates.add(new AsmFragmentTemplate(signature, fragmentCharStream.toString()));
+                try {
+                    String body = fragmentCharStream.toString();
+                    candidates.add(new AsmFragmentTemplate(signature, body));
+                } catch (StringIndexOutOfBoundsException e) {
+                    throw new RuntimeException("Problem reading fragment file " + signature, e);
+                }
                 if (log.isVerboseFragmentLog()) {
-                    log.append("Finding fragment " + this.signature.getSignature() + " - Successfully loaded fragment " + signature);
+                    log.append("Finding fragment " + this.creating + " - Successfully loaded fragment " + signature);
                 }
             }
             fragmentTemplateCache.put(signature, candidates);
@@ -215,6 +156,14 @@ public class AsmFragmentManager {
         }
     }
 
+
+    static File[] allFragmentFiles() {
+        ClassLoader classLoader = AsmFragmentManager.class.getClassLoader();
+        String path = classLoader.getResource("dk/camelot64/kickc/fragment/asm/").getPath();
+        return new File(path).listFiles((dir, name) -> name.endsWith(".asm"));
+
+    }
+
     public static class UnknownFragmentException extends RuntimeException {
 
         private String fragmentSignature;

src/main/java/dk/camelot64/kickc/fragment/AsmFragmentUsages.java

@@ -0,0 +1,153 @@
+package dk.camelot64.kickc.fragment;
+
+import dk.camelot64.kickc.CompileLog;
+
+import java.io.File;
+import java.util.*;
+
+/** Statistics for usage of the ASM fragments */
+public class AsmFragmentUsages {
+
+    /** Usage Statistics for fragment templates. */
+    private static Map<AsmFragmentTemplate, Integer> fragmentTemplateUsage = new HashMap<>();
+
+    /**
+     * Count one usage of ASM fragment templates - directly or through synthesis
+     *
+     * @param fragmentTemplate The template to increment usage of
+     */
+    static void incUsage(AsmFragmentTemplate fragmentTemplate) {
+        Integer usage = fragmentTemplateUsage.get(fragmentTemplate);
+        if (usage == null) {
+            usage = 0;
+        }
+        fragmentTemplateUsage.put(fragmentTemplate, usage + 1);
+        AsmFragmentTemplate subFragment = fragmentTemplate.getSubFragment();
+        if (subFragment != null) {
+            incUsage(subFragment);
+        }
+    }
+
+    /**
+     * Log the usage of all template fragemnts (both loaded and synthesized).
+     *
+     * @param log The compile log to add the output to
+     */
+    public static void logUsages(CompileLog log, boolean logRedundantFiles, boolean logUnusedFiles, boolean logDetails) {
+
+        Map<String, List<AsmFragmentTemplate>> fragmentTemplateCache = AsmFragmentManager.getFragmentTemplateCache();
+        ArrayList<String> signatures = new ArrayList<>(fragmentTemplateCache.keySet());
+        Collections.sort(signatures);
+        File[] files = AsmFragmentManager.allFragmentFiles();
+
+        if (logRedundantFiles) {
+            // Find all file fragments that were bested by a synthesized fragment
+            log.append("\nREDUNDANT ASM FRAGMENT FILE ANALYSIS (if found remove them from disk)");
+            for (String signature : signatures) {
+                List<AsmFragmentTemplate> templates = fragmentTemplateCache.get(signature);
+                AsmFragmentTemplate fileTemplate = null;
+                int fileUsage = 0;
+                AsmFragmentTemplate maxTemplate = null;
+                int maxUsage = 0;
+                for (AsmFragmentTemplate template : templates) {
+                    Integer usage = fragmentTemplateUsage.get(template);
+                    if (usage == null) usage = 0;
+                    if (template.isFile()) {
+                        fileTemplate = template;
+                        fileUsage = usage;
+                    }
+                    if (usage > maxUsage) {
+                        maxUsage = usage;
+                        maxTemplate = template;
+                    }
+                }
+                if (fileTemplate != null && fileUsage == 0 && maxUsage > 0) {
+                    log.append("rm " + fileTemplate.getName() + ".asm #synthesized by " + maxTemplate.getName() + " - usages: " + maxUsage);
+                }
+            }
+
+            log.append("\nMORE REDUNDANT ASM FRAGMENT FILE ANALYSIS (if found remove them from disk)");
+
+            Set<String> redundantSignatures = new LinkedHashSet<>();
+            for (File file : files) {
+                String fileName = file.getName();
+                String signature = fileName.substring(0, fileName.length() - 4);
+                // Try to synthesize the fragment - and check if the synthesis is as good as the file body
+                AsmFragmentManager.AsmFragmentTemplateSynthesizer synthesizer = new AsmFragmentManager.AsmFragmentTemplateSynthesizer(signature, log);
+                List<AsmFragmentTemplate> templates = synthesizer.loadOrSynthesizeFragment(signature);
+                AsmFragmentTemplate fileTemplate = null;
+                for (AsmFragmentTemplate template : templates) {
+                    if (template.isFile()) {
+                        fileTemplate = template;
+                    }
+                }
+                for (AsmFragmentTemplate template : templates) {
+                    if (!template.isFile() && template.getBody().equals(fileTemplate.getBody())) {
+                        // Check if the synthesis uses a file marked as redundant
+                        AsmFragmentTemplate sourceFileTemplate = template;
+                        while (!sourceFileTemplate.isFile()) {
+                            sourceFileTemplate = sourceFileTemplate.getSubFragment();
+                        }
+                        if (redundantSignatures.contains(sourceFileTemplate.getSignature())) {
+                            throw new RuntimeException("Problem in redundancy analysis! " + sourceFileTemplate.getSignature() + ".asm seems redundant but is needed for synthesis of " + signature);
+                        }
+                        log.append("rm " + fileTemplate.getName() + ".asm #synthesized same ASM by " + template.getName());
+                        redundantSignatures.add(signature);
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (logUnusedFiles) {
+            log.append("\nUNUSED ASM FRAGMENT FILES ANALYSIS (if found consider removing them from disk)");
+            for (File file : files) {
+                String fileName = file.getName();
+                String signature = fileName.substring(0, fileName.length() - 4);
+                List<AsmFragmentTemplate> templates = fragmentTemplateCache.get(signature);
+                if (templates != null && templates.size() > 0) {
+                    // The template has been loaded / synthesized - is the usage count zero?
+                    boolean allZero = true;
+                    Integer fileUsage = null;
+                    for (AsmFragmentTemplate template : templates) {
+                        Integer usage = fragmentTemplateUsage.get(template);
+                        if (usage == null) usage = 0;
+                        if (usage > 0) {
+                            allZero = false;
+                        }
+                        if (template.isFile()) {
+                            fileUsage = usage;
+                        }
+                    }
+                    if (fileUsage == null) {
+                        throw new RuntimeException("Error! Template file never loaded according to usage stats " + fileName);
+                    }
+                    if (allZero) {
+                        log.append("git mv " + fileName + " unused # Loaded but never used");
+                    }
+                } else {
+                    // The template has never been loaded
+                    log.append("git mv " + fileName + " unused # Never loaded");
+                }
+            }
+        }
+
+        if (logDetails) {
+            log.append("\nDETAILED ASM FRAGMENT USAGES");
+            for (String signature : signatures) {
+                List<AsmFragmentTemplate> templates = fragmentTemplateCache.get(signature);
+                for (AsmFragmentTemplate template : templates) {
+                    Integer usage = fragmentTemplateUsage.get(template);
+                    if (usage == null) usage = 0;
+                    log.append(String.format("%8d", usage) + "  " + template.getName());
+                }
+            }
+        }
+
+
+    }
+
+
+
+
+}

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1=pbuz2_plus_vbuaa.asm

@@ -1,6 +0,0 @@
-clc
-adc {z2}
-sta {z1}
-lda #0
-adc {z2}+1
-sta {z1}+1

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1=pbuz2_plus_vbuc1.asm

@@ -1,7 +0,0 @@
-lda #{c1}
-clc
-adc {z2}
-sta {z1}
-lda #0
-adc {z2}+1
-sta {z1}+1

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1=pbuz2_plus_vbuxx.asm

@@ -1,7 +0,0 @@
-txa
-clc
-adc {z2}
-sta {z1}
-lda #0
-adc {z2}+1
-sta {z1}+1

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1=pbuz2_plus_vbuyy.asm

@@ -1,7 +0,0 @@
-tya
-clc
-adc {z2}
-sta {z1}
-lda #0
-adc {z2}+1
-sta {z1}+1

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1=pbuz2_plus_vwuc1.asm

@@ -1,7 +0,0 @@
-lda {z2}
-clc
-adc #<{c1}
-sta {z1}
-lda {z2}+1
-adc #>{c1}
-sta {z1}+1

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1_derefidx_vbuyy=vbuc1.asm

@@ -1,2 +0,0 @@
-lda #{c1}
-sta ({z1}),y

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1_derefidx_vbuyy=vbuxx.asm

@@ -1,2 +0,0 @@
-txa
-sta ({z1}),y

src/main/java/dk/camelot64/kickc/fragment/asm/pbuz1_derefidx_vbuz2=vbuc1.asm

@@ -1,3 +0,0 @@
-lda #{c1}
-ldy {z2}
-sta ({z1}),y

src/main/java/dk/camelot64/kickc/fragment/asm/unused/vbuxx_lt_vbuxx_then_la1.asm

@@ -0,0 +1 @@
+// do nothing

src/main/java/dk/camelot64/kickc/fragment/asm/unused/vbuyy_lt_vbuyy_then_la1.asm

@@ -0,0 +1 @@
+// do nothing

src/main/java/dk/camelot64/kickc/fragment/asm/unused/vbuz1_lt_vbuz1_then_la1.asm

@@ -0,0 +1 @@
+// do nothing

src/main/java/dk/camelot64/kickc/fragment/asm/unused/vwuz1=_word_pbuz1.asm

@@ -0,0 +1 @@
+// do nothing