Added -E commandline option for running only the proprocessor. Closes #385

src/main/java/dk/camelot64/kickc/Compiler.java

@@ -11,7 +11,9 @@ import dk.camelot64.kickc.parser.CParser;
 import dk.camelot64.kickc.parser.KickCLexer;
 import dk.camelot64.kickc.parser.KickCParser;
 import dk.camelot64.kickc.passes.*;
+import dk.camelot64.kickc.preprocessor.CPreprocessor;
 import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.Token;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -54,6 +56,10 @@ public class Compiler {
       this.program = new Program();
    }
 
+   public Program getProgram() {
+      return program;
+   }
+
    public void setDisableUplift(boolean disableUplift) {
       this.disableUplift = disableUplift;
    }
@@ -138,8 +144,24 @@ public class Compiler {
       program.getImportPaths().add(path);
    }
 
-   public Program compile(List<Path> files) {
+   public void preprocess(List<Path> files) {
-      if(files.size()==0)
+      Path currentPath = new File(".").toPath();
+      CParser cParser = new CParser(program);
+      for(Path file : files) {
+         final KickCLexer fileLexer = cParser.loadCFile(file.toString(), currentPath);
+         cParser.addSourceLast(fileLexer);
+      }
+      final CPreprocessor preprocessor = cParser.getPreprocessor();
+      Token token = preprocessor.nextToken();
+      while(token.getType() != Token.EOF) {
+         System.out.print(token.getText());
+         token = preprocessor.nextToken();
+      }
+      System.out.println();
+   }
+
+   public void compile(List<Path> files) {
+      if(files.size() == 0)
          throw new CompileError("Error! You must supply at least one file to compile!");
 
       final Path primaryFile = files.get(0);
@@ -196,7 +218,6 @@ public class Compiler {
          pass3Analysis();
          pass4RegisterAllocation();
          pass5GenerateAndOptimizeAsm();
-         return program;
       } catch(Exception e) {
          throw e;
       }

src/main/java/dk/camelot64/kickc/KickC.java

@@ -61,6 +61,9 @@ public class KickC implements Callable<Void> {
    @CommandLine.Option(names = {"-d"}, description = "Debug the assembled prg file using C64Debugger. Implicitly assembles the output.")
    private boolean debug = false;
 
+   @CommandLine.Option(names = {"-E"}, description = "Only run the preprocessor. Output is sent to standard out.")
+   private boolean preprocess = false;
+
    @CommandLine.Option(names = {"-Ouplift"}, description = "Optimization Option. Number of combinations to test when uplifting variables to registers in a scope. By default 100 combinations are tested.")
    private Integer optimizeUpliftCombinations = null;
 
@@ -260,7 +263,7 @@ public class KickC implements Callable<Void> {
             outputFileNameBase = primaryFileBaseName;
          } else {
             final int extensionIdx = outputFileName.lastIndexOf('.');
-            if(extensionIdx>0)
+            if(extensionIdx > 0)
                outputFileNameBase = outputFileName.substring(0, extensionIdx);
             else
                outputFileNameBase = outputFileName;
@@ -327,10 +330,23 @@ public class KickC implements Callable<Void> {
 
          StringBuilder kcFileNames = new StringBuilder();
          kcFiles.stream().forEach(path -> kcFileNames.append(path.toString()).append(" "));
+
+         if(preprocess) {
+            System.out.println("Preprocessing " + kcFileNames);
+            try {
+               compiler.preprocess(kcFiles);
+            } catch(CompileError e) {
+               // Print the error and exit with compile error
+               System.err.println(e.getMessage());
+               System.exit(COMPILE_ERROR);
+            }
+            return null;
+         }
+
          System.out.println("Compiling " + kcFileNames);
-         Program program = null;
+         Program program = compiler.getProgram();
          try {
-            program = compiler.compile(kcFiles);
+            compiler.compile(kcFiles);
          } catch(CompileError e) {
             // Print the error and exit with compile error
             System.err.println(e.getMessage());
@@ -363,7 +379,7 @@ public class KickC implements Callable<Void> {
          }
 
          // Assemble the asm-file if instructed
-         String prgFileName = outputFileNameBase +".prg";
+         String prgFileName = outputFileNameBase + ".prg";
          Path prgPath = outputDir.resolve(prgFileName);
          if(assemble || execute || debug) {
             Path kasmLogPath = outputDir.resolve(outputFileNameBase + ".klog");

src/main/java/dk/camelot64/kickc/parser/CParser.java

@@ -32,8 +32,11 @@ public class CParser {
    /** The (single) parser. */
    private KickCParser parser;
 
+   /** The preprocessor. */
+   private CPreprocessor preprocessor;
+
    /** The token stream. */
-   private final CommonTokenStream tokenStream;
+   private final CommonTokenStream preprocessedTokenStream;
 
    /** The token source stack handling import files. */
    private CTokenSource cTokenSource;
@@ -61,9 +64,9 @@ public class CParser {
       this.program = program;
       this.cFiles = new LinkedHashMap<>();
       this.cTokenSource = new CTokenSource();
-      final CPreprocessor preprocessor = new CPreprocessor(cTokenSource, new HashMap<>());
+      this.preprocessor = new CPreprocessor(cTokenSource, new HashMap<>());
-      this.tokenStream = new CommonTokenStream(preprocessor);
+      this.preprocessedTokenStream = new CommonTokenStream(preprocessor);
-      this.parser = new KickCParser(tokenStream, this);
+      this.parser = new KickCParser(preprocessedTokenStream, this);
       this.typedefs = new ArrayList<>();
       parser.setBuildParseTree(true);
       parser.addErrorListener(new BaseErrorListener() {
@@ -89,12 +92,20 @@ public class CParser {
    }
 
    /**
-    * Get the underlying token stream.
+    * Get the preprocessor (usable for getting all preprocessed tokens).
-    *
+    * @return The preprocessor
-    * @return The token stream
     */
-   public BufferedTokenStream getTokenStream() {
+   public CPreprocessor getPreprocessor() {
-      return tokenStream;
+      return preprocessor;
+   }
+
+   /**
+    * Get the token stream containing tokens after the preprocessor.
+    *
+    * @return The preprocessed token stream
+    */
+   public BufferedTokenStream getPreprocessedTokenStream() {
+      return preprocessedTokenStream;
    }
 
    /**

src/main/java/dk/camelot64/kickc/passes/Pass0GenerateStatementSequence.java

@@ -2381,11 +2381,11 @@ public class Pass0GenerateStatementSequence extends KickCParserBaseVisitor<Objec
    private List<List<Comment>> getCommentBlocks(ParserRuleContext ctx) {
       List<List<Comment>> commentBlocks = new ArrayList<>();
       List<Comment> comments = new ArrayList<>();
-      BufferedTokenStream tokenStream = cParser.getTokenStream();
+      BufferedTokenStream preprocessedTokenStream = cParser.getPreprocessedTokenStream();
       final int startTokenIndex = ctx.start.getTokenIndex();
       if(startTokenIndex < 0)
          return commentBlocks;
-      List<Token> hiddenTokens = tokenStream.getHiddenTokensToLeft(startTokenIndex);
+      List<Token> hiddenTokens = preprocessedTokenStream.getHiddenTokensToLeft(startTokenIndex);
       if(hiddenTokens != null) {
          for(Token hiddenToken : hiddenTokens) {
             if(hiddenToken.getChannel() == CParser.CHANNEL_WHITESPACE) {

src/test/java/dk/camelot64/kickc/test/TestPrograms.java

@@ -94,6 +94,11 @@ public class TestPrograms {
       compileAndCompare("cstyle-decl-function");
    }
 
+   @Test
+   public void testPreprocessor9() throws IOException, URISyntaxException {
+      compileAndCompare("preprocessor-9");
+   }
+
    @Test
    public void testPreprocessor8() throws IOException, URISyntaxException {
       compileAndCompare("preprocessor-8");
@@ -4027,10 +4032,9 @@ public class TestPrograms {
       final ArrayList<Path> files = new ArrayList<>();
       final Path filePath = Paths.get(fileName);
       files.add(filePath);
-      Program program = compiler.compile(files);
+      compiler.compile(files);
-
+      Program program = compiler.getProgram();
       compileAsm(fileName, program);
-
       boolean success = true;
       ReferenceHelper helper = new ReferenceHelperFolder(refPath);
       success &= helper.testOutput(fileName, ".asm", program.getAsm().toString(new AsmProgram.AsmPrintState(false, true, false, false), program));

src/test/kc/preprocessor-9.kc

@@ -0,0 +1,15 @@
+// Test the preprocessor
+// Macro with parameters
+
+#define SQUARE(x) x*x
+#define DOUBLE(x) SUM(x,x)
+#define SUM(x,y) x+y
+
+char * SCREEN = 0x0400;
+
+void main() {
+    char idx = 0;
+    SCREEN[idx++] = SUM('0',4);
+    SCREEN[idx++] = DOUBLE('b');
+    SCREEN[idx++] = SQUARE('c');
+}

src/test/ref/preprocessor-9.asm

@@ -0,0 +1,19 @@
+// Test the preprocessor
+// Macro with parameters
+.pc = $801 "Basic"
+:BasicUpstart(main)
+.pc = $80d "Program"
+  .label SCREEN = $400
+main: {
+    // SCREEN[idx++] = SUM
+    lda #'0'+4
+    sta SCREEN
+    // SCREEN[idx++] = DOUBLE
+    lda #'b'+'b'
+    sta SCREEN+1
+    // SCREEN[idx++] = SQUARE
+    lda #'c'*'c'
+    sta SCREEN+2
+    // }
+    rts
+}

src/test/ref/preprocessor-9.cfg

@@ -0,0 +1,19 @@
+@begin: scope:[]  from
+  [0] phi()
+  to:@1
+@1: scope:[]  from @begin
+  [1] phi()
+  [2] call main 
+  to:@end
+@end: scope:[]  from @1
+  [3] phi()
+
+(void()) main()
+main: scope:[main]  from @1
+  [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4
+  [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b'
+  [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c'
+  to:main::@return
+main::@return: scope:[main]  from main
+  [7] return 
+  to:@return

src/test/ref/preprocessor-9.log

@@ -0,0 +1,290 @@
+Identified constant variable (byte*) SCREEN
+
+CONTROL FLOW GRAPH SSA
+@begin: scope:[]  from
+  to:@1
+
+(void()) main()
+main: scope:[main]  from @1
+  (byte) main::idx#0 ← (byte) 0
+  *((const byte*) SCREEN + (byte) main::idx#0) ← (byte) '0'+(number) 4
+  (byte) main::idx#1 ← ++ (byte) main::idx#0
+  *((const byte*) SCREEN + (byte) main::idx#1) ← (byte) 'b'+(byte) 'b'
+  (byte) main::idx#2 ← ++ (byte) main::idx#1
+  *((const byte*) SCREEN + (byte) main::idx#2) ← (byte) 'c'*(byte) 'c'
+  (byte) main::idx#3 ← ++ (byte) main::idx#2
+  to:main::@return
+main::@return: scope:[main]  from main
+  return 
+  to:@return
+@1: scope:[]  from @begin
+  call main 
+  to:@2
+@2: scope:[]  from @1
+  to:@end
+@end: scope:[]  from @2
+
+SYMBOL TABLE SSA
+(label) @1
+(label) @2
+(label) @begin
+(label) @end
+(const byte*) SCREEN = (byte*)(number) $400
+(void()) main()
+(label) main::@return
+(byte) main::idx
+(byte) main::idx#0
+(byte) main::idx#1
+(byte) main::idx#2
+(byte) main::idx#3
+
+Adding number conversion cast (unumber) '0'+4 in *((const byte*) SCREEN + (byte) main::idx#0) ← (byte) '0'+(number) 4
+Adding number conversion cast (unumber) 4 in *((const byte*) SCREEN + (byte) main::idx#0) ← ((unumber)) (byte) '0'+(number) 4
+Successful SSA optimization PassNAddNumberTypeConversions
+Inlining cast *((const byte*) SCREEN + (byte) main::idx#0) ← (unumber)(byte) '0'+(unumber)(number) 4
+Successful SSA optimization Pass2InlineCast
+Simplifying constant pointer cast (byte*) 1024
+Simplifying constant integer cast (byte) '0'+(unumber)(number) 4
+Simplifying constant integer cast 4
+Successful SSA optimization PassNCastSimplification
+Finalized unsigned number type (byte) 4
+Successful SSA optimization PassNFinalizeNumberTypeConversions
+Constant (const byte) main::idx#0 = 0
+Successful SSA optimization Pass2ConstantIdentification
+Simplifying expression containing zero SCREEN in [1] *((const byte*) SCREEN + (const byte) main::idx#0) ← (byte) '0'+(byte) 4
+Successful SSA optimization PassNSimplifyExpressionWithZero
+Eliminating unused variable (byte) main::idx#3 and assignment [5] (byte) main::idx#3 ← ++ (byte) main::idx#2
+Successful SSA optimization PassNEliminateUnusedVars
+Constant right-side identified [1] (byte) main::idx#1 ← ++ (const byte) main::idx#0
+Successful SSA optimization Pass2ConstantRValueConsolidation
+Constant (const byte) main::idx#1 = ++main::idx#0
+Successful SSA optimization Pass2ConstantIdentification
+Constant right-side identified [2] (byte) main::idx#2 ← ++ (const byte) main::idx#1
+Successful SSA optimization Pass2ConstantRValueConsolidation
+Constant (const byte) main::idx#2 = ++main::idx#1
+Successful SSA optimization Pass2ConstantIdentification
+Inlining constant with different constant siblings (const byte) main::idx#0
+Inlining constant with different constant siblings (const byte) main::idx#1
+Inlining constant with different constant siblings (const byte) main::idx#2
+Constant inlined main::idx#0 = (byte) 0
+Constant inlined main::idx#1 = ++(byte) 0
+Constant inlined main::idx#2 = ++++(byte) 0
+Successful SSA optimization Pass2ConstantInlining
+Consolidated array index constant in *(SCREEN+++0)
+Consolidated array index constant in *(SCREEN+++++0)
+Successful SSA optimization Pass2ConstantAdditionElimination
+Simplifying constant integer increment ++0
+Simplifying constant integer increment ++0
+Successful SSA optimization Pass2ConstantSimplification
+Simplifying constant integer increment ++1
+Successful SSA optimization Pass2ConstantSimplification
+Adding NOP phi() at start of @begin
+Adding NOP phi() at start of @1
+Adding NOP phi() at start of @2
+Adding NOP phi() at start of @end
+CALL GRAPH
+Calls in [] to main:2 
+
+Created 0 initial phi equivalence classes
+Coalesced down to 0 phi equivalence classes
+Culled Empty Block (label) @2
+Adding NOP phi() at start of @begin
+Adding NOP phi() at start of @1
+Adding NOP phi() at start of @end
+
+FINAL CONTROL FLOW GRAPH
+@begin: scope:[]  from
+  [0] phi()
+  to:@1
+@1: scope:[]  from @begin
+  [1] phi()
+  [2] call main 
+  to:@end
+@end: scope:[]  from @1
+  [3] phi()
+
+(void()) main()
+main: scope:[main]  from @1
+  [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4
+  [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b'
+  [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c'
+  to:main::@return
+main::@return: scope:[main]  from main
+  [7] return 
+  to:@return
+
+
+VARIABLE REGISTER WEIGHTS
+(void()) main()
+(byte) main::idx
+
+Initial phi equivalence classes
+Complete equivalence classes
+
+INITIAL ASM
+Target platform is c64basic / MOS6502X
+  // File Comments
+// Test the preprocessor
+// Macro with parameters
+  // Upstart
+.pc = $801 "Basic"
+:BasicUpstart(__bbegin)
+.pc = $80d "Program"
+  // Global Constants & labels
+  .label SCREEN = $400
+  // @begin
+__bbegin:
+  // [1] phi from @begin to @1 [phi:@begin->@1]
+__b1_from___bbegin:
+  jmp __b1
+  // @1
+__b1:
+  // [2] call main 
+  jsr main
+  // [3] phi from @1 to @end [phi:@1->@end]
+__bend_from___b1:
+  jmp __bend
+  // @end
+__bend:
+  // main
+main: {
+    // [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4 -- _deref_pbuc1=vbuc2 
+    lda #'0'+4
+    sta SCREEN
+    // [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b' -- _deref_pbuc1=vbuc2 
+    lda #'b'+'b'
+    sta SCREEN+1
+    // [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c' -- _deref_pbuc1=vbuc2 
+    lda #'c'*'c'
+    sta SCREEN+2
+    jmp __breturn
+    // main::@return
+  __breturn:
+    // [7] return 
+    rts
+}
+  // File Data
+
+REGISTER UPLIFT POTENTIAL REGISTERS
+Statement [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4 [ ] ( main:2 [ ] { }  ) always clobbers reg byte a 
+Statement [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b' [ ] ( main:2 [ ] { }  ) always clobbers reg byte a 
+Statement [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c' [ ] ( main:2 [ ] { }  ) always clobbers reg byte a 
+
+REGISTER UPLIFT SCOPES
+Uplift Scope [main] 
+Uplift Scope [] 
+
+Uplifting [main] best 39 combination 
+Uplifting [] best 39 combination 
+
+ASSEMBLER BEFORE OPTIMIZATION
+  // File Comments
+// Test the preprocessor
+// Macro with parameters
+  // Upstart
+.pc = $801 "Basic"
+:BasicUpstart(__bbegin)
+.pc = $80d "Program"
+  // Global Constants & labels
+  .label SCREEN = $400
+  // @begin
+__bbegin:
+  // [1] phi from @begin to @1 [phi:@begin->@1]
+__b1_from___bbegin:
+  jmp __b1
+  // @1
+__b1:
+  // [2] call main 
+  jsr main
+  // [3] phi from @1 to @end [phi:@1->@end]
+__bend_from___b1:
+  jmp __bend
+  // @end
+__bend:
+  // main
+main: {
+    // [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4 -- _deref_pbuc1=vbuc2 
+    lda #'0'+4
+    sta SCREEN
+    // [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b' -- _deref_pbuc1=vbuc2 
+    lda #'b'+'b'
+    sta SCREEN+1
+    // [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c' -- _deref_pbuc1=vbuc2 
+    lda #'c'*'c'
+    sta SCREEN+2
+    jmp __breturn
+    // main::@return
+  __breturn:
+    // [7] return 
+    rts
+}
+  // File Data
+
+ASSEMBLER OPTIMIZATIONS
+Removing instruction jmp __b1
+Removing instruction jmp __bend
+Removing instruction jmp __breturn
+Succesful ASM optimization Pass5NextJumpElimination
+Removing instruction __b1_from___bbegin:
+Removing instruction __b1:
+Removing instruction __bend_from___b1:
+Succesful ASM optimization Pass5RedundantLabelElimination
+Removing instruction __bend:
+Removing instruction __breturn:
+Succesful ASM optimization Pass5UnusedLabelElimination
+Updating BasicUpstart to call main directly
+Removing instruction jsr main
+Succesful ASM optimization Pass5SkipBegin
+Removing instruction __bbegin:
+Succesful ASM optimization Pass5UnusedLabelElimination
+
+FINAL SYMBOL TABLE
+(label) @1
+(label) @begin
+(label) @end
+(const byte*) SCREEN = (byte*) 1024
+(void()) main()
+(label) main::@return
+(byte) main::idx
+
+
+
+FINAL ASSEMBLER
+Score: 24
+
+  // File Comments
+// Test the preprocessor
+// Macro with parameters
+  // Upstart
+.pc = $801 "Basic"
+:BasicUpstart(main)
+.pc = $80d "Program"
+  // Global Constants & labels
+  .label SCREEN = $400
+  // @begin
+  // [1] phi from @begin to @1 [phi:@begin->@1]
+  // @1
+  // [2] call main 
+  // [3] phi from @1 to @end [phi:@1->@end]
+  // @end
+  // main
+main: {
+    // SCREEN[idx++] = SUM
+    // [4] *((const byte*) SCREEN) ← (byte) '0'+(byte) 4 -- _deref_pbuc1=vbuc2 
+    lda #'0'+4
+    sta SCREEN
+    // SCREEN[idx++] = DOUBLE
+    // [5] *((const byte*) SCREEN+(byte) 1) ← (byte) 'b'+(byte) 'b' -- _deref_pbuc1=vbuc2 
+    lda #'b'+'b'
+    sta SCREEN+1
+    // SCREEN[idx++] = SQUARE
+    // [6] *((const byte*) SCREEN+(byte) 2) ← (byte) 'c'*(byte) 'c' -- _deref_pbuc1=vbuc2 
+    lda #'c'*'c'
+    sta SCREEN+2
+    // main::@return
+    // }
+    // [7] return 
+    rts
+}
+  // File Data
+

src/test/ref/preprocessor-9.sym

@@ -0,0 +1,8 @@
+(label) @1
+(label) @begin
+(label) @end
+(const byte*) SCREEN = (byte*) 1024
+(void()) main()
+(label) main::@return
+(byte) main::idx
+