Adding basic optimization to drop statements. Closes #7.

src/main/java/com/webcodepro/applecommander/util/applesoft/Line.java

@@ -2,10 +2,9 @@ package com.webcodepro.applecommander.util.applesoft;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Consumer;
 
 /** An AppleSoft BASIC Line representation. */
-public class Line implements Consumer<Visitor> {
+public class Line {
 	public final int lineNumber;
 	public final List<Statement> statements = new ArrayList<>();
 	
@@ -13,8 +12,7 @@ public class Line implements Consumer<Visitor> {
 		this.lineNumber = lineNumber;
 	}
 	
-	@Override
-	public void accept(Visitor t) {
-		t.visit(this);
+	public Line accept(Visitor t) {
+		return t.visit(this);
 	}
 }

src/main/java/com/webcodepro/applecommander/util/applesoft/Program.java

@@ -2,14 +2,12 @@ package com.webcodepro.applecommander.util.applesoft;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Consumer;
 
 /** A Program is a series of lines. */
-public class Program implements Consumer<Visitor> {
+public class Program {
 	public final List<Line> lines = new ArrayList<>();
 	
-	@Override
-	public void accept(Visitor t) {
-		t.visit(this);
+	public Program accept(Visitor t) {
+		return t.visit(this);
 	}
 }

src/main/java/com/webcodepro/applecommander/util/applesoft/Statement.java

@@ -2,14 +2,12 @@ package com.webcodepro.applecommander.util.applesoft;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Consumer;
 
 /** A Statement is simply a series of Tokens. */
-public class Statement implements Consumer<Visitor> {
+public class Statement {
 	public final List<Token> tokens = new ArrayList<>();
 	
-	@Override
-	public void accept(Visitor t) {
-		t.visit(this);
+	public Statement accept(Visitor t) {
+		return t.visit(this);
 	}
 }

src/main/java/com/webcodepro/applecommander/util/applesoft/Token.java

@@ -1,13 +1,11 @@
 package com.webcodepro.applecommander.util.applesoft;
 
-import java.util.function.Consumer;
-
 /**
  * A Token in the classic compiler sense, in that this represents a component of the application.
  * 
  * @author rob
  */
-public class Token implements Consumer<Visitor> {
+public class Token {
 	public final int line;
 	public final Type type;
 	public final ApplesoftKeyword keyword;
@@ -21,9 +19,8 @@ public class Token implements Consumer<Visitor> {
 		this.number = number;
 		this.text = text;
 	}
-	@Override
-	public void accept(Visitor t) {
-		t.visit(this);
+	public Token accept(Visitor t) {
+		return t.visit(this);
 	}
 	@Override
 	public String toString() {

src/main/java/com/webcodepro/applecommander/util/applesoft/Visitor.java

@@ -8,14 +8,19 @@ package com.webcodepro.applecommander.util.applesoft;
  * @see Visitors
  */
 public interface Visitor {
-	default public void visit(Program program) {
+	default public Program visit(Program program) {
 		program.lines.forEach(l -> l.accept(this));
+		return program;
 	}
-	default public void visit(Line line) {
+	default public Line visit(Line line) {
 		line.statements.forEach(s -> s.accept(this));
+		return line;
 	}
-	default public void visit(Statement statement) {
+	default public Statement visit(Statement statement) {
 		statement.tokens.forEach(t -> t.accept(this));
+		return statement;
 	}
-	public void visit(Token token);
+	default public Token visit(Token token) {
+		return token;
+	};
 }

src/main/java/com/webcodepro/applecommander/util/applesoft/Visitors.java

@@ -58,7 +58,7 @@ public class Visitors {
 		}
 		
 		@Override
-		public void visit(Line line) {
+		public Line visit(Line line) {
 			boolean first = true;
 			for (Statement statement : line.statements) {
 				if (first) {
@@ -70,9 +70,10 @@ public class Visitors {
 				statement.accept(this);
 				printStream.println();
 			}
+			return line;
 		}
 		@Override
-		public void visit(Token token) {
+		public Token visit(Token token) {
 			switch (token.type) {
 			case EOL:
 				printStream.print("<EOL>");
@@ -98,6 +99,7 @@ public class Visitors {
 				}
 				break;
 			}
+			return token;
 		}
 	}
 	
@@ -109,7 +111,7 @@ public class Visitors {
 		}
 		
 		@Override
-		public void visit(Line line) {
+		public Line visit(Line line) {
 			printStream.printf("%d ", line.lineNumber);
 			boolean first = true;
 			for (Statement statement : line.statements) {
@@ -121,9 +123,10 @@ public class Visitors {
 				statement.accept(this);
 			}
 			printStream.println();
+			return line;
 		}
 		@Override
-		public void visit(Token token) {
+		public Token visit(Token token) {
 			switch (token.type) {
 			case EOL:
 				printStream.print("<EOL>");
@@ -149,6 +152,7 @@ public class Visitors {
 				}
 				break;
 			}
+			return token;
 		}
 	}
 	
@@ -175,7 +179,7 @@ public class Visitors {
 		}
 		
 		@Override
-		public void visit(Program program) {
+		public Program visit(Program program) {
 			if (stack.size() != 0) {
 				throw new RuntimeException("Please do not reuse this ByteVisitor as that is an unsafe operation.");
 			}
@@ -184,17 +188,19 @@ public class Visitors {
 			ByteArrayOutputStream os = stack.peek();
 			os.write(0x00);
 			os.write(0x00);
+			return program;
 		}
 		
 		@Override
-		public void visit(Line line) {
+		public Line visit(Line line) {
 			try {
 				stack.push(new ByteArrayOutputStream());
 				boolean first = true;
 				for (Statement statement : line.statements) {
 					if (!first) {
-						first = false;
 						stack.peek().write(':');
+					} else {
+						first = false;
 					}
 					statement.accept(this);
 				}
@@ -209,6 +215,7 @@ public class Visitors {
 				os.write(content);
 				os.write(0x00);
 				this.address = nextAddress;
+				return line;
 			} catch (IOException ex) {
 				// Hiding the IOException as ByteArrayOutputStream does not throw it
 				throw new RuntimeException(ex);
@@ -216,7 +223,7 @@ public class Visitors {
 		}
 
 		@Override
-		public void visit(Token token) {
+		public Token visit(Token token) {
 			try {
 				ByteArrayOutputStream os = stack.peek();
 				switch (token.type) {
@@ -254,6 +261,7 @@ public class Visitors {
 					}
 					break;
 				}
+				return token;
 			} catch (IOException ex) {
 				// Hiding the IOException as ByteArrayOutputStream does not throw it
 				throw new RuntimeException(ex);

src/main/java/io/github/applecommander/bastokenizer/Main.java

@@ -4,6 +4,8 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.Callable;
 
@@ -49,6 +51,9 @@ public class Main implements Callable<Void> {
 
 	@Option(names = "--tokens", description = "Dump token list to stdout for debugging.")
 	private boolean showTokens;
+	
+	@Option(names = "-O", description = "Enable optimizations.", converter = Optimization.TypeConverter.class, split = ",")
+	private List<Optimization> optimizations = new ArrayList<>();
 
 	@Parameters(index = "0", description = "AppleSoft BASIC program to process.")
 	private File sourceFile;
@@ -86,6 +91,11 @@ public class Main implements Callable<Void> {
 		}
 		Parser parser = new Parser(tokens);
 		Program program = parser.parse();
+		
+		for (Optimization optimization : optimizations) {
+			program = program.accept(optimization.visitor);
+		}
+
 		if (prettyPrint || listPrint) {
 			program.accept(Visitors.printBuilder().prettyPrint(prettyPrint).build());
 		}

src/main/java/io/github/applecommander/bastokenizer/Optimization.java

@@ -0,0 +1,77 @@
+package io.github.applecommander.bastokenizer;
+
+import com.webcodepro.applecommander.util.applesoft.Line;
+import com.webcodepro.applecommander.util.applesoft.Program;
+import com.webcodepro.applecommander.util.applesoft.Statement;
+import com.webcodepro.applecommander.util.applesoft.Token;
+import com.webcodepro.applecommander.util.applesoft.Visitor;
+
+import picocli.CommandLine.ITypeConverter;
+
+public enum Optimization {
+	REMOVE_EMPTY_STATEMENTS(new BaseVisitor() {
+		@Override
+		public Statement visit(Statement statement) {
+			return statement.tokens.isEmpty() ? null : statement;
+		}
+	})
+	;
+	
+	public final Visitor visitor;
+	
+	private Optimization(Visitor visitor) {
+		this.visitor = visitor;
+	}
+	
+	/** Add support for lower-case Optimization flags. */
+	public static class TypeConverter implements ITypeConverter<Optimization> {
+		@Override
+		public Optimization convert(String value) throws Exception {
+			try {
+				return Optimization.valueOf(value);
+			} catch (IllegalArgumentException ex) {
+				for (Optimization opt : Optimization.values()) {
+					String checkName = opt.name().replace('_', '-');
+					if (checkName.equalsIgnoreCase(value)) {
+						return opt;
+					}
+				}
+				throw ex;
+			}
+		}
+	}
+	/** Common base class for optimization visitors that allow the program tree to be rewritten. */
+	private static class BaseVisitor implements Visitor {
+		@Override
+		public Program visit(Program program) {
+			Program newProgram = new Program();
+			program.lines.forEach(l -> {
+				Line line = l.accept(this);
+				if (line != null) newProgram.lines.add(line);
+			});
+			return newProgram;
+		}
+		@Override
+		public Line visit(Line line) {
+			Line newLine = new Line(line.lineNumber);
+			line.statements.forEach(s -> {
+				Statement statement = s.accept(this);
+				if (statement != null) newLine.statements.add(statement);
+			});
+			return newLine;
+		}
+		@Override
+		public Statement visit(Statement statement) {
+			Statement newStatement = new Statement();
+			statement.tokens.forEach(t -> { 
+				Token token = t.accept(this);
+				if (token != null) newStatement.tokens.add(token);
+			});
+			return newStatement;
+		}
+		@Override
+		public Token visit(Token token) {
+			return token;
+		}
+	}
+}