Now line lengths are capped at 255 (configurable). Fixed a few glitches

in Parser and the line number collector. Closes #2.

src/main/java/com/webcodepro/applecommander/util/applesoft/Parser.java

@@ -47,7 +47,7 @@ public class Parser {
 		while (!tokens.isEmpty()) {
 			if (tokens.peek().type == Type.EOL) break;
 			Token t = tokens.remove();
-			if (":".equals(t.text)) break;
+			if (t.type == Type.SYNTAX && ":".equals(t.text)) break;
 			statement.tokens.add(t);
 		}
 		return statement;

src/main/java/com/webcodepro/applecommander/util/applesoft/Visitors.java

@@ -5,7 +5,6 @@ import java.io.IOException;
 import java.io.PrintStream;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -194,6 +193,14 @@ public class Visitors {
 			return getBytes();
 		}
 		
+		/** A convenience method to get the length of a line. */
+		public int length(Line line) {
+			stack.push(new ByteArrayOutputStream());
+			line.accept(this);
+			return stack.pop().size();
+			
+		}
+		
 		public byte[] getBytes() {
 			if (stack.size() != 1) {
 				throw new RuntimeException("Error in processing internal BASIC model!");
@@ -353,7 +360,7 @@ public class Visitors {
 	}
 
 	public static class LineNumberTargetCollector implements Visitor {
-		private Set<Integer> targets = new HashSet<>();
+		private Set<Integer> targets = new TreeSet<>();
 		
 		public Set<Integer> getTargets() {
 			return targets;
@@ -375,15 +382,18 @@ public class Visitors {
 		@Override
 		public Statement visit(Statement statement) {
 			boolean next = false;
+			boolean multiple = false;
 			for (Token t : statement.tokens) {
 				if (next) {
 					if (t.type == Type.NUMBER) {
 						targets.add(t.number.intValue());
 					}
+					next = multiple;	// preserve next based on if we have multiple line numbers or not.
 				} else {
 					next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
 						|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
 						|| t.keyword == ApplesoftKeyword.LIST;
+					multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
 				}
 			}
 			return statement;

src/main/java/io/github/applecommander/bastokenizer/Main.java

@@ -3,6 +3,7 @@ package io.github.applecommander.bastokenizer;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.PrintStream;
 import java.nio.file.Files;
 import java.util.ArrayList;
@@ -15,6 +16,7 @@ import java.util.function.BiConsumer;
 import com.webcodepro.applecommander.util.applesoft.Parser;
 import com.webcodepro.applecommander.util.applesoft.Program;
 import com.webcodepro.applecommander.util.applesoft.Token;
+import com.webcodepro.applecommander.util.applesoft.Token.Type;
 import com.webcodepro.applecommander.util.applesoft.TokenReader;
 import com.webcodepro.applecommander.util.applesoft.Visitors;
 
@@ -32,31 +34,34 @@ import picocli.CommandLine.Parameters;
 		versionProvider = Main.VersionProvider.class)
 public class Main implements Callable<Void> {
 	@Option(names = { "-o", "--output" }, description = "Write binary output to file.")
-	private File outputFile;
+	File outputFile;
 	
 	@Option(names = { "-x", "--hex"}, description = "Generate a binary hex dump for debugging.")
-	private boolean hexFormat;
+	boolean hexFormat;
 
 	@Option(names = { "-c", "--copy"}, description = "Generate a copy/paste form of output for testing in an emulator.")
-	private boolean copyFormat;
+	boolean copyFormat;
 	
 	@Option(names = { "-a", "--address" }, description = "Base address for program", showDefaultValue = Visibility.ALWAYS, converter = IntegerTypeConverter.class)
-	private int address = 0x801;
+	int address = 0x801;
 	
 	@Option(names = { "--variables" }, description = "Generate a variable report")
-	private boolean showVariableReport;
+	boolean showVariableReport;
 	
 	@Option(names = { "-p", "--pipe" }, description = "Pipe binary output to stdout.")
-	private boolean pipeOutput;
+	boolean pipeOutput;
 	
 	@Option(names = "--pretty", description = "Pretty print structure as bastokenizer understands it.")
-	private boolean prettyPrint;
+	boolean prettyPrint;
 
 	@Option(names = "--list", description = "List structure as bastokenizer understands it.")
-	private boolean listPrint;
+	boolean listPrint;
 
 	@Option(names = "--tokens", description = "Dump token list to stdout for debugging.")
-	private boolean showTokens;
+	boolean showTokens;
+	
+	@Option(names = "--max-line-length", description = "Maximum line length for generated lines.", showDefaultValue = Visibility.ALWAYS)
+	int maxLineLength = 255;
 	
 	@Option(names = "-f", converter = Optimization.TypeConverter.class, split = ",", description = {
 			"Enable specific optimizations.",
@@ -65,13 +70,22 @@ public class Main implements Callable<Void> {
 			"* @|green merge-lines|@ - Merge lines.",
 			"* @|green renumber|@ - Renumber program."
 	})
-	private List<Optimization> optimizations = new ArrayList<>();
+	List<Optimization> optimizations = new ArrayList<>();
 
 	@Option(names = { "-O", "--optimize" }, description = "Apply all optimizations.")
-	private boolean allOptimizations;
+	boolean allOptimizations;
+	
+	@Option(names = "--debug", description = "Print debug output.")
+	boolean debugFlag;
+	PrintStream debug = new PrintStream(new OutputStream() {
+			@Override
+			public void write(int b) throws IOException {
+				// Do nothing
+			}
+		});
 	
 	@Parameters(index = "0", description = "AppleSoft BASIC program to process.")
-	private File sourceFile;
+	File sourceFile;
 	
 	public static void main(String[] args) throws FileNotFoundException, IOException {
 		CommandLine.call(new Main(), args);
@@ -80,6 +94,7 @@ public class Main implements Callable<Void> {
 	@Override
 	public Void call() throws FileNotFoundException, IOException {
 		if (checkParameters()) {
+			if (debugFlag) debug = System.out;
 			process();
 		}
 		
@@ -92,10 +107,11 @@ public class Main implements Callable<Void> {
 			optimizations.clear();
 			optimizations.addAll(Arrays.asList(Optimization.values()));
 		}
-		if (pipeOutput && (hexFormat || copyFormat || prettyPrint || listPrint || showTokens || showVariableReport)) {
+		boolean hasOutput = hexFormat || copyFormat || prettyPrint || listPrint || showTokens || showVariableReport || debugFlag;
+		if (pipeOutput && hasOutput) {
 			System.err.println("The pipe option blocks any other stdout options.");
 			return false;
-		} else if (!(pipeOutput || hexFormat || copyFormat || prettyPrint || listPrint || showTokens || showVariableReport || outputFile != null)) {
+		} else if (!(pipeOutput || hasOutput || outputFile != null)) {
 			System.err.println("What do you want to do?");
 			return false;
 		}
@@ -106,13 +122,14 @@ public class Main implements Callable<Void> {
 	public void process() throws FileNotFoundException, IOException {
 		Queue<Token> tokens = TokenReader.tokenize(sourceFile);
 		if (showTokens) {
-			System.out.println(tokens.toString());
+			tokens.forEach(t -> System.out.printf("%s%s", t, t.type == Type.EOL ? "\n" : ", "));
 		}
 		Parser parser = new Parser(tokens);
 		Program program = parser.parse();
 		
 		for (Optimization optimization : optimizations) {
-			program = program.accept(optimization.visitor);
+			debug.printf("Optimization: %s\n", optimization.name());
+			program = program.accept(optimization.create(this));
 		}
 
 		if (prettyPrint || listPrint) {

src/main/java/io/github/applecommander/bastokenizer/Optimization.java

@@ -1,8 +1,10 @@
 package io.github.applecommander.bastokenizer;
 
+import java.io.PrintStream;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 
 import com.webcodepro.applecommander.util.applesoft.ApplesoftKeyword;
 import com.webcodepro.applecommander.util.applesoft.Line;
@@ -12,81 +14,26 @@ import com.webcodepro.applecommander.util.applesoft.Token;
 import com.webcodepro.applecommander.util.applesoft.Token.Type;
 import com.webcodepro.applecommander.util.applesoft.Visitor;
 import com.webcodepro.applecommander.util.applesoft.Visitors;
+import com.webcodepro.applecommander.util.applesoft.Visitors.ByteVisitor;
 import com.webcodepro.applecommander.util.applesoft.Visitors.LineNumberTargetCollector;
 
 import picocli.CommandLine.ITypeConverter;
 
 public enum Optimization {
-	REMOVE_EMPTY_STATEMENTS(new BaseVisitor() {
-		@Override
-		public Statement visit(Statement statement) {
-			return statement.tokens.isEmpty() ? null : statement;
-		}
-	}),
-	REMOVE_REM_STATEMENTS(new BaseVisitor() {
-		@Override
-		public Statement visit(Statement statement) {
-			return statement.tokens.get(0).type == Type.COMMENT ? null : statement;
-		}
-	}),
-	MERGE_LINES(new BaseVisitor() {
-		private Set<Integer> targets;
-		private Line mergeLine;
-		@Override
-		public Program visit(Program program) {
-			LineNumberTargetCollector c = Visitors.lineNumberTargetCollector();
-			program.accept(c);
-			targets = c.getTargets();
-			return super.visit(program);
-		}
-		@Override
-		public Line visit(Line line) {
-			if (mergeLine == null || targets.contains(line.lineNumber)) {
-				// merge may null out mergeLine if the this line has a "terminal".  
-				// Preserve it with newLine so it get added to the program.
-				Line newLine = new Line(line.lineNumber);
-				mergeLine = newLine;
-				merge(line);
-				return newLine;
-			}
-			merge(line);
-			// Do not preserve old line!
-			return null;
-		}
-		private void merge(Line line) {
-			mergeLine.statements.addAll(line.statements);
-			// Terminals are: IF, REM, GOTO, END, ON .. GOTO (GOTO is trigger), RESUME, RETURN, STOP
-			boolean terminal = false;
-			for (Statement s : line.statements) {
-				for (Token t : s.tokens) {
-					terminal |= t.keyword == ApplesoftKeyword.IF || t.type == Type.COMMENT /* REM */
-							|| t.keyword == ApplesoftKeyword.GOTO || t.keyword == ApplesoftKeyword.END
-							|| t.keyword == ApplesoftKeyword.RESUME || t.keyword == ApplesoftKeyword.RETURN
-							|| t.keyword == ApplesoftKeyword.STOP;
-				}
-			}
-			if (terminal) {
-				mergeLine = null;
-			}
-		}
-	}),
-	RENUMBER(new BaseVisitor() {
-		protected int lineNumber = 0;
-		@Override
-		public Line visit(Line line) {
-			Line newLine = new Line(lineNumber++);
-			newLine.statements.addAll(line.statements);
-			// Track what went where so lines can get renumbered automatically
-			reassignments.put(line.lineNumber, newLine.lineNumber);
-			return newLine;
-		}
-	})
+	REMOVE_EMPTY_STATEMENTS(opts -> new RemoveEmptyStatements()),
+	REMOVE_REM_STATEMENTS(opts -> new RemoveRemStatements()),
+	MERGE_LINES(opts -> new MergeLines(opts)),
+	RENUMBER(opts -> new Renumber())
 	;
 	
-	public final Visitor visitor;
+	private Function<Main,Visitor> factory;
 	
-	private Optimization(Visitor visitor) {
-		this.visitor = visitor;
+	private Optimization(Function<Main,Visitor> factory) {
+		this.factory = factory;
+	}
+	
+	public Visitor create(Main options) {
+		return factory.apply(options);
 	}
 	
 	/** Add support for lower-case Optimization flags. */
@@ -153,4 +100,88 @@ public enum Optimization {
 			return token;
 		}
 	}
+	private static class RemoveEmptyStatements extends BaseVisitor {
+		@Override
+		public Statement visit(Statement statement) {
+			return statement.tokens.isEmpty() ? null : statement;
+		}
+	}
+	private static class RemoveRemStatements extends BaseVisitor {
+		@Override
+		public Statement visit(Statement statement) {
+			return statement.tokens.get(0).type == Type.COMMENT ? null : statement;
+		}
+	}
+	private static class MergeLines extends BaseVisitor {
+		private Set<Integer> targets;
+		private Line mergeLine;
+		private ByteVisitor bv = Visitors.byteVisitor(0x801);
+		private int maxLineLength;
+		private PrintStream debug;
+		private MergeLines(Main options) {
+			this.maxLineLength = options.maxLineLength;
+			this.debug = options.debug;
+		}
+		@Override
+		public Program visit(Program program) {
+			LineNumberTargetCollector c = Visitors.lineNumberTargetCollector();
+			program.accept(c);
+			targets = c.getTargets();
+			debug.printf("Target lines = %s\n", targets);
+			return super.visit(program);
+		}
+		@Override
+		public Line visit(Line line) {
+			debug.printf("Line # %d : ", line.lineNumber);
+			Line newLine = new Line(line.lineNumber);
+			newLine.statements.addAll(line.statements);
+			if (mergeLine == null || targets.contains(line.lineNumber)) {
+				// Either forced to a new line or this is a GOTO type target: Ignore length
+				debug.printf("%s\n", mergeLine == null ? "mergeLine is null" : "target line #");
+			} else {
+				// Check length and decide if it merges based on that.
+				Line tmpLine = new Line(mergeLine.lineNumber);
+				tmpLine.statements.addAll(mergeLine.statements);
+				tmpLine.statements.addAll(line.statements);
+				if (bv.length(tmpLine) > maxLineLength) {
+					// It was too big, do not add
+					debug.printf("merge would exceed max line length: %d > %d\n", bv.length(tmpLine), maxLineLength);
+				} else {
+					// We can add line to mergeLine (mergeLine is already added to program, must keep that object)
+					mergeLine.statements.addAll(line.statements);
+					if (hasTerminal(line)) mergeLine = null;
+					debug.printf("line %s\n", mergeLine == null ? "had terminals" : "was added to mergeLine");
+					return null;
+				}
+			}
+			// Always reset mergeLine based on the terminal characteristics
+			mergeLine = hasTerminal(line) ? null : newLine;
+			debug.printf("line %s\n", mergeLine == null ? "had terminals" : "is now mergeLine");
+			return newLine;
+		}
+		private boolean hasTerminal(Line line) {
+			// Terminals are: IF, REM, GOTO, END, ON .. GOTO (GOTO is trigger), RESUME, RETURN, STOP
+			for (Statement s : line.statements) {
+				for (Token t : s.tokens) {
+					boolean terminal = t.keyword == ApplesoftKeyword.IF || t.type == Type.COMMENT /* REM */
+							|| t.keyword == ApplesoftKeyword.GOTO || t.keyword == ApplesoftKeyword.END
+							|| t.keyword == ApplesoftKeyword.RESUME || t.keyword == ApplesoftKeyword.RETURN
+							|| t.keyword == ApplesoftKeyword.STOP;
+					if (terminal) return true;
+				}
+			}
+			return false;
+		}
+	}
+	private static class Renumber extends BaseVisitor {
+		protected int lineNumber = 0;
+		@Override
+		public Line visit(Line line) {
+			Line newLine = new Line(lineNumber++);
+			newLine.statements.addAll(line.statements);
+			// Track what went where so lines can get renumbered automatically
+			reassignments.put(line.lineNumber, newLine.lineNumber);
+			return newLine;
+		}
+	}
 }