Migrated to Gradle; spearated into 'api' and 'tools/bt' projects. Closes #15.

.gitignore

@@ -1,7 +1,22 @@
-/target/
-/.settings/
-/.classpath
-/.project
-pom.xml.versionsBackup
+# Ignore Gradle GUI config
+gradle-app.setting
+
+# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
+!gradle-wrapper.jar
+
+# Cache of project
+.gradletasknamecache
+
+# Eclipse
+.settings/
+.classpath
+.project
+bin/
+
+# Misc. Apple II remnants
 *.bin
 *.dsk
+
+# Gradle extras
+.gradle/
+build/

api/build.gradle

@@ -0,0 +1,71 @@
+repositories {
+    jcenter()
+}
+
+apply plugin: 'java-library'
+apply plugin: 'maven'
+apply plugin: 'signing'
+
+dependencies {
+    compile group: 'net.sf.applecommander', name: 'applesingle-api', version:'1.1.0'
+    testImplementation 'junit:junit:4.12'
+}
+
+task javadocJar(type: Jar) {
+    classifier = 'javadoc'
+    from javadoc
+}
+
+task sourcesJar(type: Jar) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+}
+
+artifacts {
+    archives javadocJar, sourcesJar
+}
+
+signing {
+    sign configurations.archives
+}
+
+uploadArchives {
+  repositories {
+    mavenDeployer {
+      beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
+
+      repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") {
+        authentication(userName: findProperty('ossrhUsername'), password: findProperty('ossrhPassword'))
+      }
+
+      snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") {
+        authentication(userName: findProperty('ossrhUsername'), password: findProperty('ossrhPassword'))
+      }
+
+      pom.project {
+        name archivesBaseName
+        packaging 'jar'
+        description 'Experiments with generating an AppleSoft B/BAS tokenized "binary".'
+        url 'https://applecommander.github.io/'
+
+        scm {
+          url 'https://github.com/AppleCommander/bastokenizer'
+        }
+
+        licenses {
+          license {
+            name 'The GNU General Public License (GPL) Version 3, 29 June 2007'
+            url 'https://www.gnu.org/licenses/gpl-3.0.html'
+          }
+        }
+
+        developers {
+          developer {
+            id 'robgreene'
+            email 'robgreene@gmail.com'
+          }
+        }
+      }
+    }
+  }
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/Configuration.java

@@ -0,0 +1,61 @@
+package io.github.applecommander.bastokenizer.api;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Objects;
+
+public class Configuration {
+	public final File sourceFile;
+	public final int startAddress;
+	public final int maxLineLength;
+	public final PrintStream debugStream;
+	
+	private Configuration(Builder b) {
+		this.sourceFile = b.sourceFile;
+		this.startAddress = b.startAddress;
+		this.maxLineLength = b.maxLineLength;
+		this.debugStream = b.debugStream;
+	}
+	
+	public static Builder builder() {
+		return new Builder();
+	}
+	public static class Builder {
+		private Builder() { /* Prevent construction */ }
+		
+		private File sourceFile;
+		private int startAddress = 0x801;
+		private int maxLineLength = 255;
+		private PrintStream debugStream = new PrintStream(new OutputStream() {
+				@Override
+				public void write(int b) throws IOException {
+					// Do nothing
+				}
+			});
+
+		public Builder sourceFile(File sourceFile) {
+			this.sourceFile = sourceFile;
+			return this;
+		}
+		public Builder startAddress(int startAddress) {
+			this.startAddress = startAddress;
+			return this;
+		}
+		public Builder maxLineLength(int maxLineLength) {
+			this.maxLineLength = maxLineLength;
+			return this;
+		}
+		public Builder debugStream(PrintStream debugStream) {
+			this.debugStream = debugStream;
+			return this;
+		}
+		
+		public Configuration build() {
+			Objects.requireNonNull(sourceFile, "Please configure a sourceFile");
+			Objects.requireNonNull(debugStream, "debugStream cannot be null");
+			return new Configuration(this);
+		}
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/Directive.java

@@ -0,0 +1,84 @@
+package io.github.applecommander.bastokenizer.api;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+import io.github.applecommander.bastokenizer.api.utils.Converters;
+
+public abstract class Directive {
+	protected Configuration config;
+	protected OutputStream outputStream;
+	protected List<Token> parameters = new ArrayList<>();
+
+	protected Directive(Configuration config, OutputStream outputStream) {
+		Objects.requireNonNull(config);
+		Objects.requireNonNull(outputStream);
+		this.config = config;
+		this.outputStream = outputStream;
+	}
+	
+	public void append(Token token) {
+		// Skip the commas...
+		if (token.type == Type.SYNTAX && ",".equals(token.text)) return;
+		parameters.add(token);
+	}
+	protected Token require(Type... types) {
+		Token t = parameters.remove(0);
+		boolean matches = false;
+		for (Type type : types) {
+			matches |= type == t.type;
+		}
+		if (!matches) {
+			throw new IllegalArgumentException("Expecting a type of " + types);
+		}
+		return t;
+	}
+	protected String requiresString() {
+		Token t = require(Type.STRING);
+		return t.text;
+	}
+	protected int requiresInteger() {
+		Token t = require(Type.NUMBER, Type.STRING);
+		if (t.type == Type.NUMBER) {
+			return t.number.intValue();
+		}
+		return Converters.toInteger(t.text);
+	}
+	
+	protected void ldy(int value) throws IOException {
+		outputStream.write(0xa0);
+		outputStream.write(value);
+	}
+	protected void jmp(int address) throws IOException {
+		outputStream.write(0x4c);
+		outputStream.write(address & 0xff);
+		outputStream.write(address >> 8);
+	}
+	protected void lda(int value) throws IOException {
+		outputStream.write(0xa9);
+		outputStream.write(value);
+	}
+	protected void sta(int address) throws IOException {
+		if ((address & 0xff00) == 0) {
+			outputStream.write(0x85);
+			outputStream.write(address);
+		} else {
+			throw new RuntimeException("sta does not handle 16 bit addresses yet!");
+		}
+	}
+	protected void setAddress(int value, int address) throws IOException {
+		lda(value & 0xff);
+		sta(address);
+		lda(value >> 8);
+		sta(address+1);
+	}
+	
+	/** Write directive contents to output file. Note that address is adjusted for the line header already. */
+	public abstract void writeBytes(int startAddress, Line line) throws IOException;
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/Directives.java

@@ -0,0 +1,32 @@
+package io.github.applecommander.bastokenizer.api;
+
+import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.util.Map;
+import java.util.TreeMap;
+
+import io.github.applecommander.bastokenizer.api.directives.EmbeddedBinaryDirective;
+
+public abstract class Directives {
+	private static Map<String,Class<? extends Directive>> DIRECTIVES = 
+		new TreeMap<String,Class<? extends Directive>>(String.CASE_INSENSITIVE_ORDER) {
+			private static final long serialVersionUID = -8111460701487331592L;
+
+			{
+				put("$embed", EmbeddedBinaryDirective.class);
+			}
+		};
+
+	public static Directive find(String text, Configuration config, OutputStream outputStream) {
+		if (DIRECTIVES.containsKey(text)) {
+			try {
+				Class<? extends Directive> clazz = DIRECTIVES.get(text);
+				Constructor<? extends Directive> constructor = clazz.getConstructor(Configuration.class, OutputStream.class);
+				return constructor.newInstance(config, outputStream);
+			} catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
+				throw new IllegalArgumentException(String.format("Unable to construct directive '%s'", text), e);
+			}
+		}
+		throw new IllegalArgumentException(String.format("Unable to find directive '%s'", text));
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/Optimization.java

@@ -0,0 +1,26 @@
+package io.github.applecommander.bastokenizer.api;
+
+import java.util.function.Function;
+
+import io.github.applecommander.bastokenizer.api.optimizations.MergeLines;
+import io.github.applecommander.bastokenizer.api.optimizations.RemoveEmptyStatements;
+import io.github.applecommander.bastokenizer.api.optimizations.RemoveRemStatements;
+import io.github.applecommander.bastokenizer.api.optimizations.Renumber;
+
+public enum Optimization {
+	REMOVE_EMPTY_STATEMENTS(config -> new RemoveEmptyStatements()),
+	REMOVE_REM_STATEMENTS(config -> new RemoveRemStatements()),
+	MERGE_LINES(config -> new MergeLines(config)),
+	RENUMBER(config -> new Renumber());
+	
+	private Function<Configuration,Visitor> factory;
+	
+	private Optimization(Function<Configuration,Visitor> factory) {
+		this.factory = factory;
+	}
+	
+	public Visitor create(Configuration config) {
+		return factory.apply(config);
+	}
+	
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/Parser.java

@@ -1,9 +1,13 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api;
 
 import java.util.Objects;
 import java.util.Queue;
 
-import com.webcodepro.applecommander.util.applesoft.Token.Type;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
 
 /** 
  * The Parser will read a series of Tokens and build a Program.

api/src/main/java/io/github/applecommander/bastokenizer/api/TokenReader.java

@@ -1,4 +1,4 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -12,6 +12,9 @@ import java.util.LinkedList;
 import java.util.Optional;
 import java.util.Queue;
 
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Token;
+
 /**
  * The TokenReader, given a text file, generates a series of Tokens (in the compiler sense, 
  * not AppleSoft) for the AppleSoft program.

api/src/main/java/io/github/applecommander/bastokenizer/api/Visitor.java

@@ -1,4 +1,9 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api;
+
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
 
 /**
  * The Visitor interface allows some flexibility in what can be done with the

api/src/main/java/io/github/applecommander/bastokenizer/api/Visitors.java

@@ -0,0 +1,73 @@
+package io.github.applecommander.bastokenizer.api;
+
+import java.io.PrintStream;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+import io.github.applecommander.bastokenizer.api.visitors.ByteVisitor;
+import io.github.applecommander.bastokenizer.api.visitors.LineNumberTargetCollector;
+import io.github.applecommander.bastokenizer.api.visitors.PrettyPrintVisitor;
+import io.github.applecommander.bastokenizer.api.visitors.PrintVisitor;
+import io.github.applecommander.bastokenizer.api.visitors.ReassignmentVisitor;
+import io.github.applecommander.bastokenizer.api.visitors.VariableReportVisitor;
+
+/**
+ * This class presents all of the common Visitor implementations via builder patterns.
+ * The number is currently small enough that all the builders and visitors are defined 
+ * in this one class.
+ *  
+ * @author rob
+ */
+public class Visitors {
+	public static PrintBuilder printBuilder() {
+		return new PrintBuilder();
+	}
+	public static class PrintBuilder {
+		private PrintStream printStream = System.out;
+		private Function<PrintBuilder,Visitor> creator = PrintVisitor::new;
+		
+		public PrintBuilder printStream(PrintStream printStream) {
+			Objects.requireNonNull(printStream);
+			this.printStream = printStream;
+			return this;
+		}
+		public PrintBuilder prettyPrint(boolean flag) {
+			creator = flag ? PrettyPrintVisitor::new : PrintVisitor::new;
+			return this;
+		}
+		public PrintBuilder prettyPrint() {
+			creator = PrettyPrintVisitor::new;
+			return this;
+		}
+		public PrintBuilder print() {
+			creator = PrintVisitor::new;
+			return this;
+		}
+		
+		public Visitor build() {
+			return creator.apply(this);
+		}
+		public PrintStream getPrintStream() {
+			return printStream;
+		}
+	}
+	
+	public static ByteVisitor byteVisitor(Configuration config) {
+		return new ByteVisitor(config);
+	}
+	
+	/** Rewrite the Program tree with the line number reassignments given. */
+	public static ReassignmentVisitor reassignVisitor(Map<Integer,Integer> reassignments) {
+		return new ReassignmentVisitor(reassignments);
+	}
+	
+	/** Collect all line numbers that are a target of GOTO, GOSUB, etc. */
+	public static LineNumberTargetCollector lineNumberTargetCollector() {
+		return new LineNumberTargetCollector();
+	}
+	
+	public static Visitor variableReportVisitor() {
+		return new VariableReportVisitor();
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/directives/EmbeddedBinaryDirective.java

@@ -0,0 +1,81 @@
+package io.github.applecommander.bastokenizer.api.directives;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.Optional;
+
+import io.github.applecommander.bastokenizer.api.Configuration;
+import io.github.applecommander.bastokenizer.api.Directive;
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Line;
+
+public class EmbeddedBinaryDirective extends Directive {
+	public EmbeddedBinaryDirective(Configuration config, OutputStream outputStream) {
+		super(config, outputStream);
+	}
+	
+	@Override
+	public void writeBytes(int startAddress, Line line) throws IOException {
+		if (parameters.size() != 2) {
+			throw new IllegalArgumentException("$embed requires a name and address parameter");
+		}
+		String filename = requiresString();
+		int targetAddress = requiresInteger();
+
+		File file = new File(config.sourceFile.getParentFile(), filename);
+		byte[] bin = Files.readAllBytes(file.toPath());
+		
+		Optional<Line> nextLine = line.nextLine();
+		byte[] basicCode = nextLine.isPresent() 
+				? callAndGoto(startAddress,nextLine.get()) 
+				: callAndReturn(startAddress);
+		
+		final int moveLength = 8*3 + 2 + 3;	// LDA/STA, LDY, JMP.
+		int embeddedStart = startAddress + basicCode.length + moveLength;
+		int embeddedEnd = embeddedStart + bin.length;
+		
+		outputStream.write(basicCode);
+		setAddress(embeddedStart, 0x3c);
+		setAddress(embeddedEnd, 0x3e);
+		setAddress(targetAddress, 0x42);
+		ldy(0x00);
+		jmp(0xfe2c);
+		outputStream.write(bin);
+	}
+	// In program, "CALL <address>:GOTO line"
+	private byte[] callAndGoto(int startAddress, Line line) throws IOException {
+		// 3 for the tokens "CALL", ":", "GOTO", end of line (0x00)
+		final int tokenCount = 3 + 1;
+		int offset = Integer.toString(line.lineNumber).length() + tokenCount;
+		offset += Integer.toString(startAddress).length();
+		// Attempting to adjust if we bump from 4 digit address to a 5 digit address
+		if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
+		ByteArrayOutputStream os = new ByteArrayOutputStream();
+		os.write(ApplesoftKeyword.CALL.code);
+		os.write(Integer.toString(startAddress+offset).getBytes());
+		os.write(':');
+		os.write(ApplesoftKeyword.GOTO.code);
+		os.write(Integer.toString(line.lineNumber).getBytes());
+		os.write(0x00);
+		return os.toByteArray();
+	}
+	// At end of program, just "CALL <address>:RETURN"
+	private byte[] callAndReturn(int startAddress) throws IOException {
+		// 3 for the tokens "CALL", ":", "RETURN", end of line (0x00)
+		final int tokenCount = 3 + 1;
+		int offset = tokenCount;
+		offset += Integer.toString(startAddress).length();
+		// Attempting to adjust if we bump from 4 digit address to a 5 digit address
+		if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
+		ByteArrayOutputStream os = new ByteArrayOutputStream();
+		os.write(ApplesoftKeyword.CALL.code);
+		os.write(Integer.toString(startAddress+offset).getBytes());
+		os.write(':');
+		os.write(ApplesoftKeyword.RETURN.code);
+		os.write(0x00);
+		return os.toByteArray();
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/model/ApplesoftKeyword.java

@@ -1,4 +1,4 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api.model;
 
 import java.io.IOException;
 import java.io.Reader;

api/src/main/java/io/github/applecommander/bastokenizer/api/model/Line.java

@@ -1,10 +1,12 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api.model;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
+import io.github.applecommander.bastokenizer.api.Visitor;
+
 /** An AppleSoft BASIC Line representation. */
 public class Line {
 	public final Program program;

api/src/main/java/io/github/applecommander/bastokenizer/api/model/Program.java

@@ -1,8 +1,10 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import io.github.applecommander.bastokenizer.api.Visitor;
+
 /** A Program is a series of lines. */
 public class Program {
 	public final List<Line> lines = new ArrayList<>();

api/src/main/java/io/github/applecommander/bastokenizer/api/model/Statement.java

@@ -1,8 +1,10 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import io.github.applecommander.bastokenizer.api.Visitor;
+
 /** A Statement is simply a series of Tokens. */
 public class Statement {
 	public final List<Token> tokens = new ArrayList<>();

api/src/main/java/io/github/applecommander/bastokenizer/api/model/Token.java

@@ -1,4 +1,6 @@
-package com.webcodepro.applecommander.util.applesoft;
+package io.github.applecommander.bastokenizer.api.model;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
 
 /**
  * A Token in the classic compiler sense, in that this represents a component of the application.

api/src/main/java/io/github/applecommander/bastokenizer/api/optimizations/BaseVisitor.java

@@ -0,0 +1,64 @@
+package io.github.applecommander.bastokenizer.api.optimizations;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.Visitors;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+
+/** Common base class for optimization visitors that allow the program tree to be rewritten. */
+public class BaseVisitor implements Visitor {
+	protected Map<Integer,Integer> reassignments = new HashMap<>();
+	protected Program newProgram;
+	
+	@Override
+	public Program visit(Program program) {
+		newProgram = new Program();
+		program.lines.forEach(l -> {
+			Line line = l.accept(this);
+			boolean lineKept = line != null && !line.statements.isEmpty();
+			if (lineKept) {
+				newProgram.lines.add(line);
+				reassignments.replaceAll((k,v) -> v == null ? l.lineNumber : v);
+			} else {
+				// Make a place-holder for the reassignment; we'll patch it in once we find a line that sticks around.
+				reassignments.put(l.lineNumber, null);
+			}
+		});
+		if (!reassignments.isEmpty()) {
+			// Now, renumber based on our findings!
+			return newProgram.accept(Visitors.reassignVisitor(reassignments));
+		} else {
+			return newProgram;
+		}
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		Line newLine = new Line(line.lineNumber, this.newProgram);
+		line.statements.forEach(s -> {
+			Statement statement = s.accept(this);
+			if (statement != null) newLine.statements.add(statement);
+		});
+		return newLine;
+	}
+	
+	@Override
+	public Statement visit(Statement statement) {
+		Statement newStatement = new Statement();
+		statement.tokens.forEach(t -> { 
+			Token token = t.accept(this);
+			if (token != null) newStatement.tokens.add(token);
+		});
+		return newStatement;
+	}
+	
+	@Override
+	public Token visit(Token token) {
+		return token;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/optimizations/MergeLines.java

@@ -0,0 +1,84 @@
+package io.github.applecommander.bastokenizer.api.optimizations;
+
+import java.io.PrintStream;
+import java.util.Set;
+
+import io.github.applecommander.bastokenizer.api.Configuration;
+import io.github.applecommander.bastokenizer.api.Visitors;
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+import io.github.applecommander.bastokenizer.api.visitors.ByteVisitor;
+import io.github.applecommander.bastokenizer.api.visitors.LineNumberTargetCollector;
+
+public class MergeLines extends BaseVisitor {
+	private Set<Integer> targets;
+	private Line mergeLine;
+	private ByteVisitor bv;
+	private int maxLineLength;
+	private PrintStream debugStream;
+	
+	public MergeLines(Configuration config) {
+		this.maxLineLength = config.maxLineLength;
+		this.debugStream = config.debugStream;
+		this.bv = Visitors.byteVisitor(config);
+	}
+	
+	@Override
+	public Program visit(Program program) {
+		LineNumberTargetCollector c = Visitors.lineNumberTargetCollector();
+		program.accept(c);
+		targets = c.getTargets();
+		debugStream.printf("Target lines = %s\n", targets);
+		return super.visit(program);
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		debugStream.printf("Line # %d : ", line.lineNumber);
+		Line newLine = new Line(line.lineNumber, this.newProgram);
+		newLine.statements.addAll(line.statements);
+		if (mergeLine == null || targets.contains(line.lineNumber)) {
+			// Either forced to a new line or this is a GOTO type target: Ignore length
+			debugStream.printf("%s\n", mergeLine == null ? "mergeLine is null" : "target line #");
+		} else {
+			// Check length and decide if it merges based on that.
+			Line tmpLine = new Line(mergeLine.lineNumber, mergeLine.program);
+			tmpLine.statements.addAll(mergeLine.statements);
+			tmpLine.statements.addAll(line.statements);
+			if (bv.length(tmpLine) > maxLineLength) {
+				// It was too big, do not add
+				debugStream.printf("merge would exceed max line length: %d > %d\n", bv.length(tmpLine), maxLineLength);
+			} else {
+				// We can add line to mergeLine (mergeLine is already added to program, must keep that object)
+				mergeLine.statements.addAll(line.statements);
+				if (hasTerminal(line)) mergeLine = null;
+				debugStream.printf("line %s\n", mergeLine == null ? "had terminals" : "was added to mergeLine");
+				return null;
+			}
+		}
+		// Always reset mergeLine based on the terminal characteristics
+		mergeLine = hasTerminal(line) ? null : newLine;
+		debugStream.printf("line %s\n", mergeLine == null ? "had terminals" : "is now mergeLine");
+		return newLine;
+	}
+	
+	private boolean hasTerminal(Line line) {
+		// Terminals are: IF, REM, GOTO, END, ON .. GOTO (GOTO is trigger), RESUME, RETURN, STOP
+		// Includes directives.
+		for (Statement s : line.statements) {
+			for (Token t : s.tokens) {
+				boolean terminal = t.keyword == ApplesoftKeyword.IF || t.type == Type.COMMENT /* REM */
+						|| t.keyword == ApplesoftKeyword.GOTO || t.keyword == ApplesoftKeyword.END
+						|| t.keyword == ApplesoftKeyword.RESUME || t.keyword == ApplesoftKeyword.RETURN
+						|| t.keyword == ApplesoftKeyword.STOP
+						|| t.type == Type.DIRECTIVE;
+				if (terminal) return true;
+			}
+		}
+		return false;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/optimizations/RemoveEmptyStatements.java

@@ -0,0 +1,10 @@
+package io.github.applecommander.bastokenizer.api.optimizations;
+
+import io.github.applecommander.bastokenizer.api.model.Statement;
+
+public class RemoveEmptyStatements extends BaseVisitor {
+	@Override
+	public Statement visit(Statement statement) {
+		return statement.tokens.isEmpty() ? null : statement;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/optimizations/RemoveRemStatements.java

@@ -0,0 +1,11 @@
+package io.github.applecommander.bastokenizer.api.optimizations;
+
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+
+public class RemoveRemStatements extends BaseVisitor {
+	@Override
+	public Statement visit(Statement statement) {
+		return statement.tokens.get(0).type == Type.COMMENT ? null : statement;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/optimizations/Renumber.java

@@ -0,0 +1,15 @@
+package io.github.applecommander.bastokenizer.api.optimizations;
+
+import io.github.applecommander.bastokenizer.api.model.Line;
+
+public class Renumber extends BaseVisitor {
+	protected int lineNumber = 0;
+	@Override
+	public Line visit(Line line) {
+		Line newLine = new Line(lineNumber++, this.newProgram);
+		newLine.statements.addAll(line.statements);
+		// Track what went where so lines can get renumbered automatically
+		reassignments.put(line.lineNumber, newLine.lineNumber);
+		return newLine;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/utils/Converters.java

@@ -0,0 +1,22 @@
+package io.github.applecommander.bastokenizer.api.utils;
+
+public class Converters {
+	private Converters() { /* Prevent construction */ }
+	
+	/** 
+	 * Convert a string to an integer allowing multiple formats.  
+	 * Normal decimal, or hexadecimal with a <code>$</code> or <code>0x</code> prefix. 
+	 */
+	public static Integer toInteger(String value) {
+		if (value == null) {
+			return null;
+		} else if (value.startsWith("$")) {
+			return Integer.valueOf(value.substring(1), 16);
+		} else if (value.startsWith("0x") || value.startsWith("0X")) {
+			return Integer.valueOf(value.substring(2), 16);
+		} else {
+			return Integer.valueOf(value);
+		}
+	}
+
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/ByteVisitor.java

@@ -0,0 +1,159 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Stack;
+import java.util.TreeMap;
+
+import io.github.applecommander.bastokenizer.api.Configuration;
+import io.github.applecommander.bastokenizer.api.Directive;
+import io.github.applecommander.bastokenizer.api.Directives;
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+
+public class ByteVisitor implements Visitor {
+	private Stack<ByteArrayOutputStream> stack;
+	private Map<Integer,Integer> lineAddresses;
+	private Configuration config;
+	private int address;
+	private Directive currentDirective;
+	
+	public ByteVisitor(Configuration config) {
+		this.config = config;
+		this.address = config.startAddress;
+		this.stack = new Stack<>();
+		this.lineAddresses = new TreeMap<>();
+	}
+	
+	/** A convenience method to invoke {@link Program#accept(Visitor)} and {@link #getBytes()}. */
+	public byte[] dump(Program program) {
+		program.accept(this);
+		return getBytes();
+	}
+	
+	/** A convenience method to get the length of a line. */
+	public int length(Line line) {
+		stack.push(new ByteArrayOutputStream());
+		line.accept(this);
+		return stack.pop().size();
+	}
+	
+	public Map<Integer, Integer> getLineAddresses() {
+		return lineAddresses;
+	}
+	
+	public byte[] getBytes() {
+		if (stack.size() != 1) {
+			throw new RuntimeException("Error in processing internal BASIC model!");
+		}
+		return stack.peek().toByteArray();
+	}
+	
+	@Override
+	public Program visit(Program program) {
+		stack.clear();
+		stack.push(new ByteArrayOutputStream());
+		program.lines.forEach(line -> line.accept(this));
+		ByteArrayOutputStream os = stack.peek();
+		os.write(0x00);
+		os.write(0x00);
+		return program;
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		try {
+			stack.push(new ByteArrayOutputStream());
+			boolean first = true;
+			for (Statement statement : line.statements) {
+				if (currentDirective != null) {
+					throw new RuntimeException("No statements are allowed after a directive!");
+				}
+				if (!first) {
+					stack.peek().write(':');
+				}
+				first = false;
+				statement.accept(this);
+			}
+			if (currentDirective != null) {
+				currentDirective.writeBytes(this.address+4, line);
+				currentDirective = null;
+			}
+
+			this.lineAddresses.put(line.lineNumber, this.address);
+			byte[] content = stack.pop().toByteArray();
+			int nextAddress = address + content.length + 5;
+			ByteArrayOutputStream os = stack.peek();
+			os.write(nextAddress);
+			os.write(nextAddress >> 8);
+			os.write(line.lineNumber);
+			os.write(line.lineNumber >> 8);
+			os.write(content);
+			os.write(0x00);
+			this.address = nextAddress;
+			return line;
+		} catch (IOException ex) {
+			// Hiding the IOException as ByteArrayOutputStream does not throw it
+			throw new RuntimeException(ex);
+		}
+	}
+
+	@Override
+	public Token visit(Token token) {
+		if (currentDirective != null) {
+			currentDirective.append(token);
+			return token;
+		}
+		try {
+			ByteArrayOutputStream os = stack.peek();
+			switch (token.type) {
+			case COMMENT:
+				os.write(ApplesoftKeyword.REM.code);
+				os.write(token.text.getBytes());
+				break;
+			case EOL:
+				os.write(0x00);
+				break;
+			case IDENT:
+				os.write(token.text.getBytes());
+				break;
+			case KEYWORD:
+				os.write(token.keyword.code);
+				break;
+			case DIRECTIVE:
+				currentDirective = Directives.find(token.text, config, os);
+				break;
+			case NUMBER:
+				if (Math.rint(token.number) == token.number) {
+					os.write(Integer.toString(token.number.intValue()).getBytes());
+				} else {
+					os.write(Double.toString(token.number).getBytes());
+				}
+				break;
+			case STRING:
+				os.write('"');
+				os.write(token.text.getBytes());
+				os.write('"');
+				break;
+			case SYNTAX:
+				Optional<ApplesoftKeyword> opt = ApplesoftKeyword.find(token.text);
+				if (opt.isPresent()) {
+					os.write(opt.get().code);
+				} else {
+					os.write(token.text.getBytes());
+				}
+				break;
+			}
+			return token;
+		} catch (IOException ex) {
+			// Hiding the IOException as ByteArrayOutputStream does not throw it
+			throw new RuntimeException(ex);
+		}
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/LineNumberTargetCollector.java

@@ -0,0 +1,51 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+
+public class LineNumberTargetCollector implements Visitor {
+	private Set<Integer> targets = new TreeSet<>();
+	
+	public Set<Integer> getTargets() {
+		return targets;
+	}
+	
+	/**
+	 * We saw a trigger, collect any numbers that follow.
+	 * 
+	 * Trigger cases:
+	 * - GOSUB n
+	 * - GOTO n
+	 * - IF ... THEN n
+	 * - LIST n [ ,m ]
+	 * - ON x GOTO n, m, ...
+	 * - ON x GOSUB n, m, ...
+	 * - ONERR GOTO n
+	 * - RUN n
+	 */
+	@Override
+	public Statement visit(Statement statement) {
+		boolean next = false;
+		boolean multiple = false;
+		for (Token t : statement.tokens) {
+			if (next) {
+				if (t.type == Type.NUMBER) {
+					targets.add(t.number.intValue());
+				}
+				next = multiple;	// preserve next based on if we have multiple line numbers or not.
+			} else {
+				next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
+					|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
+					|| t.keyword == ApplesoftKeyword.LIST;
+				multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
+			}
+		}
+		return statement;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/PrettyPrintVisitor.java

@@ -0,0 +1,65 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.io.PrintStream;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.Visitors.PrintBuilder;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+
+public class PrettyPrintVisitor implements Visitor {
+	private PrintStream printStream;
+	
+	public PrettyPrintVisitor(PrintBuilder builder) {
+		this.printStream = builder.getPrintStream();
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		boolean first = true;
+		for (Statement statement : line.statements) {
+			if (first) {
+				first = false;
+				printStream.printf("%5d ", line.lineNumber);
+			} else {
+				printStream.printf("%5s ", ":");
+			}
+			statement.accept(this);
+			printStream.println();
+		}
+		return line;
+	}
+	@Override
+	public Token visit(Token token) {
+		switch (token.type) {
+		case EOL:
+			printStream.print("<EOL>");
+			break;
+		case COMMENT:
+			printStream.printf(" REM %s", token.text);
+			break;
+		case STRING:
+			printStream.printf("\"%s\"", token.text);
+			break;
+		case KEYWORD:
+			printStream.printf(" %s ", token.keyword.text);
+			break;
+		case IDENT:
+		case SYNTAX:
+			printStream.print(token.text);
+			break;
+		case DIRECTIVE:
+			printStream.printf("%s ", token.text);
+			break;
+		case NUMBER:
+			if (Math.rint(token.number) == token.number) {
+				printStream.print(token.number.intValue());
+			} else {
+				printStream.print(token.number);
+			}
+			break;
+		}
+		return token;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/PrintVisitor.java

@@ -0,0 +1,65 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.io.PrintStream;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.Visitors.PrintBuilder;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+
+public class PrintVisitor implements Visitor {
+	private PrintStream printStream;
+	
+	public PrintVisitor(PrintBuilder builder) {
+		this.printStream = builder.getPrintStream();
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		printStream.printf("%d ", line.lineNumber);
+		boolean first = true;
+		for (Statement statement : line.statements) {
+			if (first) {
+				first = false;
+			} else {
+				printStream.printf(":");
+			}
+			statement.accept(this);
+		}
+		printStream.println();
+		return line;
+	}
+	@Override
+	public Token visit(Token token) {
+		switch (token.type) {
+		case EOL:
+			printStream.print("<EOL>");
+			break;
+		case COMMENT:
+			printStream.printf("REM %s", token.text);
+			break;
+		case STRING:
+			printStream.printf("\"%s\"", token.text);
+			break;
+		case KEYWORD:
+			printStream.printf(" %s ", token.keyword.text);
+			break;
+		case IDENT:
+		case SYNTAX:
+			printStream.print(token.text);
+			break;
+		case DIRECTIVE:
+			printStream.printf("%s ", token.text);
+			break;
+		case NUMBER:
+			if (Math.rint(token.number) == token.number) {
+				printStream.print(token.number.intValue());
+			} else {
+				printStream.print(token.number);
+			}
+			break;
+		}
+		return token;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/ReassignmentVisitor.java

@@ -0,0 +1,75 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.util.Map;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.model.ApplesoftKeyword;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Statement;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+
+/** This is a mildly rewritable Visitor. */
+public class ReassignmentVisitor implements Visitor {
+	private Map<Integer,Integer> reassignments;
+	private Program newProgram;
+	
+	public ReassignmentVisitor(Map<Integer,Integer> reassignments) {
+		this.reassignments = reassignments;
+	}
+	
+	@Override
+	public Program visit(Program program) {
+		newProgram = new Program();
+		program.lines.forEach(l -> {
+			Line line = l.accept(this);
+			newProgram.lines.add(line);
+		});
+		return newProgram;
+	}
+	@Override
+	public Line visit(Line line) {
+		Line newLine = new Line(line.lineNumber, this.newProgram);
+		line.statements.forEach(s -> {
+			Statement statement = s.accept(this);
+			newLine.statements.add(statement);
+		});
+		return newLine;
+	}
+	/**
+	 * We saw a trigger, reassign any numbers that follow.
+	 * 
+	 * Trigger cases:
+	 * - GOSUB n
+	 * - GOTO n
+	 * - IF ... THEN n
+	 * - LIST n [ ,m ]
+	 * - ON x GOTO n, m, ...
+	 * - ON x GOSUB n, m, ...
+	 * - ONERR GOTO n
+	 * - RUN n
+	 */
+	@Override
+	public Statement visit(Statement statement) {
+		boolean next = false;
+		boolean multiple = false;
+		Statement newStatement = new Statement();
+		for (Token t : statement.tokens) {
+			Token newToken = t;
+			if (next) {
+				if (t.type == Type.NUMBER && reassignments.containsKey(t.number.intValue())) {
+					newToken = Token.number(t.line, reassignments.get(t.number.intValue()).doubleValue());
+				}
+				next = multiple;	// preserve next based on if we have multiple line numbers or not.
+			} else {
+				next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
+					|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
+					|| t.keyword == ApplesoftKeyword.LIST;
+				multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
+			}
+			newStatement.tokens.add(newToken);
+		}
+		return newStatement;
+	}
+}

api/src/main/java/io/github/applecommander/bastokenizer/api/visitors/VariableReportVisitor.java

@@ -0,0 +1,54 @@
+package io.github.applecommander.bastokenizer.api.visitors;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import io.github.applecommander.bastokenizer.api.Visitor;
+import io.github.applecommander.bastokenizer.api.model.Line;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+
+public class VariableReportVisitor implements Visitor {
+	private Map<String,SortedSet<Integer>> refs = new HashMap<>();
+	private int currentLineNumber = -1;
+	
+	@Override
+	public Program visit(Program program) {
+		Program p = Visitor.super.visit(program);
+		refs.entrySet().stream()
+			.sorted(Map.Entry.comparingByKey())
+			.forEach(this::print);
+		return p;
+	}
+	private void print(Map.Entry<String,SortedSet<Integer>> e) {
+		System.out.printf("%-8s  ", e.getKey());
+		int c = 0;
+		for (int i : e.getValue()) {
+			if (c > 0) System.out.print(", ");
+			if (c > 0 && c % 10 == 0) System.out.printf("\n          ");
+			System.out.print(i);
+			c += 1;
+		}
+		System.out.println();
+	}
+	
+	@Override
+	public Line visit(Line line) {
+		currentLineNumber = line.lineNumber;
+		return Visitor.super.visit(line);
+	}
+	
+	@Override
+	public Token visit(Token token) {
+		if (token.type == Type.IDENT) {
+			refs.merge(token.text, 
+					new TreeSet<>(Arrays.asList(currentLineNumber)), 
+					(a,b) -> { a.addAll(b); return a; });
+		}
+		return Visitor.super.visit(token);
+	}
+}

gradle.properties

@@ -0,0 +1,8 @@
+# Universal applesingle version number.  Used for:
+# - Naming JAR file.  
+# - The build will insert this into a file that is read at run time as well.
+version=0.2.0
+
+# Maven Central Repository G and A of GAV coordinate. :-)
+group=net.sf.applecommander
+archivesBaseName=bastokenizer

gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

pom.xml

@@ -1,67 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-	<modelVersion>4.0.0</modelVersion>
-	<groupId>net.sf.applecommander</groupId>
-	<artifactId>bastokenizer</artifactId>
-	<version>0.2.0-SNAPSHOT</version>
-	<name>AppleSoft BASIC Tokenizer</name>
-	<description>Experiments with generating an AppleSoft B/BAS tokenized "binary".</description>
-
-	<properties>
-		<maven.compiler.source>1.8</maven.compiler.source>
-		<maven.compiler.target>1.8</maven.compiler.target>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
-		<timestamp>${maven.build.timestamp}</timestamp>
-	</properties>
-
-	<dependencies>
-		<dependency>
-			<groupId>info.picocli</groupId>
-			<artifactId>picocli</artifactId>
-			<version>[3.0,3.1)</version>
-		</dependency>
-		<dependency>
-			<groupId>net.sf.applecommander</groupId>
-			<artifactId>applesingle-api</artifactId>
-			<version>1.1.0</version>
-		</dependency>
-	</dependencies>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jar-plugin</artifactId>
-				<version>3.0.2</version>
-				<configuration>
-					<archive>
-						<manifest>
-							<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
-							<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
-							<useUniqueVersions>false</useUniqueVersions>
-						</manifest>
-						<manifestEntries>
-							<Implementation-Version>${project.version} (${timestamp})</Implementation-Version>
-						</manifestEntries>
-					</archive>
-				</configuration>
-			</plugin>
-			<plugin>
-				<groupId>org.springframework.boot</groupId>
-				<artifactId>spring-boot-maven-plugin</artifactId>
-				<version>2.0.0.RELEASE</version>
-				<executions>
-					<execution>
-						<goals>
-							<goal>repackage</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-		</plugins>
-	</build>
-
-</project>

settings.gradle

@@ -0,0 +1,7 @@
+include 'api'
+include 'tools:bt'
+
+rootProject.name = 'bastokenizer'
+project(":api").name = 'bastokenizer-api'
+project(":tools").name = 'bastokenizer-tools'
+project(":tools:bt").name = 'bastokenizer-tools-bt'

src/main/java/com/webcodepro/applecommander/util/applesoft/Directive.java

@@ -1,170 +0,0 @@
-package com.webcodepro.applecommander.util.applesoft;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.TreeMap;
-
-import com.webcodepro.applecommander.util.applesoft.Token.Type;
-
-import io.github.applecommander.bastokenizer.Main;
-import io.github.applecommander.bastokenizer.Main.IntegerTypeConverter;
-
-public abstract class Directive {
-	private static Map<String,Class<? extends Directive>> directives = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);;
-	static {
-		Directive.directives.put("$embed", EmbeddedBinaryDirective.class);
-	}
-	
-	private static IntegerTypeConverter integerConverter = new IntegerTypeConverter();
-	
-	protected OutputStream outputStream;
-	protected List<Token> parameters = new ArrayList<>();
-
-	public static Directive find(String text, OutputStream outputStream) {
-		if (directives.containsKey(text)) {
-			try {
-				// Bypassing the constructor with arguments ... as that reduces code in the subclasses.
-				Directive directive = directives.get(text).newInstance();
-				directive.setOutputStream(outputStream);
-				return directive;
-			} catch (InstantiationException | IllegalAccessException e) {
-				throw new IllegalArgumentException(String.format("Unable to construct directive '%s'", text), e);
-			}
-		}
-		throw new IllegalArgumentException(String.format("Unable to find directive '%s'", text));
-	}
-	
-	public void setOutputStream(OutputStream outputStream) {
-		this.outputStream = outputStream;
-	}
-	
-	public void append(Token token) {
-		// Skip the commas...
-		if (token.type == Type.SYNTAX && ",".equals(token.text)) return;
-		parameters.add(token);
-	}
-	private Token require(Type... types) {
-		Token t = parameters.remove(0);
-		boolean matches = false;
-		for (Type type : types) {
-			matches |= type == t.type;
-		}
-		if (!matches) {
-			throw new IllegalArgumentException("Expecting a type of " + types);
-		}
-		return t;
-	}
-	protected String requiresString() {
-		Token t = require(Type.STRING);
-		return t.text;
-	}
-	protected int requiresInteger() {
-		Token t = require(Type.NUMBER, Type.STRING);
-		if (t.type == Type.NUMBER) {
-			return t.number.intValue();
-		}
-		return integerConverter.convert(t.text);
-	}
-	
-	protected void ldy(int value) throws IOException {
-		outputStream.write(0xa0);
-		outputStream.write(value);
-	}
-	protected void jmp(int address) throws IOException {
-		outputStream.write(0x4c);
-		outputStream.write(address & 0xff);
-		outputStream.write(address >> 8);
-	}
-	protected void lda(int value) throws IOException {
-		outputStream.write(0xa9);
-		outputStream.write(value);
-	}
-	protected void sta(int address) throws IOException {
-		if ((address & 0xff00) == 0) {
-			outputStream.write(0x85);
-			outputStream.write(address);
-		} else {
-			throw new RuntimeException("sta does not handle 16 bit addresses yet!");
-		}
-	}
-	protected void setAddress(int value, int address) throws IOException {
-		lda(value & 0xff);
-		sta(address);
-		lda(value >> 8);
-		sta(address+1);
-	}
-	
-	/** Write directive contents to output file. Note that address is adjusted for the line header already. */
-	public abstract void writeBytes(int startAddress, Line line) throws IOException;
-	
-	public static class EmbeddedBinaryDirective extends Directive {
-		@Override
-		public void writeBytes(int startAddress, Line line) throws IOException {
-			if (parameters.size() != 2) {
-				throw new IllegalArgumentException("$embed requires a name and address parameter");
-			}
-			String filename = requiresString();
-			int targetAddress = requiresInteger();
-
-			File file = new File(Main.configuration.sourceFile.getParentFile(), filename);
-			byte[] bin = Files.readAllBytes(file.toPath());
-			
-			Optional<Line> nextLine = line.nextLine();
-			byte[] basicCode = nextLine.isPresent() 
-					? callAndGoto(startAddress,nextLine.get()) 
-					: callAndReturn(startAddress);
-			
-			final int moveLength = 8*3 + 2 + 3;	// LDA/STA, LDY, JMP.
-			int embeddedStart = startAddress + basicCode.length + moveLength;
-			int embeddedEnd = embeddedStart + bin.length;
-			
-			outputStream.write(basicCode);
-			setAddress(embeddedStart, 0x3c);
-			setAddress(embeddedEnd, 0x3e);
-			setAddress(targetAddress, 0x42);
-			ldy(0x00);
-			jmp(0xfe2c);
-			outputStream.write(bin);
-		}
-		// In program, "CALL <address>:GOTO line"
-		private byte[] callAndGoto(int startAddress, Line line) throws IOException {
-			// 3 for the tokens "CALL", ":", "GOTO", end of line (0x00)
-			final int tokenCount = 3 + 1;
-			int offset = Integer.toString(line.lineNumber).length() + tokenCount;
-			offset += Integer.toString(startAddress).length();
-			// Attempting to adjust if we bump from 4 digit address to a 5 digit address
-			if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
-			ByteArrayOutputStream os = new ByteArrayOutputStream();
-			os.write(ApplesoftKeyword.CALL.code);
-			os.write(Integer.toString(startAddress+offset).getBytes());
-			os.write(':');
-			os.write(ApplesoftKeyword.GOTO.code);
-			os.write(Integer.toString(line.lineNumber).getBytes());
-			os.write(0x00);
-			return os.toByteArray();
-		}
-		// At end of program, just "CALL <address>:RETURN"
-		private byte[] callAndReturn(int startAddress) throws IOException {
-			// 3 for the tokens "CALL", ":", "RETURN", end of line (0x00)
-			final int tokenCount = 3 + 1;
-			int offset = tokenCount;
-			offset += Integer.toString(startAddress).length();
-			// Attempting to adjust if we bump from 4 digit address to a 5 digit address
-			if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
-			ByteArrayOutputStream os = new ByteArrayOutputStream();
-			os.write(ApplesoftKeyword.CALL.code);
-			os.write(Integer.toString(startAddress+offset).getBytes());
-			os.write(':');
-			os.write(ApplesoftKeyword.RETURN.code);
-			os.write(0x00);
-			return os.toByteArray();
-		}
-	}
-}

src/main/java/com/webcodepro/applecommander/util/applesoft/Visitors.java

@@ -1,472 +0,0 @@
-package com.webcodepro.applecommander.util.applesoft;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.function.Function;
-
-import com.webcodepro.applecommander.util.applesoft.Token.Type;
-
-/**
- * This class presents all of the common Visitor implementations via builder patterns.
- * The number is currently small enough that all the builders and visitors are defined 
- * in this one class.
- *  
- * @author rob
- */
-public class Visitors {
-	public static PrintBuilder printBuilder() {
-		return new PrintBuilder();
-	}
-	public static class PrintBuilder {
-		private PrintStream printStream = System.out;
-		private Function<PrintBuilder,Visitor> creator = PrintVisitor::new;
-		
-		public PrintBuilder printStream(PrintStream printStream) {
-			Objects.requireNonNull(printStream);
-			this.printStream = printStream;
-			return this;
-		}
-		public PrintBuilder prettyPrint(boolean flag) {
-			creator = flag ? PrettyPrintVisitor::new : PrintVisitor::new;
-			return this;
-		}
-		public PrintBuilder prettyPrint() {
-			creator = PrettyPrintVisitor::new;
-			return this;
-		}
-		public PrintBuilder print() {
-			creator = PrintVisitor::new;
-			return this;
-		}
-		
-		public Visitor build() {
-			return creator.apply(this);
-		}
-	}
-	
-	public static ByteVisitor byteVisitor(int address) {
-		return new ByteVisitor(address);
-	}
-	
-	/** Rewrite the Program tree with the line number reassignments given. */
-	public static ReassignmentVisitor reassignVisitor(Map<Integer,Integer> reassignments) {
-		return new ReassignmentVisitor(reassignments);
-	}
-	
-	/** Collect all line numbers that are a target of GOTO, GOSUB, etc. */
-	public static LineNumberTargetCollector lineNumberTargetCollector() {
-		return new LineNumberTargetCollector();
-	}
-	
-	public static Visitor variableReportVisitor() {
-		return new VariableReportVisitor();
-	}
-
-	private static class PrettyPrintVisitor implements Visitor {
-		private PrintStream printStream;
-		
-		private PrettyPrintVisitor(PrintBuilder builder) {
-			this.printStream = builder.printStream;
-		}
-		
-		@Override
-		public Line visit(Line line) {
-			boolean first = true;
-			for (Statement statement : line.statements) {
-				if (first) {
-					first = false;
-					printStream.printf("%5d ", line.lineNumber);
-				} else {
-					printStream.printf("%5s ", ":");
-				}
-				statement.accept(this);
-				printStream.println();
-			}
-			return line;
-		}
-		@Override
-		public Token visit(Token token) {
-			switch (token.type) {
-			case EOL:
-				printStream.print("<EOL>");
-				break;
-			case COMMENT:
-				printStream.printf(" REM %s", token.text);
-				break;
-			case STRING:
-				printStream.printf("\"%s\"", token.text);
-				break;
-			case KEYWORD:
-				printStream.printf(" %s ", token.keyword.text);
-				break;
-			case IDENT:
-			case SYNTAX:
-				printStream.print(token.text);
-				break;
-			case DIRECTIVE:
-				printStream.printf("%s ", token.text);
-				break;
-			case NUMBER:
-				if (Math.rint(token.number) == token.number) {
-					printStream.print(token.number.intValue());
-				} else {
-					printStream.print(token.number);
-				}
-				break;
-			}
-			return token;
-		}
-	}
-	
-	private static class PrintVisitor implements Visitor {
-		private PrintStream printStream;
-		
-		private PrintVisitor(PrintBuilder builder) {
-			this.printStream = builder.printStream;
-		}
-		
-		@Override
-		public Line visit(Line line) {
-			printStream.printf("%d ", line.lineNumber);
-			boolean first = true;
-			for (Statement statement : line.statements) {
-				if (first) {
-					first = false;
-				} else {
-					printStream.printf(":");
-				}
-				statement.accept(this);
-			}
-			printStream.println();
-			return line;
-		}
-		@Override
-		public Token visit(Token token) {
-			switch (token.type) {
-			case EOL:
-				printStream.print("<EOL>");
-				break;
-			case COMMENT:
-				printStream.printf("REM %s", token.text);
-				break;
-			case STRING:
-				printStream.printf("\"%s\"", token.text);
-				break;
-			case KEYWORD:
-				printStream.printf(" %s ", token.keyword.text);
-				break;
-			case IDENT:
-			case SYNTAX:
-				printStream.print(token.text);
-				break;
-			case DIRECTIVE:
-				printStream.printf("%s ", token.text);
-				break;
-			case NUMBER:
-				if (Math.rint(token.number) == token.number) {
-					printStream.print(token.number.intValue());
-				} else {
-					printStream.print(token.number);
-				}
-				break;
-			}
-			return token;
-		}
-	}
-	
-	public static class ByteVisitor implements Visitor {
-		private Stack<ByteArrayOutputStream> stack;
-		private Map<Integer,Integer> lineAddresses;
-		private int address;
-		private Directive currentDirective;
-		
-		private ByteVisitor(int address) {
-			this.address = address;
-			this.stack = new Stack<>();
-			this.lineAddresses = new TreeMap<>();
-		}
-		
-		/** A convenience method to invoke {@link Program#accept(Visitor)} and {@link #getBytes()}. */
-		public byte[] dump(Program program) {
-			program.accept(this);
-			return getBytes();
-		}
-		
-		/** A convenience method to get the length of a line. */
-		public int length(Line line) {
-			stack.push(new ByteArrayOutputStream());
-			line.accept(this);
-			return stack.pop().size();
-		}
-		
-		public Map<Integer, Integer> getLineAddresses() {
-			return lineAddresses;
-		}
-		
-		public byte[] getBytes() {
-			if (stack.size() != 1) {
-				throw new RuntimeException("Error in processing internal BASIC model!");
-			}
-			return stack.peek().toByteArray();
-		}
-		
-		@Override
-		public Program visit(Program program) {
-			stack.clear();
-			stack.push(new ByteArrayOutputStream());
-			program.lines.forEach(line -> line.accept(this));
-			ByteArrayOutputStream os = stack.peek();
-			os.write(0x00);
-			os.write(0x00);
-			return program;
-		}
-		
-		@Override
-		public Line visit(Line line) {
-			try {
-				stack.push(new ByteArrayOutputStream());
-				boolean first = true;
-				for (Statement statement : line.statements) {
-					if (currentDirective != null) {
-						throw new RuntimeException("No statements are allowed after a directive!");
-					}
-					if (!first) {
-						stack.peek().write(':');
-					}
-					first = false;
-					statement.accept(this);
-				}
-				if (currentDirective != null) {
-					currentDirective.writeBytes(this.address+4, line);
-					currentDirective = null;
-				}
-
-				this.lineAddresses.put(line.lineNumber, this.address);
-				byte[] content = stack.pop().toByteArray();
-				int nextAddress = address + content.length + 5;
-				ByteArrayOutputStream os = stack.peek();
-				os.write(nextAddress);
-				os.write(nextAddress >> 8);
-				os.write(line.lineNumber);
-				os.write(line.lineNumber >> 8);
-				os.write(content);
-				os.write(0x00);
-				this.address = nextAddress;
-				return line;
-			} catch (IOException ex) {
-				// Hiding the IOException as ByteArrayOutputStream does not throw it
-				throw new RuntimeException(ex);
-			}
-		}
-
-		@Override
-		public Token visit(Token token) {
-			if (currentDirective != null) {
-				currentDirective.append(token);
-				return token;
-			}
-			try {
-				ByteArrayOutputStream os = stack.peek();
-				switch (token.type) {
-				case COMMENT:
-					os.write(ApplesoftKeyword.REM.code);
-					os.write(token.text.getBytes());
-					break;
-				case EOL:
-					os.write(0x00);
-					break;
-				case IDENT:
-					os.write(token.text.getBytes());
-					break;
-				case KEYWORD:
-					os.write(token.keyword.code);
-					break;
-				case DIRECTIVE:
-					currentDirective = Directive.find(token.text, os);
-					break;
-				case NUMBER:
-					if (Math.rint(token.number) == token.number) {
-						os.write(Integer.toString(token.number.intValue()).getBytes());
-					} else {
-						os.write(Double.toString(token.number).getBytes());
-					}
-					break;
-				case STRING:
-					os.write('"');
-					os.write(token.text.getBytes());
-					os.write('"');
-					break;
-				case SYNTAX:
-					Optional<ApplesoftKeyword> opt = ApplesoftKeyword.find(token.text);
-					if (opt.isPresent()) {
-						os.write(opt.get().code);
-					} else {
-						os.write(token.text.getBytes());
-					}
-					break;
-				}
-				return token;
-			} catch (IOException ex) {
-				// Hiding the IOException as ByteArrayOutputStream does not throw it
-				throw new RuntimeException(ex);
-			}
-		}
-	}
-
-	/** This is a mildly rewritable Visitor. */
-	private static class ReassignmentVisitor implements Visitor {
-		private Map<Integer,Integer> reassignments;
-		private Program newProgram;
-		
-		private ReassignmentVisitor(Map<Integer,Integer> reassignments) {
-			this.reassignments = reassignments;
-		}
-		
-		@Override
-		public Program visit(Program program) {
-			newProgram = new Program();
-			program.lines.forEach(l -> {
-				Line line = l.accept(this);
-				newProgram.lines.add(line);
-			});
-			return newProgram;
-		}
-		@Override
-		public Line visit(Line line) {
-			Line newLine = new Line(line.lineNumber, this.newProgram);
-			line.statements.forEach(s -> {
-				Statement statement = s.accept(this);
-				newLine.statements.add(statement);
-			});
-			return newLine;
-		}
-		/**
-		 * We saw a trigger, reassign any numbers that follow.
-		 * 
-		 * Trigger cases:
-		 * - GOSUB n
-		 * - GOTO n
-		 * - IF ... THEN n
-		 * - LIST n [ ,m ]
-		 * - ON x GOTO n, m, ...
-		 * - ON x GOSUB n, m, ...
-		 * - ONERR GOTO n
-		 * - RUN n
-		 */
-		@Override
-		public Statement visit(Statement statement) {
-			boolean next = false;
-			boolean multiple = false;
-			Statement newStatement = new Statement();
-			for (Token t : statement.tokens) {
-				Token newToken = t;
-				if (next) {
-					if (t.type == Type.NUMBER && reassignments.containsKey(t.number.intValue())) {
-						newToken = Token.number(t.line, reassignments.get(t.number.intValue()).doubleValue());
-					}
-					next = multiple;	// preserve next based on if we have multiple line numbers or not.
-				} else {
-					next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
-						|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
-						|| t.keyword == ApplesoftKeyword.LIST;
-					multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
-				}
-				newStatement.tokens.add(newToken);
-			}
-			return newStatement;
-		}
-	}
-
-	public static class LineNumberTargetCollector implements Visitor {
-		private Set<Integer> targets = new TreeSet<>();
-		
-		public Set<Integer> getTargets() {
-			return targets;
-		}
-		
-		/**
-		 * We saw a trigger, collect any numbers that follow.
-		 * 
-		 * Trigger cases:
-		 * - GOSUB n
-		 * - GOTO n
-		 * - IF ... THEN n
-		 * - LIST n [ ,m ]
-		 * - ON x GOTO n, m, ...
-		 * - ON x GOSUB n, m, ...
-		 * - ONERR GOTO n
-		 * - RUN n
-		 */
-		@Override
-		public Statement visit(Statement statement) {
-			boolean next = false;
-			boolean multiple = false;
-			for (Token t : statement.tokens) {
-				if (next) {
-					if (t.type == Type.NUMBER) {
-						targets.add(t.number.intValue());
-					}
-					next = multiple;	// preserve next based on if we have multiple line numbers or not.
-				} else {
-					next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
-						|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
-						|| t.keyword == ApplesoftKeyword.LIST;
-					multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
-				}
-			}
-			return statement;
-		}
-	}
-
-	private static class VariableReportVisitor implements Visitor {
-		private Map<String,SortedSet<Integer>> refs = new HashMap<>();
-		private int currentLineNumber = -1;
-		
-		@Override
-		public Program visit(Program program) {
-			Program p = Visitor.super.visit(program);
-			refs.entrySet().stream()
-				.sorted(Map.Entry.comparingByKey())
-				.forEach(this::print);
-			return p;
-		}
-		private void print(Map.Entry<String,SortedSet<Integer>> e) {
-			System.out.printf("%-8s  ", e.getKey());
-			int c = 0;
-			for (int i : e.getValue()) {
-				if (c > 0) System.out.print(", ");
-				if (c > 0 && c % 10 == 0) System.out.printf("\n          ");
-				System.out.print(i);
-				c += 1;
-			}
-			System.out.println();
-		}
-		
-		@Override
-		public Line visit(Line line) {
-			currentLineNumber = line.lineNumber;
-			return Visitor.super.visit(line);
-		}
-		
-		@Override
-		public Token visit(Token token) {
-			if (token.type == Type.IDENT) {
-				refs.merge(token.text, 
-						new TreeSet<>(Arrays.asList(currentLineNumber)), 
-						(a,b) -> { a.addAll(b); return a; });
-			}
-			return Visitor.super.visit(token);
-		}
-	}
-}

src/main/java/com/webcodepro/applecommander/util/applesoft/package-info.java

@@ -1,8 +0,0 @@
-/**
- * This package is used by the AppleSoft text file import capabilities of AppleCommander.
- * It is separate from the other components, primarily due to it being used on the input side,
- * but also due to the fact that this is the third time AppleSoft tokens have been defined! 
- * 
- * @author rob
- */
-package com.webcodepro.applecommander.util.applesoft;

src/main/java/io/github/applecommander/bastokenizer/Optimization.java

@@ -1,190 +0,0 @@
-package io.github.applecommander.bastokenizer;
-
-import java.io.PrintStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Function;
-
-import com.webcodepro.applecommander.util.applesoft.ApplesoftKeyword;
-import com.webcodepro.applecommander.util.applesoft.Line;
-import com.webcodepro.applecommander.util.applesoft.Program;
-import com.webcodepro.applecommander.util.applesoft.Statement;
-import com.webcodepro.applecommander.util.applesoft.Token;
-import com.webcodepro.applecommander.util.applesoft.Token.Type;
-import com.webcodepro.applecommander.util.applesoft.Visitor;
-import com.webcodepro.applecommander.util.applesoft.Visitors;
-import com.webcodepro.applecommander.util.applesoft.Visitors.ByteVisitor;
-import com.webcodepro.applecommander.util.applesoft.Visitors.LineNumberTargetCollector;
-
-import picocli.CommandLine.ITypeConverter;
-
-public enum Optimization {
-	REMOVE_EMPTY_STATEMENTS(opts -> new RemoveEmptyStatements()),
-	REMOVE_REM_STATEMENTS(opts -> new RemoveRemStatements()),
-	MERGE_LINES(opts -> new MergeLines(opts)),
-	RENUMBER(opts -> new Renumber())
-	;
-	
-	private Function<Main,Visitor> factory;
-	
-	private Optimization(Function<Main,Visitor> factory) {
-		this.factory = factory;
-	}
-	
-	public Visitor create(Main options) {
-		return factory.apply(options);
-	}
-	
-	/** Add support for lower-case Optimization flags. */
-	public static class TypeConverter implements ITypeConverter<Optimization> {
-		@Override
-		public Optimization convert(String value) throws Exception {
-			try {
-				return Optimization.valueOf(value);
-			} catch (IllegalArgumentException ex) {
-				for (Optimization opt : Optimization.values()) {
-					String checkName = opt.name().replace('_', '-');
-					if (checkName.equalsIgnoreCase(value)) {
-						return opt;
-					}
-				}
-				throw ex;
-			}
-		}
-	}
-	/** Common base class for optimization visitors that allow the program tree to be rewritten. */
-	private static class BaseVisitor implements Visitor {
-		protected Map<Integer,Integer> reassignments = new HashMap<>();
-		protected Program newProgram;
-		@Override
-		public Program visit(Program program) {
-			newProgram = new Program();
-			program.lines.forEach(l -> {
-				Line line = l.accept(this);
-				boolean lineKept = line != null && !line.statements.isEmpty();
-				if (lineKept) {
-					newProgram.lines.add(line);
-					reassignments.replaceAll((k,v) -> v == null ? l.lineNumber : v);
-				} else {
-					// Make a place-holder for the reassignment; we'll patch it in once we find a line that sticks around.
-					reassignments.put(l.lineNumber, null);
-				}
-			});
-			if (!reassignments.isEmpty()) {
-				// Now, renumber based on our findings!
-				return newProgram.accept(Visitors.reassignVisitor(reassignments));
-			} else {
-				return newProgram;
-			}
-		}
-		@Override
-		public Line visit(Line line) {
-			Line newLine = new Line(line.lineNumber, this.newProgram);
-			line.statements.forEach(s -> {
-				Statement statement = s.accept(this);
-				if (statement != null) newLine.statements.add(statement);
-			});
-			return newLine;
-		}
-		@Override
-		public Statement visit(Statement statement) {
-			Statement newStatement = new Statement();
-			statement.tokens.forEach(t -> { 
-				Token token = t.accept(this);
-				if (token != null) newStatement.tokens.add(token);
-			});
-			return newStatement;
-		}
-		@Override
-		public Token visit(Token token) {
-			return token;
-		}
-	}
-	private static class RemoveEmptyStatements extends BaseVisitor {
-		@Override
-		public Statement visit(Statement statement) {
-			return statement.tokens.isEmpty() ? null : statement;
-		}
-	}
-	private static class RemoveRemStatements extends BaseVisitor {
-		@Override
-		public Statement visit(Statement statement) {
-			return statement.tokens.get(0).type == Type.COMMENT ? null : statement;
-		}
-	}
-	private static class MergeLines extends BaseVisitor {
-		private Set<Integer> targets;
-		private Line mergeLine;
-		private ByteVisitor bv = Visitors.byteVisitor(0x801);
-		private int maxLineLength;
-		private PrintStream debug;
-		private MergeLines(Main options) {
-			this.maxLineLength = options.maxLineLength;
-			this.debug = options.debug;
-		}
-		@Override
-		public Program visit(Program program) {
-			LineNumberTargetCollector c = Visitors.lineNumberTargetCollector();
-			program.accept(c);
-			targets = c.getTargets();
-			debug.printf("Target lines = %s\n", targets);
-			return super.visit(program);
-		}
-		@Override
-		public Line visit(Line line) {
-			debug.printf("Line # %d : ", line.lineNumber);
-			Line newLine = new Line(line.lineNumber, this.newProgram);
-			newLine.statements.addAll(line.statements);
-			if (mergeLine == null || targets.contains(line.lineNumber)) {
-				// Either forced to a new line or this is a GOTO type target: Ignore length
-				debug.printf("%s\n", mergeLine == null ? "mergeLine is null" : "target line #");
-			} else {
-				// Check length and decide if it merges based on that.
-				Line tmpLine = new Line(mergeLine.lineNumber, mergeLine.program);
-				tmpLine.statements.addAll(mergeLine.statements);
-				tmpLine.statements.addAll(line.statements);
-				if (bv.length(tmpLine) > maxLineLength) {
-					// It was too big, do not add
-					debug.printf("merge would exceed max line length: %d > %d\n", bv.length(tmpLine), maxLineLength);
-				} else {
-					// We can add line to mergeLine (mergeLine is already added to program, must keep that object)
-					mergeLine.statements.addAll(line.statements);
-					if (hasTerminal(line)) mergeLine = null;
-					debug.printf("line %s\n", mergeLine == null ? "had terminals" : "was added to mergeLine");
-					return null;
-				}
-			}
-			// Always reset mergeLine based on the terminal characteristics
-			mergeLine = hasTerminal(line) ? null : newLine;
-			debug.printf("line %s\n", mergeLine == null ? "had terminals" : "is now mergeLine");
-			return newLine;
-		}
-		private boolean hasTerminal(Line line) {
-			// Terminals are: IF, REM, GOTO, END, ON .. GOTO (GOTO is trigger), RESUME, RETURN, STOP
-			// Includes directives.
-			for (Statement s : line.statements) {
-				for (Token t : s.tokens) {
-					boolean terminal = t.keyword == ApplesoftKeyword.IF || t.type == Type.COMMENT /* REM */
-							|| t.keyword == ApplesoftKeyword.GOTO || t.keyword == ApplesoftKeyword.END
-							|| t.keyword == ApplesoftKeyword.RESUME || t.keyword == ApplesoftKeyword.RETURN
-							|| t.keyword == ApplesoftKeyword.STOP
-							|| t.type == Type.DIRECTIVE;
-					if (terminal) return true;
-				}
-			}
-			return false;
-		}
-	}
-	private static class Renumber extends BaseVisitor {
-		protected int lineNumber = 0;
-		@Override
-		public Line visit(Line line) {
-			Line newLine = new Line(lineNumber++, this.newProgram);
-			newLine.statements.addAll(line.statements);
-			// Track what went where so lines can get renumbered automatically
-			reassignments.put(line.lineNumber, newLine.lineNumber);
-			return newLine;
-		}
-	}
-}

tools/bt/build.gradle

@@ -0,0 +1,25 @@
+plugins {
+    id 'org.springframework.boot' version '2.0.2.RELEASE'
+}
+
+repositories {
+    jcenter()
+}
+
+apply plugin: 'application'
+
+mainClassName = "io.github.applecommander.bastokenizer.tools.bt.Main"
+
+bootJar {
+  manifest {
+    attributes(
+      'Implementation-Title': 'bastokenizer',
+      'Implementation-Version': "${version} (${new Date().format('yyyy-MM-dd HH:mm')})" 
+    )
+  }
+}
+
+dependencies {
+    compile 'info.picocli:picocli:3.0.2'
+    compile project(':bastokenizer-api')
+}

tools/bt/src/main/java/io/github/applecommander/bastokenizer/tools/bt/HexDumper.java

@@ -0,0 +1,71 @@
+package io.github.applecommander.bastokenizer.tools.bt;
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.function.BiConsumer;
+
+/** A slightly-configurable reusable hex dumping mechanism. */
+public class HexDumper {
+	private PrintStream ps = System.out;
+	private int lineWidth = 16;
+	private BiConsumer<Integer,Integer> printHeader;
+	private BiConsumer<Integer,byte[]> printLine;
+	
+	public static HexDumper standard() {
+		HexDumper hd = new HexDumper();
+		hd.printHeader = hd::emptyHeader;
+		hd.printLine = hd::standardLine;
+		return hd;
+	}
+	public static HexDumper apple2() {
+		HexDumper hd = new HexDumper();
+		hd.printHeader = hd::apple2Header;
+		hd.printLine = hd::apple2Line;
+		return hd;
+	}
+	
+	public void dump(int address, byte[] data) {
+		printHeader.accept(address, data.length);
+		int offset = 0;
+		while (offset < data.length) {
+			byte[] line = Arrays.copyOfRange(data, offset, Math.min(offset+lineWidth,data.length));
+			printLine.accept(address+offset, line);
+			offset += line.length;
+		}
+	}
+	
+	public void emptyHeader(int address, int length) {
+		// Do Nothing
+	}
+	public void apple2Header(int address, int length) {
+		int end = address + length;
+		printLine.accept(0x67, new byte[] { (byte)(address&0xff), (byte)(address>>8), (byte)(end&0xff), (byte)(end>>8) });
+		printLine.accept(address-1, new byte[] { 0x00 });
+	}
+	
+	public void standardLine(int address, byte[] data) {
+		ps.printf("%04x: ", address);
+		for (int i=0; i<lineWidth; i++) {
+			if (i < data.length) {
+				ps.printf("%02x ", data[i]);
+			} else {
+				ps.printf(".. ");
+			}
+		}
+		ps.print(" ");
+		for (int i=0; i<lineWidth; i++) {
+			char ch = ' ';
+			if (i < data.length) {
+				byte b = data[i];
+				ch = (b >= ' ') ? (char)b : '.';
+			}
+			ps.printf("%c", ch);
+		}
+		ps.printf("\n");
+	}
+	public void apple2Line(int address, byte[] data) {
+		ps.printf("%04X:", address);
+		for (byte b : data) ps.printf("%02X ", b);
+		ps.printf("\n");
+	}
+}

tools/bt/src/main/java/io/github/applecommander/bastokenizer/tools/bt/IntegerTypeConverter.java

@@ -0,0 +1,12 @@
+package io.github.applecommander.bastokenizer.tools.bt;
+
+import io.github.applecommander.bastokenizer.api.utils.Converters;
+import picocli.CommandLine.ITypeConverter;
+
+/** Add support for "$801" and "0x801" instead of just decimal like 2049. */
+public class IntegerTypeConverter implements ITypeConverter<Integer> {
+	@Override
+	public Integer convert(String value) {
+		return Converters.toInteger(value);
+	}
+}

tools/bt/src/main/java/io/github/applecommander/bastokenizer/tools/bt/Main.java

@@ -1,4 +1,4 @@
-package io.github.applecommander.bastokenizer;
+package io.github.applecommander.bastokenizer.tools.bt;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -11,22 +11,20 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.Callable;
-import java.util.function.BiConsumer;
-
-import com.webcodepro.applecommander.util.applesoft.Parser;
-import com.webcodepro.applecommander.util.applesoft.Program;
-import com.webcodepro.applecommander.util.applesoft.Token;
-import com.webcodepro.applecommander.util.applesoft.Token.Type;
-import com.webcodepro.applecommander.util.applesoft.TokenReader;
-import com.webcodepro.applecommander.util.applesoft.Visitors;
-import com.webcodepro.applecommander.util.applesoft.Visitors.ByteVisitor;
 
 import io.github.applecommander.applesingle.AppleSingle;
+import io.github.applecommander.bastokenizer.api.Configuration;
+import io.github.applecommander.bastokenizer.api.Optimization;
+import io.github.applecommander.bastokenizer.api.Parser;
+import io.github.applecommander.bastokenizer.api.TokenReader;
+import io.github.applecommander.bastokenizer.api.Visitors;
+import io.github.applecommander.bastokenizer.api.model.Program;
+import io.github.applecommander.bastokenizer.api.model.Token;
+import io.github.applecommander.bastokenizer.api.model.Token.Type;
+import io.github.applecommander.bastokenizer.api.visitors.ByteVisitor;
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Help.Visibility;
-import picocli.CommandLine.ITypeConverter;
-import picocli.CommandLine.IVersionProvider;
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 
@@ -36,62 +34,61 @@ import picocli.CommandLine.Parameters;
 		commandListHeading = "%nCommands:%n",
 		optionListHeading = "%nOptions:%n",
 		name = "bt", mixinStandardHelpOptions = true, 
-		versionProvider = Main.VersionProvider.class)
+		versionProvider = VersionProvider.class)
 public class Main implements Callable<Void> {
 	private static final int BAS = 0xfc;
-	public static Configuration configuration = new Configuration();
 	
 	@Option(names = { "-o", "--output" }, description = "Write binary output to file.")
-	File outputFile;
+	private File outputFile;
 	
 	@Option(names = { "-x", "--hex"}, description = "Generate a binary hex dump for debugging.")
-	boolean hexFormat;
+	private boolean hexFormat;
 
 	@Option(names = { "-c", "--copy"}, description = "Generate a copy/paste form of output for testing in an emulator.")
-	boolean copyFormat;
+	private boolean copyFormat;
 	
 	@Option(names = { "-a", "--address" }, description = "Base address for program", showDefaultValue = Visibility.ALWAYS, converter = IntegerTypeConverter.class)
-	int address = 0x801;
+	private int address = 0x801;
 	
 	@Option(names = { "--variables" }, description = "Generate a variable report")
-	boolean showVariableReport;
+	private boolean showVariableReport;
 	
 	@Option(names = "--stdout", description = "Send binary output to stdout.")
-	boolean stdoutFlag;
+	private boolean stdoutFlag;
 	
 	@Option(names = "--applesingle", description = "Write output in AppleSingle format")
-	boolean applesingleFlag;
+	private boolean applesingleFlag;
 	
 	@Option(names = "--pretty", description = "Pretty print structure as bastokenizer understands it.")
-	boolean prettyPrint;
+	private boolean prettyPrint;
 
 	@Option(names = "--list", description = "List structure as bastokenizer understands it.")
-	boolean listPrint;
+	private boolean listPrint;
 
 	@Option(names = "--tokens", description = "Dump token list to stdout for debugging.")
-	boolean showTokens;
+	private boolean showTokens;
 	
 	@Option(names = "--addresses", description = "Dump line number addresses out.")
-	boolean showLineAddresses;
+	private boolean showLineAddresses;
 	
 	@Option(names = "--max-line-length", description = "Maximum line length for generated lines.", showDefaultValue = Visibility.ALWAYS)
-	int maxLineLength = 255;
+	private int maxLineLength = 255;
 	
-	@Option(names = "-f", converter = Optimization.TypeConverter.class, split = ",", description = {
+	@Option(names = "-f", converter = OptimizationTypeConverter.class, split = ",", description = {
 			"Enable specific optimizations.",
 			"* @|green remove-empty-statements|@ - Strip out all '::'-like statements.",
 			"* @|green remove-rem-statements|@ - Remove all REM statements.",
 			"* @|green merge-lines|@ - Merge lines.",
 			"* @|green renumber|@ - Renumber program."
 	})
-	List<Optimization> optimizations = new ArrayList<>();
+	private List<Optimization> optimizations = new ArrayList<>();
 
 	@Option(names = { "-O", "--optimize" }, description = "Apply all optimizations.")
-	boolean allOptimizations;
+	private boolean allOptimizations;
 	
 	@Option(names = "--debug", description = "Print debug output.")
-	boolean debugFlag;
-	PrintStream debug = new PrintStream(new OutputStream() {
+	private boolean debugFlag;
+	private PrintStream debug = new PrintStream(new OutputStream() {
 			@Override
 			public void write(int b) throws IOException {
 				// Do nothing
@@ -99,7 +96,7 @@ public class Main implements Callable<Void> {
 		});
 	
 	@Parameters(index = "0", description = "AppleSoft BASIC program to process.")
-	File sourceFile;
+	private File sourceFile;
 	
 	public static void main(String[] args) throws FileNotFoundException, IOException {
 		CommandLine.call(new Main(), args);
@@ -108,9 +105,12 @@ public class Main implements Callable<Void> {
 	@Override
 	public Void call() throws FileNotFoundException, IOException {
 		if (checkParameters()) {
-			if (debugFlag) debug = System.out;
-			configuration.sourceFile = this.sourceFile;
-			process();
+			Configuration.Builder builder = Configuration.builder()
+					.maxLineLength(this.maxLineLength)
+					.sourceFile(this.sourceFile)
+					.startAddress(this.address);
+			if (debugFlag) builder.debugStream(System.out);
+			process(builder.build());
 		}
 		
 		return null;		// To satisfy object "Void"
@@ -135,7 +135,7 @@ public class Main implements Callable<Void> {
 	}
 	
 	/** General CLI processing. */
-	public void process() throws FileNotFoundException, IOException {
+	public void process(Configuration config) throws FileNotFoundException, IOException {
 		Queue<Token> tokens = TokenReader.tokenize(sourceFile);
 		if (showTokens) {
 			tokens.forEach(t -> System.out.printf("%s%s", t, t.type == Type.EOL ? "\n" : ", "));
@@ -145,7 +145,7 @@ public class Main implements Callable<Void> {
 		
 		for (Optimization optimization : optimizations) {
 			debug.printf("Optimization: %s\n", optimization.name());
-			program = program.accept(optimization.create(this));
+			program = program.accept(optimization.create(config));
 		}
 
 		if (prettyPrint || listPrint) {
@@ -155,7 +155,7 @@ public class Main implements Callable<Void> {
 			program.accept(Visitors.variableReportVisitor());
 		}
 
-		ByteVisitor byteVisitor = Visitors.byteVisitor(address);
+		ByteVisitor byteVisitor = Visitors.byteVisitor(config);
 		byte[] data = byteVisitor.dump(program);
 		if (showLineAddresses) {
 			byteVisitor.getLineAddresses().forEach((l,a) -> System.out.printf("%5d ... $%04x\n", l, a));
@@ -204,96 +204,4 @@ public class Main implements Callable<Void> {
 			}
 		}
 	}
-	
-	/** A slightly-configurable reusable hex dumping mechanism. */
-	public static class HexDumper {
-		private PrintStream ps = System.out;
-		private int lineWidth = 16;
-		private BiConsumer<Integer,Integer> printHeader;
-		private BiConsumer<Integer,byte[]> printLine;
-		
-		public static HexDumper standard() {
-			HexDumper hd = new HexDumper();
-			hd.printHeader = hd::emptyHeader;
-			hd.printLine = hd::standardLine;
-			return hd;
-		}
-		public static HexDumper apple2() {
-			HexDumper hd = new HexDumper();
-			hd.printHeader = hd::apple2Header;
-			hd.printLine = hd::apple2Line;
-			return hd;
-		}
-		
-		public void dump(int address, byte[] data) {
-			printHeader.accept(address, data.length);
-			int offset = 0;
-			while (offset < data.length) {
-				byte[] line = Arrays.copyOfRange(data, offset, Math.min(offset+lineWidth,data.length));
-				printLine.accept(address+offset, line);
-				offset += line.length;
-			}
-		}
-		
-		public void emptyHeader(int address, int length) {
-			// Do Nothing
-		}
-		public void apple2Header(int address, int length) {
-			int end = address + length;
-			printLine.accept(0x67, new byte[] { (byte)(address&0xff), (byte)(address>>8), (byte)(end&0xff), (byte)(end>>8) });
-			printLine.accept(address-1, new byte[] { 0x00 });
-		}
-		
-		public void standardLine(int address, byte[] data) {
-			ps.printf("%04x: ", address);
-			for (int i=0; i<lineWidth; i++) {
-				if (i < data.length) {
-					ps.printf("%02x ", data[i]);
-				} else {
-					ps.printf(".. ");
-				}
-			}
-			ps.print(" ");
-			for (int i=0; i<lineWidth; i++) {
-				char ch = ' ';
-				if (i < data.length) {
-					byte b = data[i];
-					ch = (b >= ' ') ? (char)b : '.';
-				}
-				ps.printf("%c", ch);
-			}
-			ps.printf("\n");
-		}
-		public void apple2Line(int address, byte[] data) {
-			ps.printf("%04X:", address);
-			for (byte b : data) ps.printf("%02X ", b);
-			ps.printf("\n");
-		}
-	}
-
-	/** Display version information.  Note that this is dependent on Maven configuration. */
-	public static class VersionProvider implements IVersionProvider {
-        public String[] getVersion() {
-        	return new String[] { Main.class.getPackage().getImplementationVersion() };
-        }
-	}
-	/** Add support for "$801" and "0x801" instead of just decimal like 2049. */
-	public static class IntegerTypeConverter implements ITypeConverter<Integer> {
-		@Override
-		public Integer convert(String value) {
-			if (value == null) {
-				return null;
-			} else if (value.startsWith("$")) {
-				return Integer.valueOf(value.substring(1), 16);
-			} else if (value.startsWith("0x") || value.startsWith("0X")) {
-				return Integer.valueOf(value.substring(2), 16);
-			} else {
-				return Integer.valueOf(value);
-			}
-		}
-	}
-	/** Expose configuration details for other components to see. */
-	public static class Configuration {
-		public File sourceFile;
-	}
 }

tools/bt/src/main/java/io/github/applecommander/bastokenizer/tools/bt/OptimizationTypeConverter.java

@@ -0,0 +1,22 @@
+package io.github.applecommander.bastokenizer.tools.bt;
+
+import io.github.applecommander.bastokenizer.api.Optimization;
+import picocli.CommandLine.ITypeConverter;
+
+/** Add support for lower-case Optimization flags. */
+public class OptimizationTypeConverter implements ITypeConverter<Optimization> {
+	@Override
+	public Optimization convert(String value) throws Exception {
+		try {
+			return Optimization.valueOf(value);
+		} catch (IllegalArgumentException ex) {
+			for (Optimization opt : Optimization.values()) {
+				String checkName = opt.name().replace('_', '-');
+				if (checkName.equalsIgnoreCase(value)) {
+					return opt;
+				}
+			}
+			throw ex;
+		}
+	}
+}

tools/bt/src/main/java/io/github/applecommander/bastokenizer/tools/bt/VersionProvider.java

@@ -0,0 +1,10 @@
+package io.github.applecommander.bastokenizer.tools.bt;
+
+import picocli.CommandLine.IVersionProvider;
+
+/** Display version information.  Note that this is dependent on Gradle configuration. */
+public class VersionProvider implements IVersionProvider {
+    public String[] getVersion() {
+    	return new String[] { Main.class.getPackage().getImplementationVersion() };
+    }
+}