Adding directives and the '$embed' directive. Closes #11.

src/main/java/com/webcodepro/applecommander/util/applesoft/Directive.java

@@ -0,0 +1,170 @@
+package com.webcodepro.applecommander.util.applesoft;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+
+import com.webcodepro.applecommander.util.applesoft.Token.Type;
+
+import io.github.applecommander.bastokenizer.Main.IntegerTypeConverter;
+
+public abstract class Directive {
+	private static Map<String,Class<? extends Directive>> directives = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);;
+	static {
+		Directive.directives.put("$embed", EmbeddedBinaryDirective.class);
+	}
+	
+	private static IntegerTypeConverter integerConverter = new IntegerTypeConverter();
+	
+	protected OutputStream outputStream;
+	protected List<Token> parameters = new ArrayList<>();
+
+	public static Directive find(String text, OutputStream outputStream) {
+		if (directives.containsKey(text)) {
+			try {
+				// Bypassing the constructor with arguments ... as that reduces code in the subclasses.
+				Directive directive = directives.get(text).newInstance();
+				directive.setOutputStream(outputStream);
+				return directive;
+			} catch (InstantiationException | IllegalAccessException e) {
+				throw new IllegalArgumentException(String.format("Unable to construct directive '%s'", text), e);
+			}
+		}
+		throw new IllegalArgumentException(String.format("Unable to find directive '%s'", text));
+	}
+	
+	public void setOutputStream(OutputStream outputStream) {
+		this.outputStream = outputStream;
+	}
+	
+	public void append(Token token) {
+		// Skip the commas...
+		if (token.type == Type.SYNTAX && ",".equals(token.text)) return;
+		parameters.add(token);
+	}
+	private Token require(Type... types) {
+		Token t = parameters.remove(0);
+		boolean matches = false;
+		for (Type type : types) {
+			matches |= type == t.type;
+		}
+		if (!matches) {
+			throw new IllegalArgumentException("Expecting a type of " + types);
+		}
+		return t;
+	}
+	protected String requiresString() {
+		Token t = require(Type.STRING);
+		return t.text;
+	}
+	protected int requiresInteger() {
+		Token t = require(Type.NUMBER, Type.STRING);
+		if (t.type == Type.NUMBER) {
+			return t.number.intValue();
+		}
+		return integerConverter.convert(t.text);
+	}
+	
+	protected void ldy(int value) throws IOException {
+		outputStream.write(0xa0);
+		outputStream.write(value);
+	}
+	protected void jmp(int address) throws IOException {
+		outputStream.write(0x4c);
+		outputStream.write(address & 0xff);
+		outputStream.write(address >> 8);
+	}
+	protected void lda(int value) throws IOException {
+		outputStream.write(0xa9);
+		outputStream.write(value);
+	}
+	protected void sta(int address) throws IOException {
+		if ((address & 0xff00) == 0) {
+			outputStream.write(0x85);
+			outputStream.write(address);
+		} else {
+			throw new RuntimeException("sta does not handle 16 bit addresses yet!");
+		}
+	}
+	protected void setAddress(int value, int address) throws IOException {
+		lda(value & 0xff);
+		sta(address);
+		lda(value >> 8);
+		sta(address+1);
+	}
+	
+	/** Write directive contents to output file. Note that address is adjusted for the line header already. */
+	public abstract void writeBytes(int startAddress, Line line) throws IOException;
+	
+	public static class EmbeddedBinaryDirective extends Directive {
+		@Override
+		public void writeBytes(int startAddress, Line line) throws IOException {
+			if (parameters.size() != 2) {
+				throw new IllegalArgumentException("$embed requires a name and address parameter");
+			}
+			String filename = requiresString();
+			int targetAddress = requiresInteger();
+			
+			Path path = Paths.get(filename);
+			byte[] bin = Files.readAllBytes(path);
+			
+			Optional<Line> nextLine = line.nextLine();
+			byte[] basicCode = nextLine.isPresent() 
+					? callAndGoto(startAddress,nextLine.get()) 
+					: callAndReturn(startAddress);
+			
+			final int moveLength = 8*3 + 2 + 3;	// LDA/STA, LDY, JMP.
+			int embeddedStart = startAddress + basicCode.length + moveLength;
+			int embeddedEnd = embeddedStart + bin.length;
+			
+			outputStream.write(basicCode);
+			setAddress(embeddedStart, 0x3c);
+			setAddress(embeddedEnd, 0x3e);
+			setAddress(targetAddress, 0x42);
+			ldy(0x00);
+			jmp(0xfe2c);
+			outputStream.write(bin);
+		}
+		// In program, "CALL <address>:GOTO line"
+		private byte[] callAndGoto(int startAddress, Line line) throws IOException {
+			// 3 for the tokens "CALL", ":", "GOTO", end of line (0x00)
+			final int tokenCount = 3 + 1;
+			int offset = Integer.toString(line.lineNumber).length() + tokenCount;
+			offset += Integer.toString(startAddress).length();
+			// Attempting to adjust if we bump from 4 digit address to a 5 digit address
+			if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
+			ByteArrayOutputStream os = new ByteArrayOutputStream();
+			os.write(ApplesoftKeyword.CALL.code);
+			os.write(Integer.toString(startAddress+offset).getBytes());
+			os.write(':');
+			os.write(ApplesoftKeyword.GOTO.code);
+			os.write(Integer.toString(line.lineNumber).getBytes());
+			os.write(0x00);
+			return os.toByteArray();
+		}
+		// At end of program, just "CALL <address>:RETURN"
+		private byte[] callAndReturn(int startAddress) throws IOException {
+			// 3 for the tokens "CALL", ":", "RETURN", end of line (0x00)
+			final int tokenCount = 3 + 1;
+			int offset = tokenCount;
+			offset += Integer.toString(startAddress).length();
+			// Attempting to adjust if we bump from 4 digit address to a 5 digit address
+			if (startAddress < 10000 && startAddress + offset >= 10000) offset += 1;
+			ByteArrayOutputStream os = new ByteArrayOutputStream();
+			os.write(ApplesoftKeyword.CALL.code);
+			os.write(Integer.toString(startAddress+offset).getBytes());
+			os.write(':');
+			os.write(ApplesoftKeyword.RETURN.code);
+			os.write(0x00);
+			return os.toByteArray();
+		}
+	}
+}

src/main/java/com/webcodepro/applecommander/util/applesoft/Line.java

@@ -2,14 +2,27 @@ package com.webcodepro.applecommander.util.applesoft;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 
 /** An AppleSoft BASIC Line representation. */
 public class Line {
+	public final Program program;
 	public final int lineNumber;
 	public final List<Statement> statements = new ArrayList<>();
 	
-	public Line(int lineNumber) {
+	public Line(int lineNumber, Program program) {
+		Objects.requireNonNull(program);
 		this.lineNumber = lineNumber;
+		this.program = program;
+	}
+	
+	public Optional<Line> nextLine() {
+		int i = program.lines.indexOf(this);
+		if (i == -1 || i+1 >= program.lines.size()) {
+			return Optional.empty();
+		}
+		return Optional.of(program.lines.get(i+1));
 	}
 	
 	public Line accept(Visitor t) {

src/main/java/com/webcodepro/applecommander/util/applesoft/Parser.java

@@ -20,14 +20,14 @@ public class Parser {
 	public Program parse() {
 		Program program = new Program();
 		while (!tokens.isEmpty()) {
-			Line line = readLine();
+			Line line = readLine(program);
 			program.lines.add(line);
 		}
 		return program;
 	}
 	
-	public Line readLine() {
-		Line line = new Line(expectNumber());
+	public Line readLine(Program program) {
+		Line line = new Line(expectNumber(), program);
 		while (!tokens.isEmpty() && tokens.peek().type != Type.EOL) {
 			Statement statement = readStatement();
 			if (statement != null) {

src/main/java/com/webcodepro/applecommander/util/applesoft/Token.java

@@ -58,8 +58,11 @@ public class Token {
 	public static Token syntax(int line, int ch) {
 		return new Token(line, Type.SYNTAX, null, null, String.format("%c", ch));
 	}
+	public static Token directive(int line, String text) {
+		return new Token(line, Type.DIRECTIVE, null, null, text);
+	}
 	
 	public static enum Type {
-		EOL, NUMBER, IDENT, COMMENT, STRING, KEYWORD, SYNTAX
+		EOL, NUMBER, IDENT, COMMENT, STRING, KEYWORD, SYNTAX, DIRECTIVE
 	}
 }

src/main/java/com/webcodepro/applecommander/util/applesoft/TokenReader.java

@@ -106,17 +106,20 @@ public class TokenReader {
 							    .orElseThrow(() -> new IOException("Expecting: " + opt.get().parts));
 						}
 						return Optional.of(Token.keyword(line, opt.get()));
-					} else {
-						// Found an identifier (A, A$, A%).  Test if it is an array ('A(', 'A$(', 'A%(').
-						String sval = tokenizer.sval;
-						tokenizer.nextToken();
-						if (tokenizer.ttype == '(') {
-							sval += (char)tokenizer.ttype;
-						} else {
-							tokenizer.pushBack();
-						}
-						return Optional.of(Token.ident(line, sval));
 					}
+					// Check if we found a directive
+					if (tokenizer.sval.startsWith("$")) {
+						return Optional.of(Token.directive(line, tokenizer.sval));
+					}
+					// Found an identifier (A, A$, A%).  Test if it is an array ('A(', 'A$(', 'A%(').
+					String sval = tokenizer.sval;
+					tokenizer.nextToken();
+					if (tokenizer.ttype == '(') {
+						sval += (char)tokenizer.ttype;
+					} else {
+						tokenizer.pushBack();
+					}
+					return Optional.of(Token.ident(line, sval));
 				case '"':
 					return Optional.of(Token.string(line, tokenizer.sval));
 				case '(':

src/main/java/com/webcodepro/applecommander/util/applesoft/Visitors.java

@@ -114,6 +114,9 @@ public class Visitors {
 			case SYNTAX:
 				printStream.print(token.text);
 				break;
+			case DIRECTIVE:
+				printStream.printf("%s ", token.text);
+				break;
 			case NUMBER:
 				if (Math.rint(token.number) == token.number) {
 					printStream.print(token.number.intValue());
@@ -167,6 +170,9 @@ public class Visitors {
 			case SYNTAX:
 				printStream.print(token.text);
 				break;
+			case DIRECTIVE:
+				printStream.printf("%s ", token.text);
+				break;
 			case NUMBER:
 				if (Math.rint(token.number) == token.number) {
 					printStream.print(token.number.intValue());
@@ -183,6 +189,7 @@ public class Visitors {
 		private Stack<ByteArrayOutputStream> stack;
 		private Map<Integer,Integer> lineAddresses;
 		private int address;
+		private Directive currentDirective;
 		
 		private ByteVisitor(int address) {
 			this.address = address;
@@ -231,13 +238,19 @@ public class Visitors {
 				stack.push(new ByteArrayOutputStream());
 				boolean first = true;
 				for (Statement statement : line.statements) {
+					if (currentDirective != null) {
+						throw new RuntimeException("No statements are allowed after a directive!");
+					}
 					if (!first) {
 						stack.peek().write(':');
-					} else {
-						first = false;
 					}
+					first = false;
 					statement.accept(this);
 				}
+				if (currentDirective != null) {
+					currentDirective.writeBytes(this.address+4, line);
+					currentDirective = null;
+				}
 
 				this.lineAddresses.put(line.lineNumber, this.address);
 				byte[] content = stack.pop().toByteArray();
@@ -259,6 +272,10 @@ public class Visitors {
 
 		@Override
 		public Token visit(Token token) {
+			if (currentDirective != null) {
+				currentDirective.append(token);
+				return token;
+			}
 			try {
 				ByteArrayOutputStream os = stack.peek();
 				switch (token.type) {
@@ -275,6 +292,9 @@ public class Visitors {
 				case KEYWORD:
 					os.write(token.keyword.code);
 					break;
+				case DIRECTIVE:
+					currentDirective = Directive.find(token.text, os);
+					break;
 				case NUMBER:
 					if (Math.rint(token.number) == token.number) {
 						os.write(Integer.toString(token.number.intValue()).getBytes());
@@ -307,6 +327,7 @@ public class Visitors {
 	/** This is a mildly rewritable Visitor. */
 	private static class ReassignmentVisitor implements Visitor {
 		private Map<Integer,Integer> reassignments;
+		private Program newProgram;
 		
 		private ReassignmentVisitor(Map<Integer,Integer> reassignments) {
 			this.reassignments = reassignments;
@@ -314,7 +335,7 @@ public class Visitors {
 		
 		@Override
 		public Program visit(Program program) {
-			Program newProgram = new Program();
+			newProgram = new Program();
 			program.lines.forEach(l -> {
 				Line line = l.accept(this);
 				newProgram.lines.add(line);
@@ -323,7 +344,7 @@ public class Visitors {
 		}
 		@Override
 		public Line visit(Line line) {
-			Line newLine = new Line(line.lineNumber);
+			Line newLine = new Line(line.lineNumber, this.newProgram);
 			line.statements.forEach(s -> {
 				Statement statement = s.accept(this);
 				newLine.statements.add(statement);
@@ -346,6 +367,7 @@ public class Visitors {
 		@Override
 		public Statement visit(Statement statement) {
 			boolean next = false;
+			boolean multiple = false;
 			Statement newStatement = new Statement();
 			for (Token t : statement.tokens) {
 				Token newToken = t;
@@ -353,10 +375,12 @@ public class Visitors {
 					if (t.type == Type.NUMBER && reassignments.containsKey(t.number.intValue())) {
 						newToken = Token.number(t.line, reassignments.get(t.number.intValue()).doubleValue());
 					}
+					next = multiple;	// preserve next based on if we have multiple line numbers or not.
 				} else {
 					next = t.keyword == ApplesoftKeyword.GOSUB || t.keyword == ApplesoftKeyword.GOTO 
 						|| t.keyword == ApplesoftKeyword.THEN || t.keyword == ApplesoftKeyword.RUN
 						|| t.keyword == ApplesoftKeyword.LIST;
+					multiple |= t.keyword == ApplesoftKeyword.LIST || t.keyword == ApplesoftKeyword.ON;
 				}
 				newStatement.tokens.add(newToken);
 			}

src/main/java/io/github/applecommander/bastokenizer/Main.java

@@ -238,7 +238,7 @@ public class Main implements Callable<Void> {
 	/** Add support for "$801" and "0x801" instead of just decimal like 2049. */
 	public static class IntegerTypeConverter implements ITypeConverter<Integer> {
 		@Override
-		public Integer convert(String value) throws Exception {
+		public Integer convert(String value) {
 			if (value == null) {
 				return null;
 			} else if (value.startsWith("$")) {

src/main/java/io/github/applecommander/bastokenizer/Optimization.java

@@ -56,9 +56,10 @@ public enum Optimization {
 	/** Common base class for optimization visitors that allow the program tree to be rewritten. */
 	private static class BaseVisitor implements Visitor {
 		protected Map<Integer,Integer> reassignments = new HashMap<>();
+		protected Program newProgram;
 		@Override
 		public Program visit(Program program) {
-			final Program newProgram = new Program();
+			newProgram = new Program();
 			program.lines.forEach(l -> {
 				Line line = l.accept(this);
 				boolean lineKept = line != null && !line.statements.isEmpty();
@@ -79,7 +80,7 @@ public enum Optimization {
 		}
 		@Override
 		public Line visit(Line line) {
-			Line newLine = new Line(line.lineNumber);
+			Line newLine = new Line(line.lineNumber, this.newProgram);
 			line.statements.forEach(s -> {
 				Statement statement = s.accept(this);
 				if (statement != null) newLine.statements.add(statement);
@@ -133,14 +134,14 @@ public enum Optimization {
 		@Override
 		public Line visit(Line line) {
 			debug.printf("Line # %d : ", line.lineNumber);
-			Line newLine = new Line(line.lineNumber);
+			Line newLine = new Line(line.lineNumber, this.newProgram);
 			newLine.statements.addAll(line.statements);
 			if (mergeLine == null || targets.contains(line.lineNumber)) {
 				// Either forced to a new line or this is a GOTO type target: Ignore length
 				debug.printf("%s\n", mergeLine == null ? "mergeLine is null" : "target line #");
 			} else {
 				// Check length and decide if it merges based on that.
-				Line tmpLine = new Line(mergeLine.lineNumber);
+				Line tmpLine = new Line(mergeLine.lineNumber, mergeLine.program);
 				tmpLine.statements.addAll(mergeLine.statements);
 				tmpLine.statements.addAll(line.statements);
 				if (bv.length(tmpLine) > maxLineLength) {
@@ -161,12 +162,14 @@ public enum Optimization {
 		}
 		private boolean hasTerminal(Line line) {
 			// Terminals are: IF, REM, GOTO, END, ON .. GOTO (GOTO is trigger), RESUME, RETURN, STOP
+			// Includes directives.
 			for (Statement s : line.statements) {
 				for (Token t : s.tokens) {
 					boolean terminal = t.keyword == ApplesoftKeyword.IF || t.type == Type.COMMENT /* REM */
 							|| t.keyword == ApplesoftKeyword.GOTO || t.keyword == ApplesoftKeyword.END
 							|| t.keyword == ApplesoftKeyword.RESUME || t.keyword == ApplesoftKeyword.RETURN
-							|| t.keyword == ApplesoftKeyword.STOP;
+							|| t.keyword == ApplesoftKeyword.STOP
+							|| t.type == Type.DIRECTIVE;
 					if (terminal) return true;
 				}
 			}
@@ -177,7 +180,7 @@ public enum Optimization {
 		protected int lineNumber = 0;
 		@Override
 		public Line visit(Line line) {
-			Line newLine = new Line(lineNumber++);
+			Line newLine = new Line(lineNumber++, this.newProgram);
 			newLine.statements.addAll(line.statements);
 			// Track what went where so lines can get renumbered automatically
 			reassignments.put(line.lineNumber, newLine.lineNumber);

src/test/resources/circles-timing.bas

@@ -0,0 +1,48 @@
+0 REM Provide a somewhat consistent runtime and take out the randomness
+1 REM to allow for timing and verify that the "optimizations" improve runtime
+2 REM to some degree... 
+
+5 $embed "src/test/resources/read.time.bin", "0x0260"
+
+10 call 608: rem initialize clock routine
+15 call 768,t1$:goto 100
+
+20 rem draw circle routine
+30 for a = 0 to pt
+40 x = x(a) * r:y = y(a) * r
+50 hplot xo + x,yo + y
+60 hplot xo - x,yo + y
+70 hplot xo + x,yo - y
+80 hplot xo - x,yo - y
+90 next a
+95 return
+
+100 rem main program
+110 hgr:hcolor=3
+120 home : vtab 21: inverse : print "JUST A MOMENT": normal
+130 pi = 3.14159
+140 pt = 30: dim x(pt),y(pt)
+150 for a = 0 to pt
+160 b = pi * (a / (pt * 2))
+170 x(a) = sin (b)
+180 y(a) = cos (b)
+190 next a
+
+199 rem draw code...
+200 d = 160
+210 home : vtab 21 : print "DIAMETER = ";d
+220 for xx = 0 to 279 step d
+225 if xx+d > 280 then 270
+230 for yy = 0 to 159 step d
+235 if yy+d > 160 then 260
+240 r=d/2:xo=xx + r:yo=yy + r:gosub 30
+250 next yy
+260 next xx
+270 d = d / 2
+280 if d > 10 then 210 
+
+300 rem display time
+310 call 768,t2$
+320 home : vtab 21
+330 print "START TIME = ";t1$
+340 print "END TIME =   ";t2$