From 4c7e1ca5303d18a366a4586aace4f8ddde0cee55 Mon Sep 17 00:00:00 2001
From: Rob Greene <robgreene@gmail.com>
Date: Sun, 13 May 2018 11:55:36 -0500
Subject: [PATCH] Pulling in AppleCommander tweaks.

---
 .../util/applesoft/TokenReader.java           | 27 +++++++++++++------
 .../util/applesoft/package-info.java          |  8 ++++++
 2 files changed, 27 insertions(+), 8 deletions(-)
 create mode 100644 bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/package-info.java

diff --git a/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/TokenReader.java b/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/TokenReader.java
index c9f138e..62e4650 100644
--- a/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/TokenReader.java
+++ b/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/TokenReader.java
@@ -3,6 +3,8 @@ package com.webcodepro.applecommander.util.applesoft;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StreamTokenizer;
 import java.util.LinkedList;
@@ -25,16 +27,25 @@ public class TokenReader {
 	/** A handy method to generate a list of Tokens from a file. */
 	public static Queue<Token> tokenize(String filename) throws FileNotFoundException, IOException {
 		try (FileReader fileReader = new FileReader(filename)) {
-			TokenReader tokenReader = new TokenReader(fileReader);
-			LinkedList<Token> tokens = new LinkedList<>();
-			while (tokenReader.hasMore()) {
-				// Magic number: maximum number of pieces from the StreamTokenizer that may be combined.
-				tokenReader.next(2)
-						   .ifPresent(tokens::add);
-			}
-			return tokens;
+			return tokenize(fileReader);
 		}
 	}
+	/** A handy method to generate a list of Tokens from an InputStream. */
+	public static Queue<Token> tokenize(InputStream inputStream) throws IOException {
+		try (InputStreamReader streamReader = new InputStreamReader(inputStream)) {
+			return tokenize(streamReader);
+		}
+	}
+	private static Queue<Token> tokenize(Reader reader) throws IOException {
+		TokenReader tokenReader = new TokenReader(reader);
+		LinkedList<Token> tokens = new LinkedList<>();
+		while (tokenReader.hasMore()) {
+			// Magic number: maximum number of pieces from the StreamTokenizer that may be combined.
+			tokenReader.next(2)
+					   .ifPresent(tokens::add);
+		}
+		return tokens;
+	}
 
 	public TokenReader(Reader reader) {
 		this.reader = reader;
diff --git a/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/package-info.java b/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/package-info.java
new file mode 100644
index 0000000..2674183
--- /dev/null
+++ b/bastokenizer/src/main/java/com/webcodepro/applecommander/util/applesoft/package-info.java
@@ -0,0 +1,8 @@
+/**
+ * This package is used by the AppleSoft text file import capabilities of AppleCommander.
+ * It is separate from the other components, primarily due to it being used on the input side,
+ * but also due to the fact that this is the third time AppleSoft tokens have been defined! 
+ * 
+ * @author rob
+ */
+package com.webcodepro.applecommander.util.applesoft;
\ No newline at end of file