add some unit tests

Rez/CMakeLists.txt

@@ -17,7 +17,7 @@
 
 cmake_minimum_required(VERSION 2.8)
 
-set(CMAKE_CXX_FLAGS "--std=c++11")
+set(CMAKE_CXX_FLAGS "--std=c++11 -Wall")
 
 find_package(Boost COMPONENTS wave filesystem system thread regex program_options)
 
@@ -34,9 +34,7 @@ add_custom_command(
   COMMENT "Generating parser.cpp"
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.hh
 )
-add_executable(Rez
+add_library(RezLib
-		Rez.cc
-
 		RezParser.yy RezParser.generated.hh RezParser.generated.cc
 
 		RezLexer.h
@@ -58,6 +56,13 @@ add_executable(Rez
 
 		ResSpec.h
 	)
-target_link_libraries(Rez ResourceFiles ${Boost_LIBRARIES})
+target_link_libraries(RezLib ResourceFiles ${Boost_LIBRARIES})
+
+add_executable(Rez
+		Rez.cc
+	)
+target_link_libraries(Rez RezLib ResourceFiles ${Boost_LIBRARIES})
 
 install(TARGETS Rez RUNTIME DESTINATION bin)
+
+add_subdirectory(Test)

Rez/ResourceCompiler.cc

@@ -6,19 +6,32 @@ ResourceCompiler::ResourceCompiler(
 		TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag)
 	: typeDefinition(type),
 	  body(body),
-	  currentOffset(0),
+	  currentField(nullptr)
-	  currentField(nullptr),
-	  verboseFlag(verboseFlag)
 {
-
+	this->verboseFlag = verboseFlag;
 }
 
-std::string ResourceCompiler::resourceData()
+BinaryOutput::BinaryOutput()
+	: verboseFlag(false)
+{
+	reset(true);
+}
+
+void BinaryOutput::reset(bool prePass)
+{
+	currentOffset = 0;
+	if(!prePass)
+		prePassData = std::move(data);
+	data.clear();
+	this->prePass = prePass;
+}
+
+std::string BinaryOutput::resourceData()
 {
 	return std::string(data.begin(), data.end());
 }
 
-void ResourceCompiler::write(int nBits, int value)
+void BinaryOutput::write(int nBits, int value)
 {
 	if(verboseFlag)
 		std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl;
@@ -41,17 +54,17 @@ void ResourceCompiler::write(int nBits, int value)
 	//currentOffset += nBits;
 }
 
-int ResourceCompiler::peek(int bitPos, int size)
+int BinaryOutput::peek(int bitPos, int size)
 {
-	int bytePos = bitPos / 8;
+	unsigned bytePos = bitPos / 8;
-	int endBytePos = (bitPos + size - 1) / 8 + 1;
+	unsigned endBytePos = (bitPos + size - 1) / 8 + 1;
 
 	unsigned bitPosInByte = bitPos % 8;
 	unsigned outPos = 32 - size;
 
 	unsigned val = 0;
 
-	for(int i = bytePos; i != endBytePos; ++i)
+	for(unsigned i = bytePos; i != endBytePos; ++i)
 	{
 		unsigned byte;
 		if(i < data.size())
@@ -99,18 +112,13 @@ void ResourceCompiler::defineLabel(const std::string &name)
 void ResourceCompiler::compile()
 {
 	if(verboseFlag) std::cout << "(first pass)\n";
-	currentOffset = 0;
+	reset(true);
-	data.clear();
-	prePass = true;
 	typeDefinition->compile(body, this, true);
 	if(verboseFlag) std::cout << "(second pass)\n";
-	currentOffset = 0;
+
-	prePassData = std::move(data);
+	reset(false);
-	data.clear();	// ###
-	prePass = false;
 	typeDefinition->compile(body, this, false);
 	if(verboseFlag) std::cout << "(done)\n";
-
 }
 
 int ResourceCompiler::getArrayCount(const std::string &name)

Rez/ResourceCompiler.h

@@ -20,25 +20,17 @@ public:
 	bool empty() const { return subscripts.empty(); }
 };
 
-class ResourceCompiler
+class BinaryOutput
 {
-	TypeDefinitionPtr typeDefinition;
+protected:
-	CompoundExprPtr body;
-	std::map<std::pair<std::string, Subscripts>, ExprPtr> labelValues;
-	std::map<std::pair<std::string, Subscripts>, int> arrayCounts;
-	std::map<std::string, int> curArrayIndices;
 	int currentOffset;
-	Field* currentField;
+	std::vector<unsigned char> data;
-	Subscripts currentSubscripts;
+	std::vector<unsigned char> prePassData;
-
-	std::vector<unsigned char>	data, prePassData;
 	bool verboseFlag;
-
-	void beginArrayScope(std::string& arrayName, int index);
-
 	bool prePass;
 public:
-	ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag);
+	BinaryOutput();
+	void reset(bool prePass);
 
 	std::string resourceData();
 
@@ -48,6 +40,27 @@ public:
 
 	int peek(int bitPos, int size);
 
+	bool isPrePass() { return prePass; }
+};
+
+class ResourceCompiler : public BinaryOutput
+{
+	TypeDefinitionPtr typeDefinition;
+	CompoundExprPtr body;
+	std::map<std::pair<std::string, Subscripts>, ExprPtr> labelValues;
+	std::map<std::pair<std::string, Subscripts>, int> arrayCounts;
+	std::map<std::string, int> curArrayIndices;
+	Field* currentField;
+	Subscripts currentSubscripts;
+
+
+
+	void beginArrayScope(std::string& arrayName, int index);
+
+public:
+	ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag);
+
+
 	ExprPtr lookupIdentifier(std::string name, const Subscripts& sub = Subscripts());
 
 	void defineLabel(const std::string& name);
@@ -56,7 +69,6 @@ public:
 	int getArrayCount(const std::string& arrayName);
 	int getArrayIndex(const std::string& arrayName);
 
-	bool isPrePass() { return prePass; }
 
 	class FieldScope
 	{

Rez/RezLexer.cc

@@ -11,6 +11,25 @@ namespace wave = boost::wave;
 
 using namespace boost::wave;
 
+static std::string readContents(std::istream&& instream)
+{
+	instream.unsetf(std::ios::skipws);
+
+	return std::string(std::istreambuf_iterator<char>(instream.rdbuf()),
+					   std::istreambuf_iterator<char>());
+}
+
+static std::string preFilter(std::string str)
+{
+	boost::regex endif("#endif[^\r\n]*");
+	str = boost::regex_replace(str, endif, "#endif");
+
+	boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])");
+	str = boost::regex_replace(str, dollar_escape, "\\\\0x$1");
+
+	return str;
+}
+
 struct load_file_to_string_filtered
 {
 	template <typename IterContextT>
@@ -30,19 +49,8 @@ struct load_file_to_string_filtered
 					bad_include_file, iter_ctx.filename.c_str(), act_pos);
 				return;
 			}
-			instream.unsetf(std::ios::skipws);
 
-			std::string str(std::istreambuf_iterator<char>(instream.rdbuf()),
+			iter_ctx.instring = preFilter(readContents(std::move(instream)));
-							std::istreambuf_iterator<char>());
-
-			boost::regex endif("#endif[^\r\n]*");
-			str = boost::regex_replace(str, endif, "#endif");
-
-			boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])");
-			str = boost::regex_replace(str, dollar_escape, "\\x$1");
-
-
-			iter_ctx.instring = str;
 
 			iter_ctx.first = iterator_type(
 				iter_ctx.instring.begin(), iter_ctx.instring.end(),
@@ -79,13 +87,13 @@ struct RezLexer::Priv
 };
 
 RezLexer::RezLexer(std::string filename)
+	: RezLexer(filename, readContents(std::ifstream(filename)))
 {
-	std::ifstream instream(filename);
+}
 
-	pImpl.reset(new Priv(std::string(
+RezLexer::RezLexer(std::string filename, const std::string &data)
-							 std::istreambuf_iterator<char>(instream.rdbuf()),
+{
-							 std::istreambuf_iterator<char>()),
+	pImpl.reset(new Priv(preFilter(data), filename));
-						  filename));
 
 	pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes");
 	pImpl->ctx.add_macro_definition("DeRez=0");

Rez/RezLexer.h

@@ -20,6 +20,7 @@ class RezLexer
 
 public:
 	RezLexer(std::string filename);
+	RezLexer(std::string filename, const std::string& data);
 	~RezLexer();
 
 	RezSymbol nextToken();

Rez/Test/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(CMAKE_CXX_FLAGS "--std=c++11 -Wall -Wno-multichar")
+find_package(Boost COMPONENTS unit_test_framework)
+add_executable(RezUnitTests UnitTests.cc)
+target_link_libraries(RezUnitTests RezLib ${Boost_LIBRARIES})

Rez/Test/UnitTests.cc

@@ -0,0 +1,134 @@
+#define BOOST_TEST_MODULE UnitTests
+#define BOOST_TEST_DYN_LINK
+#define BOOST_TEST_MAIN
+#include <boost/test/unit_test.hpp>
+
+#include "RezLexer.h"
+#include "RezParser.generated.hh"
+#include "ResourceCompiler.h"
+
+#include <iostream>
+
+BOOST_AUTO_TEST_SUITE(LexSuite)
+
+#define CHECKSYM(TOKEN, TYPE, VAL)	\
+	do {	\
+		RezSymbol t = lex.nextToken();	\
+		BOOST_CHECK_EQUAL(t.token(), TOKEN);	\
+		if(t.token() == TOKEN)	\
+			BOOST_CHECK_EQUAL(t.value.as<TYPE>(), VAL);	\
+	} while(0)
+#define CHECKSYM_(TOKEN)	\
+	do {	\
+		RezSymbol t = lex.nextToken();	\
+		BOOST_CHECK_EQUAL(t.token(), TOKEN);	\
+	} while(0)
+
+BOOST_AUTO_TEST_CASE(basicInt)
+{
+	RezLexer lex("test", "123 0x456 0xaBcd9\n");
+
+	CHECKSYM(RezParser::token::INTLIT, int, 123);
+	CHECKSYM(RezParser::token::INTLIT, int, 0x456);
+	CHECKSYM(RezParser::token::INTLIT, int, 0xabcd9);
+	CHECKSYM_(0);
+}
+
+BOOST_AUTO_TEST_CASE(alternateHex)
+{
+	RezLexer lex("test", "$456 $aBcd9\n");
+
+	CHECKSYM(RezParser::token::INTLIT, int, 0x456);
+	CHECKSYM(RezParser::token::INTLIT, int, 0xabcd9);
+	CHECKSYM_(0);
+}
+
+BOOST_AUTO_TEST_CASE(noNewlineAtEOF)
+{
+	RezLexer lex("test", "123 456");
+	CHECKSYM(RezParser::token::INTLIT, int, 123);
+	CHECKSYM(RezParser::token::INTLIT, int, 456);
+	CHECKSYM_(0);
+}
+
+BOOST_AUTO_TEST_CASE(strings)
+{
+	RezLexer lex("test", R"rez(
+		"Hello, world."
+		"Foo \n"
+		"\r Quux"
+		"\001\002\003"
+		"\0x42\0x43"
+		"Blah \$5F"
+	)rez" "\n");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "Hello, world.");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "Foo \n");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "\r Quux");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "\001\002\003");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "\x42\x43");
+	CHECKSYM(RezParser::token::STRINGLIT, std::string, "Blah \x5F");
+	CHECKSYM_(0);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+BOOST_AUTO_TEST_SUITE(BinarySuite)
+
+BOOST_AUTO_TEST_CASE(bytes)
+{
+	BinaryOutput out;
+	out.write(8, 'a');
+	out.write(8, 'b');
+	out.write(8, 'c');
+	BOOST_CHECK_EQUAL(out.resourceData(), "abc");
+}
+
+BOOST_AUTO_TEST_CASE(multibyte)
+{
+	BinaryOutput out;
+	out.write(32, 'abcd');
+	BOOST_CHECK_EQUAL(out.resourceData(), "abcd");
+}
+
+BOOST_AUTO_TEST_CASE(subbyte)
+{
+	BinaryOutput out;
+	out.write(4, 6);
+	out.write(4, 1);
+
+	out.write(2, 1);
+	out.write(2, 2);
+	out.write(4, 2);
+
+	out.write(3, 3);
+	out.write(2, 0);
+	out.write(3, 3);
+	BOOST_CHECK_EQUAL(out.resourceData(), "abc");
+}
+
+
+BOOST_AUTO_TEST_CASE(peek)
+{
+	BinaryOutput out;
+	for(char c : "Hello, world.")
+		if(c != 0)
+			out.write(8, c);
+
+	BOOST_CHECK_EQUAL(out.resourceData(), "Hello, world.");
+
+	BOOST_CHECK_EQUAL(out.peek(0,8), 'H');
+	BOOST_CHECK_EQUAL(out.peek(32,8), 'o');
+
+	BOOST_CHECK_EQUAL(out.peek(0,32), 'Hell');
+	BOOST_CHECK_EQUAL(out.peek(40,32), ', wo');
+
+	BOOST_CHECK_EQUAL(out.peek(1,8), 'H' * 2);
+	BOOST_CHECK_EQUAL(out.peek(2,8), 0x21);
+
+	BOOST_CHECK_EQUAL(out.peek(2,30), 'Hell' & 0x3FFFFFFF);
+	BOOST_CHECK_EQUAL(out.peek(2,32), ('Hell' & 0x3FFFFFFF) << 2 | ('o' >> 6) );
+
+	BOOST_CHECK_EQUAL(out.peek(4,3), 4);
+}
+
+BOOST_AUTO_TEST_SUITE_END()