From cb554ed40f838624c8aad01aac25de4407b7018a Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Sun, 5 Oct 2014 23:52:34 +0200 Subject: [PATCH 01/29] Rez: successful parse. --- CMakeLists.txt | 1 + Rez/CMakeLists.txt | 49 +++++++ Rez/Rez.cc | 67 ++++++++++ Rez/RezLexer.cc | 116 +++++++++++++++++ Rez/RezLexer.h | 28 ++++ Rez/RezLexerNextToken.cc | 192 +++++++++++++++++++++++++++ Rez/RezLexerWaveToken.h | 15 +++ Rez/RezParser.yy | 271 +++++++++++++++++++++++++++++++++++++++ Rez/Test.r | 11 ++ 9 files changed, 750 insertions(+) create mode 100644 Rez/CMakeLists.txt create mode 100644 Rez/Rez.cc create mode 100644 Rez/RezLexer.cc create mode 100644 Rez/RezLexer.h create mode 100644 Rez/RezLexerNextToken.cc create mode 100644 Rez/RezLexerWaveToken.h create mode 100644 Rez/RezParser.yy create mode 100644 Rez/Test.r diff --git a/CMakeLists.txt b/CMakeLists.txt index 67f1b96c41..f46cdec399 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,4 +69,5 @@ add_subdirectory(Launcher) else() add_subdirectory(MakeAPPL) add_subdirectory(ASFilter) +add_subdirectory(Rez) endif() diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt new file mode 100644 index 0000000000..ae7c33bf55 --- /dev/null +++ b/Rez/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright 2012 Wolfgang Thaller. +# +# This file is part of Retro68. +# +# Retro68 is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Retro68 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Retro68. If not, see . + +cmake_minimum_required(VERSION 2.8) + +set(CMAKE_CXX_FLAGS "--std=c++11") + +find_package(Boost COMPONENTS wave filesystem system thread regex) + + +find_package(BISON REQUIRED) + +include_directories(. ${CMAKE_CURRENT_BINARY_DIR}) + +add_custom_command( + DEPENDS RezParser.yy + COMMAND ${BISON_EXECUTABLE} + ARGS -o ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc + ${CMAKE_CURRENT_SOURCE_DIR}/RezParser.yy --graph + COMMENT "Generating parser.cpp" + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.hh +) +add_executable(Rez + Rez.cc + + RezParser.yy RezParser.generated.hh RezParser.generated.cc + + RezLexer.h + RezLexer.cc + RezLexerWaveToken.h + RezLexerNextToken.cc + ) +target_link_libraries(Rez ${Boost_LIBRARIES}) + +install(TARGETS Rez RUNTIME DESTINATION bin) diff --git a/Rez/Rez.cc b/Rez/Rez.cc new file mode 100644 index 0000000000..91560796d7 --- /dev/null +++ b/Rez/Rez.cc @@ -0,0 +1,67 @@ +#include +#include + + +#include "RezParser.generated.hh" +#include "RezLexer.h" + + +int main() +{ + //RezLexer lexer("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); + RezLexer lexer("/home/wolfgang/Projects/Retro68/Rez/Test.r"); + + RezParser parser(lexer); + + parser.parse(); + + /* + // The following preprocesses a given input file. + // Open the file and read it into a string variable + std::ifstream instream("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); + + std::string input( + std::istreambuf_iterator(instream.rdbuf()), + std::istreambuf_iterator()); + + context_type ctx(input.begin(), input.end(), "Types.r"); + + // At this point you may want to set the parameters of the + // preprocessing as include paths and/or predefined macros. + ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); + // ctx.add_macro_definition(...); + + auto first = ctx.begin(); + auto last = ctx.end(); + + std::ostringstream out; + + try + { + while(first != last) + { + out << (*first).get_value(); + ++first; + } + } + catch(boost::wave::preprocess_exception& e) + { + std::cout << e.file_name() << ":" << e.line_no() << ": "; + std::cout << e.description() << std::endl; + } + + std::string str = out.str(); + + + std::cout << str.substr(0,100) << std::endl;*/ + +/* + int i = 0; + while (first != last) { + std::cout << i << ": " << get_token_name(token_id(*first)) << " <<" << (*first).get_value() << ">>\n"; + ++first; + if(++i > 10) + break; + }*/ + return 0; +} diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc new file mode 100644 index 0000000000..739664b524 --- /dev/null +++ b/Rez/RezLexer.cc @@ -0,0 +1,116 @@ +#include "RezLexer.h" + +#include +#include +#include +#include + +#include "RezLexerWaveToken.h" + +namespace wave = boost::wave; + +using namespace boost::wave; + +struct load_file_to_string_filtered +{ + template + class inner + { + public: + template + static void init_iterators(IterContextT &iter_ctx, + PositionT const &act_pos, language_support language) + { + typedef typename IterContextT::iterator_type iterator_type; + + // read in the file + std::ifstream instream(iter_ctx.filename.c_str()); + if (!instream.is_open()) { + BOOST_WAVE_THROW_CTX(iter_ctx.ctx, preprocess_exception, + bad_include_file, iter_ctx.filename.c_str(), act_pos); + return; + } + instream.unsetf(std::ios::skipws); + + std::string str(std::istreambuf_iterator(instream.rdbuf()), + std::istreambuf_iterator()); + + boost::regex endif("#endif[^\r\n]*"); + str = boost::regex_replace(str, endif, "#endif"); + + boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])"); + str = boost::regex_replace(str, dollar_escape, "\\x$1"); + + + iter_ctx.instring = str; + + iter_ctx.first = iterator_type( + iter_ctx.instring.begin(), iter_ctx.instring.end(), + PositionT(iter_ctx.filename), language); + iter_ctx.last = iterator_type(); + } + + private: + std::string instring; + }; +}; + + + +typedef wave::cpplexer::lex_iterator< + wave::cpplexer::lex_token<> > + lex_iterator_type; +typedef wave::context< + std::string::iterator, lex_iterator_type, + load_file_to_string_filtered> + context_type; +typedef context_type::iterator_type pp_iterator_type; + +struct RezLexer::Priv +{ + std::string input; + context_type ctx; + pp_iterator_type iter; + + Priv(std::string data, std::string name) + : input(data), ctx(input.begin(), input.end(), name.c_str()) + { + } +}; + +RezLexer::RezLexer(std::string filename) +{ + std::ifstream instream(filename); + + pImpl.reset(new Priv(std::string( + std::istreambuf_iterator(instream.rdbuf()), + std::istreambuf_iterator()), + filename)); + + pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); + // ctx.add_macro_definition(...); + pImpl->ctx.add_macro_definition("DeRez", "0"); + + pImpl->iter = pImpl->ctx.begin(); +} + +RezLexer::~RezLexer() +{ + +} + +bool RezLexer::atEnd() +{ + return pImpl->iter == pImpl->ctx.end(); +} + +RezLexer::WaveToken RezLexer::nextWave() +{ + return pImpl->iter == pImpl->ctx.end() ? WaveToken() : (*pImpl->iter++); +} + +RezLexer::WaveToken RezLexer::peekWave() +{ + return pImpl->iter == pImpl->ctx.end() ? WaveToken() : *pImpl->iter; +} + diff --git a/Rez/RezLexer.h b/Rez/RezLexer.h new file mode 100644 index 0000000000..c0e7f12cea --- /dev/null +++ b/Rez/RezLexer.h @@ -0,0 +1,28 @@ +#ifndef REZLEXER_H +#define REZLEXER_H + +#include + +class RezSymbol; + +class RezLexer +{ + struct Priv; + std::unique_ptr pImpl; + + std::string curFile; + + class WaveToken; + + bool atEnd(); + WaveToken nextWave(); + WaveToken peekWave(); + +public: + RezLexer(std::string filename); + ~RezLexer(); + + RezSymbol nextToken(); +}; + +#endif // REZLEXER_H diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc new file mode 100644 index 0000000000..f862edbf02 --- /dev/null +++ b/Rez/RezLexerNextToken.cc @@ -0,0 +1,192 @@ +#include "RezLexer.h" +#include "RezLexerWaveToken.h" +#include "RezParser.generated.hh" +#include + +#include + +using namespace boost::wave; + +static int readInt(const char *str) +{ + int x = 0; + + int base = 10; + + if(*str == '0') + { + base = 8; + ++str; + if(*str == 'x' || *str == 'X') + { + base = 16; + ++str; + } + if(*str == 'b' || *str == 'B') + { + base = 2; + ++str; + } + } + else if(*str == 'b' || *str == 'B') + { + base = 2; + ++str; + } + + while(*str) + { + x *= base; + if(*str >= 'a' && *str <= 'z') + x += *str - 'a'; + else if(*str >= 'A' && *str <= 'Z') + x += *str - 'A' + 10; + else if(*str >= '0' && *str <= '9') + x += *str - '0'; + *str++; + } + + return x; +} + +RezSymbol RezLexer::nextToken() +{ + for(auto tok = nextWave(); tok != T_EOI && tok != T_EOF; tok = nextWave()) + { + if(IS_CATEGORY(tok, WhiteSpaceTokenType)) + continue; + else if(IS_CATEGORY(tok, EOLTokenType)) + continue; + else if(tok == T_PP_LINE) + { + while(tok != T_EOI && tok != T_EOF && !IS_CATEGORY(tok, EOLTokenType)) + tok = nextWave(); + continue; + } + else + { + //std::cout << "{" << std::hex << (token_id)tok << std::dec << "|" << tok.get_value() << "}\n"; + + auto pos = tok.get_position(); + curFile = pos.get_file().c_str(); + auto yypos = yy::position(&curFile, pos.get_line(), pos.get_column()); + yy::location loc(yypos); + + if(tok == (UnknownTokenType | '"')) + { + return RezParser::make_STRINGLIT("Hello, world.", loc); + } + else if(IS_CATEGORY(tok, IdentifierTokenType) || IS_CATEGORY(tok, KeywordTokenType) || IS_CATEGORY(tok, BoolLiteralTokenType)) + { + typedef decltype(&RezParser::make_TYPE) memfun; +#define KEYWORD(upper, lower) \ +{ lower, &RezParser::make_ ## upper } + + static std::unordered_map keywords = { + KEYWORD(TYPE, "type"), + KEYWORD(RESOURCE, "resource"), + + KEYWORD(ARRAY,"array"), + KEYWORD(SWITCH, "switch"), + KEYWORD(CASE, "case"), + KEYWORD(AS, "as"), + KEYWORD(FILL,"fill"), + KEYWORD(ALIGN, "align"), + KEYWORD(HEX,"hex"), + KEYWORD(KEY, "key"), + KEYWORD(WIDE,"wide"), + KEYWORD(UNSIGNED, "unsigned"), + KEYWORD(LITERAL, "literal"), + KEYWORD(BOOLEAN, "boolean"), + KEYWORD(BIT, "bit"), + KEYWORD(BYTE, "byte"), + KEYWORD(CHAR, "char"), + KEYWORD(WORD, "word"), + KEYWORD(INTEGER, "integer"), + KEYWORD(LONG, "long"), + KEYWORD(LONGINT, "longint"), + KEYWORD(PSTRING, "pstring"), + KEYWORD(PSTRING, "wstring"), + KEYWORD(STRING, "string"), + KEYWORD(POINT, "point"), + KEYWORD(RECT, "rect"), + KEYWORD(BITSTRING, "bitstring"), + + KEYWORD(INTEGER, "int"), + + }; + + std::string s = tok.get_value().c_str(); + std::string lower = s; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + auto p = keywords.find(lower); + if(p == keywords.end()) + { + //std::cout << "id: " << s << std::endl; + return RezParser::make_IDENTIFIER(lower, loc); + } + else + { + //std::cout << "key: " << s << std::endl; + return (*p->second)(loc); + } + } + else if(tok == T_INTLIT) + { + if(tok.get_value() == "0") + { + auto tok2 = peekWave(); + while(tok2 != T_EOI && tok2 != T_EOF && IS_CATEGORY(tok2, WhiteSpaceTokenType)) + nextWave(), tok2 = peekWave(); + + //std::cout << "!" << std::hex << (token_id)tok2 << std::dec << "|" << tok2.get_value() << "!\n"; + static boost::regex binlit("[bB][01]+"); + if(tok2 == T_IDENTIFIER && boost::regex_match(tok2.get_value().c_str(), binlit)) + tok = nextWave(); + } + return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc); + } + else + { +#define NOVAL_TOK(name) \ +case T_ ## name: /*std::cout << #name << std::endl;*/ return RezParser::make_ ## name(loc) + switch(token_id(tok)) + { + case T_INTLIT: return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc); + + case T_CHARLIT: return RezParser::make_CHARLIT(tok.get_value().c_str(), loc); + case T_STRINGLIT: return RezParser::make_STRINGLIT(tok.get_value().c_str(), loc); + + NOVAL_TOK(LEFTBRACE); + NOVAL_TOK(RIGHTBRACE); + NOVAL_TOK(LEFTBRACKET); + NOVAL_TOK(RIGHTBRACKET); + NOVAL_TOK(LEFTPAREN); + NOVAL_TOK(RIGHTPAREN); + NOVAL_TOK(SEMICOLON); + NOVAL_TOK(COMMA); + NOVAL_TOK(PLUS); + NOVAL_TOK(MINUS); + NOVAL_TOK(DIVIDE); + NOVAL_TOK(STAR); + NOVAL_TOK(ASSIGN); + NOVAL_TOK(COLON); + NOVAL_TOK(SHIFTLEFT); + NOVAL_TOK(SHIFTRIGHT); + NOVAL_TOK(EQUAL); + NOVAL_TOK(NOTEQUAL); + NOVAL_TOK(AND); + NOVAL_TOK(OR); + NOVAL_TOK(XOR); + NOVAL_TOK(COMPL); + + default: + + return RezParser::make_BADTOKEN(tok.get_value().c_str(), loc); + } + + } + } + } + return RezSymbol(); +} diff --git a/Rez/RezLexerWaveToken.h b/Rez/RezLexerWaveToken.h new file mode 100644 index 0000000000..3d11903b0b --- /dev/null +++ b/Rez/RezLexerWaveToken.h @@ -0,0 +1,15 @@ +#ifndef REZLEXERWAVETOKEN_H +#define REZLEXERWAVETOKEN_H + +#include "RezLexer.h" + +#include + +class RezLexer::WaveToken : public boost::wave::cpplexer::lex_token<> +{ +public: + WaveToken() = default; + WaveToken(const boost::wave::cpplexer::lex_token<> & o) : boost::wave::cpplexer::lex_token<>(o) {} +}; + +#endif // REZLEXERWAVETOKEN_H diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy new file mode 100644 index 0000000000..ed5dacad8e --- /dev/null +++ b/Rez/RezParser.yy @@ -0,0 +1,271 @@ +%require "3.0.2" +%defines +%define parser_class_name {RezParser} +%skeleton "lalr1.cc" + +%locations; + +%define api.token.constructor +%define api.value.type variant +%define parse.assert + +%token IDENTIFIER; +%token CHARLIT; +%token STRINGLIT; +%token INTLIT; + +%token BADTOKEN; + + +%token LEFTBRACE "{"; +%token RIGHTBRACE "}"; +%token LEFTBRACKET "["; +%token RIGHTBRACKET "]"; +%token LEFTPAREN "("; +%token RIGHTPAREN ")"; +%token SEMICOLON ";"; +%token COMMA ","; +%token PLUS "+"; +%token MINUS "-"; +%token DIVIDE "/"; +%token STAR "*"; +%token ASSIGN "="; +%token COLON ":"; +%token SHIFTLEFT "<<"; +%token SHIFTRIGHT ">>"; +%token EQUAL "=="; +%token NOTEQUAL "!="; +%token AND "&"; +%token OR "|"; +%token XOR "^"; +%token COMPL "~"; + + +%token TYPE "type"; +%token RESOURCE "resource"; +%token ARRAY "array"; +%token SWITCH "switch"; +%token CASE "case"; +%token AS "as"; +%token FILL "fill"; +%token ALIGN "align"; +%token HEX "hex"; +%token KEY "key"; +%token WIDE "wide"; +%token LITERAL "literal"; +%token UNSIGNED "unsigned"; + +%token BOOLEAN "boolean"; +%token BIT "bit"; +%token BYTE "byte"; +%token CHAR "char"; +%token WORD "word"; +%token INTEGER "integer"; +%token LONG "long"; +%token LONGINT "longint"; +%token PSTRING "pstring"; +%token WSTRING "wstring"; +%token STRING "string"; +%token POINT "point"; +%token RECT "rect"; +%token BITSTRING "bitstring"; + +/* +%left "|"; +%left "^"; +%left "&"; +%left "==" "!="; +%left ">>" "<<"; +%left "+" "-"; +%left "*" "/"; +*/ + +%param { RezLexer& lexer } + +%code requires { + #define YY_NULLPTR nullptr + class RezLexer; +} + +%code provides { + using yy::RezParser; + //using RezSymbol = yy::RezParser::symbol_type; + + class RezSymbol : public yy::RezParser::symbol_type + { + public: + RezSymbol() = default; + RezSymbol(const yy::RezParser::symbol_type& x) : yy::RezParser::symbol_type(x) {} + }; +} + +%code { + #include "RezLexer.h" + static yy::RezParser::symbol_type yylex(RezLexer& lexer) + { + return lexer.nextToken(); + } + + void yy::RezParser::error(const location_type& loc, std::string const& err) + { + std::cerr << loc << ": " << err << std::endl; + } +} + +%% +%start rez; + +rez : %empty + | rez type_definition ";" + | rez resource ";" + ; + +simpletype : "boolean" + | "bit" | "byte" | "word" | "integer" | "long" | "longint" | "rect" + | "point" + | "char" + | "pstring" array_count_opt + | "wstring" array_count_opt + | "string" array_count_opt; + | "bitstring" "[" expression "]"; + +type_definition : "type" type_spec "{" field_definitions "}" + { std::cout << "TYPE " << $2 << std::endl; } + | "type" type_spec "as" type_spec + { std::cout << "TYPE " << $2 << std::endl; } + ; + +%type type_spec; +type_spec : CHARLIT { $$ = $1; } + | CHARLIT "(" INTLIT ")" { $$ = $1; } + ; + +field_definitions : %empty + | field_definitions IDENTIFIER ":" + | field_definitions ";" + | field_definitions field_definition ";" ; + +field_definition: simple_field_definition + | array_definition + | switch_definition + | fill_statement + | align_statement; + +fill_statement : "fill" fill_unit array_count_opt; +align_statement : "align" fill_unit; + +fill_unit : "bit" | "byte" | "word" | "long"; + +simple_field_definition: field_attributes simpletype value_spec; + +value_spec : %empty + | named_values + | "=" expression; + +named_values: named_value + | named_values "," named_value + | named_values named_value; + +named_value : IDENTIFIER + | IDENTIFIER "=" expression ; + +array_definition: array_attributes "array" array_name_opt array_count_opt "{" field_definitions "}" ; + +array_count : "[" expression "]" ; +array_count_opt : %empty | array_count ; + +array_name_opt : %empty | IDENTIFIER ; + +array_attributes: %empty | "wide" ; +field_attributes: %empty | field_attributes field_attribute; +field_attribute : "hex" | "key" | "unsigned" | "literal"; + +switch_definition: "switch" "{" + switch_cases + "}" ; + +switch_cases : %empty | switch_cases switch_case ; + +switch_case : "case" IDENTIFIER ":" field_definitions ; + +/* +expression + | expression "^" expression + | expression "&" expression + | expression "|" expression + | "~" expression + | expression "==" expression + | expression "!=" expression + | expression ">>" expression + | expression "<<" expression + | expression "+" expression + | expression "-" expression + | "-" expression + | expression "/" expression + | expression "*" expression + ; +*/ + +expression : expression1 + | expression "^" expression1 + ; + +expression1 : expression2 + | expression1 "&" expression2 + ; + +expression2 : expression3 + | expression2 "|" expression3 + ; + +expression3 : expression4 + | expression3 "==" expression4 + | expression3 "!=" expression4 + ; + +expression4 : expression5 + | expression4 ">>" expression5 + | expression4 "<<" expression5 + ; + +expression5 : expression6 + | expression5 "+" expression6 + | expression5 "-" expression6 + ; + +expression6 : expression7 + | expression6 "*" expression7 + | expression6 "/" expression7 + ; +expression7 : expression8 + | "-" expression7 + | "+" expression7 + | "~" expression7 + ; + +expression8 : INTLIT + | CHARLIT + | STRINGLIT + | IDENTIFIER + | IDENTIFIER "(" function_argument_list ")" + | IDENTIFIER "[" function_argument_list1 "]" + | "(" expression ")" + | "{" resource_body "}" + ; + +function_argument_list : %empty | function_argument_list1 ; +function_argument_list1 : expression | function_argument_list "," expression ; + +resource: "resource" CHARLIT "(" function_argument_list ")" "{" resource_body "}" + { std::cout << "RESOURCE " << $2 << std::endl; } + +resource_body : %empty | resource_body1 ; +resource_body1 : resource_item + | resource_body1 "," resource_item + | resource_body1 ";" resource_item + | resource_body1 ";" + ; + +resource_item : expression | IDENTIFIER "{" resource_body "}" ; + +%% diff --git a/Rez/Test.r b/Rez/Test.r new file mode 100644 index 0000000000..173917529b --- /dev/null +++ b/Rez/Test.r @@ -0,0 +1,11 @@ +/*#include "Types.r" + + +*/ + +#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r" + +type 'TEST' { +boolean itemUnlocked = false, // defined attributes bits... + itemLocked = true; +}; From 2b3e533937d2ce98b2122b5c3fde3d7d959b346c Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Mon, 6 Oct 2014 17:03:25 +0200 Subject: [PATCH 02/29] start with semantics --- Rez/CMakeLists.txt | 3 + Rez/ResourceDefinitions.cc | 27 +++++++++ Rez/ResourceDefinitions.h | 112 +++++++++++++++++++++++++++++++++++++ Rez/RezLexerNextToken.cc | 20 ++++++- Rez/RezParser.yy | 112 +++++++++++++++++++++++++------------ 5 files changed, 237 insertions(+), 37 deletions(-) create mode 100644 Rez/ResourceDefinitions.cc create mode 100644 Rez/ResourceDefinitions.h diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index ae7c33bf55..c5e070d5ca 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -43,6 +43,9 @@ add_executable(Rez RezLexer.cc RezLexerWaveToken.h RezLexerNextToken.cc + + ResourceDefinitions.cc + ResourceDefinitions.h ) target_link_libraries(Rez ${Boost_LIBRARIES}) diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc new file mode 100644 index 0000000000..6d0e481485 --- /dev/null +++ b/Rez/ResourceDefinitions.cc @@ -0,0 +1,27 @@ +#include "ResourceDefinitions.h" +#include + +ResourceDefinitions::ResourceDefinitions() +{ +} + + + +std::ostream &operator<<(std::ostream &out, ResType t) +{ + char c1 = static_cast((int)t >> 24); + char c2 = static_cast((int)t >> 16); + char c3 = static_cast((int)t >> 8); + char c4 = static_cast((int)t); + + out << "'" << c1 << c2 << c3 << c4 << "'"; + return out; +} + +std::ostream &operator<<(std::ostream &out, TypeSpec ts) +{ + out << ts.getType(); + if(ts.hasID()) + out << " (" << ts.getID() << ")"; + return out; +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h new file mode 100644 index 0000000000..06c239c4e5 --- /dev/null +++ b/Rez/ResourceDefinitions.h @@ -0,0 +1,112 @@ +#ifndef RESOURCEDEFINITIONS_H +#define RESOURCEDEFINITIONS_H + +#include +#include +#include + +class ResType +{ + int x; +public: + ResType() : x(0) {} + ResType(int x) : x(x) {} + operator int() const { return x; } +}; + +std::ostream& operator<<(std::ostream& out, ResType t); + +class TypeSpec +{ + ResType type; + int id; +public: + static const int noID = 65536; + + TypeSpec() : id(noID) {} + TypeSpec(ResType type) : type(type), id(noID) {} + TypeSpec(ResType type, int id) : type(type), id(id) {} + + ResType getType() const { return type; } + int getID() const { return id; } + + bool hasID() const { return id != noID; } +}; + +std::ostream& operator<<(std::ostream& out, TypeSpec ts); + +class Context +{ + +}; + +class Expression +{ +public: + //virtual int evaluateInt(Context *ctx); + +}; + +class StringExpr : public Expression +{ + std::string str; +public: + StringExpr(const std::string& str) : str(str) {} +}; + +class IntExpr : public Expression +{ + int val; +public: + IntExpr(int val) : val(val) {} +}; + +typedef std::shared_ptr ExprPtr; + +class Field +{ +public: + virtual bool needsValue() { return true; } +}; +typedef std::shared_ptr FieldPtr; + +class SimpleField : public Field +{ +public: + enum class Type + { + boolean, byte, integer, longint, rect, point, char_, + pstring, wstring, string, bitstring + }; + + enum class Attrs + { + none = 0, hex = 1, key = 2, unsigned_ = 4, literal = 8 + }; + + Type type; + Attrs attrs = Attrs::none; + ExprPtr arrayCount; + + ExprPtr value; + std::map namedValues; + + void addNamedValue(std::string n) {} + void addNamedValue(std::string n, ExprPtr val) {} +}; +typedef std::shared_ptr SimpleFieldPtr; + + +inline SimpleField::Attrs operator|(SimpleField::Attrs a, SimpleField::Attrs b) +{ + return SimpleField::Attrs( int(a) | int(b) ); +} + + +class ResourceDefinitions +{ +public: + ResourceDefinitions(); +}; + +#endif // RESOURCEDEFINITIONS_H diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index f862edbf02..2297705ed4 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -49,6 +49,24 @@ static int readInt(const char *str) return x; } +static int readCharLit(const char *str) +{ + const char *p = str + 1; + const char *e = str + strlen(str) - 1; + + if(e - p != 4) + std::cout << "warning: CHAR LITERAL " << str << "\n"; + + int x = 0; + while(p != e) + { + x <<= 8; + x |= (*p) & 0xFF; + ++p; + } + return x; +} + RezSymbol RezLexer::nextToken() { for(auto tok = nextWave(); tok != T_EOI && tok != T_EOF; tok = nextWave()) @@ -154,7 +172,7 @@ case T_ ## name: /*std::cout << #name << std::endl;*/ return RezParser::make_ ## { case T_INTLIT: return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc); - case T_CHARLIT: return RezParser::make_CHARLIT(tok.get_value().c_str(), loc); + case T_CHARLIT: return RezParser::make_CHARLIT(readCharLit(tok.get_value().c_str()), loc); case T_STRINGLIT: return RezParser::make_STRINGLIT(tok.get_value().c_str(), loc); NOVAL_TOK(LEFTBRACE); diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index ed5dacad8e..d8005168ca 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -10,7 +10,7 @@ %define parse.assert %token IDENTIFIER; -%token CHARLIT; +%token CHARLIT; %token STRINGLIT; %token INTLIT; @@ -83,6 +83,8 @@ %param { RezLexer& lexer } %code requires { + #include "ResourceDefinitions.h" + #define YY_NULLPTR nullptr class RezLexer; } @@ -120,14 +122,6 @@ rez : %empty | rez resource ";" ; -simpletype : "boolean" - | "bit" | "byte" | "word" | "integer" | "long" | "longint" | "rect" - | "point" - | "char" - | "pstring" array_count_opt - | "wstring" array_count_opt - | "string" array_count_opt; - | "bitstring" "[" expression "]"; type_definition : "type" type_spec "{" field_definitions "}" { std::cout << "TYPE " << $2 << std::endl; } @@ -135,9 +129,12 @@ type_definition : "type" type_spec "{" field_definitions "}" { std::cout << "TYPE " << $2 << std::endl; } ; -%type type_spec; -type_spec : CHARLIT { $$ = $1; } - | CHARLIT "(" INTLIT ")" { $$ = $1; } +%type res_type; +res_type : CHARLIT { $$ = ResType($1); } ; + +%type type_spec; +type_spec : res_type { $$ = TypeSpec($res_type); } + | res_type "(" INTLIT ")" { $$ = TypeSpec($res_type, $INTLIT); } ; field_definitions : %empty @@ -145,40 +142,78 @@ field_definitions : %empty | field_definitions ";" | field_definitions field_definition ";" ; -field_definition: simple_field_definition +%type field_definition; +field_definition: simple_field_definition { $$ = $1; } | array_definition | switch_definition | fill_statement | align_statement; +%type simple_field_definition; +simple_field_definition: field_attributes simpletype array_count_opt value_spec_opt + { + $$ = std::make_shared(); + $$->attrs = $field_attributes; + $$->type = $simpletype; + $$->arrayCount = $array_count_opt; + $$->value = $value_spec_opt; + } + | simple_field_definition IDENTIFIER + { $$ = $1; $$->addNamedValue($IDENTIFIER); } + | simple_field_definition IDENTIFIER "=" value + { $$ = $1; $$->addNamedValue($IDENTIFIER, $value); } + | simple_field_definition "," IDENTIFIER + { $$ = $1; $$->addNamedValue($IDENTIFIER); } + | simple_field_definition "," IDENTIFIER "=" value + { $$ = $1; $$->addNamedValue($IDENTIFIER, $value); } + ; + +%type array_count array_count_opt value_spec_opt value ; +%type expression expression1 expression2 ; +%type expression3 expression4 expression5 expression6; +%type expression7 expression8; + +value_spec_opt : %empty { $$ = nullptr; } | "=" value { $$ = $2; } ; + +%type simpletype; +simpletype : "boolean" { $$ = SimpleField::Type::boolean; } + | "byte" { $$ = SimpleField::Type::byte; } + | "integer" { $$ = SimpleField::Type::integer; } + | "longint" { $$ = SimpleField::Type::longint; } + | "rect" { $$ = SimpleField::Type::rect; } + | "point" { $$ = SimpleField::Type::point; } + | "char" { $$ = SimpleField::Type::char_; } + | "pstring" { $$ = SimpleField::Type::pstring; } + | "wstring" { $$ = SimpleField::Type::wstring; } + | "string" { $$ = SimpleField::Type::string; } + | "bitstring" { $$ = SimpleField::Type::bitstring; } + ; + fill_statement : "fill" fill_unit array_count_opt; align_statement : "align" fill_unit; fill_unit : "bit" | "byte" | "word" | "long"; -simple_field_definition: field_attributes simpletype value_spec; - -value_spec : %empty - | named_values - | "=" expression; - -named_values: named_value - | named_values "," named_value - | named_values named_value; - -named_value : IDENTIFIER - | IDENTIFIER "=" expression ; array_definition: array_attributes "array" array_name_opt array_count_opt "{" field_definitions "}" ; -array_count : "[" expression "]" ; -array_count_opt : %empty | array_count ; +array_count : "[" expression "]" { $$ = $2; } +array_count_opt : %empty { $$ = nullptr; } | array_count; array_name_opt : %empty | IDENTIFIER ; array_attributes: %empty | "wide" ; -field_attributes: %empty | field_attributes field_attribute; -field_attribute : "hex" | "key" | "unsigned" | "literal"; + +%type field_attributes field_attribute; +field_attributes: %empty { $$ = SimpleField::Attrs::none; } + | field_attributes field_attribute { $$ = $1 | $2; } + ; + +field_attribute : "hex" { $$ = SimpleField::Attrs::hex; } + | "key" { $$ = SimpleField::Attrs::key; } + | "unsigned" { $$ = SimpleField::Attrs::unsigned_; } + | "literal" { $$ = SimpleField::Attrs::literal; } + ; switch_definition: "switch" "{" switch_cases @@ -206,6 +241,11 @@ expression ; */ +value : expression + | "{" resource_body "}" + | STRINGLIT { $$ = std::make_shared($1); } + ; + expression : expression1 | expression "^" expression1 ; @@ -243,20 +283,20 @@ expression7 : expression8 | "~" expression7 ; -expression8 : INTLIT - | CHARLIT - | STRINGLIT +expression8 : INTLIT { $$ = std::make_shared($1); } + | CHARLIT { $$ = std::make_shared($1); } + | IDENTIFIER | IDENTIFIER "(" function_argument_list ")" | IDENTIFIER "[" function_argument_list1 "]" - | "(" expression ")" - | "{" resource_body "}" + | "(" expression ")" { $$ = $2; } + ; function_argument_list : %empty | function_argument_list1 ; function_argument_list1 : expression | function_argument_list "," expression ; -resource: "resource" CHARLIT "(" function_argument_list ")" "{" resource_body "}" +resource: "resource" res_type "(" function_argument_list ")" "{" resource_body "}" { std::cout << "RESOURCE " << $2 << std::endl; } resource_body : %empty | resource_body1 ; @@ -266,6 +306,6 @@ resource_body1 : resource_item | resource_body1 ";" ; -resource_item : expression | IDENTIFIER "{" resource_body "}" ; +resource_item : value | IDENTIFIER "{" resource_body "}" ; %% From e5d2ab752afd27853d76468a22fc18f712a91b31 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Tue, 7 Oct 2014 20:15:46 +0200 Subject: [PATCH 03/29] can layout (but not really compile) resources consisting of integers only --- Rez/CMakeLists.txt | 9 +++ Rez/Expression.cc | 87 ++++++++++++++++++++++ Rez/Expression.h | 94 ++++++++++++++++++++++++ Rez/ResourceCompiler.cc | 11 +++ Rez/ResourceCompiler.h | 15 ++++ Rez/ResourceDefinitions.cc | 86 ++++++++++++++++++++-- Rez/ResourceDefinitions.h | 62 ++++++++-------- Rez/Rez.cc | 55 +------------- Rez/RezParser.yy | 145 +++++++++++++++++++++---------------- Rez/RezWorld.cc | 22 ++++++ Rez/RezWorld.h | 23 ++++++ 11 files changed, 459 insertions(+), 150 deletions(-) create mode 100644 Rez/Expression.cc create mode 100644 Rez/Expression.h create mode 100644 Rez/ResourceCompiler.cc create mode 100644 Rez/ResourceCompiler.h create mode 100644 Rez/RezWorld.cc create mode 100644 Rez/RezWorld.h diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index c5e070d5ca..4b59cb4f7b 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -44,8 +44,17 @@ add_executable(Rez RezLexerWaveToken.h RezLexerNextToken.cc + RezWorld.cc + RezWorld.h + ResourceDefinitions.cc ResourceDefinitions.h + + Expression.cc + Expression.h + + ResourceCompiler.cc + ResourceCompiler.h ) target_link_libraries(Rez ${Boost_LIBRARIES}) diff --git a/Rez/Expression.cc b/Rez/Expression.cc new file mode 100644 index 0000000000..dfa3208764 --- /dev/null +++ b/Rez/Expression.cc @@ -0,0 +1,87 @@ +#include "Expression.h" + + + +int Expression::evaluateInt(Context *ctx) +{ + throw TypeError(); +} + +Expression::~Expression() +{ +} + + +StringExpr::~StringExpr() +{ +} + + +IntExpr::~IntExpr() +{ +} + +int IntExpr::evaluateInt(Context *ctx) +{ + return val; +} + + +void CompoundExpr::addItem(ExprPtr item) +{ + items.push_back(item); +} + +CompoundExpr::~CompoundExpr() +{ +} + + +BinaryExpr::~BinaryExpr() +{ +} + +int BinaryExpr::evaluateInt(Context *ctx) +{ + switch(op) + { + case BinaryOp::XOR: + return a->evaluateInt(ctx) ^ b->evaluateInt(ctx); + case BinaryOp::OR: + return a->evaluateInt(ctx) | b->evaluateInt(ctx); + case BinaryOp::AND: + return a->evaluateInt(ctx) & b->evaluateInt(ctx); + case BinaryOp::SHIFTLEFT: + return a->evaluateInt(ctx) << b->evaluateInt(ctx); + case BinaryOp::SHIFTRIGHT: + return a->evaluateInt(ctx) >> b->evaluateInt(ctx); + case BinaryOp::EQUAL: + return a->evaluateInt(ctx) == b->evaluateInt(ctx); + case BinaryOp::NOTEQUAL: + return a->evaluateInt(ctx) != b->evaluateInt(ctx); + case BinaryOp::PLUS: + return a->evaluateInt(ctx) + b->evaluateInt(ctx); + case BinaryOp::MINUS: + return a->evaluateInt(ctx) - b->evaluateInt(ctx); + case BinaryOp::MULTIPLY: + return a->evaluateInt(ctx) * b->evaluateInt(ctx); + case BinaryOp::DIVIDE: + return a->evaluateInt(ctx) / b->evaluateInt(ctx); + } +} + + +UnaryExpr::~UnaryExpr() +{ +} + +int UnaryExpr::evaluateInt(Context *ctx) +{ + switch(op) + { + case UnaryOp::MINUS: + return -a->evaluateInt(ctx); + case UnaryOp::COMPLEMENT: + return ~a->evaluateInt(ctx); + } +} diff --git a/Rez/Expression.h b/Rez/Expression.h new file mode 100644 index 0000000000..072b634a82 --- /dev/null +++ b/Rez/Expression.h @@ -0,0 +1,94 @@ +#ifndef EXPRESSION_H +#define EXPRESSION_H + +#include +#include + +class Context +{ + +}; + + + +class Expression; +class CompoundExpr; +typedef std::shared_ptr ExprPtr; +typedef std::shared_ptr CompoundExprPtr; + + +enum class BinaryOp +{ + XOR, OR, AND, SHIFTLEFT, SHIFTRIGHT, EQUAL, NOTEQUAL, PLUS, MINUS, MULTIPLY, DIVIDE +}; + +enum class UnaryOp +{ + MINUS, COMPLEMENT +}; + +class TypeError +{ +}; + +class Expression +{ +public: + virtual int evaluateInt(Context *ctx); + virtual ~Expression(); +}; + +class StringExpr : public Expression +{ + std::string str; +public: + StringExpr(const std::string& str) : str(str) {} + ~StringExpr(); +}; + +class IntExpr : public Expression +{ + int val; +public: + IntExpr(int val) : val(val) {} + ~IntExpr(); + + virtual int evaluateInt(Context *ctx); +}; + +class CompoundExpr : public Expression +{ + std::vector items; +public: + void addItem(ExprPtr item); + ExprPtr getItem(int i) { return items[i]; } + ~CompoundExpr(); +}; + +class BinaryExpr : public Expression +{ + BinaryOp op; + ExprPtr a, b; +public: + BinaryExpr(BinaryOp op, ExprPtr a, ExprPtr b) + : op(op), a(a), b(b) {} + ~BinaryExpr(); + + virtual int evaluateInt(Context *ctx); +}; + +class UnaryExpr : public Expression +{ + UnaryOp op; + ExprPtr a; +public: + UnaryExpr(UnaryOp op, ExprPtr a) + : op(op), a(a) {} + ~UnaryExpr(); + + virtual int evaluateInt(Context *ctx); +}; + + + +#endif // EXPRESSION_H diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc new file mode 100644 index 0000000000..533624448f --- /dev/null +++ b/Rez/ResourceCompiler.cc @@ -0,0 +1,11 @@ +#include "ResourceCompiler.h" +#include + +ResourceCompiler::ResourceCompiler() +{ +} + +void ResourceCompiler::write(int nBits, int value) +{ + std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; +} diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h new file mode 100644 index 0000000000..4523557183 --- /dev/null +++ b/Rez/ResourceCompiler.h @@ -0,0 +1,15 @@ +#ifndef RESOURCECOMPILER_H +#define RESOURCECOMPILER_H + +#include "Expression.h" + +class ResourceCompiler : public Context +{ +public: + ResourceCompiler(); + + void reserve(int nBits) { write(nBits, 0); } + void write(int nBits, int value); +}; + +#endif // RESOURCECOMPILER_H diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 6d0e481485..695419627a 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -1,11 +1,8 @@ #include "ResourceDefinitions.h" #include +#include -ResourceDefinitions::ResourceDefinitions() -{ -} - - +#include "ResourceCompiler.h" std::ostream &operator<<(std::ostream &out, ResType t) { @@ -25,3 +22,82 @@ std::ostream &operator<<(std::ostream &out, TypeSpec ts) out << " (" << ts.getID() << ")"; return out; } + + +FieldList::~FieldList() +{ + +} + +void FieldList::addField(FieldPtr field) +{ + fields.push_back(field); +} + +void FieldList::addLabel(std::string name) +{ + // ### TODO +} + +void FieldList::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + CompoundExprPtr compound = std::dynamic_pointer_cast(expr); + assert(compound); + + int i = 0; + for(FieldPtr f : fields) + { + if(f->needsValue()) + f->compile(compound->getItem(i++), compiler, prePass); + else + f->compile(nullptr, compiler, prePass); + } +} + + + + +bool SimpleField::needsValue() +{ + return !value; +} + +void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + int bitSize; + + switch(type) + { + case Type::bitstring: + bitSize = arrayCount->evaluateInt(compiler); + break; + case Type::boolean: + bitSize = 1; + break; + case Type::byte: + bitSize = 8; + break; + case Type::integer: + bitSize = 16; + break; + case Type::longint: + bitSize = 32; + break; + } + + int actualValue = 0; + if(!prePass) + { + if(value) + { + actualValue = value->evaluateInt(compiler); + } + else + { + // TODO: add alternatives to context + actualValue = expr->evaluateInt(compiler); + } + } + + compiler->write(bitSize, actualValue); +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index 06c239c4e5..a19985b341 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -5,6 +5,8 @@ #include #include +#include "Expression.h" + class ResType { int x; @@ -12,6 +14,7 @@ public: ResType() : x(0) {} ResType(int x) : x(x) {} operator int() const { return x; } + bool operator<(ResType y) const { return x < y.x; } }; std::ostream& operator<<(std::ostream& out, ResType t); @@ -31,42 +34,29 @@ public: int getID() const { return id; } bool hasID() const { return id != noID; } + + bool operator<(TypeSpec y) const + { + if(type < y.type) + return true; + else if(y.type < type) + return false; + else + return id < y.id; + } }; std::ostream& operator<<(std::ostream& out, TypeSpec ts); -class Context -{ -}; - -class Expression -{ -public: - //virtual int evaluateInt(Context *ctx); - -}; - -class StringExpr : public Expression -{ - std::string str; -public: - StringExpr(const std::string& str) : str(str) {} -}; - -class IntExpr : public Expression -{ - int val; -public: - IntExpr(int val) : val(val) {} -}; - -typedef std::shared_ptr ExprPtr; +class ResourceCompiler; class Field { public: virtual bool needsValue() { return true; } + + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) = 0; }; typedef std::shared_ptr FieldPtr; @@ -93,6 +83,9 @@ public: void addNamedValue(std::string n) {} void addNamedValue(std::string n, ExprPtr val) {} + + virtual bool needsValue(); + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); }; typedef std::shared_ptr SimpleFieldPtr; @@ -102,11 +95,22 @@ inline SimpleField::Attrs operator|(SimpleField::Attrs a, SimpleField::Attrs b) return SimpleField::Attrs( int(a) | int(b) ); } - -class ResourceDefinitions +class FieldList : public Field { +protected: + std::vector fields; public: - ResourceDefinitions(); + virtual ~FieldList(); + void addField(FieldPtr field); + void addLabel(std::string name); + + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); }; +typedef std::shared_ptr FieldListPtr; + +class TypeDefinition : public FieldList +{ +}; +typedef std::shared_ptr TypeDefinitionPtr; #endif // RESOURCEDEFINITIONS_H diff --git a/Rez/Rez.cc b/Rez/Rez.cc index 91560796d7..ee617ec62d 100644 --- a/Rez/Rez.cc +++ b/Rez/Rez.cc @@ -4,64 +4,15 @@ #include "RezParser.generated.hh" #include "RezLexer.h" - +#include "RezWorld.h" int main() { //RezLexer lexer("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); RezLexer lexer("/home/wolfgang/Projects/Retro68/Rez/Test.r"); - - RezParser parser(lexer); + RezWorld world; + RezParser parser(lexer, world); parser.parse(); - - /* - // The following preprocesses a given input file. - // Open the file and read it into a string variable - std::ifstream instream("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); - - std::string input( - std::istreambuf_iterator(instream.rdbuf()), - std::istreambuf_iterator()); - - context_type ctx(input.begin(), input.end(), "Types.r"); - - // At this point you may want to set the parameters of the - // preprocessing as include paths and/or predefined macros. - ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); - // ctx.add_macro_definition(...); - - auto first = ctx.begin(); - auto last = ctx.end(); - - std::ostringstream out; - - try - { - while(first != last) - { - out << (*first).get_value(); - ++first; - } - } - catch(boost::wave::preprocess_exception& e) - { - std::cout << e.file_name() << ":" << e.line_no() << ": "; - std::cout << e.description() << std::endl; - } - - std::string str = out.str(); - - - std::cout << str.substr(0,100) << std::endl;*/ - -/* - int i = 0; - while (first != last) { - std::cout << i << ": " << get_token_name(token_id(*first)) << " <<" << (*first).get_value() << ">>\n"; - ++first; - if(++i > 10) - break; - }*/ return 0; } diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index d8005168ca..c2ec800ff1 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -81,12 +81,15 @@ */ %param { RezLexer& lexer } +%param { RezWorld& world } %code requires { #include "ResourceDefinitions.h" + #include "Expression.h" #define YY_NULLPTR nullptr class RezLexer; + class RezWorld; } %code provides { @@ -103,7 +106,10 @@ %code { #include "RezLexer.h" - static yy::RezParser::symbol_type yylex(RezLexer& lexer) + #include "RezWorld.h" + #include "ResourceCompiler.h" + + static yy::RezParser::symbol_type yylex(RezLexer& lexer, RezWorld&) { return lexer.nextToken(); } @@ -122,9 +128,14 @@ rez : %empty | rez resource ";" ; - -type_definition : "type" type_spec "{" field_definitions "}" - { std::cout << "TYPE " << $2 << std::endl; } +type_definition : "type" type_spec + { + TypeDefinitionPtr td = std::make_shared(); + world.addTypeDefinition($type_spec, td); + world.fieldLists.push(td); + } + "{" field_definitions "}" + { world.fieldLists.pop(); std::cout << "TYPE " << $2 << std::endl; } | "type" type_spec "as" type_spec { std::cout << "TYPE " << $2 << std::endl; } ; @@ -138,16 +149,18 @@ type_spec : res_type { $$ = TypeSpec($res_type); } ; field_definitions : %empty - | field_definitions IDENTIFIER ":" + | field_definitions IDENTIFIER ":" { world.fieldLists.top()->addLabel($2); } | field_definitions ";" - | field_definitions field_definition ";" ; + | field_definitions field_definition ";" { world.fieldLists.top()->addField($2); } + ; %type field_definition; field_definition: simple_field_definition { $$ = $1; } - | array_definition - | switch_definition - | fill_statement - | align_statement; + | array_definition { $$ = nullptr; } + | switch_definition { $$ = nullptr; } + | fill_statement { $$ = nullptr; } + | align_statement { $$ = nullptr; } + ; %type simple_field_definition; simple_field_definition: field_attributes simpletype array_count_opt value_spec_opt @@ -168,10 +181,10 @@ simple_field_definition: field_attributes simpletype array_count_opt value_spec_ { $$ = $1; $$->addNamedValue($IDENTIFIER, $value); } ; -%type array_count array_count_opt value_spec_opt value ; +%type array_count array_count_opt value_spec_opt value resource_item; %type expression expression1 expression2 ; -%type expression3 expression4 expression5 expression6; -%type expression7 expression8; +%type expression3 expression4 expression5 ; +%type expression6 expression7 expression8; value_spec_opt : %empty { $$ = nullptr; } | "=" value { $$ = $2; } ; @@ -223,70 +236,52 @@ switch_cases : %empty | switch_cases switch_case ; switch_case : "case" IDENTIFIER ":" field_definitions ; -/* -expression - | expression "^" expression - | expression "&" expression - | expression "|" expression - | "~" expression - | expression "==" expression - | expression "!=" expression - | expression ">>" expression - | expression "<<" expression - | expression "+" expression - | expression "-" expression - | "-" expression - | expression "/" expression - | expression "*" expression - ; -*/ - -value : expression - | "{" resource_body "}" +value : expression { $$ = $1; } + | "{" resource_body "}" { $$ = $2; } | STRINGLIT { $$ = std::make_shared($1); } ; -expression : expression1 - | expression "^" expression1 +expression : expression1 { $$ = $1; } + | expression "^" expression1 { $$ = std::make_shared(BinaryOp::XOR, $1, $3); } ; -expression1 : expression2 - | expression1 "&" expression2 +expression1 : expression2 { $$ = $1; } + | expression1 "&" expression2 { $$ = std::make_shared(BinaryOp::AND, $1, $3); } ; -expression2 : expression3 - | expression2 "|" expression3 +expression2 : expression3 { $$ = $1; } + | expression2 "|" expression3 { $$ = std::make_shared(BinaryOp::OR, $1, $3); } ; -expression3 : expression4 - | expression3 "==" expression4 - | expression3 "!=" expression4 +expression3 : expression4 { $$ = $1; } + | expression3 "==" expression4 { $$ = std::make_shared(BinaryOp::EQUAL, $1, $3); } + | expression3 "!=" expression4 { $$ = std::make_shared(BinaryOp::NOTEQUAL, $1, $3); } ; -expression4 : expression5 - | expression4 ">>" expression5 - | expression4 "<<" expression5 +expression4 : expression5 { $$ = $1; } + | expression4 ">>" expression5 { $$ = std::make_shared(BinaryOp::SHIFTRIGHT, $1, $3); } + | expression4 "<<" expression5 { $$ = std::make_shared(BinaryOp::SHIFTLEFT, $1, $3); } ; -expression5 : expression6 - | expression5 "+" expression6 - | expression5 "-" expression6 +expression5 : expression6 { $$ = $1; } + | expression5 "+" expression6 { $$ = std::make_shared(BinaryOp::PLUS, $1, $3); } + | expression5 "-" expression6 { $$ = std::make_shared(BinaryOp::MINUS, $1, $3); } ; -expression6 : expression7 - | expression6 "*" expression7 - | expression6 "/" expression7 +expression6 : expression7 { $$ = $1; } + | expression6 "*" expression7 { $$ = std::make_shared(BinaryOp::MULTIPLY, $1, $3); } + | expression6 "/" expression7 { $$ = std::make_shared(BinaryOp::DIVIDE, $1, $3); } ; -expression7 : expression8 - | "-" expression7 - | "+" expression7 - | "~" expression7 +expression7 : expression8 { $$ = $1; } + | "-" expression7 { $$ = std::make_shared(UnaryOp::MINUS, $2); } + | "+" expression7 { $$ = $2; } + | "~" expression7 { $$ = std::make_shared(UnaryOp::COMPLEMENT, $2); } ; expression8 : INTLIT { $$ = std::make_shared($1); } | CHARLIT { $$ = std::make_shared($1); } - | IDENTIFIER + | IDENTIFIER /* ### */ | IDENTIFIER "(" function_argument_list ")" | IDENTIFIER "[" function_argument_list1 "]" | "(" expression ")" { $$ = $2; } @@ -296,16 +291,38 @@ expression8 : INTLIT { $$ = std::make_shared($1); } function_argument_list : %empty | function_argument_list1 ; function_argument_list1 : expression | function_argument_list "," expression ; -resource: "resource" res_type "(" function_argument_list ")" "{" resource_body "}" - { std::cout << "RESOURCE " << $2 << std::endl; } +resource : "resource" res_type "(" expression resource_attributes ")" "{" resource_body "}" + { + int id = $expression->evaluateInt(nullptr); + std::cout << "RESOURCE " << $2 << "(" << id << ")" << std::endl; + TypeDefinitionPtr type = world.getTypeDefinition($res_type, id); + ResourceCompiler compiler; + std::cout << "(first pass)\n"; + type->compile($resource_body, &compiler, true); + std::cout << "(second pass)\n"; + type->compile($resource_body, &compiler, false); + std::cout << "(done)\n"; + } + ; -resource_body : %empty | resource_body1 ; -resource_body1 : resource_item - | resource_body1 "," resource_item - | resource_body1 ";" resource_item - | resource_body1 ";" +%type resource_attributes resource_attribute; +resource_attributes : %empty { $$ = 0; } + | resource_attributes "," resource_attribute { $$ = $1 | $3; } + ; +resource_attribute : IDENTIFIER { $$ = 0; } /* ### */ + +%type resource_body resource_body1; +resource_body : %empty { $$ = std::make_shared(); } + | resource_body1 { $$ = $1; } + ; +resource_body1 : resource_item { $$ = std::make_shared(); $$->addItem($1); } + | resource_body1 "," resource_item { $$ = $1; $$->addItem($3); } + | resource_body1 ";" resource_item { $$ = $1; $$->addItem($3); } + | resource_body1 ";" { $$ = $1; } ; -resource_item : value | IDENTIFIER "{" resource_body "}" ; +resource_item : value { $$ = $1; } + | IDENTIFIER "{" resource_body "}" // ### + ; %% diff --git a/Rez/RezWorld.cc b/Rez/RezWorld.cc new file mode 100644 index 0000000000..90e6148b1a --- /dev/null +++ b/Rez/RezWorld.cc @@ -0,0 +1,22 @@ +#include "RezWorld.h" + +RezWorld::RezWorld() +{ +} + +void RezWorld::addTypeDefinition(TypeSpec spec, TypeDefinitionPtr type) +{ + types[spec] = type; +} + +TypeDefinitionPtr RezWorld::getTypeDefinition(ResType type, int id) +{ + auto p = types.find(TypeSpec(type, id)); + if(p != types.end()) + return p->second; + p = types.find(TypeSpec(type)); + if(p != types.end()) + return p->second; + + return nullptr; +} diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h new file mode 100644 index 0000000000..63f205bcfb --- /dev/null +++ b/Rez/RezWorld.h @@ -0,0 +1,23 @@ +#ifndef REZWORLD_H +#define REZWORLD_H + +#include +#include +#include "ResourceDefinitions.h" +#include "Expression.h" + +class RezWorld +{ + friend class RezParser; + + std::map types; + std::stack fieldLists; +public: + RezWorld(); + void addTypeDefinition(TypeSpec spec, TypeDefinitionPtr type); + + TypeDefinitionPtr getTypeDefinition(ResType type, int id); +}; + + +#endif // REZWORLD_H From 523a68d3de1d14894bdcfe82d6470042ee2f7bb0 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Tue, 7 Oct 2014 20:44:40 +0200 Subject: [PATCH 04/29] primitive array support --- Rez/Expression.h | 4 +++- Rez/ResourceDefinitions.cc | 38 ++++++++++++++++++++++++++++++++++++++ Rez/ResourceDefinitions.h | 12 ++++++++++++ Rez/RezParser.yy | 22 +++++++++++++++++----- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/Rez/Expression.h b/Rez/Expression.h index 072b634a82..ec10db222e 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -61,7 +61,9 @@ class CompoundExpr : public Expression std::vector items; public: void addItem(ExprPtr item); - ExprPtr getItem(int i) { return items[i]; } + ExprPtr getItem(int i) const { return items[i]; } + int size() const { return items.size(); } + ~CompoundExpr(); }; diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 695419627a..3af85e9524 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -101,3 +101,41 @@ void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass compiler->write(bitSize, actualValue); } + + +ArrayField::ArrayField(std::string name, ExprPtr count) + : name(name), arrayCount(count) +{ +} + +void ArrayField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + CompoundExprPtr compound = std::dynamic_pointer_cast(expr); + assert(compound); + + + int i = 0; + int n = compound->size(); + + int iterations = 0; + while(i < n) + { + ++iterations; + for(FieldPtr f : fields) + { + if(f->needsValue()) + { + assert(i < n); + f->compile(compound->getItem(i++), compiler, prePass); + } + else + f->compile(nullptr, compiler, prePass); + } + } + + if(arrayCount) + { + int expected = arrayCount->evaluateInt(compiler); + assert(expected == iterations); + } +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index a19985b341..085c1cf629 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -108,6 +108,18 @@ public: }; typedef std::shared_ptr FieldListPtr; + +class ArrayField : public FieldList +{ + std::string name; + ExprPtr arrayCount; +public: + ArrayField(std::string name /* or empty */, ExprPtr count /* may be null*/); + + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); +}; +typedef std::shared_ptr ArrayFieldPtr; + class TypeDefinition : public FieldList { }; diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index c2ec800ff1..8875ad5da5 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -156,7 +156,7 @@ field_definitions : %empty %type field_definition; field_definition: simple_field_definition { $$ = $1; } - | array_definition { $$ = nullptr; } + | array_definition { $$ = $1; } | switch_definition { $$ = nullptr; } | fill_statement { $$ = nullptr; } | align_statement { $$ = nullptr; } @@ -207,13 +207,25 @@ align_statement : "align" fill_unit; fill_unit : "bit" | "byte" | "word" | "long"; - -array_definition: array_attributes "array" array_name_opt array_count_opt "{" field_definitions "}" ; +%type array_definition; +array_definition: + array_attributes "array" array_name_opt array_count_opt + { + ArrayFieldPtr af = std::make_shared($array_name_opt, $array_count_opt); + world.fieldLists.push(af); + } + "{" field_definitions "}" + { + $$ = world.fieldLists.top(); + world.fieldLists.pop(); + } + ; array_count : "[" expression "]" { $$ = $2; } -array_count_opt : %empty { $$ = nullptr; } | array_count; +array_count_opt : %empty { $$ = nullptr; } | array_count { $$ = $1; }; -array_name_opt : %empty | IDENTIFIER ; +%type array_name_opt; +array_name_opt : %empty { $$ = ""; } | IDENTIFIER { $$ = $1; } ; array_attributes: %empty | "wide" ; From a02d56ba1c92bea977cf3fb85295ee85b5678536 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 00:41:40 +0200 Subject: [PATCH 05/29] Rez: Labels, $$CountOf, $$ArrayIndex --- Rez/Expression.cc | 52 +++++++++++++++--- Rez/Expression.h | 27 ++++++---- Rez/ResourceCompiler.cc | 107 ++++++++++++++++++++++++++++++++++++- Rez/ResourceCompiler.h | 59 +++++++++++++++++++- Rez/ResourceDefinitions.cc | 46 +++++++++++++++- Rez/ResourceDefinitions.h | 21 +++++++- Rez/RezParser.yy | 26 +++++---- Rez/RezWorld.h | 1 + Rez/Test.r | 23 ++++++-- 9 files changed, 326 insertions(+), 36 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index dfa3208764..0f5ff22ee5 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -1,8 +1,9 @@ #include "Expression.h" +#include "ResourceCompiler.h" +#include +#include - - -int Expression::evaluateInt(Context *ctx) +int Expression::evaluateInt(ResourceCompiler *ctx) { throw TypeError(); } @@ -21,7 +22,7 @@ IntExpr::~IntExpr() { } -int IntExpr::evaluateInt(Context *ctx) +int IntExpr::evaluateInt(ResourceCompiler *ctx) { return val; } @@ -41,7 +42,7 @@ BinaryExpr::~BinaryExpr() { } -int BinaryExpr::evaluateInt(Context *ctx) +int BinaryExpr::evaluateInt(ResourceCompiler *ctx) { switch(op) { @@ -75,7 +76,7 @@ UnaryExpr::~UnaryExpr() { } -int UnaryExpr::evaluateInt(Context *ctx) +int UnaryExpr::evaluateInt(ResourceCompiler *ctx) { switch(op) { @@ -85,3 +86,42 @@ int UnaryExpr::evaluateInt(Context *ctx) return ~a->evaluateInt(ctx); } } + + +IdentifierExpr::IdentifierExpr(std::string id, bool isFunction) + : id(id), isFunction(isFunction) +{ +} + +void IdentifierExpr::addArgument(ExprPtr e) +{ + arguments.push_back(e); +} + +int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) +{ + if(isFunction) + { + if(id == "$$countof" || id == "$$arrayindex") + { + assert(arguments.size() == 1); + IdentifierExprPtr arr = std::dynamic_pointer_cast(arguments[0]); + assert(arr); + if(id == "$$countof") + return ctx->getArrayCount(arr->id); + else + return ctx->getArrayIndex(arr->id); + } + else + { + std::cout << id << std::endl; + assert(false); + } + } + else + { + ExprPtr val = ctx->lookupIdentifier(id); + assert(val); + return val->evaluateInt(ctx); + } +} diff --git a/Rez/Expression.h b/Rez/Expression.h index ec10db222e..697f694592 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -4,17 +4,14 @@ #include #include -class Context -{ - -}; - - +class ResourceCompiler; class Expression; class CompoundExpr; +class IdentifierExpr; typedef std::shared_ptr ExprPtr; typedef std::shared_ptr CompoundExprPtr; +typedef std::shared_ptr IdentifierExprPtr; enum class BinaryOp @@ -34,7 +31,7 @@ class TypeError class Expression { public: - virtual int evaluateInt(Context *ctx); + virtual int evaluateInt(ResourceCompiler *ctx); virtual ~Expression(); }; @@ -53,7 +50,7 @@ public: IntExpr(int val) : val(val) {} ~IntExpr(); - virtual int evaluateInt(Context *ctx); + virtual int evaluateInt(ResourceCompiler *ctx); }; class CompoundExpr : public Expression @@ -76,7 +73,7 @@ public: : op(op), a(a), b(b) {} ~BinaryExpr(); - virtual int evaluateInt(Context *ctx); + virtual int evaluateInt(ResourceCompiler *ctx); }; class UnaryExpr : public Expression @@ -88,9 +85,19 @@ public: : op(op), a(a) {} ~UnaryExpr(); - virtual int evaluateInt(Context *ctx); + virtual int evaluateInt(ResourceCompiler *ctx); }; +class IdentifierExpr : public Expression +{ + std::string id; + std::vector arguments; + bool isFunction; +public: + IdentifierExpr(std::string id, bool isFunction = false); + void addArgument(ExprPtr e); + virtual int evaluateInt(ResourceCompiler *ctx); +}; #endif // EXPRESSION_H diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc index 533624448f..d54798d05e 100644 --- a/Rez/ResourceCompiler.cc +++ b/Rez/ResourceCompiler.cc @@ -1,11 +1,116 @@ #include "ResourceCompiler.h" #include +#include "ResourceDefinitions.h" -ResourceCompiler::ResourceCompiler() +ResourceCompiler::ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body) + : typeDefinition(type), body(body), currentOffset(0), currentField(nullptr) { + } void ResourceCompiler::write(int nBits, int value) { std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; + currentOffset += nBits; +} + +ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &sub) +{ + if(currentField) + { + if(ExprPtr val = currentField->lookupNamedValue(name)) + return val; + } + + auto p = labelValues.find(std::make_pair(name, sub)); + if(p != labelValues.end()) + return p->second; + + return nullptr; +} + +void ResourceCompiler::defineLabel(const std::string &name) +{ + labelValues[std::make_pair(name,currentSubscripts)] = std::make_shared(currentOffset); +} + +void ResourceCompiler::compile() +{ + std::cout << "(first pass)\n"; + currentOffset = 0; + typeDefinition->compile(body, this, true); + std::cout << "(second pass)\n"; + currentOffset = 0; + typeDefinition->compile(body, this, false); + std::cout << "(done)\n"; + +} + +int ResourceCompiler::getArrayCount(const std::string &name) +{ + Subscripts sub = currentSubscripts; + for(;;) + { + auto p = arrayCounts.find(std::make_pair(name, sub)); + if(p != arrayCounts.end()) + return p->second; + + + if(sub.empty()) + return 0; /* ### */ + sub.popSubscript(); + } +} + +int ResourceCompiler::getArrayIndex(const std::string &arrayName) +{ + return curArrayIndices[arrayName]; +} + +void ResourceCompiler::beginArrayScope(std::string &arrayName, int index) +{ + if(arrayName != "") + { + curArrayIndices[arrayName] = index; + int& count = arrayCounts[std::make_pair(arrayName, currentSubscripts)]; + if(count < index) + count = index; + arrayCounts[std::make_pair(arrayName, Subscripts())] = count; + //std::cout << "count for " << arrayName << " is " << count << std::endl; + } + currentSubscripts.addSubscript(index); +} + +Subscripts::Subscripts() +{ +} + +Subscripts::~Subscripts() +{ +} + +void Subscripts::addSubscript(int x) +{ + subscripts.push_back(x); +} + +void Subscripts::popSubscript() +{ + subscripts.pop_back(); +} + +bool Subscripts::operator<(const Subscripts &other) const +{ + if(subscripts.size() < other.subscripts.size()) + return true; + if(other.subscripts.size() < subscripts.size()) + return false; + for(int i = 0, n = subscripts.size(); i < n; i++) + { + if(subscripts[i] < other.subscripts[i]) + return true; + else if(subscripts[i] > other.subscripts[i]) + return false; + } + return false; } diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index 4523557183..95f3f60cbf 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -2,14 +2,69 @@ #define RESOURCECOMPILER_H #include "Expression.h" +#include "ResourceDefinitions.h" -class ResourceCompiler : public Context +class Field; + + +class Subscripts { + std::vector subscripts; public: - ResourceCompiler(); + Subscripts(); + ~Subscripts(); + + void addSubscript(int x); + void popSubscript(); + bool operator<(const Subscripts& other) const; + bool empty() const { return subscripts.empty(); } +}; + +class ResourceCompiler +{ + TypeDefinitionPtr typeDefinition; + CompoundExprPtr body; + std::map, ExprPtr> labelValues; + std::map, int> arrayCounts; + std::map curArrayIndices; + int currentOffset; + Field* currentField; + Subscripts currentSubscripts; + + void beginArrayScope(std::string& arrayName, int index); +public: + ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body); void reserve(int nBits) { write(nBits, 0); } void write(int nBits, int value); + + ExprPtr lookupIdentifier(std::string name, const Subscripts& sub = Subscripts()); + + void defineLabel(const std::string& name); + void compile(); + + int getArrayCount(const std::string& arrayName); + int getArrayIndex(const std::string& arrayName); + + class FieldScope + { + ResourceCompiler *compiler; + public: + FieldScope(ResourceCompiler* compiler, Field *field) + : compiler(compiler) { compiler->currentField = field; } + ~FieldScope() { compiler->currentField = nullptr; } + }; + + class ArrayScope + { + ResourceCompiler *compiler; + public: + ArrayScope(ResourceCompiler* compiler, std::string& arrayName, int index) + : compiler(compiler) { compiler->beginArrayScope(arrayName, index); } + ~ArrayScope() { compiler->currentSubscripts.popSubscript(); } + }; }; + + #endif // RESOURCECOMPILER_H diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 3af85e9524..3cdc84f48d 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -36,7 +36,7 @@ void FieldList::addField(FieldPtr field) void FieldList::addLabel(std::string name) { - // ### TODO + addField(std::make_shared(name)); } void FieldList::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) @@ -57,6 +57,30 @@ void FieldList::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +void SimpleField::addNamedValue(std::string n) +{ + if(lastNamedValue) + addNamedValue(n, std::make_shared( + BinaryOp::PLUS, lastNamedValue, std::make_shared(1))); + else + addNamedValue(n, std::make_shared(0)); +} + +void SimpleField::addNamedValue(std::string n, ExprPtr val) +{ + namedValues[n] = val; + lastNamedValue = val; +} + +ExprPtr SimpleField::lookupNamedValue(std::string n) +{ + auto p = namedValues.find(n); + if(p != namedValues.end()) + return p->second; + else + return nullptr; +} + bool SimpleField::needsValue() { return !value; @@ -88,6 +112,7 @@ void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass int actualValue = 0; if(!prePass) { + ResourceCompiler::FieldScope scope(compiler, this); if(value) { actualValue = value->evaluateInt(compiler); @@ -121,6 +146,7 @@ void ArrayField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) while(i < n) { ++iterations; + ResourceCompiler::ArrayScope scope(compiler, name, iterations); for(FieldPtr f : fields) { if(f->needsValue()) @@ -133,9 +159,25 @@ void ArrayField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) } } - if(arrayCount) + if(!prePass && arrayCount) { int expected = arrayCount->evaluateInt(compiler); assert(expected == iterations); } } + + +LabelField::LabelField(std::string name) + : name(name) +{ +} + +bool LabelField::needsValue() +{ + return false; +} + +void LabelField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + compiler->defineLabel(name); +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index 085c1cf629..093346db40 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -57,6 +57,8 @@ public: virtual bool needsValue() { return true; } virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) = 0; + + virtual ExprPtr lookupNamedValue(std::string) { return nullptr; } }; typedef std::shared_ptr FieldPtr; @@ -80,9 +82,11 @@ public: ExprPtr value; std::map namedValues; + ExprPtr lastNamedValue; - void addNamedValue(std::string n) {} - void addNamedValue(std::string n, ExprPtr val) {} + void addNamedValue(std::string n); + void addNamedValue(std::string n, ExprPtr val); + ExprPtr lookupNamedValue(std::string); virtual bool needsValue(); virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); @@ -95,6 +99,19 @@ inline SimpleField::Attrs operator|(SimpleField::Attrs a, SimpleField::Attrs b) return SimpleField::Attrs( int(a) | int(b) ); } + +class LabelField : public Field +{ + std::string name; +public: + LabelField(std::string name); + + virtual bool needsValue(); + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); +}; +typedef std::shared_ptr LabelFieldPtr; + + class FieldList : public Field { protected: diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 8875ad5da5..e566f4efef 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -293,27 +293,33 @@ expression7 : expression8 { $$ = $1; } expression8 : INTLIT { $$ = std::make_shared($1); } | CHARLIT { $$ = std::make_shared($1); } - | IDENTIFIER /* ### */ - | IDENTIFIER "(" function_argument_list ")" - | IDENTIFIER "[" function_argument_list1 "]" + | IDENTIFIER { $$ = std::make_shared($1); } + | IDENTIFIER + { world.functionCalls.push(std::make_shared($1,true)); } + "(" function_argument_list ")" + { $$ = world.functionCalls.top(); world.functionCalls.pop(); } + | IDENTIFIER + { world.functionCalls.push(std::make_shared($1,false)); } + "[" function_argument_list1 "]" + { $$ = world.functionCalls.top(); world.functionCalls.pop(); } | "(" expression ")" { $$ = $2; } ; function_argument_list : %empty | function_argument_list1 ; -function_argument_list1 : expression | function_argument_list "," expression ; +function_argument_list1 : expression + { world.functionCalls.top()->addArgument($expression); } + | function_argument_list "," expression + { world.functionCalls.top()->addArgument($expression); } + ; resource : "resource" res_type "(" expression resource_attributes ")" "{" resource_body "}" { int id = $expression->evaluateInt(nullptr); std::cout << "RESOURCE " << $2 << "(" << id << ")" << std::endl; TypeDefinitionPtr type = world.getTypeDefinition($res_type, id); - ResourceCompiler compiler; - std::cout << "(first pass)\n"; - type->compile($resource_body, &compiler, true); - std::cout << "(second pass)\n"; - type->compile($resource_body, &compiler, false); - std::cout << "(done)\n"; + ResourceCompiler compiler(type, $resource_body); + compiler.compile(); } ; diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h index 63f205bcfb..ae42589fc8 100644 --- a/Rez/RezWorld.h +++ b/Rez/RezWorld.h @@ -12,6 +12,7 @@ class RezWorld std::map types; std::stack fieldLists; + std::stack functionCalls; public: RezWorld(); void addTypeDefinition(TypeSpec spec, TypeDefinitionPtr type); diff --git a/Rez/Test.r b/Rez/Test.r index 173917529b..6679348422 100644 --- a/Rez/Test.r +++ b/Rez/Test.r @@ -3,9 +3,26 @@ */ -#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r" +//#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r" type 'TEST' { -boolean itemUnlocked = false, // defined attributes bits... - itemLocked = true; + integer zero, one, two, answer = 42, missed; + longint; + //integer = (after - before) / 32; + integer = $$CountOf(foo); + before: + array foo { + integer = $$ArrayIndex(foo); + integer; + integer; + }; + after: + ; }; + +resource 'TEST' (128) { + answer, + 0x1234, + { 1, 2; 3, 4; } +}; + From 399131a8a6d5ec3af27bd192dffda050a101f174 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:17:17 +0200 Subject: [PATCH 06/29] Rez: switch --- Rez/Expression.cc | 6 ++++++ Rez/Expression.h | 11 +++++++++++ Rez/ResourceDefinitions.cc | 17 +++++++++++++++++ Rez/ResourceDefinitions.h | 10 ++++++++++ Rez/RezParser.yy | 28 +++++++++++++++++++++++----- Rez/RezWorld.h | 1 + 6 files changed, 68 insertions(+), 5 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index 0f5ff22ee5..4c98dd4384 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -125,3 +125,9 @@ int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) return val->evaluateInt(ctx); } } + + +CaseExpr::CaseExpr(const std::string &tag, CompoundExprPtr expr) + : tag(tag), expr(expr) +{ +} diff --git a/Rez/Expression.h b/Rez/Expression.h index 697f694592..140f2ea0d9 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -9,9 +9,11 @@ class ResourceCompiler; class Expression; class CompoundExpr; class IdentifierExpr; +class CaseExpr; typedef std::shared_ptr ExprPtr; typedef std::shared_ptr CompoundExprPtr; typedef std::shared_ptr IdentifierExprPtr; +typedef std::shared_ptr CaseExprPtr; enum class BinaryOp @@ -64,6 +66,15 @@ public: ~CompoundExpr(); }; +class CaseExpr : public Expression +{ + std::string tag; + CompoundExprPtr expr; + friend class SwitchField; +public: + CaseExpr(const std::string& tag, CompoundExprPtr expr); +}; + class BinaryExpr : public Expression { BinaryOp op; diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 3cdc84f48d..fea09f43c2 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -181,3 +181,20 @@ void LabelField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) { compiler->defineLabel(name); } + + +void SwitchField::addCase(const std::string name, FieldListPtr alternative) +{ + cases[name] = alternative; +} + +void SwitchField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + CaseExprPtr caseExpr = std::dynamic_pointer_cast(expr); + assert(caseExpr); + + FieldListPtr caseDefinition = cases[caseExpr->tag]; + assert(caseDefinition); + + caseDefinition->compile(caseExpr->expr, compiler, prePass); +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index 093346db40..b7ab67c450 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -137,6 +137,16 @@ public: }; typedef std::shared_ptr ArrayFieldPtr; +class SwitchField : public Field +{ + std::map cases; +public: + void addCase(const std::string name, FieldListPtr alternative); + + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); +}; +typedef std::shared_ptr SwitchFieldPtr; + class TypeDefinition : public FieldList { }; diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index e566f4efef..9eebd20513 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -157,7 +157,7 @@ field_definitions : %empty %type field_definition; field_definition: simple_field_definition { $$ = $1; } | array_definition { $$ = $1; } - | switch_definition { $$ = nullptr; } + | switch_definition { $$ = $1; } | fill_statement { $$ = nullptr; } | align_statement { $$ = nullptr; } ; @@ -240,13 +240,31 @@ field_attribute : "hex" { $$ = SimpleField::Attrs::hex; } | "literal" { $$ = SimpleField::Attrs::literal; } ; -switch_definition: "switch" "{" +%type switch_definition; +switch_definition: + "switch" + { world.switches.push(std::make_shared()); } + "{" switch_cases - "}" ; + "}" + { + $$ = world.switches.top(); + world.switches.pop(); + } + ; switch_cases : %empty | switch_cases switch_case ; -switch_case : "case" IDENTIFIER ":" field_definitions ; +switch_case : "case" IDENTIFIER ":" + { + world.fieldLists.push(std::make_shared()); + } + field_definitions + { + world.switches.top()->addCase($IDENTIFIER, world.fieldLists.top()); + world.fieldLists.pop(); + } + ; value : expression { $$ = $1; } | "{" resource_body "}" { $$ = $2; } @@ -340,7 +358,7 @@ resource_body1 : resource_item { $$ = std::make_shared(); $$->addI ; resource_item : value { $$ = $1; } - | IDENTIFIER "{" resource_body "}" // ### + | IDENTIFIER "{" resource_body "}" { $$ = std::make_shared($IDENTIFIER, $resource_body); } ; %% diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h index ae42589fc8..ec44502369 100644 --- a/Rez/RezWorld.h +++ b/Rez/RezWorld.h @@ -13,6 +13,7 @@ class RezWorld std::map types; std::stack fieldLists; std::stack functionCalls; + std::stack switches; public: RezWorld(); void addTypeDefinition(TypeSpec spec, TypeDefinitionPtr type); From e0ab85c1c4fc1ffe285869ae160fd3a36d0f27b2 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:17:39 +0200 Subject: [PATCH 07/29] Rez: strings --- Rez/Expression.cc | 10 ++++++ Rez/Expression.h | 2 ++ Rez/ResourceDefinitions.cc | 74 ++++++++++++++++++++++++++++++++------ Rez/ResourceDefinitions.h | 4 +++ 4 files changed, 80 insertions(+), 10 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index 4c98dd4384..e93bec695a 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -8,6 +8,11 @@ int Expression::evaluateInt(ResourceCompiler *ctx) throw TypeError(); } +std::string Expression::evaluateString(ResourceCompiler *ctx) +{ + throw TypeError(); +} + Expression::~Expression() { } @@ -17,6 +22,11 @@ StringExpr::~StringExpr() { } +std::string StringExpr::evaluateString(ResourceCompiler *ctx) +{ + return str; +} + IntExpr::~IntExpr() { diff --git a/Rez/Expression.h b/Rez/Expression.h index 140f2ea0d9..9b4ec6b444 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -34,6 +34,7 @@ class Expression { public: virtual int evaluateInt(ResourceCompiler *ctx); + virtual std::string evaluateString(ResourceCompiler *ctx); virtual ~Expression(); }; @@ -43,6 +44,7 @@ class StringExpr : public Expression public: StringExpr(const std::string& str) : str(str) {} ~StringExpr(); + virtual std::string evaluateString(ResourceCompiler *ctx); }; class IntExpr : public Expression diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index fea09f43c2..46c4233cc0 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -88,7 +88,69 @@ bool SimpleField::needsValue() void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) { - int bitSize; + switch(type) + { + case Type::bitstring: + case Type::boolean: + case Type::byte: + case Type::integer: + case Type::longint: + compileInt(expr, compiler, prePass); + break; + case Type::string: + case Type::wstring: + case Type::pstring: + case Type::char_: + compileString(expr, compiler, prePass); + break; + } +} + +void SimpleField::compileString(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + std::string str; + { + ResourceCompiler::FieldScope scope(compiler, this); + str = (value ? value : expr)->evaluateString(compiler); + } + + if(arrayCount || type == Type::char_) + { + int requestedSize = type == Type::char_ ? 1 : arrayCount->evaluateInt(compiler); + if(requestedSize < str.size()) + str.erase(str.begin() + requestedSize, str.end()); + else if(requestedSize > str.size()) + str.insert(str.end(),requestedSize - str.size(), '\0'); + } + + int count = str.size(); + + if(type == Type::pstring) + { + if(count > 255) + { + str.erase(str.begin() + 255, str.end()); + count = 255; + } + compiler->write(8, count); + } + else if(type == Type::wstring) + { + if(count > 65535) + { + str.erase(str.begin() + 65535, str.end()); + count = 65535; + } + compiler->write(16, count); + } + + for(char c : str) + compiler->write(8, c); +} + +void SimpleField::compileInt(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + int bitSize = 0; switch(type) { @@ -113,15 +175,7 @@ void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass if(!prePass) { ResourceCompiler::FieldScope scope(compiler, this); - if(value) - { - actualValue = value->evaluateInt(compiler); - } - else - { - // TODO: add alternatives to context - actualValue = expr->evaluateInt(compiler); - } + actualValue = (value ? value : expr)->evaluateInt(compiler); } compiler->write(bitSize, actualValue); diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index b7ab67c450..fc52cd11b4 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -90,6 +90,10 @@ public: virtual bool needsValue(); virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); + +private: + void compileString(ExprPtr expr, ResourceCompiler *compiler, bool prePass); + void compileInt(ExprPtr expr, ResourceCompiler *compiler, bool prePass); }; typedef std::shared_ptr SimpleFieldPtr; From 50eb0f53731686341ab71a0cef65e9a94bccf191 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:37:28 +0200 Subject: [PATCH 08/29] Rez: fill and align --- Rez/ResourceCompiler.h | 1 + Rez/ResourceDefinitions.cc | 43 ++++++++++++++++++++++++++++++++++++++ Rez/ResourceDefinitions.h | 16 ++++++++++++++ Rez/RezLexerNextToken.cc | 1 + Rez/RezParser.yy | 22 ++++++++++++++----- 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index 95f3f60cbf..697dc9e556 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -37,6 +37,7 @@ public: void reserve(int nBits) { write(nBits, 0); } void write(int nBits, int value); + int tell() { return currentOffset; } ExprPtr lookupIdentifier(std::string name, const Subscripts& sub = Subscripts()); diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 46c4233cc0..65a17fc98c 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -252,3 +252,46 @@ void SwitchField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass caseDefinition->compile(caseExpr->expr, compiler, prePass); } + + +FillAlignField::FillAlignField(FillAlignField::Type type, bool isAlign, ExprPtr count) + : type(type), isAlign(isAlign), count(count) +{ + +} + +bool FillAlignField::needsValue() +{ + return false; +} + +void FillAlignField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + int bitSize; + switch(type) + { + case Type::bit: bitSize = 1; break; + case Type::nibble: bitSize = 4; break; + case Type::byte: bitSize = 8; break; + case Type::word: bitSize = 16; break; + case Type::long_: bitSize = 32; break; + } + + int actualCount = 1; + if(count) + actualCount = count->evaluateInt(compiler); + + for(int i = 0; i < actualCount; i++) + { + int n; + if(isAlign) + { + int mask = bitSize - 1; + int pos = compiler->tell(); + n = ((pos + mask) & ~mask) - pos; + } + else + n = bitSize; + compiler->write(n, 0); + } +} diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index fc52cd11b4..baa2dd059e 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -97,6 +97,22 @@ private: }; typedef std::shared_ptr SimpleFieldPtr; +class FillAlignField : public Field +{ +public: + enum class Type + { + bit, nibble, byte, word, long_ + }; + + FillAlignField(Type type, bool isAlign, ExprPtr count = ExprPtr()); + virtual bool needsValue(); + virtual void compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass); +private: + Type type; + ExprPtr count; + bool isAlign; +}; inline SimpleField::Attrs operator|(SimpleField::Attrs a, SimpleField::Attrs b) { diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index 2297705ed4..26dcc9ae8d 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -117,6 +117,7 @@ RezSymbol RezLexer::nextToken() KEYWORD(LITERAL, "literal"), KEYWORD(BOOLEAN, "boolean"), KEYWORD(BIT, "bit"), + KEYWORD(NIBBLE, "nibble"), KEYWORD(BYTE, "byte"), KEYWORD(CHAR, "char"), KEYWORD(WORD, "word"), diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 9eebd20513..6aa71657b7 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -57,6 +57,7 @@ %token BOOLEAN "boolean"; %token BIT "bit"; +%token NIBBLE "nibble"; %token BYTE "byte"; %token CHAR "char"; %token WORD "word"; @@ -158,8 +159,8 @@ field_definitions : %empty field_definition: simple_field_definition { $$ = $1; } | array_definition { $$ = $1; } | switch_definition { $$ = $1; } - | fill_statement { $$ = nullptr; } - | align_statement { $$ = nullptr; } + | fill_statement { $$ = $1; } + | align_statement { $$ = $1; } ; %type simple_field_definition; @@ -202,10 +203,21 @@ simpletype : "boolean" { $$ = SimpleField::Type::boolean; } | "bitstring" { $$ = SimpleField::Type::bitstring; } ; -fill_statement : "fill" fill_unit array_count_opt; -align_statement : "align" fill_unit; +%type fill_statement align_statement; +fill_statement : "fill" fill_unit array_count_opt + { $$ = std::make_shared($fill_unit, false, $array_count_opt); } + ; +align_statement : "align" fill_unit + { $$ = std::make_shared($fill_unit, true); } + ; -fill_unit : "bit" | "byte" | "word" | "long"; +%type fill_unit; +fill_unit : "bit" { $$ = FillAlignField::Type::bit; } + | "nibble" { $$ = FillAlignField::Type::nibble; } + | "byte" { $$ = FillAlignField::Type::byte; } + | "word" { $$ = FillAlignField::Type::word; } + | "long" { $$ = FillAlignField::Type::long_; } + ; %type array_definition; array_definition: From 340db62480ab80c55406fe5cad781e0a2f6a91da Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:37:51 +0200 Subject: [PATCH 09/29] Rez: IdentifierExpr: strings and subscripts --- Rez/Expression.cc | 16 +++++++++++++++- Rez/Expression.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index e93bec695a..259cde21f3 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -130,12 +130,26 @@ int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) } else { - ExprPtr val = ctx->lookupIdentifier(id); + Subscripts sub; + for(auto arg : arguments) + sub.addSubscript(arg->evaluateInt(ctx)); + ExprPtr val = ctx->lookupIdentifier(id, sub); assert(val); return val->evaluateInt(ctx); } } +std::string IdentifierExpr::evaluateString(ResourceCompiler *ctx) +{ + assert(!isFunction); + Subscripts sub; + for(auto arg : arguments) + sub.addSubscript(arg->evaluateInt(ctx)); + ExprPtr val = ctx->lookupIdentifier(id, sub); + assert(val); + return val->evaluateString(ctx); +} + CaseExpr::CaseExpr(const std::string &tag, CompoundExprPtr expr) : tag(tag), expr(expr) diff --git a/Rez/Expression.h b/Rez/Expression.h index 9b4ec6b444..acc417e759 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -111,6 +111,7 @@ public: void addArgument(ExprPtr e); virtual int evaluateInt(ResourceCompiler *ctx); + virtual std::string evaluateString(ResourceCompiler *ctx); }; #endif // EXPRESSION_H From e9edbb2ffac8d9ed0b0980cf185226fc1c99a6ba Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:38:00 +0200 Subject: [PATCH 10/29] predefine true and false --- Rez/RezLexer.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc index 739664b524..201bad5723 100644 --- a/Rez/RezLexer.cc +++ b/Rez/RezLexer.cc @@ -90,6 +90,9 @@ RezLexer::RezLexer(std::string filename) pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); // ctx.add_macro_definition(...); pImpl->ctx.add_macro_definition("DeRez", "0"); + pImpl->ctx.add_macro_definition("Rez", "1"); + pImpl->ctx.add_macro_definition("true", "1"); + pImpl->ctx.add_macro_definition("false", "0"); pImpl->iter = pImpl->ctx.begin(); } From cb25b8106c04238f26e5c52cae3276599d6556f6 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 01:55:54 +0200 Subject: [PATCH 11/29] Rez: rect and point --- Rez/Expression.cc | 24 ++++++++++++------------ Rez/Expression.h | 1 + Rez/ResourceDefinitions.cc | 38 ++++++++++++++++++++++++++++++++++++++ Rez/ResourceDefinitions.h | 1 + 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index 259cde21f3..e8d1b72ced 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -108,6 +108,16 @@ void IdentifierExpr::addArgument(ExprPtr e) arguments.push_back(e); } +ExprPtr IdentifierExpr::lookup(ResourceCompiler *ctx) +{ + Subscripts sub; + for(auto arg : arguments) + sub.addSubscript(arg->evaluateInt(ctx)); + ExprPtr val = ctx->lookupIdentifier(id, sub); + assert(val); + return val; +} + int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) { if(isFunction) @@ -130,24 +140,14 @@ int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) } else { - Subscripts sub; - for(auto arg : arguments) - sub.addSubscript(arg->evaluateInt(ctx)); - ExprPtr val = ctx->lookupIdentifier(id, sub); - assert(val); - return val->evaluateInt(ctx); + return lookup(ctx)->evaluateInt(ctx); } } std::string IdentifierExpr::evaluateString(ResourceCompiler *ctx) { assert(!isFunction); - Subscripts sub; - for(auto arg : arguments) - sub.addSubscript(arg->evaluateInt(ctx)); - ExprPtr val = ctx->lookupIdentifier(id, sub); - assert(val); - return val->evaluateString(ctx); + return lookup(ctx)->evaluateString(ctx); } diff --git a/Rez/Expression.h b/Rez/Expression.h index acc417e759..be57fc31e8 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -110,6 +110,7 @@ public: IdentifierExpr(std::string id, bool isFunction = false); void addArgument(ExprPtr e); + ExprPtr lookup(ResourceCompiler *ctx); virtual int evaluateInt(ResourceCompiler *ctx); virtual std::string evaluateString(ResourceCompiler *ctx); }; diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 65a17fc98c..36078bd934 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -103,6 +103,12 @@ void SimpleField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass case Type::char_: compileString(expr, compiler, prePass); break; + + case Type::rect: + case Type::point: + compileCompound(expr, compiler, prePass); + break; + } } @@ -181,6 +187,38 @@ void SimpleField::compileInt(ExprPtr expr, ResourceCompiler *compiler, bool preP compiler->write(bitSize, actualValue); } +void SimpleField::compileCompound(ExprPtr expr, ResourceCompiler *compiler, bool prePass) +{ + ExprPtr val = value ? value : expr; + if(IdentifierExprPtr id = std::dynamic_pointer_cast(val)) + { + ResourceCompiler::FieldScope scope(compiler, this); + val = id->lookup(compiler); + } + + int count = 0; + switch(type) + { + case Type::rect: + count = 4; + break; + case Type::point: + count = 2; + break; + } + + CompoundExprPtr compound = std::dynamic_pointer_cast(val); + assert(compound); + + assert(compound->size() == count); + + for(int i = 0; i < count; i++) + { + int x = compound->getItem(i)->evaluateInt(compiler); + compiler->write(16, x); + } +} + ArrayField::ArrayField(std::string name, ExprPtr count) : name(name), arrayCount(count) diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index baa2dd059e..d2456e2419 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -94,6 +94,7 @@ public: private: void compileString(ExprPtr expr, ResourceCompiler *compiler, bool prePass); void compileInt(ExprPtr expr, ResourceCompiler *compiler, bool prePass); + void compileCompound(ExprPtr expr, ResourceCompiler *compiler, bool prePass); }; typedef std::shared_ptr SimpleFieldPtr; From a9c399ff2af276cfea29b5ada2ccb6c8d76f2b78 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 02:52:15 +0200 Subject: [PATCH 12/29] MakeAPPL: factor out resource file code --- MakeAPPL/BinaryIO.cc | 52 ++++++++++ MakeAPPL/BinaryIO.h | 17 ++++ MakeAPPL/CMakeLists.txt | 4 + MakeAPPL/ResType.cc | 38 +++++++ MakeAPPL/ResType.h | 23 +++++ MakeAPPL/ResourceFiles.cc | 129 +++++++++++++++++++++++ MakeAPPL/ResourceFiles.h | 42 ++++++++ MakeAPPL/main.cc | 209 +------------------------------------- 8 files changed, 308 insertions(+), 206 deletions(-) create mode 100644 MakeAPPL/BinaryIO.cc create mode 100644 MakeAPPL/BinaryIO.h create mode 100644 MakeAPPL/ResType.cc create mode 100644 MakeAPPL/ResType.h create mode 100644 MakeAPPL/ResourceFiles.cc create mode 100644 MakeAPPL/ResourceFiles.h diff --git a/MakeAPPL/BinaryIO.cc b/MakeAPPL/BinaryIO.cc new file mode 100644 index 0000000000..32070edbd4 --- /dev/null +++ b/MakeAPPL/BinaryIO.cc @@ -0,0 +1,52 @@ +#include "BinaryIO.h" +#include +#include + +void byte(std::ostream& out, int byte) +{ + out.put((unsigned char)byte); +} +void word(std::ostream& out, int word) +{ + byte(out,(word >> 8) & 0xFF); + byte(out,word & 0xFF); +} +void ostype(std::ostream& out, std::string type) +{ + assert(type.size() == 4); + out << type; +} +void longword(std::ostream& out, int longword) +{ + byte(out,(longword >> 24) & 0xFF); + byte(out,(longword >> 16) & 0xFF); + byte(out,(longword >> 8) & 0xFF); + byte(out,longword & 0xFF); +} + +int byte(std::istream& in) +{ + return in.get() & 0xFF; +} +int word(std::istream& in) +{ + int a = byte(in); + int b = byte(in); + return (a << 8) | b; +} +std::string ostype(std::istream& in) +{ + char s[5]; + in.read(s,4); + s[4] = 0; + return s; +} +int longword(std::istream& in) +{ + int a = byte(in); + int b = byte(in); + int c = byte(in); + int d = byte(in); + return (a << 24) | (b << 16) | (c << 8) | d; +} + diff --git a/MakeAPPL/BinaryIO.h b/MakeAPPL/BinaryIO.h new file mode 100644 index 0000000000..e4b84d4336 --- /dev/null +++ b/MakeAPPL/BinaryIO.h @@ -0,0 +1,17 @@ +#ifndef BINARYIO_H +#define BINARYIO_H + +#include +#include + +void byte(std::ostream& out, int byte); +void word(std::ostream& out, int word); +void ostype(std::ostream& out, std::string type); +void longword(std::ostream& out, int longword); + +int byte(std::istream& in); +int word(std::istream& in); +std::string ostype(std::istream& in); +int longword(std::istream& in); + +#endif // BINARYIO_H diff --git a/MakeAPPL/CMakeLists.txt b/MakeAPPL/CMakeLists.txt index 8bd5489b1e..11347f2356 100644 --- a/MakeAPPL/CMakeLists.txt +++ b/MakeAPPL/CMakeLists.txt @@ -16,7 +16,11 @@ # along with Retro68. If not, see . cmake_minimum_required(VERSION 2.8) +set(CMAKE_CXX_FLAGS "--std=c++0x") +add_library(ResourceFiles ResourceFiles.h ResourceFiles.cc BinaryIO.h BinaryIO.cc ResType.h ResType.cc) +target_include_directories(ResourceFiles PUBLIC .) add_executable(MakeAPPL main.cc) +target_link_libraries(MakeAPPL ResourceFiles) install(TARGETS MakeAPPL RUNTIME DESTINATION bin) diff --git a/MakeAPPL/ResType.cc b/MakeAPPL/ResType.cc new file mode 100644 index 0000000000..add8a6302a --- /dev/null +++ b/MakeAPPL/ResType.cc @@ -0,0 +1,38 @@ +#include "ResType.h" +#include + +ResType::ResType(const std::string &str) +{ + auto p = str.begin(); + auto e = str.end(); + + x = 0; + while(p != e) + { + x <<= 8; + x |= (*p) & 0xFF; + ++p; + } +} + +ResType::operator std::string() +{ + char c1 = static_cast(x >> 24); + char c2 = static_cast(x >> 16); + char c3 = static_cast(x >> 8); + char c4 = static_cast(x); + + return std::string{ c1, c2, c3, c4 }; +} + +std::ostream &operator<<(std::ostream &out, ResType t) +{ + char c1 = static_cast((int)t >> 24); + char c2 = static_cast((int)t >> 16); + char c3 = static_cast((int)t >> 8); + char c4 = static_cast((int)t); + + out << "'" << c1 << c2 << c3 << c4 << "'"; + return out; +} + diff --git a/MakeAPPL/ResType.h b/MakeAPPL/ResType.h new file mode 100644 index 0000000000..324d94c002 --- /dev/null +++ b/MakeAPPL/ResType.h @@ -0,0 +1,23 @@ +#ifndef RESTYPE_H +#define RESTYPE_H + +#include +#include + +class ResType +{ + int x; +public: + ResType() : x(0) {} + ResType(int x) : x(x) {} + ResType(const std::string& s); + + operator int() const { return x; } + bool operator<(ResType y) const { return x < y.x; } + + operator std::string(); +}; + +std::ostream& operator<<(std::ostream& out, ResType t); + +#endif // RESTYPE_H diff --git a/MakeAPPL/ResourceFiles.cc b/MakeAPPL/ResourceFiles.cc new file mode 100644 index 0000000000..9f84426d7f --- /dev/null +++ b/MakeAPPL/ResourceFiles.cc @@ -0,0 +1,129 @@ +#include "ResourceFiles.h" +#include "BinaryIO.h" + +#include +#include + +void Resources::addResources(const Resources& res) +{ + resources.insert(resources.end(),res.resources.begin(), res.resources.end()); +} + +void Resources::writeFork(std::ostream& out) const +{ + std::streampos start = out.tellp(); + longword(out,0x100); + longword(out,0); + longword(out,0); + longword(out,0); + out.seekp(start + std::streampos(0x100)); + std::map< std::string, std::map > resourceInfos; + std::streampos datastart = out.tellp(); + for(std::vector::const_iterator p = resources.begin(); p != resources.end(); ++p) + { + const std::string& data = p->getData(); + resourceInfos[ p->getType() ][ p->getID() ] = out.tellp() - datastart; + longword(out, data.size()); + out << data; + } + std::streampos dataend = out.tellp(); +// while(out.tellp() % 0x100) +// out.put(0); + std::streampos resmap = out.tellp(); + out.seekp(16+4+2+2, std::ios::cur); + word(out,16+4+2+2+2+2); // offset to resource type list + std::streampos resnameOffset = out.tellp(); + word(out,0); + std::streampos typelist = out.tellp(); + word(out,resourceInfos.size() - 1); + for(std::map< std::string, std::map >::iterator p = resourceInfos.begin(); + p != resourceInfos.end(); ++p) + { + if(p->second.size()) + { + ostype(out,p->first); + word(out,p->second.size()-1); + word(out,0); // replaced later + } + } + int typeIndex = 0; + for(std::map< std::string, std::map >::iterator p = resourceInfos.begin(); + p != resourceInfos.end(); ++p) + { + if(p->second.size()) + { + std::streampos pos = out.tellp(); + out.seekp((int)typelist + 2 + 8 * typeIndex + 6); + word(out, pos - typelist); + out.seekp(pos); + typeIndex++; + + for(std::map::iterator q = p->second.begin(); q != p->second.end(); ++q) + { + word(out,q->first); + word(out,-1); + longword(out,q->second); + longword(out,0); + } + } + } + std::streampos resnames = out.tellp(); + out.seekp(resnameOffset); + word(out, resnames - resmap); + out.seekp(resnames); + std::streampos end = out.tellp(); + out.seekp(start + std::streampos(4)); + longword(out, resmap - start); + longword(out, dataend - start - std::streampos(0x100)); + longword(out, end - resmap); + out.seekp(end); +} + +Resources::Resources(std::istream &in) +{ + std::streampos start = in.tellg(); + int resdataOffset = longword(in); + int resmapOffset = longword(in); + + in.seekg(start + std::streampos(resmapOffset + 16 + 4 + 2 + 2)); + int typeListOffset = word(in); + int nameListOffset = word(in); + int nTypes = (word(in) + 1) & 0xFFFF; + + for(int i = 0; i < nTypes; i++) + { + in.seekg(start + std::streampos(resmapOffset + typeListOffset + 2 + i * 8)); + std::string type = ostype(in); + int nRes = (word(in) + 1) & 0xFFFF; + int refListOffset = word(in); + + for(int j = 0; j < nRes; j++) + { + in.seekg(start + std::streampos(resmapOffset + typeListOffset + refListOffset + j * 12)); + int id = word(in); + int nameOffset = word(in); + int attr = byte(in); + int off1 = byte(in); + int off2 = byte(in); + int off3 = byte(in); + int offset = (off1 << 16) | (off2 << 8) | off3; + std::string name; + if(nameOffset != 0xFFFF) + { + in.seekg(start + std::streampos(resmapOffset + nameListOffset + nameOffset)); + int nameLen = byte(in); + char buf[256]; + in.read(buf, nameLen); + name = std::string(buf, nameLen); + } + + in.seekg(start + std::streampos(resdataOffset + offset)); + int size = longword(in); + std::vector tmp(size); + in.read(tmp.data(), size); + std::string data(tmp.data(), size); + + addResource(Resource(type, id, data, name, attr)); + } + } +} diff --git a/MakeAPPL/ResourceFiles.h b/MakeAPPL/ResourceFiles.h new file mode 100644 index 0000000000..8e24ca80cb --- /dev/null +++ b/MakeAPPL/ResourceFiles.h @@ -0,0 +1,42 @@ +#ifndef RESOURCEFILES_H +#define RESOURCEFILES_H + +#include +#include + +class Resource +{ + std::string type; + int id; + std::string name; + std::string data; + int attr; +public: + Resource(std::string type, int id, std::string data, std::string name = "", int attr = 0) + : type(type), id(id), data(data), name(name), attr(attr) {} + + const std::string& getData() const { return data; } + inline std::string getType() const { return type; } + inline int getID() const { return id; } +}; + +class Fork +{ +public: + virtual void writeFork(std::ostream& out) const { } + virtual ~Fork() {} +}; + +class Resources : public Fork +{ + std::vector resources; +public: + Resources() {} + Resources(std::istream& in); + void writeFork(std::ostream& out) const; + void addResource(Resource res) { resources.push_back(res); } + + void addResources(const Resources& res); +}; + +#endif // RESOURCEFILES_H diff --git a/MakeAPPL/main.cc b/MakeAPPL/main.cc index 79bc813083..e2baaa3d5d 100644 --- a/MakeAPPL/main.cc +++ b/MakeAPPL/main.cc @@ -28,6 +28,9 @@ #include #include +#include "ResourceFiles.h" +#include "BinaryIO.h" + std::string commandPath; void wrapMacBinary(std::string macBinaryFile, std::string diskImagePath) @@ -45,212 +48,6 @@ void wrapMacBinary(std::string macBinaryFile, std::string diskImagePath) std::system((commandPath + "hcopy -m " + macBinaryFile + " :").c_str()); } -class Resource -{ - std::string type; - int id; - std::string name; - std::string data; - int attr; -public: - Resource(std::string type, int id, std::string data, std::string name = "", int attr = 0) - : type(type), id(id), data(data), name(name), attr(attr) {} - - const std::string& getData() const { return data; } - inline std::string getType() const { return type; } - inline int getID() const { return id; } -}; - -class Fork -{ -public: - virtual void writeFork(std::ostream& out) const { } - virtual ~Fork() {} -}; - -class Resources : public Fork -{ - std::vector resources; -public: - Resources() {} - Resources(std::istream& in); - void writeFork(std::ostream& out) const; - void addResource(Resource res) { resources.push_back(res); } - - void addResources(const Resources& res); -}; - -void byte(std::ostream& out, int byte) -{ - out.put((unsigned char)byte); -} -void word(std::ostream& out, int word) -{ - byte(out,(word >> 8) & 0xFF); - byte(out,word & 0xFF); -} -void ostype(std::ostream& out, std::string type) -{ - assert(type.size() == 4); - out << type; -} -void longword(std::ostream& out, int longword) -{ - byte(out,(longword >> 24) & 0xFF); - byte(out,(longword >> 16) & 0xFF); - byte(out,(longword >> 8) & 0xFF); - byte(out,longword & 0xFF); -} - -int byte(std::istream& in) -{ - return in.get() & 0xFF; -} -int word(std::istream& in) -{ - int a = byte(in); - int b = byte(in); - return (a << 8) | b; -} -std::string ostype(std::istream& in) -{ - char s[5]; - in.read(s,4); - s[4] = 0; - return s; -} -int longword(std::istream& in) -{ - int a = byte(in); - int b = byte(in); - int c = byte(in); - int d = byte(in); - return (a << 24) | (b << 16) | (c << 8) | d; -} - -void Resources::addResources(const Resources& res) -{ - resources.insert(resources.end(),res.resources.begin(), res.resources.end()); -} - -void Resources::writeFork(std::ostream& out) const -{ - std::streampos start = out.tellp(); - longword(out,0x100); - longword(out,0); - longword(out,0); - longword(out,0); - out.seekp(start + std::streampos(0x100)); - std::map< std::string, std::map > resourceInfos; - std::streampos datastart = out.tellp(); - for(std::vector::const_iterator p = resources.begin(); p != resources.end(); ++p) - { - const std::string& data = p->getData(); - resourceInfos[ p->getType() ][ p->getID() ] = out.tellp() - datastart; - longword(out, data.size()); - out << data; - } - std::streampos dataend = out.tellp(); -// while(out.tellp() % 0x100) -// out.put(0); - std::streampos resmap = out.tellp(); - out.seekp(16+4+2+2, std::ios::cur); - word(out,16+4+2+2+2+2); // offset to resource type list - std::streampos resnameOffset = out.tellp(); - word(out,0); - std::streampos typelist = out.tellp(); - word(out,resourceInfos.size() - 1); - for(std::map< std::string, std::map >::iterator p = resourceInfos.begin(); - p != resourceInfos.end(); ++p) - { - if(p->second.size()) - { - ostype(out,p->first); - word(out,p->second.size()-1); - word(out,0); // replaced later - } - } - int typeIndex = 0; - for(std::map< std::string, std::map >::iterator p = resourceInfos.begin(); - p != resourceInfos.end(); ++p) - { - if(p->second.size()) - { - std::streampos pos = out.tellp(); - out.seekp((int)typelist + 2 + 8 * typeIndex + 6); - word(out, pos - typelist); - out.seekp(pos); - typeIndex++; - - for(std::map::iterator q = p->second.begin(); q != p->second.end(); ++q) - { - word(out,q->first); - word(out,-1); - longword(out,q->second); - longword(out,0); - } - } - } - std::streampos resnames = out.tellp(); - out.seekp(resnameOffset); - word(out, resnames - resmap); - out.seekp(resnames); - std::streampos end = out.tellp(); - out.seekp(start + std::streampos(4)); - longword(out, resmap - start); - longword(out, dataend - start - std::streampos(0x100)); - longword(out, end - resmap); - out.seekp(end); -} - -Resources::Resources(std::istream &in) -{ - std::streampos start = in.tellg(); - int resdataOffset = longword(in); - int resmapOffset = longword(in); - - in.seekg(start + std::streampos(resmapOffset + 16 + 4 + 2 + 2)); - int typeListOffset = word(in); - int nameListOffset = word(in); - int nTypes = (word(in) + 1) & 0xFFFF; - - for(int i = 0; i < nTypes; i++) - { - in.seekg(start + std::streampos(resmapOffset + typeListOffset + 2 + i * 8)); - std::string type = ostype(in); - int nRes = (word(in) + 1) & 0xFFFF; - int refListOffset = word(in); - - for(int j = 0; j < nRes; j++) - { - in.seekg(start + std::streampos(resmapOffset + typeListOffset + refListOffset + j * 12)); - int id = word(in); - int nameOffset = word(in); - int attr = byte(in); - int off1 = byte(in); - int off2 = byte(in); - int off3 = byte(in); - int offset = (off1 << 16) | (off2 << 8) | off3; - std::string name; - if(nameOffset != 0xFFFF) - { - in.seekg(start + std::streampos(resmapOffset + nameListOffset + nameOffset)); - int nameLen = byte(in); - char buf[256]; - in.read(buf, nameLen); - name = std::string(buf, nameLen); - } - - in.seekg(start + std::streampos(resdataOffset + offset)); - int size = longword(in); - std::vector tmp(size); - in.read(tmp.data(), size); - std::string data(tmp.data(), size); - - addResource(Resource(type, id, data, name, attr)); - } - } -} // CRC 16 table lookup array static unsigned short CRC16Table[256] = From 70b5b36d479e1d9c5a104fec904162f89fbefcf1 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Wed, 8 Oct 2014 02:52:34 +0200 Subject: [PATCH 13/29] Rez: write actual resource files --- Rez/CMakeLists.txt | 2 +- Rez/ResourceCompiler.cc | 30 +++++++++++++++++++++++++++++- Rez/ResourceCompiler.h | 4 ++++ Rez/ResourceDefinitions.cc | 9 --------- Rez/ResourceDefinitions.h | 12 +----------- Rez/Rez.cc | 16 ++++++++++++++++ Rez/RezLexer.cc | 18 ++++++++++++------ Rez/RezParser.yy | 5 +---- Rez/RezWorld.cc | 14 ++++++++++++++ Rez/RezWorld.h | 7 +++++++ 10 files changed, 85 insertions(+), 32 deletions(-) diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index 4b59cb4f7b..2e7059a25e 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -56,6 +56,6 @@ add_executable(Rez ResourceCompiler.cc ResourceCompiler.h ) -target_link_libraries(Rez ${Boost_LIBRARIES}) +target_link_libraries(Rez ResourceFiles ${Boost_LIBRARIES}) install(TARGETS Rez RUNTIME DESTINATION bin) diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc index d54798d05e..bfb73cfd53 100644 --- a/Rez/ResourceCompiler.cc +++ b/Rez/ResourceCompiler.cc @@ -8,10 +8,31 @@ ResourceCompiler::ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body) } +std::string ResourceCompiler::resourceData() +{ + return std::string(data.begin(), data.end()); +} + void ResourceCompiler::write(int nBits, int value) { std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; - currentOffset += nBits; + + unsigned mask = 1 << (nBits-1); + + for(int i = 0; i < nBits; i++) + { + bool bit = (value & mask) != 0; + + if(currentOffset % 8 == 0) + data.push_back(bit ? 0x80 : 0); + else if(bit) + data.back() |= (0x80 >> (currentOffset % 8)); + ++currentOffset; + + mask >>= 1; + } + + //currentOffset += nBits; } ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &sub) @@ -19,13 +40,18 @@ ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &s if(currentField) { if(ExprPtr val = currentField->lookupNamedValue(name)) + { + std::cout << "current field: " << name << " found."; return val; + } } auto p = labelValues.find(std::make_pair(name, sub)); if(p != labelValues.end()) return p->second; + std::cout << "ID lookup failed: " << name << std::endl; + return nullptr; } @@ -38,9 +64,11 @@ void ResourceCompiler::compile() { std::cout << "(first pass)\n"; currentOffset = 0; + data.clear(); typeDefinition->compile(body, this, true); std::cout << "(second pass)\n"; currentOffset = 0; + data.clear(); typeDefinition->compile(body, this, false); std::cout << "(done)\n"; diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index 697dc9e556..bafb10e5ce 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -31,10 +31,14 @@ class ResourceCompiler Field* currentField; Subscripts currentSubscripts; + std::vector data; + void beginArrayScope(std::string& arrayName, int index); public: ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body); + std::string resourceData(); + void reserve(int nBits) { write(nBits, 0); } void write(int nBits, int value); int tell() { return currentOffset; } diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 36078bd934..08c8ce44d8 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -4,16 +4,7 @@ #include "ResourceCompiler.h" -std::ostream &operator<<(std::ostream &out, ResType t) -{ - char c1 = static_cast((int)t >> 24); - char c2 = static_cast((int)t >> 16); - char c3 = static_cast((int)t >> 8); - char c4 = static_cast((int)t); - out << "'" << c1 << c2 << c3 << c4 << "'"; - return out; -} std::ostream &operator<<(std::ostream &out, TypeSpec ts) { diff --git a/Rez/ResourceDefinitions.h b/Rez/ResourceDefinitions.h index d2456e2419..e2d5bb828f 100644 --- a/Rez/ResourceDefinitions.h +++ b/Rez/ResourceDefinitions.h @@ -6,18 +6,8 @@ #include #include "Expression.h" +#include "ResType.h" -class ResType -{ - int x; -public: - ResType() : x(0) {} - ResType(int x) : x(x) {} - operator int() const { return x; } - bool operator<(ResType y) const { return x < y.x; } -}; - -std::ostream& operator<<(std::ostream& out, ResType t); class TypeSpec { diff --git a/Rez/Rez.cc b/Rez/Rez.cc index ee617ec62d..69a75e2a38 100644 --- a/Rez/Rez.cc +++ b/Rez/Rez.cc @@ -6,6 +6,9 @@ #include "RezLexer.h" #include "RezWorld.h" +#include "ResourceFiles.h" +#include "BinaryIO.h" + int main() { //RezLexer lexer("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); @@ -14,5 +17,18 @@ int main() RezParser parser(lexer, world); parser.parse(); + { + std::ofstream dataOut("test.rsrc"); + system("mkdir -p .rsrc"); + std::ofstream rsrcOut(".rsrc/test.rsrc"); + + world.getResources().writeFork(rsrcOut); + system("mkdir -p .finf"); + std::ofstream finfOut(".finf/test.rsrc"); + ostype(finfOut, "rsrc"); + ostype(finfOut, "RSED"); + for(int i = 8; i < 32; i++) + byte(finfOut, 0); + } return 0; } diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc index 201bad5723..a53a2cdd04 100644 --- a/Rez/RezLexer.cc +++ b/Rez/RezLexer.cc @@ -88,11 +88,10 @@ RezLexer::RezLexer(std::string filename) filename)); pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); - // ctx.add_macro_definition(...); - pImpl->ctx.add_macro_definition("DeRez", "0"); - pImpl->ctx.add_macro_definition("Rez", "1"); - pImpl->ctx.add_macro_definition("true", "1"); - pImpl->ctx.add_macro_definition("false", "0"); + pImpl->ctx.add_macro_definition("DeRez=0"); + pImpl->ctx.add_macro_definition("Rez=1"); + pImpl->ctx.add_macro_definition("true=1"); + pImpl->ctx.add_macro_definition("false=0"); pImpl->iter = pImpl->ctx.begin(); } @@ -109,7 +108,14 @@ bool RezLexer::atEnd() RezLexer::WaveToken RezLexer::nextWave() { - return pImpl->iter == pImpl->ctx.end() ? WaveToken() : (*pImpl->iter++); + if(pImpl->iter == pImpl->ctx.end()) + return WaveToken(); + else + { + WaveToken tok = *pImpl->iter++; + std::cout << tok.get_value(); + return tok; + } } RezLexer::WaveToken RezLexer::peekWave() diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 6aa71657b7..68a3ff69e5 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -346,10 +346,7 @@ function_argument_list1 : expression resource : "resource" res_type "(" expression resource_attributes ")" "{" resource_body "}" { int id = $expression->evaluateInt(nullptr); - std::cout << "RESOURCE " << $2 << "(" << id << ")" << std::endl; - TypeDefinitionPtr type = world.getTypeDefinition($res_type, id); - ResourceCompiler compiler(type, $resource_body); - compiler.compile(); + world.addResource($res_type, id, $resource_body); } ; diff --git a/Rez/RezWorld.cc b/Rez/RezWorld.cc index 90e6148b1a..68b77897ea 100644 --- a/Rez/RezWorld.cc +++ b/Rez/RezWorld.cc @@ -1,4 +1,8 @@ #include "RezWorld.h" +#include "ResourceCompiler.h" +#include "ResourceFiles.h" + +#include RezWorld::RezWorld() { @@ -20,3 +24,13 @@ TypeDefinitionPtr RezWorld::getTypeDefinition(ResType type, int id) return nullptr; } + +void RezWorld::addResource(ResType type, int id, CompoundExprPtr body) +{ + std::cout << "RESOURCE " << type << "(" << id << ")" << std::endl; + TypeDefinitionPtr def = getTypeDefinition(type, id); + ResourceCompiler compiler(def, body); + compiler.compile(); + + resources.addResource(Resource(type, id, compiler.resourceData())); +} diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h index ec44502369..fa0cf24608 100644 --- a/Rez/RezWorld.h +++ b/Rez/RezWorld.h @@ -5,6 +5,7 @@ #include #include "ResourceDefinitions.h" #include "Expression.h" +#include "ResourceFiles.h" class RezWorld { @@ -14,11 +15,17 @@ class RezWorld std::stack fieldLists; std::stack functionCalls; std::stack switches; + + Resources resources; public: RezWorld(); void addTypeDefinition(TypeSpec spec, TypeDefinitionPtr type); TypeDefinitionPtr getTypeDefinition(ResType type, int id); + + void addResource(ResType type, int id, CompoundExprPtr body); + + Resources& getResources() { return resources; } }; From 6763bf3b4dcd80cb1fbefe8e58edb77901fe2346 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 9 Oct 2014 00:52:17 +0200 Subject: [PATCH 14/29] handle strings properly (remove quotes, handle escapes) --- Rez/RezLexerNextToken.cc | 74 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index 26dcc9ae8d..6a7b16d269 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -7,13 +7,15 @@ using namespace boost::wave; -static int readInt(const char *str) +static int readInt(const char *str, const char *end = NULL, int baseOverride = 0) { int x = 0; int base = 10; - if(*str == '0') + if(baseOverride) + base = baseOverride; + else if(*str == '0') { base = 8; ++str; @@ -34,7 +36,7 @@ static int readInt(const char *str) ++str; } - while(*str) + while(str != end && *str) { x *= base; if(*str >= 'a' && *str <= 'z') @@ -67,6 +69,70 @@ static int readCharLit(const char *str) return x; } +static std::string readStringLit(const char *str) +{ + const char *p = str + 1; + const char *e = str + strlen(str) - 1; + + std::ostringstream out; + + while(p != e) + { + if(*p == '\\') + { + ++p; + if(p != e) + { + switch(*p) + { + case 'n': + out << '\n'; + break; + case 'r': + out << '\r'; + break; + case 't': + out << '\t'; + break; + case '0': + case '1': + case '2': + case '3': + if(p + 3 > e) + continue; + if(p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) + { + if(p + 4 > e) + continue; + out << (char)readInt(p+2, p+4, 16); + p += 4; + } + else + { + out << (char)readInt(p, p+3, 8); + p += 3; + } + break; + case '$': + { + if(p + 3 > e) + continue; + out << (char)readInt(p+1, p+3, 16); + p += 3; + } + break; + } + } + } + else + { + out << *p++; + } + } + + return out.str(); +} + RezSymbol RezLexer::nextToken() { for(auto tok = nextWave(); tok != T_EOI && tok != T_EOF; tok = nextWave()) @@ -174,7 +240,7 @@ case T_ ## name: /*std::cout << #name << std::endl;*/ return RezParser::make_ ## case T_INTLIT: return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc); case T_CHARLIT: return RezParser::make_CHARLIT(readCharLit(tok.get_value().c_str()), loc); - case T_STRINGLIT: return RezParser::make_STRINGLIT(tok.get_value().c_str(), loc); + case T_STRINGLIT: return RezParser::make_STRINGLIT(readStringLit(tok.get_value().c_str()), loc); NOVAL_TOK(LEFTBRACE); NOVAL_TOK(RIGHTBRACE); From 2b5c415db3c33c30b47811cc62262e95bc69611b Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 9 Oct 2014 00:52:39 +0200 Subject: [PATCH 15/29] add Rez/README.md --- Rez/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Rez/README.md diff --git a/Rez/README.md b/Rez/README.md new file mode 100644 index 0000000000..807dc0cffd --- /dev/null +++ b/Rez/README.md @@ -0,0 +1,12 @@ +Rez Resource Compiler +===================== + +A reimplementation of the classic Rez resource compiler. + + +Known Bugs & Limitations +----------------- + +* `$ABCD` alternate syntax for hex numbers is not supported +* hex strings not supported +* Other than in Apple Rez, the preprocessor is case sensitive. From 2b64cb707d584e5efc16504e33ce2774191093e8 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 9 Oct 2014 21:41:41 +0200 Subject: [PATCH 16/29] Rez: string literal concatenation, hex strings --- Rez/RezLexerNextToken.cc | 2 +- Rez/RezParser.yy | 43 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index 6a7b16d269..21cf1a7a12 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -198,7 +198,7 @@ RezSymbol RezLexer::nextToken() KEYWORD(BITSTRING, "bitstring"), KEYWORD(INTEGER, "int"), - + KEYWORD(DOLLAR, "$") }; std::string s = tok.get_value().c_str(); diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 68a3ff69e5..f383d60ca0 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -39,7 +39,7 @@ %token OR "|"; %token XOR "^"; %token COMPL "~"; - +%token DOLLAR "$"; %token TYPE "type"; %token RESOURCE "resource"; @@ -119,6 +119,37 @@ { std::cerr << loc << ": " << err << std::endl; } + + static std::string fromHex(std::string hex) + { + std::string bin; + int nibble; + bool haveNibble = false; + for(std::string::iterator p = hex.begin(); p != hex.end(); ++p) + { + if(std::isspace(*p)) + continue; + assert(isdigit(*p) || (tolower(*p) >= 'a' && tolower(*p) <= 'f')); + int digit; + if(isdigit(*p)) + digit = *p - '0'; + else + digit = tolower(*p) - 'a' + 0xA; + + if(haveNibble) + { + bin += (char) ((nibble << 4) | digit); + haveNibble = false; + } + else + { + nibble = digit; + haveNibble = true; + } + } + return bin; + } + } %% @@ -278,9 +309,17 @@ switch_case : "case" IDENTIFIER ":" } ; +%type string onestring; +string : onestring { $$ = $1; } + | string onestring { $$ = $1 + $2; } + ; +onestring : STRINGLIT { $$ = $1; } + | DOLLAR STRINGLIT { $$ = fromHex($2); } + ; + value : expression { $$ = $1; } | "{" resource_body "}" { $$ = $2; } - | STRINGLIT { $$ = std::make_shared($1); } + | string { $$ = std::make_shared($1); } ; expression : expression1 { $$ = $1; } From 127db48a266e9fbab925fe64788bf8319fe57468 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 9 Oct 2014 22:15:13 +0200 Subject: [PATCH 17/29] Rez: command line options --- Rez/CMakeLists.txt | 2 +- Rez/ResourceCompiler.cc | 21 ++++++---- Rez/ResourceCompiler.h | 3 +- Rez/Rez.cc | 92 ++++++++++++++++++++++++++++++++++------- Rez/RezLexer.cc | 1 - Rez/RezParser.yy | 4 +- Rez/RezWorld.cc | 6 ++- Rez/RezWorld.h | 2 + Rez/Test.r | 11 +++-- 9 files changed, 108 insertions(+), 34 deletions(-) diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index 2e7059a25e..cd200af5fb 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -19,7 +19,7 @@ cmake_minimum_required(VERSION 2.8) set(CMAKE_CXX_FLAGS "--std=c++11") -find_package(Boost COMPONENTS wave filesystem system thread regex) +find_package(Boost COMPONENTS wave filesystem system thread regex program_options) find_package(BISON REQUIRED) diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc index bfb73cfd53..b22419b727 100644 --- a/Rez/ResourceCompiler.cc +++ b/Rez/ResourceCompiler.cc @@ -2,8 +2,13 @@ #include #include "ResourceDefinitions.h" -ResourceCompiler::ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body) - : typeDefinition(type), body(body), currentOffset(0), currentField(nullptr) +ResourceCompiler::ResourceCompiler( + TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag) + : typeDefinition(type), + body(body), + currentOffset(0), + currentField(nullptr), + verboseFlag(verboseFlag) { } @@ -15,7 +20,8 @@ std::string ResourceCompiler::resourceData() void ResourceCompiler::write(int nBits, int value) { - std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; + if(verboseFlag) + std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; unsigned mask = 1 << (nBits-1); @@ -41,7 +47,6 @@ ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &s { if(ExprPtr val = currentField->lookupNamedValue(name)) { - std::cout << "current field: " << name << " found."; return val; } } @@ -50,7 +55,7 @@ ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &s if(p != labelValues.end()) return p->second; - std::cout << "ID lookup failed: " << name << std::endl; + std::cerr << "ID lookup failed: " << name << std::endl; return nullptr; } @@ -62,15 +67,15 @@ void ResourceCompiler::defineLabel(const std::string &name) void ResourceCompiler::compile() { - std::cout << "(first pass)\n"; + if(verboseFlag) std::cout << "(first pass)\n"; currentOffset = 0; data.clear(); typeDefinition->compile(body, this, true); - std::cout << "(second pass)\n"; + if(verboseFlag) std::cout << "(second pass)\n"; currentOffset = 0; data.clear(); typeDefinition->compile(body, this, false); - std::cout << "(done)\n"; + if(verboseFlag) std::cout << "(done)\n"; } diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index bafb10e5ce..2ae56d4c58 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -32,10 +32,11 @@ class ResourceCompiler Subscripts currentSubscripts; std::vector data; + bool verboseFlag; void beginArrayScope(std::string& arrayName, int index); public: - ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body); + ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag); std::string resourceData(); diff --git a/Rez/Rez.cc b/Rez/Rez.cc index 69a75e2a38..cae9d57e63 100644 --- a/Rez/Rez.cc +++ b/Rez/Rez.cc @@ -1,6 +1,7 @@ -#include #include - +#include "boost/program_options.hpp" +#include "boost/filesystem.hpp" +#include "boost/filesystem/fstream.hpp" #include "RezParser.generated.hh" #include "RezLexer.h" @@ -9,24 +10,83 @@ #include "ResourceFiles.h" #include "BinaryIO.h" -int main() -{ - //RezLexer lexer("/home/wolfgang/Projects/Retro68/RIncludes/Types.r"); - RezLexer lexer("/home/wolfgang/Projects/Retro68/Rez/Test.r"); - RezWorld world; - RezParser parser(lexer, world); +namespace po = boost::program_options; +namespace fs = boost::filesystem; - parser.parse(); +static po::options_description desc; + +static void usage() +{ + std::cerr << "Usage: " << "Rez [options] input-file\n"; + std::cerr << desc << std::endl; +} + +int main(int argc, const char *argv[]) +{ + desc.add_options() + ("help,h", "show this help message") + ("output,o", po::value()->default_value("rez.output.rsrc"), "output file") + ("type,t", po::value()->default_value("rsrc"), "output file finder type code") + ("creator,c", po::value()->default_value("RSED"), "output file finder creator code") + ("debug,d", "debug logging") + ; + po::options_description hidden, alldesc; + hidden.add_options() + ("input", po::value>(), "input file" ) + ; + alldesc.add(desc).add(hidden); + + po::variables_map options; + try { - std::ofstream dataOut("test.rsrc"); - system("mkdir -p .rsrc"); - std::ofstream rsrcOut(".rsrc/test.rsrc"); + auto parsed = po::command_line_parser(argc, argv) + .options(alldesc) + .positional(po::positional_options_description().add("input", -1)) + .style(po::command_line_style::default_style | + po::command_line_style::allow_long_disguise) + .run(); + + po::store(parsed, options); + } + catch(po::error& e) + { + std::cerr << "ERROR: " << e.what() << std::endl << std::endl; + usage(); + return 1; + } + + po::notify(options); + + if(options.count("help") || !options.count("input")) + { + usage(); + return 0; + } + + RezWorld world; + + if(options.count("debug")) + world.verboseFlag = true; + + for(std::string fn : options["input"].as>()) + { + RezLexer lexer(fn); + RezParser parser(lexer, world); + parser.parse(); + } + + std::string outfile = options["output"].as(); + { + fs::path dataPath(outfile); + fs::create_directory(dataPath.parent_path() / ".rsrc"); + fs::create_directory(dataPath.parent_path() / ".finf"); + fs::ofstream dataOut(dataPath); + fs::ofstream rsrcOut(dataPath.parent_path() / ".rsrc" / dataPath.filename()); + fs::ofstream finfOut(dataPath.parent_path() / ".finf" / dataPath.filename()); world.getResources().writeFork(rsrcOut); - system("mkdir -p .finf"); - std::ofstream finfOut(".finf/test.rsrc"); - ostype(finfOut, "rsrc"); - ostype(finfOut, "RSED"); + ostype(finfOut, options["type"].as()); + ostype(finfOut, options["creator"].as()); for(int i = 8; i < 32; i++) byte(finfOut, 0); } diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc index a53a2cdd04..07f23a2861 100644 --- a/Rez/RezLexer.cc +++ b/Rez/RezLexer.cc @@ -113,7 +113,6 @@ RezLexer::WaveToken RezLexer::nextWave() else { WaveToken tok = *pImpl->iter++; - std::cout << tok.get_value(); return tok; } } diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index f383d60ca0..952f9a8ba8 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -167,9 +167,9 @@ type_definition : "type" type_spec world.fieldLists.push(td); } "{" field_definitions "}" - { world.fieldLists.pop(); std::cout << "TYPE " << $2 << std::endl; } + { world.fieldLists.pop(); if(world.verboseFlag) std::cout << "TYPE " << $2 << std::endl; } | "type" type_spec "as" type_spec - { std::cout << "TYPE " << $2 << std::endl; } + { if(world.verboseFlag) std::cout << "TYPE " << $2 << std::endl; } ; %type res_type; diff --git a/Rez/RezWorld.cc b/Rez/RezWorld.cc index 68b77897ea..7c33d5345b 100644 --- a/Rez/RezWorld.cc +++ b/Rez/RezWorld.cc @@ -5,6 +5,7 @@ #include RezWorld::RezWorld() + : verboseFlag(false) { } @@ -27,9 +28,10 @@ TypeDefinitionPtr RezWorld::getTypeDefinition(ResType type, int id) void RezWorld::addResource(ResType type, int id, CompoundExprPtr body) { - std::cout << "RESOURCE " << type << "(" << id << ")" << std::endl; + if(verboseFlag) + std::cout << "RESOURCE " << type << "(" << id << ")" << std::endl; TypeDefinitionPtr def = getTypeDefinition(type, id); - ResourceCompiler compiler(def, body); + ResourceCompiler compiler(def, body, verboseFlag); compiler.compile(); resources.addResource(Resource(type, id, compiler.resourceData())); diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h index fa0cf24608..50277b55ea 100644 --- a/Rez/RezWorld.h +++ b/Rez/RezWorld.h @@ -26,6 +26,8 @@ public: void addResource(ResType type, int id, CompoundExprPtr body); Resources& getResources() { return resources; } + + bool verboseFlag; }; diff --git a/Rez/Test.r b/Rez/Test.r index 6679348422..a0b7903b5a 100644 --- a/Rez/Test.r +++ b/Rez/Test.r @@ -3,12 +3,12 @@ */ -//#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r" +#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r" type 'TEST' { integer zero, one, two, answer = 42, missed; longint; - //integer = (after - before) / 32; + integer = (after - before) / 8; integer = $$CountOf(foo); before: array foo { @@ -16,6 +16,7 @@ type 'TEST' { integer; integer; }; + string; after: ; }; @@ -23,6 +24,10 @@ type 'TEST' { resource 'TEST' (128) { answer, 0x1234, - { 1, 2; 3, 4; } + { 1, 2; 3, 4; }, + "Hello, " + "world: " + $"Abcd 1234"; }; + From df0e042120d453426215ba8750876130560d2fff Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Sun, 12 Oct 2014 19:12:10 +0200 Subject: [PATCH 18/29] use ResType class in Resources lib, prevent dupicate resources --- MakeAPPL/BinaryIO.cc | 14 ++++++-------- MakeAPPL/BinaryIO.h | 6 ++++-- MakeAPPL/ResType.cc | 20 ++++++++++++++++++++ MakeAPPL/ResType.h | 12 ++++++++++++ MakeAPPL/ResourceFiles.cc | 12 ++++++++---- MakeAPPL/ResourceFiles.h | 16 +++++++++------- MakeAPPL/main.cc | 2 +- 7 files changed, 60 insertions(+), 22 deletions(-) diff --git a/MakeAPPL/BinaryIO.cc b/MakeAPPL/BinaryIO.cc index 32070edbd4..b8bf73b3ec 100644 --- a/MakeAPPL/BinaryIO.cc +++ b/MakeAPPL/BinaryIO.cc @@ -2,6 +2,8 @@ #include #include +#include "ResType.h" + void byte(std::ostream& out, int byte) { out.put((unsigned char)byte); @@ -11,10 +13,9 @@ void word(std::ostream& out, int word) byte(out,(word >> 8) & 0xFF); byte(out,word & 0xFF); } -void ostype(std::ostream& out, std::string type) +void ostype(std::ostream& out, ResType type) { - assert(type.size() == 4); - out << type; + longword(out, type); } void longword(std::ostream& out, int longword) { @@ -34,12 +35,9 @@ int word(std::istream& in) int b = byte(in); return (a << 8) | b; } -std::string ostype(std::istream& in) +ResType ostype(std::istream& in) { - char s[5]; - in.read(s,4); - s[4] = 0; - return s; + return longword(in); } int longword(std::istream& in) { diff --git a/MakeAPPL/BinaryIO.h b/MakeAPPL/BinaryIO.h index e4b84d4336..a23f004bfa 100644 --- a/MakeAPPL/BinaryIO.h +++ b/MakeAPPL/BinaryIO.h @@ -4,14 +4,16 @@ #include #include +class ResType; + void byte(std::ostream& out, int byte); void word(std::ostream& out, int word); -void ostype(std::ostream& out, std::string type); +void ostype(std::ostream& out, ResType type); void longword(std::ostream& out, int longword); int byte(std::istream& in); int word(std::istream& in); -std::string ostype(std::istream& in); +ResType ostype(std::istream& in); int longword(std::istream& in); #endif // BINARYIO_H diff --git a/MakeAPPL/ResType.cc b/MakeAPPL/ResType.cc index add8a6302a..87ece75f3e 100644 --- a/MakeAPPL/ResType.cc +++ b/MakeAPPL/ResType.cc @@ -1,11 +1,14 @@ #include "ResType.h" #include +#include ResType::ResType(const std::string &str) { auto p = str.begin(); auto e = str.end(); + assert(str.size() == 4); + x = 0; while(p != e) { @@ -15,6 +18,23 @@ ResType::ResType(const std::string &str) } } +ResType::ResType(const char *s) +{ + auto p = s; + auto e = s + 4; + + assert(s[0] && s[1] && s[2] && s[3] && !s[4]); + + x = 0; + while(p != e) + { + x <<= 8; + x |= (*p) & 0xFF; + ++p; + } +} + + ResType::operator std::string() { char c1 = static_cast(x >> 24); diff --git a/MakeAPPL/ResType.h b/MakeAPPL/ResType.h index 324d94c002..0b44d48e75 100644 --- a/MakeAPPL/ResType.h +++ b/MakeAPPL/ResType.h @@ -11,6 +11,7 @@ public: ResType() : x(0) {} ResType(int x) : x(x) {} ResType(const std::string& s); + ResType(const char* s); operator int() const { return x; } bool operator<(ResType y) const { return x < y.x; } @@ -20,4 +21,15 @@ public: std::ostream& operator<<(std::ostream& out, ResType t); +struct ResRef : public std::pair +{ + ResRef() : std::pair(ResType(), 0) {} + ResRef(ResType t, int id) : std::pair(t,id) {} + + ResType& type() { return first; } + ResType type() const { return first; } + int& id() { return second; } + int id() const { return second; } +}; + #endif // RESTYPE_H diff --git a/MakeAPPL/ResourceFiles.cc b/MakeAPPL/ResourceFiles.cc index 9f84426d7f..ad6ce0ca1c 100644 --- a/MakeAPPL/ResourceFiles.cc +++ b/MakeAPPL/ResourceFiles.cc @@ -3,10 +3,13 @@ #include #include +#include void Resources::addResources(const Resources& res) { - resources.insert(resources.end(),res.resources.begin(), res.resources.end()); + for(auto& rr : res.resources) + resources.insert(rr); +// resources.insert(resources.end(),res.resources.begin(), res.resources.end()); } void Resources::writeFork(std::ostream& out) const @@ -19,10 +22,11 @@ void Resources::writeFork(std::ostream& out) const out.seekp(start + std::streampos(0x100)); std::map< std::string, std::map > resourceInfos; std::streampos datastart = out.tellp(); - for(std::vector::const_iterator p = resources.begin(); p != resources.end(); ++p) + for(auto& rr : resources) { - const std::string& data = p->getData(); - resourceInfos[ p->getType() ][ p->getID() ] = out.tellp() - datastart; + const Resource& r = rr.second; + const std::string& data = r.getData(); + resourceInfos[ r.getType() ][ r.getID() ] = out.tellp() - datastart; longword(out, data.size()); out << data; } diff --git a/MakeAPPL/ResourceFiles.h b/MakeAPPL/ResourceFiles.h index 8e24ca80cb..7ea6b96e73 100644 --- a/MakeAPPL/ResourceFiles.h +++ b/MakeAPPL/ResourceFiles.h @@ -2,22 +2,25 @@ #define RESOURCEFILES_H #include -#include +#include +#include "ResType.h" class Resource { - std::string type; + ResType type; int id; std::string name; std::string data; int attr; public: - Resource(std::string type, int id, std::string data, std::string name = "", int attr = 0) + Resource() {} + Resource(ResType type, int id, std::string data, std::string name = "", int attr = 0) : type(type), id(id), data(data), name(name), attr(attr) {} const std::string& getData() const { return data; } - inline std::string getType() const { return type; } + inline ResType getType() const { return type; } inline int getID() const { return id; } + inline ResRef getTypeAndID() const { return ResRef(type, id); } }; class Fork @@ -29,13 +32,12 @@ public: class Resources : public Fork { - std::vector resources; + std::map resources; public: Resources() {} Resources(std::istream& in); void writeFork(std::ostream& out) const; - void addResource(Resource res) { resources.push_back(res); } - + void addResource(Resource res) { resources[res.getTypeAndID()] = res; } void addResources(const Resources& res); }; diff --git a/MakeAPPL/main.cc b/MakeAPPL/main.cc index e2baaa3d5d..0da7a782e3 100644 --- a/MakeAPPL/main.cc +++ b/MakeAPPL/main.cc @@ -233,7 +233,7 @@ int main(int argc, char *argv[]) std::string fn = argv[i++]; std::string flt = readfile(fn); - rsrc.addResource(Resource("CODE", 0, + rsrc.addResource(Resource(ResType("CODE"), 0, fromhex( "00000028 00000000 00000008 00000020" "0000 3F3C 0001 A9F0" From 3aed54a672e757d4b141aae9b8fa80843d048079 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Sun, 12 Oct 2014 19:16:02 +0200 Subject: [PATCH 19/29] string concat, $$read function; make function names keywords --- Rez/Expression.cc | 81 +++++++++++++++++++++++++++------------- Rez/Expression.h | 42 +++++++++++++++++++-- Rez/RezLexerNextToken.cc | 10 ++++- Rez/RezParser.yy | 63 ++++++++++++++++++++++--------- 4 files changed, 149 insertions(+), 47 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index e8d1b72ced..02aa7a4f21 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -2,6 +2,7 @@ #include "ResourceCompiler.h" #include #include +#include int Expression::evaluateInt(ResourceCompiler *ctx) { @@ -78,6 +79,21 @@ int BinaryExpr::evaluateInt(ResourceCompiler *ctx) return a->evaluateInt(ctx) * b->evaluateInt(ctx); case BinaryOp::DIVIDE: return a->evaluateInt(ctx) / b->evaluateInt(ctx); + default: + throw TypeError(); + break; + } +} + +std::string BinaryExpr::evaluateString(ResourceCompiler *ctx) +{ + switch(op) + { + case BinaryOp::CONCAT: + return a->evaluateString(ctx) + b->evaluateString(ctx); + default: + throw TypeError(); + break; } } @@ -98,8 +114,8 @@ int UnaryExpr::evaluateInt(ResourceCompiler *ctx) } -IdentifierExpr::IdentifierExpr(std::string id, bool isFunction) - : id(id), isFunction(isFunction) +IdentifierExpr::IdentifierExpr(std::string id) + : id(id) { } @@ -120,33 +136,11 @@ ExprPtr IdentifierExpr::lookup(ResourceCompiler *ctx) int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) { - if(isFunction) - { - if(id == "$$countof" || id == "$$arrayindex") - { - assert(arguments.size() == 1); - IdentifierExprPtr arr = std::dynamic_pointer_cast(arguments[0]); - assert(arr); - if(id == "$$countof") - return ctx->getArrayCount(arr->id); - else - return ctx->getArrayIndex(arr->id); - } - else - { - std::cout << id << std::endl; - assert(false); - } - } - else - { - return lookup(ctx)->evaluateInt(ctx); - } + return lookup(ctx)->evaluateInt(ctx); } std::string IdentifierExpr::evaluateString(ResourceCompiler *ctx) { - assert(!isFunction); return lookup(ctx)->evaluateString(ctx); } @@ -155,3 +149,40 @@ CaseExpr::CaseExpr(const std::string &tag, CompoundExprPtr expr) : tag(tag), expr(expr) { } + + +int CountOfExpr::evaluateInt(ResourceCompiler *ctx) +{ + assert(arg->arguments.size() == 0); + return ctx->getArrayCount(arg->id); +} + + +int ArrayIndexExpr::evaluateInt(ResourceCompiler *ctx) +{ + assert(arg->arguments.size() == 0); + return ctx->getArrayIndex(arg->id); +} + + +std::string ReadExpr::evaluateString(ResourceCompiler *ctx) +{ + std::string filename = arg->evaluateString(ctx); + std::ifstream instream(filename); + // ### TODO: check error + return std::string(std::istreambuf_iterator(instream.rdbuf()), + std::istreambuf_iterator()); +} + + +int UnimplementedExpr::evaluateInt(ResourceCompiler *ctx) +{ + std::cerr << msg << std::endl; + return 0; +} + +std::string UnimplementedExpr::evaluateString(ResourceCompiler *ctx) +{ + std::cerr << msg << std::endl; + return ""; +} diff --git a/Rez/Expression.h b/Rez/Expression.h index be57fc31e8..2e950c26b9 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -18,7 +18,7 @@ typedef std::shared_ptr CaseExprPtr; enum class BinaryOp { - XOR, OR, AND, SHIFTLEFT, SHIFTRIGHT, EQUAL, NOTEQUAL, PLUS, MINUS, MULTIPLY, DIVIDE + XOR, OR, AND, SHIFTLEFT, SHIFTRIGHT, EQUAL, NOTEQUAL, PLUS, MINUS, MULTIPLY, DIVIDE, CONCAT }; enum class UnaryOp @@ -87,6 +87,7 @@ public: ~BinaryExpr(); virtual int evaluateInt(ResourceCompiler *ctx); + virtual std::string evaluateString(ResourceCompiler *ctx); }; class UnaryExpr : public Expression @@ -103,11 +104,10 @@ public: class IdentifierExpr : public Expression { +public: std::string id; std::vector arguments; - bool isFunction; -public: - IdentifierExpr(std::string id, bool isFunction = false); + IdentifierExpr(std::string id); void addArgument(ExprPtr e); ExprPtr lookup(ResourceCompiler *ctx); @@ -115,4 +115,38 @@ public: virtual std::string evaluateString(ResourceCompiler *ctx); }; +class CountOfExpr : public Expression +{ + IdentifierExprPtr arg; +public: + CountOfExpr(IdentifierExprPtr arg) : arg(arg) {} + virtual int evaluateInt(ResourceCompiler *ctx); +}; + +class ArrayIndexExpr : public Expression +{ + IdentifierExprPtr arg; +public: + ArrayIndexExpr(IdentifierExprPtr arg) : arg(arg) {} + virtual int evaluateInt(ResourceCompiler *ctx); +}; + +class ReadExpr : public Expression +{ + ExprPtr arg; +public: + ReadExpr(ExprPtr arg) : arg(arg) {} + virtual std::string evaluateString(ResourceCompiler *ctx); +}; + +class UnimplementedExpr : public Expression +{ + std::string msg; +public: + UnimplementedExpr(std::string msg) : msg(msg) {} + virtual int evaluateInt(ResourceCompiler *ctx); + virtual std::string evaluateString(ResourceCompiler *ctx); +}; + + #endif // EXPRESSION_H diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index 21cf1a7a12..da1df02f7f 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -198,7 +198,15 @@ RezSymbol RezLexer::nextToken() KEYWORD(BITSTRING, "bitstring"), KEYWORD(INTEGER, "int"), - KEYWORD(DOLLAR, "$") + KEYWORD(DOLLAR, "$"), + + KEYWORD(FUN_COUNTOF, "$$countof"), + KEYWORD(FUN_ARRAYINDEX, "$$arrayindex"), + KEYWORD(FUN_READ, "$$read"), + KEYWORD(FUN_BITFIELD, "$$bitfield"), + KEYWORD(FUN_WORD, "$$word"), + KEYWORD(FUN_BYTE, "$$byte"), + KEYWORD(FUN_LONG, "$$long"), }; std::string s = tok.get_value().c_str(); diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 952f9a8ba8..2384fa6ef6 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -71,6 +71,14 @@ %token RECT "rect"; %token BITSTRING "bitstring"; +%token FUN_COUNTOF "$$countof"; +%token FUN_ARRAYINDEX "$$arrayindex"; +%token FUN_READ "$$read"; +%token FUN_BITFIELD "$$bitfield"; +%token FUN_WORD "$$word"; +%token FUN_BYTE "$$byte"; +%token FUN_LONG "$$long"; + /* %left "|"; %left "^"; @@ -309,17 +317,10 @@ switch_case : "case" IDENTIFIER ":" } ; -%type string onestring; -string : onestring { $$ = $1; } - | string onestring { $$ = $1 + $2; } - ; -onestring : STRINGLIT { $$ = $1; } - | DOLLAR STRINGLIT { $$ = fromHex($2); } - ; value : expression { $$ = $1; } | "{" resource_body "}" { $$ = $2; } - | string { $$ = std::make_shared($1); } + | string_expression { $$ = $1; } ; expression : expression1 { $$ = $1; } @@ -362,19 +363,31 @@ expression7 : expression8 { $$ = $1; } expression8 : INTLIT { $$ = std::make_shared($1); } | CHARLIT { $$ = std::make_shared($1); } - | IDENTIFIER { $$ = std::make_shared($1); } - | IDENTIFIER - { world.functionCalls.push(std::make_shared($1,true)); } - "(" function_argument_list ")" - { $$ = world.functionCalls.top(); world.functionCalls.pop(); } - | IDENTIFIER - { world.functionCalls.push(std::make_shared($1,false)); } - "[" function_argument_list1 "]" - { $$ = world.functionCalls.top(); world.functionCalls.pop(); } + | identifier_expression { $$ = $1; } | "(" expression ")" { $$ = $2; } + | "$$countof" "(" identifier_expression ")" + { $$ = std::make_shared($identifier_expression); } + | "$$arrayindex" "(" identifier_expression ")" + { $$ = std::make_shared($identifier_expression); } + | "$$bitfield" "(" expression "," expression "," expression ")" + { $$ = std::make_shared("$$bitfield"); } + | "$$word" "(" expression ")" + { $$ = std::make_shared("$$word"); } + | "$$byte" "(" expression ")" + { $$ = std::make_shared("$$byte"); } + | "$$long" "(" expression ")" + { $$ = std::make_shared("$$long"); } ; +%type identifier_expression; +identifier_expression : IDENTIFIER { $$ = std::make_shared($1); } + | IDENTIFIER + { world.functionCalls.push(std::make_shared($1)); } + "[" function_argument_list1 "]" + { $$ = world.functionCalls.top(); world.functionCalls.pop(); } + ; + function_argument_list : %empty | function_argument_list1 ; function_argument_list1 : expression { world.functionCalls.top()->addArgument($expression); } @@ -382,6 +395,22 @@ function_argument_list1 : expression { world.functionCalls.top()->addArgument($expression); } ; +%type string_expression string_expression1; +string_expression : string_expression1 { $$ = $1; } + | string_expression string_expression1 + { $$ = std::make_shared(BinaryOp::CONCAT, $1, $2); } + ; + +%type stringlit; +stringlit : STRINGLIT { $$ = $1; } + | DOLLAR STRINGLIT { $$ = fromHex($2); } + ; + +string_expression1 : stringlit { $$ = std::make_shared($1); } + | "$$read" "(" string_expression ")" + { $$ = std::make_shared($string_expression); } + ; + resource : "resource" res_type "(" expression resource_attributes ")" "{" resource_body "}" { int id = $expression->evaluateInt(nullptr); From 6a298ca3bbfabdbb734d0c1bc3d2444df44c31a2 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Sun, 12 Oct 2014 19:17:04 +0200 Subject: [PATCH 20/29] add Rez to toolchain file --- retro68.toolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/retro68.toolchain.cmake b/retro68.toolchain.cmake index 5d1c110b7c..a2478b5ddd 100644 --- a/retro68.toolchain.cmake +++ b/retro68.toolchain.cmake @@ -22,6 +22,7 @@ set( RETRO68_ROOT "" CACHE PATH "path to root of Retro68 Toolchain" ) set( CMAKE_INSTALL_PREFIX "${RETRO68_ROOT}/m68k-unknown-elf/" CACHE PATH "installation prefix" ) set( MAKE_APPL "${RETRO68_ROOT}/bin/MakeAPPL" ) +set( REZ "${RETRO68_ROOT}/bin/Rez" ) set( CMAKE_C_COMPILER "${RETRO68_ROOT}/bin/m68k-unknown-elf-gcc" ) set( CMAKE_CXX_COMPILER "${RETRO68_ROOT}/bin/m68k-unknown-elf-g++" ) From 959846f0938e836524416f951bf7a8d0c73f5179 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Mon, 13 Oct 2014 22:59:14 +0200 Subject: [PATCH 21/29] fix bug in hex numbers --- Rez/RezLexerNextToken.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index da1df02f7f..bffe017cc0 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -40,7 +40,7 @@ static int readInt(const char *str, const char *end = NULL, int baseOverride = 0 { x *= base; if(*str >= 'a' && *str <= 'z') - x += *str - 'a'; + x += *str - 'a' + 10; else if(*str >= 'A' && *str <= 'Z') x += *str - 'A' + 10; else if(*str >= '0' && *str <= '9') From c93cb8feaecb274aab9a4e176aa108f20bc8b1a5 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Tue, 14 Oct 2014 00:11:21 +0200 Subject: [PATCH 22/29] implement $$byte, $$word, $$long, $$bitfield --- Rez/Expression.cc | 23 +++++++++++++++++++++++ Rez/Expression.h | 10 ++++++++++ Rez/ResourceCompiler.cc | 36 +++++++++++++++++++++++++++++++++++- Rez/ResourceCompiler.h | 8 +++++++- Rez/ResourceDefinitions.cc | 7 ++----- Rez/RezParser.yy | 8 ++++---- 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/Rez/Expression.cc b/Rez/Expression.cc index 02aa7a4f21..6994541c39 100644 --- a/Rez/Expression.cc +++ b/Rez/Expression.cc @@ -136,6 +136,8 @@ ExprPtr IdentifierExpr::lookup(ResourceCompiler *ctx) int IdentifierExpr::evaluateInt(ResourceCompiler *ctx) { + if(ctx->isPrePass()) + return 0; return lookup(ctx)->evaluateInt(ctx); } @@ -186,3 +188,24 @@ std::string UnimplementedExpr::evaluateString(ResourceCompiler *ctx) std::cerr << msg << std::endl; return ""; } + + +PeekExpr::PeekExpr(ExprPtr addr, ExprPtr offset, ExprPtr size) + : addr(addr), offset(offset), size(size) +{ +} + +PeekExpr::PeekExpr(ExprPtr addr, int size) + : addr(addr), + offset(std::make_shared(0)), + size(std::make_shared(size)) +{ +} + +int PeekExpr::evaluateInt(ResourceCompiler *ctx) +{ + int p = addr->evaluateInt(ctx) + offset->evaluateInt(ctx); + int s = size->evaluateInt(ctx); + + return ctx->peek(p, s); +} diff --git a/Rez/Expression.h b/Rez/Expression.h index 2e950c26b9..0300ec6aec 100644 --- a/Rez/Expression.h +++ b/Rez/Expression.h @@ -148,5 +148,15 @@ public: virtual std::string evaluateString(ResourceCompiler *ctx); }; +class PeekExpr : public Expression +{ + ExprPtr addr; + ExprPtr offset; + ExprPtr size; +public: + PeekExpr(ExprPtr addr, ExprPtr offset, ExprPtr size); + PeekExpr(ExprPtr addr, int size); + virtual int evaluateInt(ResourceCompiler *ctx); +}; #endif // EXPRESSION_H diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc index b22419b727..ebc86a8e92 100644 --- a/Rez/ResourceCompiler.cc +++ b/Rez/ResourceCompiler.cc @@ -41,6 +41,37 @@ void ResourceCompiler::write(int nBits, int value) //currentOffset += nBits; } +int ResourceCompiler::peek(int bitPos, int size) +{ + int bytePos = bitPos / 8; + int endBytePos = (bitPos + size - 1) / 8 + 1; + + unsigned bitPosInByte = bitPos % 8; + unsigned outPos = 32 - size; + + unsigned val = 0; + + for(int i = bytePos; i != endBytePos; ++i) + { + unsigned byte; + if(i < data.size()) + byte = data[i]; + else if(i < prePassData.size()) + byte = prePassData[i]; + else + byte = 0; + + unsigned read = byte << (bitPosInByte + 24); + val |= (read >> outPos); + + outPos += 8 - bitPosInByte; + + bitPosInByte = 0; + } + + return val; +} + ExprPtr ResourceCompiler::lookupIdentifier(std::string name, const Subscripts &sub) { if(currentField) @@ -70,10 +101,13 @@ void ResourceCompiler::compile() if(verboseFlag) std::cout << "(first pass)\n"; currentOffset = 0; data.clear(); + prePass = true; typeDefinition->compile(body, this, true); if(verboseFlag) std::cout << "(second pass)\n"; currentOffset = 0; - data.clear(); + prePassData = std::move(data); + data.clear(); // ### + prePass = false; typeDefinition->compile(body, this, false); if(verboseFlag) std::cout << "(done)\n"; diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index 2ae56d4c58..056f55da91 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -31,10 +31,12 @@ class ResourceCompiler Field* currentField; Subscripts currentSubscripts; - std::vector data; + std::vector data, prePassData; bool verboseFlag; void beginArrayScope(std::string& arrayName, int index); + + bool prePass; public: ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag); @@ -44,6 +46,8 @@ public: void write(int nBits, int value); int tell() { return currentOffset; } + int peek(int bitPos, int size); + ExprPtr lookupIdentifier(std::string name, const Subscripts& sub = Subscripts()); void defineLabel(const std::string& name); @@ -52,6 +56,8 @@ public: int getArrayCount(const std::string& arrayName); int getArrayIndex(const std::string& arrayName); + bool isPrePass() { return prePass; } + class FieldScope { ResourceCompiler *compiler; diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 08c8ce44d8..6d55f9ad0b 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -169,11 +169,8 @@ void SimpleField::compileInt(ExprPtr expr, ResourceCompiler *compiler, bool preP } int actualValue = 0; - if(!prePass) - { - ResourceCompiler::FieldScope scope(compiler, this); - actualValue = (value ? value : expr)->evaluateInt(compiler); - } + ResourceCompiler::FieldScope scope(compiler, this); + actualValue = (value ? value : expr)->evaluateInt(compiler); compiler->write(bitSize, actualValue); } diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index 2384fa6ef6..ff4d421476 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -371,13 +371,13 @@ expression8 : INTLIT { $$ = std::make_shared($1); } | "$$arrayindex" "(" identifier_expression ")" { $$ = std::make_shared($identifier_expression); } | "$$bitfield" "(" expression "," expression "," expression ")" - { $$ = std::make_shared("$$bitfield"); } + { $$ = std::make_shared($3, $5, $7); } | "$$word" "(" expression ")" - { $$ = std::make_shared("$$word"); } + { $$ = std::make_shared($3, 16); } | "$$byte" "(" expression ")" - { $$ = std::make_shared("$$byte"); } + { $$ = std::make_shared($3, 8); } | "$$long" "(" expression ")" - { $$ = std::make_shared("$$long"); } + { $$ = std::make_shared($3, 32); } ; %type identifier_expression; From 701c6d21bce19986b4fa520214c673c2dd476a53 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Tue, 14 Oct 2014 02:19:26 +0200 Subject: [PATCH 23/29] Rez: "data" declarations --- Rez/CMakeLists.txt | 2 ++ Rez/ResSpec.h | 24 ++++++++++++++++++++++++ Rez/RezLexerNextToken.cc | 5 +++++ Rez/RezParser.yy | 35 ++++++++++++++++++++++++++++------- Rez/RezWorld.cc | 15 +++++++++++---- Rez/RezWorld.h | 5 ++++- 6 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 Rez/ResSpec.h diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index cd200af5fb..402ce60a18 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -55,6 +55,8 @@ add_executable(Rez ResourceCompiler.cc ResourceCompiler.h + + ResSpec.h ) target_link_libraries(Rez ResourceFiles ${Boost_LIBRARIES}) diff --git a/Rez/ResSpec.h b/Rez/ResSpec.h new file mode 100644 index 0000000000..2aa648a061 --- /dev/null +++ b/Rez/ResSpec.h @@ -0,0 +1,24 @@ +#ifndef REZSPEC_H +#define REZSPEC_H + +#include "ResType.h" +#include + +class ResSpec : public ResRef +{ + int attr_; + std::string name_; + +public: + ResSpec() {} + ResSpec(ResType type, int id, int attr = 0, std::string name = "") + : ResRef(type, id), attr_(attr), name_(name) + {} + + int& attr() { return attr_; } + int attr() const { return attr_; } + std::string& name() { return name_; } + const std::string& name() const { return name_; } +}; + +#endif // REZSPEC_H diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index bffe017cc0..14bed3c566 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -169,6 +169,11 @@ RezSymbol RezLexer::nextToken() static std::unordered_map keywords = { KEYWORD(TYPE, "type"), KEYWORD(RESOURCE, "resource"), + KEYWORD(DATA, "data"), + KEYWORD(READ, "read"), + KEYWORD(INCLUDE, "include"), + KEYWORD(CHANGE, "change"), + KEYWORD(DELETE, "delete"), KEYWORD(ARRAY,"array"), KEYWORD(SWITCH, "switch"), diff --git a/Rez/RezParser.yy b/Rez/RezParser.yy index ff4d421476..0b6e9d49af 100644 --- a/Rez/RezParser.yy +++ b/Rez/RezParser.yy @@ -43,6 +43,13 @@ %token TYPE "type"; %token RESOURCE "resource"; +%token DATA "data"; +%token READ "read"; +%token INCLUDE "include"; +%token CHANGE "change"; +%token DELETE "delete"; + + %token ARRAY "array"; %token SWITCH "switch"; %token CASE "case"; @@ -95,6 +102,7 @@ %code requires { #include "ResourceDefinitions.h" #include "Expression.h" + #include "ResSpec.h" #define YY_NULLPTR nullptr class RezLexer; @@ -166,6 +174,7 @@ rez : %empty | rez type_definition ";" | rez resource ";" + | rez data ";" ; type_definition : "type" type_spec @@ -411,18 +420,23 @@ string_expression1 : stringlit { $$ = std::make_shared($1); } { $$ = std::make_shared($string_expression); } ; -resource : "resource" res_type "(" expression resource_attributes ")" "{" resource_body "}" +resource : "resource" res_spec "{" resource_body "}" { - int id = $expression->evaluateInt(nullptr); - world.addResource($res_type, id, $resource_body); + world.addResource($res_spec, $resource_body); } ; -%type resource_attributes resource_attribute; -resource_attributes : %empty { $$ = 0; } - | resource_attributes "," resource_attribute { $$ = $1 | $3; } +%type res_spec; + +res_spec : res_type "(" expression resource_attributes ")" + { $$ = $resource_attributes( ResSpec($res_type, $expression->evaluateInt(nullptr)) ); } + +%type > resource_attributes ; +resource_attributes : %empty { $$ = [](ResSpec s){ return s; }; } + | resource_attributes "," IDENTIFIER { $$ = $1; } + | resource_attributes "," string_expression { $$ = $1; } + ; -resource_attribute : IDENTIFIER { $$ = 0; } /* ### */ %type resource_body resource_body1; resource_body : %empty { $$ = std::make_shared(); } @@ -438,4 +452,11 @@ resource_item : value { $$ = $1; } | IDENTIFIER "{" resource_body "}" { $$ = std::make_shared($IDENTIFIER, $resource_body); } ; + +data : "data" res_spec "{" string_expression "}" +{ + world.addData($res_spec, $string_expression->evaluateString(nullptr)); +} +; + %% diff --git a/Rez/RezWorld.cc b/Rez/RezWorld.cc index 7c33d5345b..93d0d5bfb3 100644 --- a/Rez/RezWorld.cc +++ b/Rez/RezWorld.cc @@ -26,13 +26,20 @@ TypeDefinitionPtr RezWorld::getTypeDefinition(ResType type, int id) return nullptr; } -void RezWorld::addResource(ResType type, int id, CompoundExprPtr body) +void RezWorld::addResource(ResSpec spec, CompoundExprPtr body) { if(verboseFlag) - std::cout << "RESOURCE " << type << "(" << id << ")" << std::endl; - TypeDefinitionPtr def = getTypeDefinition(type, id); + std::cout << "RESOURCE " << spec.type() << "(" << spec.id() << ", " << "\"" << spec.name() << "\"" << spec.attr() << ")" << std::endl; + TypeDefinitionPtr def = getTypeDefinition(spec.type(), spec.id()); ResourceCompiler compiler(def, body, verboseFlag); compiler.compile(); - resources.addResource(Resource(type, id, compiler.resourceData())); + resources.addResource(Resource(spec.type(), spec.id(), compiler.resourceData(), spec.name(), spec.attr())); +} + +void RezWorld::addData(ResSpec spec, const std::string &data) +{ + if(verboseFlag) + std::cout << "DATA " << spec.type() << "(" << spec.id() << ", " << "\"" << spec.name() << "\"" << spec.attr() << ")" << std::endl; + resources.addResource(Resource(spec.type(), spec.id(), data, spec.name(), spec.attr())); } diff --git a/Rez/RezWorld.h b/Rez/RezWorld.h index 50277b55ea..5c047f196a 100644 --- a/Rez/RezWorld.h +++ b/Rez/RezWorld.h @@ -3,9 +3,11 @@ #include #include +#include #include "ResourceDefinitions.h" #include "Expression.h" #include "ResourceFiles.h" +#include "ResSpec.h" class RezWorld { @@ -23,7 +25,8 @@ public: TypeDefinitionPtr getTypeDefinition(ResType type, int id); - void addResource(ResType type, int id, CompoundExprPtr body); + void addResource(ResSpec spec, CompoundExprPtr body); + void addData(ResSpec spec, const std::string& data); Resources& getResources() { return resources; } From 30abda2087773217c1016e827f9d731bee79477c Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 16 Oct 2014 02:25:23 +0200 Subject: [PATCH 24/29] silence warnings --- MakeAPPL/ResourceFiles.h | 2 +- Rez/ResourceDefinitions.cc | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/MakeAPPL/ResourceFiles.h b/MakeAPPL/ResourceFiles.h index 7ea6b96e73..308df74d1d 100644 --- a/MakeAPPL/ResourceFiles.h +++ b/MakeAPPL/ResourceFiles.h @@ -15,7 +15,7 @@ class Resource public: Resource() {} Resource(ResType type, int id, std::string data, std::string name = "", int attr = 0) - : type(type), id(id), data(data), name(name), attr(attr) {} + : type(type), id(id), name(name), data(data), attr(attr) {} const std::string& getData() const { return data; } inline ResType getType() const { return type; } diff --git a/Rez/ResourceDefinitions.cc b/Rez/ResourceDefinitions.cc index 6d55f9ad0b..50d663abcb 100644 --- a/Rez/ResourceDefinitions.cc +++ b/Rez/ResourceDefinitions.cc @@ -113,7 +113,7 @@ void SimpleField::compileString(ExprPtr expr, ResourceCompiler *compiler, bool p if(arrayCount || type == Type::char_) { - int requestedSize = type == Type::char_ ? 1 : arrayCount->evaluateInt(compiler); + unsigned requestedSize = type == Type::char_ ? 1 : arrayCount->evaluateInt(compiler); if(requestedSize < str.size()) str.erase(str.begin() + requestedSize, str.end()); else if(requestedSize > str.size()) @@ -166,6 +166,8 @@ void SimpleField::compileInt(ExprPtr expr, ResourceCompiler *compiler, bool preP case Type::longint: bitSize = 32; break; + default: + assert(false); } int actualValue = 0; @@ -193,6 +195,8 @@ void SimpleField::compileCompound(ExprPtr expr, ResourceCompiler *compiler, bool case Type::point: count = 2; break; + default: + assert(false); } CompoundExprPtr compound = std::dynamic_pointer_cast(val); @@ -281,7 +285,7 @@ void SwitchField::compile(ExprPtr expr, ResourceCompiler *compiler, bool prePass FillAlignField::FillAlignField(FillAlignField::Type type, bool isAlign, ExprPtr count) - : type(type), isAlign(isAlign), count(count) + : type(type), count(count), isAlign(isAlign) { } From 8b723e5685e7bfa8e268fe58a4391cb1f63d5880 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 16 Oct 2014 02:26:41 +0200 Subject: [PATCH 25/29] -I and -D command line options (specify include paths/defines) --- Rez/Rez.cc | 9 +++++++++ Rez/RezLexer.cc | 12 ++++++++++++ Rez/RezLexer.h | 3 +++ 3 files changed, 24 insertions(+) diff --git a/Rez/Rez.cc b/Rez/Rez.cc index cae9d57e63..268d700ccd 100644 --- a/Rez/Rez.cc +++ b/Rez/Rez.cc @@ -28,6 +28,8 @@ int main(int argc, const char *argv[]) ("output,o", po::value()->default_value("rez.output.rsrc"), "output file") ("type,t", po::value()->default_value("rsrc"), "output file finder type code") ("creator,c", po::value()->default_value("RSED"), "output file finder creator code") + ("define,D", po::value>(), "predefine preprocessor symbol") + ("include,I", po::value>(), "add include file path") ("debug,d", "debug logging") ; po::options_description hidden, alldesc; @@ -71,6 +73,13 @@ int main(int argc, const char *argv[]) for(std::string fn : options["input"].as>()) { RezLexer lexer(fn); + + for(std::string define : options["define"].as>()) + lexer.addDefine(define); + for(std::string path : options["include"].as>()) + lexer.addIncludePath(path); + + RezParser parser(lexer, world); parser.parse(); } diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc index 07f23a2861..70d344f07c 100644 --- a/Rez/RezLexer.cc +++ b/Rez/RezLexer.cc @@ -101,6 +101,18 @@ RezLexer::~RezLexer() } + + +void RezLexer::addDefine(std::string str) +{ + pImpl->ctx.add_macro_definition(str); +} + +void RezLexer::addIncludePath(std::string path) +{ + pImpl->ctx.add_include_path(path.c_str()); +} + bool RezLexer::atEnd() { return pImpl->iter == pImpl->ctx.end(); diff --git a/Rez/RezLexer.h b/Rez/RezLexer.h index c0e7f12cea..a3c6b97a0c 100644 --- a/Rez/RezLexer.h +++ b/Rez/RezLexer.h @@ -23,6 +23,9 @@ public: ~RezLexer(); RezSymbol nextToken(); + + void addDefine(std::string str); + void addIncludePath(std::string path); }; #endif // REZLEXER_H From 75f6ed32f51874e86cd864e4d0f3dbb5f41a5a47 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 16 Oct 2014 02:27:12 +0200 Subject: [PATCH 26/29] Rez: --append option --- Rez/Rez.cc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Rez/Rez.cc b/Rez/Rez.cc index 268d700ccd..d902e95ca0 100644 --- a/Rez/Rez.cc +++ b/Rez/Rez.cc @@ -26,6 +26,7 @@ int main(int argc, const char *argv[]) desc.add_options() ("help,h", "show this help message") ("output,o", po::value()->default_value("rez.output.rsrc"), "output file") + ("append,a", "append to existing output file") ("type,t", po::value()->default_value("rsrc"), "output file finder type code") ("creator,c", po::value()->default_value("RSED"), "output file finder creator code") ("define,D", po::value>(), "predefine preprocessor symbol") @@ -70,6 +71,19 @@ int main(int argc, const char *argv[]) if(options.count("debug")) world.verboseFlag = true; + std::string outfile = options["output"].as(); + fs::path dataPath = outfile; + fs::create_directory(dataPath.parent_path() / ".rsrc"); + fs::create_directory(dataPath.parent_path() / ".finf"); + fs::path rsrcPath = dataPath.parent_path() / ".rsrc" / dataPath.filename(); + fs::path finfPath = dataPath.parent_path() / ".finf" / dataPath.filename(); + + if(options.count("append")) + { + fs::ifstream rsrcIn(rsrcPath); + world.getResources().addResources(Resources(rsrcIn)); + } + for(std::string fn : options["input"].as>()) { RezLexer lexer(fn); @@ -84,14 +98,10 @@ int main(int argc, const char *argv[]) parser.parse(); } - std::string outfile = options["output"].as(); { - fs::path dataPath(outfile); - fs::create_directory(dataPath.parent_path() / ".rsrc"); - fs::create_directory(dataPath.parent_path() / ".finf"); fs::ofstream dataOut(dataPath); - fs::ofstream rsrcOut(dataPath.parent_path() / ".rsrc" / dataPath.filename()); - fs::ofstream finfOut(dataPath.parent_path() / ".finf" / dataPath.filename()); + fs::ofstream rsrcOut(rsrcPath); + fs::ofstream finfOut(finfPath); world.getResources().writeFork(rsrcOut); ostype(finfOut, options["type"].as()); From eadbe38cfcf3ed3dc4e7f9d57feb528cbf69e07d Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 16 Oct 2014 02:29:12 +0200 Subject: [PATCH 27/29] add some unit tests --- Rez/CMakeLists.txt | 15 +++-- Rez/ResourceCompiler.cc | 44 +++++++------ Rez/ResourceCompiler.h | 42 ++++++++----- Rez/RezLexer.cc | 42 ++++++++----- Rez/RezLexer.h | 1 + Rez/Test/CMakeLists.txt | 4 ++ Rez/Test/UnitTests.cc | 134 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 227 insertions(+), 55 deletions(-) create mode 100644 Rez/Test/CMakeLists.txt create mode 100644 Rez/Test/UnitTests.cc diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index 402ce60a18..50e41253fd 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -17,7 +17,7 @@ cmake_minimum_required(VERSION 2.8) -set(CMAKE_CXX_FLAGS "--std=c++11") +set(CMAKE_CXX_FLAGS "--std=c++11 -Wall") find_package(Boost COMPONENTS wave filesystem system thread regex program_options) @@ -34,9 +34,7 @@ add_custom_command( COMMENT "Generating parser.cpp" OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.hh ) -add_executable(Rez - Rez.cc - +add_library(RezLib RezParser.yy RezParser.generated.hh RezParser.generated.cc RezLexer.h @@ -58,6 +56,13 @@ add_executable(Rez ResSpec.h ) -target_link_libraries(Rez ResourceFiles ${Boost_LIBRARIES}) +target_link_libraries(RezLib ResourceFiles ${Boost_LIBRARIES}) + +add_executable(Rez + Rez.cc + ) +target_link_libraries(Rez RezLib ResourceFiles ${Boost_LIBRARIES}) install(TARGETS Rez RUNTIME DESTINATION bin) + +add_subdirectory(Test) diff --git a/Rez/ResourceCompiler.cc b/Rez/ResourceCompiler.cc index ebc86a8e92..f332d91d4e 100644 --- a/Rez/ResourceCompiler.cc +++ b/Rez/ResourceCompiler.cc @@ -6,19 +6,32 @@ ResourceCompiler::ResourceCompiler( TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag) : typeDefinition(type), body(body), - currentOffset(0), - currentField(nullptr), - verboseFlag(verboseFlag) + currentField(nullptr) { - + this->verboseFlag = verboseFlag; } -std::string ResourceCompiler::resourceData() +BinaryOutput::BinaryOutput() + : verboseFlag(false) +{ + reset(true); +} + +void BinaryOutput::reset(bool prePass) +{ + currentOffset = 0; + if(!prePass) + prePassData = std::move(data); + data.clear(); + this->prePass = prePass; +} + +std::string BinaryOutput::resourceData() { return std::string(data.begin(), data.end()); } -void ResourceCompiler::write(int nBits, int value) +void BinaryOutput::write(int nBits, int value) { if(verboseFlag) std::cout << "[" << nBits << " bits] = " << std::hex << value << std::dec << std::endl; @@ -41,17 +54,17 @@ void ResourceCompiler::write(int nBits, int value) //currentOffset += nBits; } -int ResourceCompiler::peek(int bitPos, int size) +int BinaryOutput::peek(int bitPos, int size) { - int bytePos = bitPos / 8; - int endBytePos = (bitPos + size - 1) / 8 + 1; + unsigned bytePos = bitPos / 8; + unsigned endBytePos = (bitPos + size - 1) / 8 + 1; unsigned bitPosInByte = bitPos % 8; unsigned outPos = 32 - size; unsigned val = 0; - for(int i = bytePos; i != endBytePos; ++i) + for(unsigned i = bytePos; i != endBytePos; ++i) { unsigned byte; if(i < data.size()) @@ -99,18 +112,13 @@ void ResourceCompiler::defineLabel(const std::string &name) void ResourceCompiler::compile() { if(verboseFlag) std::cout << "(first pass)\n"; - currentOffset = 0; - data.clear(); - prePass = true; + reset(true); typeDefinition->compile(body, this, true); if(verboseFlag) std::cout << "(second pass)\n"; - currentOffset = 0; - prePassData = std::move(data); - data.clear(); // ### - prePass = false; + + reset(false); typeDefinition->compile(body, this, false); if(verboseFlag) std::cout << "(done)\n"; - } int ResourceCompiler::getArrayCount(const std::string &name) diff --git a/Rez/ResourceCompiler.h b/Rez/ResourceCompiler.h index 056f55da91..6b11777dac 100644 --- a/Rez/ResourceCompiler.h +++ b/Rez/ResourceCompiler.h @@ -20,25 +20,17 @@ public: bool empty() const { return subscripts.empty(); } }; -class ResourceCompiler +class BinaryOutput { - TypeDefinitionPtr typeDefinition; - CompoundExprPtr body; - std::map, ExprPtr> labelValues; - std::map, int> arrayCounts; - std::map curArrayIndices; +protected: int currentOffset; - Field* currentField; - Subscripts currentSubscripts; - - std::vector data, prePassData; + std::vector data; + std::vector prePassData; bool verboseFlag; - - void beginArrayScope(std::string& arrayName, int index); - bool prePass; public: - ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag); + BinaryOutput(); + void reset(bool prePass); std::string resourceData(); @@ -48,6 +40,27 @@ public: int peek(int bitPos, int size); + bool isPrePass() { return prePass; } +}; + +class ResourceCompiler : public BinaryOutput +{ + TypeDefinitionPtr typeDefinition; + CompoundExprPtr body; + std::map, ExprPtr> labelValues; + std::map, int> arrayCounts; + std::map curArrayIndices; + Field* currentField; + Subscripts currentSubscripts; + + + + void beginArrayScope(std::string& arrayName, int index); + +public: + ResourceCompiler(TypeDefinitionPtr type, CompoundExprPtr body, bool verboseFlag); + + ExprPtr lookupIdentifier(std::string name, const Subscripts& sub = Subscripts()); void defineLabel(const std::string& name); @@ -56,7 +69,6 @@ public: int getArrayCount(const std::string& arrayName); int getArrayIndex(const std::string& arrayName); - bool isPrePass() { return prePass; } class FieldScope { diff --git a/Rez/RezLexer.cc b/Rez/RezLexer.cc index 70d344f07c..fa0703bb80 100644 --- a/Rez/RezLexer.cc +++ b/Rez/RezLexer.cc @@ -11,6 +11,25 @@ namespace wave = boost::wave; using namespace boost::wave; +static std::string readContents(std::istream&& instream) +{ + instream.unsetf(std::ios::skipws); + + return std::string(std::istreambuf_iterator(instream.rdbuf()), + std::istreambuf_iterator()); +} + +static std::string preFilter(std::string str) +{ + boost::regex endif("#endif[^\r\n]*"); + str = boost::regex_replace(str, endif, "#endif"); + + boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])"); + str = boost::regex_replace(str, dollar_escape, "\\\\0x$1"); + + return str; +} + struct load_file_to_string_filtered { template @@ -30,19 +49,8 @@ struct load_file_to_string_filtered bad_include_file, iter_ctx.filename.c_str(), act_pos); return; } - instream.unsetf(std::ios::skipws); - std::string str(std::istreambuf_iterator(instream.rdbuf()), - std::istreambuf_iterator()); - - boost::regex endif("#endif[^\r\n]*"); - str = boost::regex_replace(str, endif, "#endif"); - - boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])"); - str = boost::regex_replace(str, dollar_escape, "\\x$1"); - - - iter_ctx.instring = str; + iter_ctx.instring = preFilter(readContents(std::move(instream))); iter_ctx.first = iterator_type( iter_ctx.instring.begin(), iter_ctx.instring.end(), @@ -79,13 +87,13 @@ struct RezLexer::Priv }; RezLexer::RezLexer(std::string filename) + : RezLexer(filename, readContents(std::ifstream(filename))) { - std::ifstream instream(filename); +} - pImpl.reset(new Priv(std::string( - std::istreambuf_iterator(instream.rdbuf()), - std::istreambuf_iterator()), - filename)); +RezLexer::RezLexer(std::string filename, const std::string &data) +{ + pImpl.reset(new Priv(preFilter(data), filename)); pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes"); pImpl->ctx.add_macro_definition("DeRez=0"); diff --git a/Rez/RezLexer.h b/Rez/RezLexer.h index a3c6b97a0c..4bc4891374 100644 --- a/Rez/RezLexer.h +++ b/Rez/RezLexer.h @@ -20,6 +20,7 @@ class RezLexer public: RezLexer(std::string filename); + RezLexer(std::string filename, const std::string& data); ~RezLexer(); RezSymbol nextToken(); diff --git a/Rez/Test/CMakeLists.txt b/Rez/Test/CMakeLists.txt new file mode 100644 index 0000000000..b323384653 --- /dev/null +++ b/Rez/Test/CMakeLists.txt @@ -0,0 +1,4 @@ +set(CMAKE_CXX_FLAGS "--std=c++11 -Wall -Wno-multichar") +find_package(Boost COMPONENTS unit_test_framework) +add_executable(RezUnitTests UnitTests.cc) +target_link_libraries(RezUnitTests RezLib ${Boost_LIBRARIES}) diff --git a/Rez/Test/UnitTests.cc b/Rez/Test/UnitTests.cc new file mode 100644 index 0000000000..e7fefd1f27 --- /dev/null +++ b/Rez/Test/UnitTests.cc @@ -0,0 +1,134 @@ +#define BOOST_TEST_MODULE UnitTests +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MAIN +#include + +#include "RezLexer.h" +#include "RezParser.generated.hh" +#include "ResourceCompiler.h" + +#include + +BOOST_AUTO_TEST_SUITE(LexSuite) + +#define CHECKSYM(TOKEN, TYPE, VAL) \ + do { \ + RezSymbol t = lex.nextToken(); \ + BOOST_CHECK_EQUAL(t.token(), TOKEN); \ + if(t.token() == TOKEN) \ + BOOST_CHECK_EQUAL(t.value.as(), VAL); \ + } while(0) +#define CHECKSYM_(TOKEN) \ + do { \ + RezSymbol t = lex.nextToken(); \ + BOOST_CHECK_EQUAL(t.token(), TOKEN); \ + } while(0) + +BOOST_AUTO_TEST_CASE(basicInt) +{ + RezLexer lex("test", "123 0x456 0xaBcd9\n"); + + CHECKSYM(RezParser::token::INTLIT, int, 123); + CHECKSYM(RezParser::token::INTLIT, int, 0x456); + CHECKSYM(RezParser::token::INTLIT, int, 0xabcd9); + CHECKSYM_(0); +} + +BOOST_AUTO_TEST_CASE(alternateHex) +{ + RezLexer lex("test", "$456 $aBcd9\n"); + + CHECKSYM(RezParser::token::INTLIT, int, 0x456); + CHECKSYM(RezParser::token::INTLIT, int, 0xabcd9); + CHECKSYM_(0); +} + +BOOST_AUTO_TEST_CASE(noNewlineAtEOF) +{ + RezLexer lex("test", "123 456"); + CHECKSYM(RezParser::token::INTLIT, int, 123); + CHECKSYM(RezParser::token::INTLIT, int, 456); + CHECKSYM_(0); +} + +BOOST_AUTO_TEST_CASE(strings) +{ + RezLexer lex("test", R"rez( + "Hello, world." + "Foo \n" + "\r Quux" + "\001\002\003" + "\0x42\0x43" + "Blah \$5F" + )rez" "\n"); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "Hello, world."); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "Foo \n"); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "\r Quux"); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "\001\002\003"); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "\x42\x43"); + CHECKSYM(RezParser::token::STRINGLIT, std::string, "Blah \x5F"); + CHECKSYM_(0); +} + +BOOST_AUTO_TEST_SUITE_END() + +BOOST_AUTO_TEST_SUITE(BinarySuite) + +BOOST_AUTO_TEST_CASE(bytes) +{ + BinaryOutput out; + out.write(8, 'a'); + out.write(8, 'b'); + out.write(8, 'c'); + BOOST_CHECK_EQUAL(out.resourceData(), "abc"); +} + +BOOST_AUTO_TEST_CASE(multibyte) +{ + BinaryOutput out; + out.write(32, 'abcd'); + BOOST_CHECK_EQUAL(out.resourceData(), "abcd"); +} + +BOOST_AUTO_TEST_CASE(subbyte) +{ + BinaryOutput out; + out.write(4, 6); + out.write(4, 1); + + out.write(2, 1); + out.write(2, 2); + out.write(4, 2); + + out.write(3, 3); + out.write(2, 0); + out.write(3, 3); + BOOST_CHECK_EQUAL(out.resourceData(), "abc"); +} + + +BOOST_AUTO_TEST_CASE(peek) +{ + BinaryOutput out; + for(char c : "Hello, world.") + if(c != 0) + out.write(8, c); + + BOOST_CHECK_EQUAL(out.resourceData(), "Hello, world."); + + BOOST_CHECK_EQUAL(out.peek(0,8), 'H'); + BOOST_CHECK_EQUAL(out.peek(32,8), 'o'); + + BOOST_CHECK_EQUAL(out.peek(0,32), 'Hell'); + BOOST_CHECK_EQUAL(out.peek(40,32), ', wo'); + + BOOST_CHECK_EQUAL(out.peek(1,8), 'H' * 2); + BOOST_CHECK_EQUAL(out.peek(2,8), 0x21); + + BOOST_CHECK_EQUAL(out.peek(2,30), 'Hell' & 0x3FFFFFFF); + BOOST_CHECK_EQUAL(out.peek(2,32), ('Hell' & 0x3FFFFFFF) << 2 | ('o' >> 6) ); + + BOOST_CHECK_EQUAL(out.peek(4,3), 4); +} + +BOOST_AUTO_TEST_SUITE_END() From 8cd19154a579b384853658cd9b790ba9ca9dadea Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Thu, 16 Oct 2014 02:29:41 +0200 Subject: [PATCH 28/29] fix handling of \n, \r, \t escape sequences in strings --- Rez/RezLexerNextToken.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Rez/RezLexerNextToken.cc b/Rez/RezLexerNextToken.cc index 14bed3c566..b00d3e2b4f 100644 --- a/Rez/RezLexerNextToken.cc +++ b/Rez/RezLexerNextToken.cc @@ -45,7 +45,7 @@ static int readInt(const char *str, const char *end = NULL, int baseOverride = 0 x += *str - 'A' + 10; else if(*str >= '0' && *str <= '9') x += *str - '0'; - *str++; + str++; } return x; @@ -86,13 +86,13 @@ static std::string readStringLit(const char *str) switch(*p) { case 'n': - out << '\n'; + out << '\n'; ++p; break; case 'r': - out << '\r'; + out << '\r'; ++p; break; case 't': - out << '\t'; + out << '\t'; ++p; break; case '0': case '1': From 37077e3706ac9adf8f972d7219de8089cb925bd2 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Mon, 27 Oct 2014 23:39:35 +0100 Subject: [PATCH 29/29] Don't build Rez if boost is unavailable --- Rez/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Rez/CMakeLists.txt b/Rez/CMakeLists.txt index 50e41253fd..0c7a08f08a 100644 --- a/Rez/CMakeLists.txt +++ b/Rez/CMakeLists.txt @@ -21,6 +21,7 @@ set(CMAKE_CXX_FLAGS "--std=c++11 -Wall") find_package(Boost COMPONENTS wave filesystem system thread regex program_options) +if(Boost_FOUND) find_package(BISON REQUIRED) @@ -66,3 +67,5 @@ target_link_libraries(Rez RezLib ResourceFiles ${Boost_LIBRARIES}) install(TARGETS Rez RUNTIME DESTINATION bin) add_subdirectory(Test) + +endif(Boost_FOUND)