Rez: successful parse.

This commit is contained in:
Wolfgang Thaller 2014-10-05 23:52:34 +02:00
parent 7581140436
commit cb554ed40f
9 changed files with 750 additions and 0 deletions

View File

@ -69,4 +69,5 @@ add_subdirectory(Launcher)
else()
add_subdirectory(MakeAPPL)
add_subdirectory(ASFilter)
add_subdirectory(Rez)
endif()

49
Rez/CMakeLists.txt Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2012 Wolfgang Thaller.
#
# This file is part of Retro68.
#
# Retro68 is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Retro68 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Retro68. If not, see <http://www.gnu.org/licenses/>.
cmake_minimum_required(VERSION 2.8)
set(CMAKE_CXX_FLAGS "--std=c++11")
find_package(Boost COMPONENTS wave filesystem system thread regex)
find_package(BISON REQUIRED)
include_directories(. ${CMAKE_CURRENT_BINARY_DIR})
add_custom_command(
DEPENDS RezParser.yy
COMMAND ${BISON_EXECUTABLE}
ARGS -o ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc
${CMAKE_CURRENT_SOURCE_DIR}/RezParser.yy --graph
COMMENT "Generating parser.cpp"
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.cc ${CMAKE_CURRENT_BINARY_DIR}/RezParser.generated.hh
)
add_executable(Rez
Rez.cc
RezParser.yy RezParser.generated.hh RezParser.generated.cc
RezLexer.h
RezLexer.cc
RezLexerWaveToken.h
RezLexerNextToken.cc
)
target_link_libraries(Rez ${Boost_LIBRARIES})
install(TARGETS Rez RUNTIME DESTINATION bin)

67
Rez/Rez.cc Normal file
View File

@ -0,0 +1,67 @@
#include <fstream>
#include <iostream>
#include "RezParser.generated.hh"
#include "RezLexer.h"
int main()
{
//RezLexer lexer("/home/wolfgang/Projects/Retro68/RIncludes/Types.r");
RezLexer lexer("/home/wolfgang/Projects/Retro68/Rez/Test.r");
RezParser parser(lexer);
parser.parse();
/*
// The following preprocesses a given input file.
// Open the file and read it into a string variable
std::ifstream instream("/home/wolfgang/Projects/Retro68/RIncludes/Types.r");
std::string input(
std::istreambuf_iterator<char>(instream.rdbuf()),
std::istreambuf_iterator<char>());
context_type ctx(input.begin(), input.end(), "Types.r");
// At this point you may want to set the parameters of the
// preprocessing as include paths and/or predefined macros.
ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes");
// ctx.add_macro_definition(...);
auto first = ctx.begin();
auto last = ctx.end();
std::ostringstream out;
try
{
while(first != last)
{
out << (*first).get_value();
++first;
}
}
catch(boost::wave::preprocess_exception& e)
{
std::cout << e.file_name() << ":" << e.line_no() << ": ";
std::cout << e.description() << std::endl;
}
std::string str = out.str();
std::cout << str.substr(0,100) << std::endl;*/
/*
int i = 0;
while (first != last) {
std::cout << i << ": " << get_token_name(token_id(*first)) << " <<" << (*first).get_value() << ">>\n";
++first;
if(++i > 10)
break;
}*/
return 0;
}

116
Rez/RezLexer.cc Normal file
View File

@ -0,0 +1,116 @@
#include "RezLexer.h"
#include <boost/wave.hpp>
#include <boost/wave/cpplexer/cpp_lex_iterator.hpp>
#include <boost/wave/token_ids.hpp>
#include <boost/regex.hpp>
#include "RezLexerWaveToken.h"
namespace wave = boost::wave;
using namespace boost::wave;
struct load_file_to_string_filtered
{
template <typename IterContextT>
class inner
{
public:
template <typename PositionT>
static void init_iterators(IterContextT &iter_ctx,
PositionT const &act_pos, language_support language)
{
typedef typename IterContextT::iterator_type iterator_type;
// read in the file
std::ifstream instream(iter_ctx.filename.c_str());
if (!instream.is_open()) {
BOOST_WAVE_THROW_CTX(iter_ctx.ctx, preprocess_exception,
bad_include_file, iter_ctx.filename.c_str(), act_pos);
return;
}
instream.unsetf(std::ios::skipws);
std::string str(std::istreambuf_iterator<char>(instream.rdbuf()),
std::istreambuf_iterator<char>());
boost::regex endif("#endif[^\r\n]*");
str = boost::regex_replace(str, endif, "#endif");
boost::regex dollar_escape("\\\\\\$([a-zA-Z0-9][a-zA-Z0-9])");
str = boost::regex_replace(str, dollar_escape, "\\x$1");
iter_ctx.instring = str;
iter_ctx.first = iterator_type(
iter_ctx.instring.begin(), iter_ctx.instring.end(),
PositionT(iter_ctx.filename), language);
iter_ctx.last = iterator_type();
}
private:
std::string instring;
};
};
typedef wave::cpplexer::lex_iterator<
wave::cpplexer::lex_token<> >
lex_iterator_type;
typedef wave::context<
std::string::iterator, lex_iterator_type,
load_file_to_string_filtered>
context_type;
typedef context_type::iterator_type pp_iterator_type;
struct RezLexer::Priv
{
std::string input;
context_type ctx;
pp_iterator_type iter;
Priv(std::string data, std::string name)
: input(data), ctx(input.begin(), input.end(), name.c_str())
{
}
};
RezLexer::RezLexer(std::string filename)
{
std::ifstream instream(filename);
pImpl.reset(new Priv(std::string(
std::istreambuf_iterator<char>(instream.rdbuf()),
std::istreambuf_iterator<char>()),
filename));
pImpl->ctx.add_include_path("/home/wolfgang/Projects/Retro68/RIncludes");
// ctx.add_macro_definition(...);
pImpl->ctx.add_macro_definition("DeRez", "0");
pImpl->iter = pImpl->ctx.begin();
}
RezLexer::~RezLexer()
{
}
bool RezLexer::atEnd()
{
return pImpl->iter == pImpl->ctx.end();
}
RezLexer::WaveToken RezLexer::nextWave()
{
return pImpl->iter == pImpl->ctx.end() ? WaveToken() : (*pImpl->iter++);
}
RezLexer::WaveToken RezLexer::peekWave()
{
return pImpl->iter == pImpl->ctx.end() ? WaveToken() : *pImpl->iter;
}

28
Rez/RezLexer.h Normal file
View File

@ -0,0 +1,28 @@
#ifndef REZLEXER_H
#define REZLEXER_H
#include <memory>
class RezSymbol;
class RezLexer
{
struct Priv;
std::unique_ptr<Priv> pImpl;
std::string curFile;
class WaveToken;
bool atEnd();
WaveToken nextWave();
WaveToken peekWave();
public:
RezLexer(std::string filename);
~RezLexer();
RezSymbol nextToken();
};
#endif // REZLEXER_H

192
Rez/RezLexerNextToken.cc Normal file
View File

@ -0,0 +1,192 @@
#include "RezLexer.h"
#include "RezLexerWaveToken.h"
#include "RezParser.generated.hh"
#include <unordered_map>
#include <boost/regex.hpp>
using namespace boost::wave;
static int readInt(const char *str)
{
int x = 0;
int base = 10;
if(*str == '0')
{
base = 8;
++str;
if(*str == 'x' || *str == 'X')
{
base = 16;
++str;
}
if(*str == 'b' || *str == 'B')
{
base = 2;
++str;
}
}
else if(*str == 'b' || *str == 'B')
{
base = 2;
++str;
}
while(*str)
{
x *= base;
if(*str >= 'a' && *str <= 'z')
x += *str - 'a';
else if(*str >= 'A' && *str <= 'Z')
x += *str - 'A' + 10;
else if(*str >= '0' && *str <= '9')
x += *str - '0';
*str++;
}
return x;
}
RezSymbol RezLexer::nextToken()
{
for(auto tok = nextWave(); tok != T_EOI && tok != T_EOF; tok = nextWave())
{
if(IS_CATEGORY(tok, WhiteSpaceTokenType))
continue;
else if(IS_CATEGORY(tok, EOLTokenType))
continue;
else if(tok == T_PP_LINE)
{
while(tok != T_EOI && tok != T_EOF && !IS_CATEGORY(tok, EOLTokenType))
tok = nextWave();
continue;
}
else
{
//std::cout << "{" << std::hex << (token_id)tok << std::dec << "|" << tok.get_value() << "}\n";
auto pos = tok.get_position();
curFile = pos.get_file().c_str();
auto yypos = yy::position(&curFile, pos.get_line(), pos.get_column());
yy::location loc(yypos);
if(tok == (UnknownTokenType | '"'))
{
return RezParser::make_STRINGLIT("Hello, world.", loc);
}
else if(IS_CATEGORY(tok, IdentifierTokenType) || IS_CATEGORY(tok, KeywordTokenType) || IS_CATEGORY(tok, BoolLiteralTokenType))
{
typedef decltype(&RezParser::make_TYPE) memfun;
#define KEYWORD(upper, lower) \
{ lower, &RezParser::make_ ## upper }
static std::unordered_map<std::string, memfun> keywords = {
KEYWORD(TYPE, "type"),
KEYWORD(RESOURCE, "resource"),
KEYWORD(ARRAY,"array"),
KEYWORD(SWITCH, "switch"),
KEYWORD(CASE, "case"),
KEYWORD(AS, "as"),
KEYWORD(FILL,"fill"),
KEYWORD(ALIGN, "align"),
KEYWORD(HEX,"hex"),
KEYWORD(KEY, "key"),
KEYWORD(WIDE,"wide"),
KEYWORD(UNSIGNED, "unsigned"),
KEYWORD(LITERAL, "literal"),
KEYWORD(BOOLEAN, "boolean"),
KEYWORD(BIT, "bit"),
KEYWORD(BYTE, "byte"),
KEYWORD(CHAR, "char"),
KEYWORD(WORD, "word"),
KEYWORD(INTEGER, "integer"),
KEYWORD(LONG, "long"),
KEYWORD(LONGINT, "longint"),
KEYWORD(PSTRING, "pstring"),
KEYWORD(PSTRING, "wstring"),
KEYWORD(STRING, "string"),
KEYWORD(POINT, "point"),
KEYWORD(RECT, "rect"),
KEYWORD(BITSTRING, "bitstring"),
KEYWORD(INTEGER, "int"),
};
std::string s = tok.get_value().c_str();
std::string lower = s;
std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
auto p = keywords.find(lower);
if(p == keywords.end())
{
//std::cout << "id: " << s << std::endl;
return RezParser::make_IDENTIFIER(lower, loc);
}
else
{
//std::cout << "key: " << s << std::endl;
return (*p->second)(loc);
}
}
else if(tok == T_INTLIT)
{
if(tok.get_value() == "0")
{
auto tok2 = peekWave();
while(tok2 != T_EOI && tok2 != T_EOF && IS_CATEGORY(tok2, WhiteSpaceTokenType))
nextWave(), tok2 = peekWave();
//std::cout << "!" << std::hex << (token_id)tok2 << std::dec << "|" << tok2.get_value() << "!\n";
static boost::regex binlit("[bB][01]+");
if(tok2 == T_IDENTIFIER && boost::regex_match(tok2.get_value().c_str(), binlit))
tok = nextWave();
}
return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc);
}
else
{
#define NOVAL_TOK(name) \
case T_ ## name: /*std::cout << #name << std::endl;*/ return RezParser::make_ ## name(loc)
switch(token_id(tok))
{
case T_INTLIT: return RezParser::make_INTLIT(readInt(tok.get_value().c_str()), loc);
case T_CHARLIT: return RezParser::make_CHARLIT(tok.get_value().c_str(), loc);
case T_STRINGLIT: return RezParser::make_STRINGLIT(tok.get_value().c_str(), loc);
NOVAL_TOK(LEFTBRACE);
NOVAL_TOK(RIGHTBRACE);
NOVAL_TOK(LEFTBRACKET);
NOVAL_TOK(RIGHTBRACKET);
NOVAL_TOK(LEFTPAREN);
NOVAL_TOK(RIGHTPAREN);
NOVAL_TOK(SEMICOLON);
NOVAL_TOK(COMMA);
NOVAL_TOK(PLUS);
NOVAL_TOK(MINUS);
NOVAL_TOK(DIVIDE);
NOVAL_TOK(STAR);
NOVAL_TOK(ASSIGN);
NOVAL_TOK(COLON);
NOVAL_TOK(SHIFTLEFT);
NOVAL_TOK(SHIFTRIGHT);
NOVAL_TOK(EQUAL);
NOVAL_TOK(NOTEQUAL);
NOVAL_TOK(AND);
NOVAL_TOK(OR);
NOVAL_TOK(XOR);
NOVAL_TOK(COMPL);
default:
return RezParser::make_BADTOKEN(tok.get_value().c_str(), loc);
}
}
}
}
return RezSymbol();
}

15
Rez/RezLexerWaveToken.h Normal file
View File

@ -0,0 +1,15 @@
#ifndef REZLEXERWAVETOKEN_H
#define REZLEXERWAVETOKEN_H
#include "RezLexer.h"
#include <boost/wave/cpplexer/cpp_lex_iterator.hpp>
class RezLexer::WaveToken : public boost::wave::cpplexer::lex_token<>
{
public:
WaveToken() = default;
WaveToken(const boost::wave::cpplexer::lex_token<> & o) : boost::wave::cpplexer::lex_token<>(o) {}
};
#endif // REZLEXERWAVETOKEN_H

271
Rez/RezParser.yy Normal file
View File

@ -0,0 +1,271 @@
%require "3.0.2"
%defines
%define parser_class_name {RezParser}
%skeleton "lalr1.cc"
%locations;
%define api.token.constructor
%define api.value.type variant
%define parse.assert
%token<std::string> IDENTIFIER;
%token<std::string> CHARLIT;
%token<std::string> STRINGLIT;
%token<int> INTLIT;
%token<std::string> BADTOKEN;
%token LEFTBRACE "{";
%token RIGHTBRACE "}";
%token LEFTBRACKET "[";
%token RIGHTBRACKET "]";
%token LEFTPAREN "(";
%token RIGHTPAREN ")";
%token SEMICOLON ";";
%token COMMA ",";
%token PLUS "+";
%token MINUS "-";
%token DIVIDE "/";
%token STAR "*";
%token ASSIGN "=";
%token COLON ":";
%token SHIFTLEFT "<<";
%token SHIFTRIGHT ">>";
%token EQUAL "==";
%token NOTEQUAL "!=";
%token AND "&";
%token OR "|";
%token XOR "^";
%token COMPL "~";
%token TYPE "type";
%token RESOURCE "resource";
%token ARRAY "array";
%token SWITCH "switch";
%token CASE "case";
%token AS "as";
%token FILL "fill";
%token ALIGN "align";
%token HEX "hex";
%token KEY "key";
%token WIDE "wide";
%token LITERAL "literal";
%token UNSIGNED "unsigned";
%token BOOLEAN "boolean";
%token BIT "bit";
%token BYTE "byte";
%token CHAR "char";
%token WORD "word";
%token INTEGER "integer";
%token LONG "long";
%token LONGINT "longint";
%token PSTRING "pstring";
%token WSTRING "wstring";
%token STRING "string";
%token POINT "point";
%token RECT "rect";
%token BITSTRING "bitstring";
/*
%left "|";
%left "^";
%left "&";
%left "==" "!=";
%left ">>" "<<";
%left "+" "-";
%left "*" "/";
*/
%param { RezLexer& lexer }
%code requires {
#define YY_NULLPTR nullptr
class RezLexer;
}
%code provides {
using yy::RezParser;
//using RezSymbol = yy::RezParser::symbol_type;
class RezSymbol : public yy::RezParser::symbol_type
{
public:
RezSymbol() = default;
RezSymbol(const yy::RezParser::symbol_type& x) : yy::RezParser::symbol_type(x) {}
};
}
%code {
#include "RezLexer.h"
static yy::RezParser::symbol_type yylex(RezLexer& lexer)
{
return lexer.nextToken();
}
void yy::RezParser::error(const location_type& loc, std::string const& err)
{
std::cerr << loc << ": " << err << std::endl;
}
}
%%
%start rez;
rez : %empty
| rez type_definition ";"
| rez resource ";"
;
simpletype : "boolean"
| "bit" | "byte" | "word" | "integer" | "long" | "longint" | "rect"
| "point"
| "char"
| "pstring" array_count_opt
| "wstring" array_count_opt
| "string" array_count_opt;
| "bitstring" "[" expression "]";
type_definition : "type" type_spec "{" field_definitions "}"
{ std::cout << "TYPE " << $2 << std::endl; }
| "type" type_spec "as" type_spec
{ std::cout << "TYPE " << $2 << std::endl; }
;
%type <std::string> type_spec;
type_spec : CHARLIT { $$ = $1; }
| CHARLIT "(" INTLIT ")" { $$ = $1; }
;
field_definitions : %empty
| field_definitions IDENTIFIER ":"
| field_definitions ";"
| field_definitions field_definition ";" ;
field_definition: simple_field_definition
| array_definition
| switch_definition
| fill_statement
| align_statement;
fill_statement : "fill" fill_unit array_count_opt;
align_statement : "align" fill_unit;
fill_unit : "bit" | "byte" | "word" | "long";
simple_field_definition: field_attributes simpletype value_spec;
value_spec : %empty
| named_values
| "=" expression;
named_values: named_value
| named_values "," named_value
| named_values named_value;
named_value : IDENTIFIER
| IDENTIFIER "=" expression ;
array_definition: array_attributes "array" array_name_opt array_count_opt "{" field_definitions "}" ;
array_count : "[" expression "]" ;
array_count_opt : %empty | array_count ;
array_name_opt : %empty | IDENTIFIER ;
array_attributes: %empty | "wide" ;
field_attributes: %empty | field_attributes field_attribute;
field_attribute : "hex" | "key" | "unsigned" | "literal";
switch_definition: "switch" "{"
switch_cases
"}" ;
switch_cases : %empty | switch_cases switch_case ;
switch_case : "case" IDENTIFIER ":" field_definitions ;
/*
expression
| expression "^" expression
| expression "&" expression
| expression "|" expression
| "~" expression
| expression "==" expression
| expression "!=" expression
| expression ">>" expression
| expression "<<" expression
| expression "+" expression
| expression "-" expression
| "-" expression
| expression "/" expression
| expression "*" expression
;
*/
expression : expression1
| expression "^" expression1
;
expression1 : expression2
| expression1 "&" expression2
;
expression2 : expression3
| expression2 "|" expression3
;
expression3 : expression4
| expression3 "==" expression4
| expression3 "!=" expression4
;
expression4 : expression5
| expression4 ">>" expression5
| expression4 "<<" expression5
;
expression5 : expression6
| expression5 "+" expression6
| expression5 "-" expression6
;
expression6 : expression7
| expression6 "*" expression7
| expression6 "/" expression7
;
expression7 : expression8
| "-" expression7
| "+" expression7
| "~" expression7
;
expression8 : INTLIT
| CHARLIT
| STRINGLIT
| IDENTIFIER
| IDENTIFIER "(" function_argument_list ")"
| IDENTIFIER "[" function_argument_list1 "]"
| "(" expression ")"
| "{" resource_body "}"
;
function_argument_list : %empty | function_argument_list1 ;
function_argument_list1 : expression | function_argument_list "," expression ;
resource: "resource" CHARLIT "(" function_argument_list ")" "{" resource_body "}"
{ std::cout << "RESOURCE " << $2 << std::endl; }
resource_body : %empty | resource_body1 ;
resource_body1 : resource_item
| resource_body1 "," resource_item
| resource_body1 ";" resource_item
| resource_body1 ";"
;
resource_item : expression | IDENTIFIER "{" resource_body "}" ;
%%

11
Rez/Test.r Normal file
View File

@ -0,0 +1,11 @@
/*#include "Types.r"
*/
#include "/home/wolfgang/Projects/Retro68/CExamples/Sample.r"
type 'TEST' {
boolean itemUnlocked = false, // defined attributes bits...
itemLocked = true;
};