From 12c3ddd2079152560a8ff6086a70abe9d251af83 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 7 Jan 2018 23:45:42 +0100 Subject: [PATCH] renames --- il65/__init__.py | 7 +--- il65/{plycompiler.py => compiler.py} | 60 ++++++++++++++++++++-------- il65/handwritten/__init__.py | 1 + il65/{ => handwritten}/codegen.py | 0 il65/{ => handwritten}/exprparse.py | 0 il65/{ => handwritten}/optimize.py | 0 il65/{ => handwritten}/parse.py | 19 +++++---- il65/{ => handwritten}/preprocess.py | 0 il65/{ => handwritten}/symbols.py | 0 {lib => il65/lib}/c64lib.ill | 0 {lib => il65/lib}/il65lib.ill | 0 {lib => il65/lib}/mathlib.ill | 0 il65/main.py | 8 ++-- il65/{lexer.py => plylexer.py} | 27 +++++++++---- il65/plyparser.py | 12 +++--- tests/test_floats.py | 2 +- tests/test_zp.py | 2 +- todo.ill | 1 + 18 files changed, 87 insertions(+), 52 deletions(-) rename il65/{plycompiler.py => compiler.py} (74%) create mode 100644 il65/handwritten/__init__.py rename il65/{ => handwritten}/codegen.py (100%) rename il65/{ => handwritten}/exprparse.py (100%) rename il65/{ => handwritten}/optimize.py (100%) rename il65/{ => handwritten}/parse.py (99%) rename il65/{ => handwritten}/preprocess.py (100%) rename il65/{ => handwritten}/symbols.py (100%) rename {lib => il65/lib}/c64lib.ill (100%) rename {lib => il65/lib}/il65lib.ill (100%) rename {lib => il65/lib}/mathlib.ill (100%) rename il65/{lexer.py => plylexer.py} (91%) diff --git a/il65/__init__.py b/il65/__init__.py index 04f92b363..5bb534f79 100644 --- a/il65/__init__.py +++ b/il65/__init__.py @@ -1,6 +1 @@ -""" -Programming Language for 6502/6510 microprocessors - -Written by Irmen de Jong (irmen@razorvine.net) -License: GNU GPL 3.0, see LICENSE -""" +# package diff --git a/il65/plycompiler.py b/il65/compiler.py similarity index 74% rename from il65/plycompiler.py rename to il65/compiler.py index e95daca10..eb03ccf6c 100644 --- a/il65/plycompiler.py +++ b/il65/compiler.py @@ -3,26 +3,42 @@ import sys import linecache from typing import Optional, Generator, Tuple, Set from .plyparser import parse_file, Module, Directive, Block, Subroutine, AstNode -from .parse import ParseError -from .symbols import SourceRef +from .plylexer import SourceRef + + +class ParseError(Exception): + def __init__(self, message: str, sourcetext: Optional[str], sourceref: SourceRef) -> None: + self.sourceref = sourceref + self.msg = message + self.sourcetext = sourcetext + + def __str__(self): + return "{} {:s}".format(self.sourceref, self.msg) class PlyParser: - def __init__(self): + def __init__(self, parsing_import: bool=False) -> None: self.parse_errors = 0 - self.parsing_import = False + self.parsing_import = parsing_import def parse_file(self, filename: str) -> Module: print("parsing:", filename) - module = parse_file(filename) + module = parse_file(filename, self.lexer_error) try: self.check_directives(module) self.remove_empty_blocks(module) self.process_imports(module) except ParseError as x: self.handle_parse_error(x) + if self.parse_errors: + self.print_bold("\nNo output; there were {:d} errors.\n".format(self.parse_errors)) + raise SystemExit(1) return module + def lexer_error(self, sourceref: SourceRef, fmtstring: str, *args: str) -> None: + self.parse_errors += 1 + self.print_bold("ERROR: {}: {}".format(sourceref, fmtstring.format(*args))) + def remove_empty_blocks(self, module: Module) -> None: # remove blocks without name and without address, or that are empty for scope, parent in self.recurse_scopes(module): @@ -87,27 +103,39 @@ class PlyParser: filename = self.find_import_file(arg, directive.sourceref.file) if not filename: raise ParseError("imported file not found", None, directive.sourceref) - imported_module = self.import_file(filename) + imported_module, import_parse_errors = self.import_file(filename) imported_module.scope.parent_scope = module.scope imported.append(imported_module) + self.parse_errors += import_parse_errors + if not self.parsing_import: + # compiler support library is always imported (in main parser) + filename = self.find_import_file("il65lib", module.sourceref.file) + if filename: + imported_module, import_parse_errors = self.import_file(filename) + imported_module.scope.parent_scope = module.scope + imported.append(imported_module) + self.parse_errors += import_parse_errors + else: + raise FileNotFoundError("missing il65lib") # append the imported module's contents (blocks) at the end of the current module for imported_module in imported: for block in imported_module.scope.filter_nodes(Block): module.scope.nodes.append(block) - def import_file(self, filename: str) -> Module: - sub_parser = PlyParser() - return sub_parser.parse_file(filename) + def import_file(self, filename: str) -> Tuple[Module, int]: + sub_parser = PlyParser(parsing_import=True) + return sub_parser.parse_file(filename), sub_parser.parse_errors def find_import_file(self, modulename: str, sourcefile: str) -> Optional[str]: + candidates = [modulename+".ill", modulename] filename_at_source_location = os.path.join(os.path.split(sourcefile)[0], modulename) - filename_at_libs_location = os.path.join(os.getcwd(), "lib", modulename) - candidates = [modulename, - filename_at_source_location, - filename_at_libs_location, - modulename+".ill", - filename_at_source_location+".ill", - filename_at_libs_location+".ill"] + if filename_at_source_location not in candidates: + candidates.append(filename_at_source_location+".ill") + candidates.append(filename_at_source_location) + filename_at_libs_location = os.path.join(os.path.split(__file__)[0], "lib", modulename) + if filename_at_libs_location not in candidates: + candidates.append(filename_at_libs_location+".ill") + candidates.append(filename_at_libs_location) for filename in candidates: if os.path.isfile(filename): return filename diff --git a/il65/handwritten/__init__.py b/il65/handwritten/__init__.py new file mode 100644 index 000000000..5bb534f79 --- /dev/null +++ b/il65/handwritten/__init__.py @@ -0,0 +1 @@ +# package diff --git a/il65/codegen.py b/il65/handwritten/codegen.py similarity index 100% rename from il65/codegen.py rename to il65/handwritten/codegen.py diff --git a/il65/exprparse.py b/il65/handwritten/exprparse.py similarity index 100% rename from il65/exprparse.py rename to il65/handwritten/exprparse.py diff --git a/il65/optimize.py b/il65/handwritten/optimize.py similarity index 100% rename from il65/optimize.py rename to il65/handwritten/optimize.py diff --git a/il65/parse.py b/il65/handwritten/parse.py similarity index 99% rename from il65/parse.py rename to il65/handwritten/parse.py index 5db56f623..9ced08794 100644 --- a/il65/parse.py +++ b/il65/handwritten/parse.py @@ -1,6 +1,6 @@ """ Programming Language for 6502/6510 microprocessors -This is the parser of the IL65 code, that generates a parse tree. +This is the hand-written parser of the IL65 code, that generates a parse tree. Written by Irmen de Jong (irmen@razorvine.net) License: GNU GPL 3.0, see LICENSE @@ -10,9 +10,7 @@ import re import os import sys import shutil -import attr from collections import defaultdict -from typing import Set, List, Tuple, Optional, Dict, Union, Generator from .exprparse import ParseError, parse_expr_as_int, parse_expr_as_number, parse_expr_as_primitive,\ parse_expr_as_string, parse_arguments, parse_expr_as_comparison from .symbols import * @@ -513,14 +511,15 @@ class Parser: self._parse_import_file(filename) def _parse_import_file(self, filename: str) -> None: + candidates = [filename+".ill", filename] filename_at_source_location = os.path.join(os.path.split(self.sourceref.file)[0], filename) - filename_at_libs_location = os.path.join(os.getcwd(), "lib", filename) - candidates = [filename, - filename_at_source_location, - filename_at_libs_location, - filename+".ill", - filename_at_source_location+".ill", - filename_at_libs_location+".ill"] + if filename_at_source_location not in candidates: + candidates.append(filename_at_source_location+".ill") + candidates.append(filename_at_source_location) + filename_at_libs_location = os.path.join(os.path.split(__file__)[0], "../lib", filename) + if filename_at_libs_location not in candidates: + candidates.append(filename_at_libs_location+".ill") + candidates.append(filename_at_libs_location) for filename in candidates: if os.path.isfile(filename): if not self.check_import_okay(filename): diff --git a/il65/preprocess.py b/il65/handwritten/preprocess.py similarity index 100% rename from il65/preprocess.py rename to il65/handwritten/preprocess.py diff --git a/il65/symbols.py b/il65/handwritten/symbols.py similarity index 100% rename from il65/symbols.py rename to il65/handwritten/symbols.py diff --git a/lib/c64lib.ill b/il65/lib/c64lib.ill similarity index 100% rename from lib/c64lib.ill rename to il65/lib/c64lib.ill diff --git a/lib/il65lib.ill b/il65/lib/il65lib.ill similarity index 100% rename from lib/il65lib.ill rename to il65/lib/il65lib.ill diff --git a/lib/mathlib.ill b/il65/lib/mathlib.ill similarity index 100% rename from lib/mathlib.ill rename to il65/lib/mathlib.ill diff --git a/il65/main.py b/il65/main.py index f1a8c5c53..970cf3fbf 100644 --- a/il65/main.py +++ b/il65/main.py @@ -12,10 +12,10 @@ import time import os import argparse import subprocess -from .parse import Parser -from .optimize import Optimizer -from .preprocess import PreprocessingParser -from .codegen import CodeGenerator, Assembler64Tass +from .handwritten.parse import Parser +from .handwritten.optimize import Optimizer +from .handwritten.preprocess import PreprocessingParser +from .handwritten.codegen import CodeGenerator, Assembler64Tass def main() -> None: diff --git a/il65/lexer.py b/il65/plylexer.py similarity index 91% rename from il65/lexer.py rename to il65/plylexer.py index 4da60b33f..62673a29a 100644 --- a/il65/lexer.py +++ b/il65/plylexer.py @@ -8,8 +8,21 @@ License: GNU GPL 3.0, see LICENSE import sys import ply.lex -from .symbols import SourceRef -from .parse import ParseError +import attr + + +@attr.s(slots=True, frozen=True) +class SourceRef: + file = attr.ib(type=str) + line = attr.ib(type=int) + column = attr.ib(type=int, default=0) + + def __str__(self) -> str: + if self.column: + return "{:s}:{:d}:{:d}".format(self.file, self.line, self.column) + if self.line: + return "{:s}:{:d}".format(self.file, self.line) + return self.file # token names @@ -294,7 +307,10 @@ def t_error(t): line, col = t.lineno, find_tok_column(t) filename = getattr(t.lexer, "source_filename", "") sref = SourceRef(filename, line, col) - t.lexer.error_function("{}: Illegal character '{:s}'", sref, t.value[0], sourceref=sref) + if hasattr(t.lexer, "error_function"): + t.lexer.error_function(sref, "illegal character '{:s}'", t.value[0]) + else: + print("{}: illegal character '{:s}'".format(sref, t.value[0]), file=sys.stderr) t.lexer.skip(1) @@ -304,12 +320,7 @@ def find_tok_column(token): return token.lexpos - last_cr -def error_function(fmtstring, *args, sourceref: SourceRef=None) -> None: - raise ParseError(fmtstring.format(*args), None, sourceref=sourceref) - - lexer = ply.lex.lex() -lexer.error_function = error_function # can override this if __name__ == "__main__": diff --git a/il65/plyparser.py b/il65/plyparser.py index 7687a5716..58eac34e9 100644 --- a/il65/plyparser.py +++ b/il65/plyparser.py @@ -8,9 +8,8 @@ License: GNU GPL 3.0, see LICENSE import attr from ply.yacc import yacc -from typing import Union, Type, Generator -from .symbols import SourceRef -from .lexer import tokens, lexer, find_tok_column # get the lexer tokens. required. +from typing import Union, Generator +from .plylexer import SourceRef, tokens, lexer, find_tok_column start = "start" @@ -800,9 +799,9 @@ def p_error(p): print('\n[ERROR DEBUG: parser state={:d} stack: {} . {} ]'.format(parser.state, stack_state_str, p)) if p: sref = SourceRef(p.lexer.source_filename, p.lineno, find_tok_column(p)) - p.lexer.error_function("syntax error before '{:.20s}'", str(p.value), sourceref=sref) + p.lexer.error_function(sref, "syntax error before '{:.20s}'", str(p.value)) else: - lexer.error_function("syntax error at end of input", lexer.source_filename, sourceref=None) + lexer.error_function(None, "syntax error at end of input", lexer.source_filename) def _token_sref(p, token_idx): @@ -840,7 +839,8 @@ class TokenFilter: parser = yacc(write_tables=True) -def parse_file(filename: str) -> Module: +def parse_file(filename: str, lexer_error_func=None) -> Module: + lexer.error_function = lexer_error_func lexer.lineno = 1 lexer.source_filename = filename tfilter = TokenFilter(lexer) diff --git a/tests/test_floats.py b/tests/test_floats.py index 5ab633a99..86927a1fb 100644 --- a/tests/test_floats.py +++ b/tests/test_floats.py @@ -1,5 +1,5 @@ import pytest -from il65 import codegen, symbols +from il65.handwritten import symbols, codegen def test_float_to_mflpt5(): diff --git a/tests/test_zp.py b/tests/test_zp.py index 0502c9242..167d32f9a 100644 --- a/tests/test_zp.py +++ b/tests/test_zp.py @@ -1,5 +1,5 @@ import pytest -from il65.symbols import Zeropage, SymbolError, DataType +from il65.handwritten.symbols import Zeropage, SymbolError, DataType def test_zp_configure_onlyonce(): diff --git a/todo.ill b/todo.ill index 1e08cdc1f..96e2b69b3 100644 --- a/todo.ill +++ b/todo.ill @@ -172,6 +172,7 @@ loop : sub sub1 () -> () { + %breakpoint %saveregisters off %breakpoint